Skip to main content

hyprstream_core/services/
inference.rs

1//! ZMQ-based inference service for text generation
2//!
3//! This service wraps TorchEngine and provides a ZMQ interface for inference operations.
4//! It uses:
5//! - REQ/REP for standard requests (generate, model_info, lora operations, etc.)
6//! - PUB (via StreamService) for streaming generation with JWT authorization
7//!
8//! # Thread Model
9//!
10//! The service runs on a dedicated thread with its own TorchEngine because:
11//! - tch-rs types contain raw pointers (not Send)
12//! - GPU operations benefit from thread isolation
13//!
14//! # Streaming Architecture
15//!
16//! ```text
17//! Client                    InferenceService         StreamService
18//!   │                            │                         │
19//!   │─ REQ: GenerateStream ─────►│                         │
20//!   │◄─ REP: {stream_id,endpoint}│                         │
21//!   │                            │                         │
22//!   │                            │── PUB: chunks ─────────►│ (validates JWT)
23//!   │                            │                         │
24//!   │  SUB: stream-{id}|{jwt} ───────────────────────────►│
25//!   │◄────────────────── chunks (JWT stripped) ───────────│
26//! ```
27//!
28//! InferenceService publishes to StreamService's XSUB (PUB socket).
29//! StreamService validates JWT at subscription and forwards to clients.
30//!
31//! # Authorization
32//!
33//! Uses `InferenceHandler::authorize()` via generated dispatch for policy-backed
34//! authorization on all requests. The handler delegates to PolicyClient.
35
36use crate::config::TrainingMode;
37use crate::runtime::kv_cache::CacheOwner;
38use crate::runtime::model_config::ModelConfig;
39use crate::runtime::{create_engine, RuntimeConfig, RuntimeEngine, TorchEngine};
40use hyprstream_rpc_std::inference_client::GenerationRequest;
41use hyprstream_rpc_std::inference_client::ModelInfo;
42use hyprstream_rpc_std::policy_client::PolicyClient;
43
44use crate::services::EnvelopeContext;
45use crate::services::WorktreeClientExt;
46use crate::training::serialize_state_dict_to_bytes;
47use crate::training::{DeltaPool, TTTConfig, TenantDeltaConfig, TestTimeTrainer};
48use anyhow::{anyhow, Result};
49use hyprstream_rpc::prelude::*;
50use hyprstream_rpc::StreamChannel;
51use hyprstream_rpc::Subject;
52use hyprstream_rpc_std::registry_client::WorktreeClient;
53use parking_lot::Mutex;
54use std::path::{Path, PathBuf};
55use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
56use std::sync::Arc;
57use tokenizers::Tokenizer;
58use tokio::runtime::Handle;
59use tracing::{debug, error, info, trace, warn};
60
61// #1264/#1265: completion-time ledger spend (behind the `ledger` feature,
62// default off).
63#[cfg(feature = "ledger")]
64use crate::services::ledger::{
65    observe_spend_result, InferenceSpendEmitter, SpendInput, SpendResult,
66};
67
68/// Pending work to be executed after REP response is sent.
69///
70/// This solves the streaming deadlock where the service waits for subscription
71/// before returning the response, but the client can't subscribe without
72/// the stream_id from the response.
73///
74/// Wraps `StreamContext` from hyprstream-rpc with operation-specific data.
75enum PendingWork {
76    /// Streaming text generation
77    Generation {
78        /// Stream context with DH-derived keys (from hyprstream-rpc)
79        stream_ctx: hyprstream_rpc::StreamContext,
80        /// Generation request to execute
81        request: GenerationRequest,
82        /// Subject identity for tenant-aware TTT (deferred to execute_stream)
83        subject: Subject,
84        /// Per-request TTT overrides (deferred to execute_stream)
85        ttt_overrides: crate::training::ttt::TTTOverrides,
86        /// The verified caller's self-certifying pairwise DID — the ledger
87        /// account owner for the #1264 completion spend. `None` ⇒ anonymous ⇒
88        /// fail closed (no spend, no leak). Captured at admission from the
89        /// verified envelope and spent at completion. Read only under the
90        /// `ledger` feature.
91        #[cfg_attr(not(feature = "ledger"), allow(dead_code))]
92        owner_did: Option<String>,
93    },
94    /// Streaming training step (avoids REQ/REP timeout on backward pass compilation)
95    Training {
96        /// Stream context with DH-derived keys (from hyprstream-rpc)
97        stream_ctx: hyprstream_rpc::StreamContext,
98        /// Subject identity for tenant-aware TTT
99        subject: Subject,
100        /// Text to train on
101        input: String,
102        /// Number of gradient steps (None = use model default)
103        gradient_steps: Option<u32>,
104        /// Learning rate override (None = use model default)
105        learning_rate: Option<f32>,
106        /// How to handle the adaptation result
107        adaptation_strategy: crate::training::adaptation_state::AdaptationStrategy,
108    },
109}
110
111/// Default endpoint for the inference service
112pub const INFERENCE_ENDPOINT: &str = "inproc://hyprstream/inference";
113
114// Note: Stream endpoints are now dynamically generated per InferenceService instance
115// using UUIDs (e.g., inproc://hyprstream/inference/stream/{uuid}).
116// Each service binds to its own unique endpoint to prevent bind conflicts.
117
118/// ZMQ-based inference service
119///
120/// Wraps TorchEngine and provides a Cap'n Proto interface over ZMQ.
121/// Thread-safe via RwLock for multi-threaded access.
122///
123/// # Security
124///
125/// All requests must be wrapped in `SignedEnvelope` and are verified before processing.
126/// The service logs the identity for audit trails.
127///
128/// # Streaming Architecture
129///
130/// Uses a single PUB socket connected to StreamService:
131/// - Topic-based multiplexing: `stream-{id}` prefix on each message
132/// - StreamService validates JWT at subscription (prevents hijacking)
133/// - InferenceService just publishes chunks (no authorization logic)
134/// - Clients subscribe via StreamService with JWT tokens
135///
136/// Inner state of InferenceService, shared via Arc for continuations.
137///
138/// All methods are defined on this inner type. `InferenceService` wraps it
139/// in `Arc` and implements `Deref` so all field/method access is transparent.
140/// Streaming continuations clone the `Arc` to own the state across await points.
141pub struct InferenceServiceInner {
142    engine: parking_lot::RwLock<TorchEngine>,
143    /// Immutable base inputs, including reads deferred until after startup.
144    model_path: PathBuf,
145    /// Writable adaptation state is never placed in the sealed base tree.
146    state_path: PathBuf,
147    /// Retains verified read-only descriptors and their paths through lazy reads.
148    _pinned_artifact: Option<Arc<crate::storage::pinned_model::PinnedModelArtifact>>,
149    /// Current session ID for events
150    session_id: parking_lot::RwLock<Option<String>>,
151    /// Runtime handle for async operations (reused instead of creating new runtimes)
152    #[allow(dead_code)] // Reserved for future async operations
153    runtime_handle: Handle,
154    /// Stream channel for streaming generation (connects to StreamService)
155    /// Handles DH key exchange, pre-authorization, and publishing.
156    stream_channel: Option<StreamChannel>,
157    /// Server's Ed25519 verifying key for signature verification
158    #[allow(dead_code)]
159    server_pubkey: VerifyingKey,
160    /// Service signing key for stream registration (generated at init)
161    #[allow(dead_code)]
162    signing_key: SigningKey,
163    /// Nonce cache for replay protection
164    #[allow(dead_code)]
165    nonce_cache: Arc<InMemoryNonceCache>,
166    /// Policy client for authorization checks (async via TMQ)
167    policy_client: PolicyClient,
168    /// Optional TTT trainer (initialized from config.json)
169    ttt_trainer: Option<Arc<TestTimeTrainer>>,
170    /// Tokenizer for TTT adaptation
171    tokenizer: Option<Arc<Tokenizer>>,
172    /// Per-tenant delta pool for isolated TTT adaptations
173    delta_pool: Option<Arc<DeltaPool>>,
174    /// Base LoRA delta loaded from a .safetensors adapter file.
175    /// Applied to all tenants (composed with per-tenant delta if both exist).
176    base_delta: Mutex<Option<Arc<Mutex<crate::training::TenantDelta>>>>,
177    /// Optional WorktreeClient for worktree-scoped file operations.
178    /// When present, adapter/snapshot writes use contained-root access.
179    fs: Option<WorktreeClient>,
180    /// Transport configuration for the service endpoint
181    #[allow(dead_code)] // Standard mode passes transport to InferenceZmqAdapter
182    transport: hyprstream_rpc::transport::TransportConfig,
183    /// LoRA generation counter — incremented on create/load/unload.
184    /// Checked before generation to detect LoRA reconfiguration mid-stream.
185    lora_generation: Arc<AtomicU64>,
186    /// Authority-verified tenant that owns this engine instance.
187    tenant_domain: String,
188    /// Verified key of the ModelService controller allowed to bridge a local
189    /// request into this tenant-bound instance.
190    controller_pubkey: VerifyingKey,
191    /// True only while the network stream plane is bound and accepting.
192    network_ready: Arc<AtomicBool>,
193    /// Rejects new generation streams once lifecycle drain begins.
194    draining: Arc<AtomicBool>,
195    /// Process/service shutdown notification.
196    shutdown: Arc<tokio::sync::Notify>,
197    /// Optional #1264 completion-spend emitter. `None` (default) ⇒ inert: the
198    /// completion path posts no spend. Behind a `RwLock` so an operator can
199    /// attach it after the service thread constructs the inner (the emitter's
200    /// `LedgerHandle` comes from a sibling ledger service whose startup ordering
201    /// is itself a follow-up). Gated behind the `ledger` feature.
202    #[cfg(feature = "ledger")]
203    ledger: parking_lot::RwLock<Option<Arc<InferenceSpendEmitter>>>,
204    object_label: hyprstream_rpc::auth::mac::SecurityLabel,
205}
206
207/// ZMQ-based inference service
208///
209/// Wraps `InferenceServiceInner` in `Arc` for continuation-based streaming.
210/// All field and method access is transparent via `Deref`.
211pub struct InferenceService {
212    inner: Arc<InferenceServiceInner>,
213}
214
215impl Clone for InferenceService {
216    fn clone(&self) -> Self {
217        Self {
218            inner: Arc::clone(&self.inner),
219        }
220    }
221}
222
223impl std::ops::Deref for InferenceService {
224    type Target = InferenceServiceInner;
225    fn deref(&self) -> &Self::Target {
226        &self.inner
227    }
228}
229
230// SAFETY: InferenceServiceInner contains tch-rs types (raw pointers) behind
231// parking_lot::RwLock/Mutex, providing proper synchronization. This is consistent
232// with the codebase pattern for tch-rs wrappers (LlamaModel, TenantDelta,
233// TestTimeTrainer, etc.) which all have `unsafe impl Send + Sync`.
234unsafe impl Send for InferenceServiceInner {}
235unsafe impl Sync for InferenceServiceInner {}
236
237fn resolve_instance_domain(
238    tenant_domain: &str,
239    controller_pubkey: &VerifyingKey,
240    ctx: &EnvelopeContext,
241) -> Result<String> {
242    let domain = ctx.domain()?;
243    if domain == tenant_domain {
244        return Ok(domain);
245    }
246    if domain == "local" && ctx.cnf == controller_pubkey.to_bytes() {
247        return Ok(tenant_domain.to_owned());
248    }
249    anyhow::bail!(
250        "authorization denied: inference instance is bound to a different verified tenant"
251    )
252}
253pub(crate) fn enforce_inference_mac(
254    ctx: &EnvelopeContext,
255    object_label: &hyprstream_rpc::auth::mac::SecurityLabel,
256) -> Result<()> {
257    enforce_inference_security_context(ctx.security_context(), object_label)
258}
259/// Canonical content label for the inference engine boundary.
260pub(crate) fn inference_object_label() -> hyprstream_rpc::auth::mac::SecurityLabel {
261    crate::mac::genesis::SitePolicy::conservative().label_for("/srv/inference")
262}
263fn enforce_inference_security_context(
264    security_context: Option<hyprstream_rpc::auth::mac::SecurityContext>,
265    object_label: &hyprstream_rpc::auth::mac::SecurityLabel,
266) -> Result<()> {
267    let security_context = security_context.ok_or_else(|| {
268        anyhow!("authorization denied: inference caller has no verified MAC clearance")
269    })?;
270    anyhow::ensure!(
271        security_context.can_access(object_label),
272        "authorization denied: inference caller clearance does not dominate engine label {object_label}"
273    );
274    Ok(())
275}
276
277// Intermediate response structs (DeltaStatusInfo, SaveAdaptationResult, SnapshotDeltaResult,
278// ExportPeftResult) eliminated — handlers return generated types directly.
279
280// ────────────────────────────────────────────────────────────────────────────
281// #1265: generation-loop accounting primitives (module-level, testable).
282//
283// `drive_generation_loop` consumes a generation output stream to a terminal
284// outcome (normal exhaustion / cancellation / stream error / publish failure).
285// It is the seam that makes "account for the work actually done" testable
286// without a live engine: `execute_stream` reads `completion_stats()` on the
287// SINGLE common exit after it returns, so no terminal path — including cancel
288// and publish-failure, which previously returned before the stats copy — can
289// skip the spend (#1265 blocker).
290// ────────────────────────────────────────────────────────────────────────────
291
292/// The terminal outcome of one generation output loop (#1265).
293#[derive(Debug)]
294enum GenLoopOutcome {
295    /// The stream was exhausted normally.
296    Exhausted,
297    /// The client cancelled / disconnected mid-stream.
298    Cancelled,
299    /// The generation stream yielded an error.
300    StreamError(anyhow::Error),
301    /// Publishing a generated chunk failed.
302    PublishFailed(anyhow::Error),
303}
304
305/// Abstraction over the stream publisher the generation loop writes to (#1265).
306/// A trait — not the concrete `AnyStreamPublisher` — so the loop can be driven by
307/// a fake sink in tests for the cancel / publish-failure / stream-error
308/// accounting paths without a live engine.
309trait GenSink {
310    /// Forward one generated chunk; an `Err` is a terminal publish-failure.
311    fn publish_chunk(
312        &mut self,
313        data: &str,
314        rate: f32,
315    ) -> impl std::future::Future<Output = Result<()>>;
316}
317
318impl GenSink for hyprstream_rpc::moq_stream::AnyStreamPublisher {
319    fn publish_chunk(
320        &mut self,
321        data: &str,
322        rate: f32,
323    ) -> impl std::future::Future<Output = Result<()>> {
324        self.publish_data_with_rate(data.as_bytes(), rate)
325    }
326}
327
328/// Abstraction over a generation output stream the loop consumes (#1265).
329/// Decouples the loop (and its tests) from `TextStream` / the PyTorch engine.
330trait GenOutput: futures::Stream<Item = anyhow::Result<String>> + Unpin {
331    /// Current EMA token rate (the per-chunk publish rate hint).
332    fn ema_rate(&self) -> f32;
333    /// Final completion stats (prefill + generated) — read on the common exit.
334    fn completion_stats(&self) -> crate::runtime::GenerationStats;
335}
336
337impl<'a> GenOutput for crate::runtime::TextStream<'a> {
338    fn ema_rate(&self) -> f32 {
339        self.stats().inference_tokens_per_sec_ema
340    }
341    fn completion_stats(&self) -> crate::runtime::GenerationStats {
342        self.stats()
343    }
344}
345
346/// Drive a generation output stream to a terminal outcome. Does NOT capture
347/// stats — the caller snapshots `output.completion_stats()` after this returns
348/// on the single common exit, so cancel / stream-error / publish-failure all
349/// account for the work actually done (#1265). Publish failures are terminal
350/// (returned to the caller, which lets the framework emit the Error frame,
351/// matching the prior behavior).
352async fn drive_generation_loop<S: GenOutput>(
353    stream: &mut S,
354    cancel: &tokio_util::sync::CancellationToken,
355    sink: &mut impl GenSink,
356) -> GenLoopOutcome {
357    use futures::StreamExt;
358    loop {
359        let next = tokio::select! {
360            biased;
361            _ = cancel.cancelled() => return GenLoopOutcome::Cancelled,
362            n = stream.next() => n,
363        };
364        match next {
365            None => return GenLoopOutcome::Exhausted,
366            Some(Err(e)) => return GenLoopOutcome::StreamError(e),
367            Some(Ok(text)) => {
368                let rate = stream.ema_rate();
369                if let Err(e) = sink.publish_chunk(&text, rate).await {
370                    return GenLoopOutcome::PublishFailed(e);
371                }
372            }
373        }
374    }
375}
376
377/// The content-free error frame emitted when a generation is denied because the
378/// caller has no self-certifying billing identity while a spend emitter is
379/// attached (#1265). Carries no subject DID and no prompt material.
380#[cfg_attr(not(feature = "ledger"), allow(dead_code))] // only read on the ledger gate path
381const LEDGER_DENY_NO_IDENTITY: &str =
382    "generation denied: no verified billing identity for token-spend accounting";
383
384/// #1265: decide whether a generation must fail CLOSED before it starts because
385/// a #1264 spend emitter is attached but the caller has no self-certifying
386/// pairwise DID — the account owner the spend would debit. When `true`, the
387/// service denies the request with NO model output (see the gate in
388/// `execute_stream`, which precedes `engine.generate_with_delta`).
389///
390/// With no emitter attached the subsystem is inert — anonymous callers are
391/// unaffected. An explicit fail-open / debt policy for anonymous callers is a
392/// separate decision and is intentionally NOT implemented here.
393#[cfg_attr(not(feature = "ledger"), allow(dead_code))] // only read on the ledger gate path
394fn ledger_requires_identity(emitter_attached: bool, owner_did: Option<&str>) -> bool {
395    emitter_attached && owner_did.is_none()
396}
397
398/// #1265: post the completion token-spend for the generation work actually done.
399/// Pure decision over the captured completion stats and the verified owner DID,
400/// delegating the single-phase debit to `emitter`. Returns:
401/// - `Some(SpendResult)` when a spend was attempted (Posted / Declined / Failed)
402///   — observe it as a content-free signal;
403/// - `None` when there is nothing to account for (no stats — generation never
404///   began — or no verified owner). A DID-less caller cannot reach the stats arm
405///   here in production: `execute_stream` gates it closed pre-generation.
406#[cfg(feature = "ledger")]
407async fn post_completion_spend(
408    emitter: &InferenceSpendEmitter,
409    stats: Option<&crate::runtime::GenerationStats>,
410    owner_did: Option<&str>,
411    stream_id: &str,
412) -> Option<SpendResult> {
413    let stats = stats?;
414    let owner = owner_did?;
415    let res = emitter
416        .post_generation_spend(SpendInput {
417            owner_did: owner,
418            stream_id,
419            prompt_tokens: stats.prefill_tokens as u64,
420            generated_tokens: stats.tokens_generated as u64,
421        })
422        .await;
423    Some(res)
424}
425
426impl InferenceService {
427    /// Drain-time export hook (#869): force-snapshot every resident per-tenant delta
428    /// with accumulated steps to `adapters/.snapshots/`, so a graceful shutdown / pod
429    /// preStop / scale-in does not drop uncommitted TTT adaptation.
430    ///
431    /// This is the reachable seam for the #865 serving-controller drain path to call.
432    /// It is exposed rather than auto-wired into `Spawnable::run`'s shutdown `Notify`
433    /// because the delta pool lives inside this `!Send` service behind the
434    /// `LocalServiceBridge` — there is no post-shutdown hook on the bridge lifecycle
435    /// today to invoke it from (adding one is part of the #865 rework, not this change).
436    /// Returns the number of deltas persisted (a `None` snapshot slot is a loss event and
437    /// is logged by the pool).
438    pub async fn drain_export(&self) -> anyhow::Result<usize> {
439        let Some(pool) = self.delta_pool.as_ref() else {
440            return Ok(0);
441        };
442        let exported = pool.export_all().await?;
443        let persisted = exported.iter().filter(|(_, p)| p.is_some()).count();
444        tracing::info!(
445            "drain_export: persisted {}/{} resident delta(s) to snapshots",
446            persisted,
447            exported.len()
448        );
449        Ok(persisted)
450    }
451
452    /// Attach the #1264 completion-spend emitter. Once attached, every completed
453    /// generation posts a single-phase token spend to the cell ledger for the
454    /// verified caller's account (best-effort, fail-safe). When the emitter is
455    /// absent (`None`, the default), the completion path posts no spend — the
456    /// accounting subsystem is inert until an operator opts in.
457    ///
458    /// Behind the `ledger` feature. The emitter's [`LedgerHandle`] is cheap to
459    /// clone; attaching shares it with the service thread.
460    #[cfg(feature = "ledger")]
461    pub fn attach_ledger_spend(&self, emitter: Arc<InferenceSpendEmitter>) {
462        *self.ledger.write() = Some(emitter);
463    }
464
465    /// Run invariant checks before any TTT-scoped operation.
466    /// Returns GuardStatus encoding expired, capacity, and generation state.
467    /// The result must be passed to delta.adaptation_state.resolve().
468    #[allow(dead_code)] // Reserved for dispatch-layer callers; inline guards kept where delta lock is held
469    fn ttt_guard(&self, subject: &Subject) -> crate::training::GuardStatus {
470        let (expired, at_capacity) = if let Some(pool) = &self.delta_pool {
471            if let Some(delta_arc) = pool.get(subject) {
472                let delta = delta_arc.lock();
473                (delta.adaptation_state.is_expired(), delta.is_at_capacity())
474            } else {
475                (false, false)
476            }
477        } else {
478            (false, false)
479        };
480        crate::training::GuardStatus {
481            expired,
482            at_capacity,
483            lora_generation: self
484                .lora_generation
485                .load(std::sync::atomic::Ordering::Acquire),
486        }
487    }
488
489    /// Initialize the service
490    async fn initialize(
491        model_path: PathBuf,
492        state_path: PathBuf,
493        pinned_artifact: Option<Arc<crate::storage::pinned_model::PinnedModelArtifact>>,
494        config: RuntimeConfig,
495        server_pubkey: VerifyingKey,
496        signing_key: SigningKey,
497        nonce_cache: Arc<InMemoryNonceCache>,
498        policy_client: PolicyClient,
499        fs: Option<WorktreeClient>,
500        tenant_domain: String,
501        controller_pubkey: VerifyingKey,
502        producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
503        moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
504        network_ready: Arc<AtomicBool>,
505        draining: Arc<AtomicBool>,
506        shutdown: Arc<tokio::sync::Notify>,
507    ) -> Result<Self> {
508        // Capture runtime handle for reuse in handlers
509        let runtime_handle = Handle::current();
510
511        #[cfg(test)]
512        crate::services::restart_diag::phase("engine.create.enter");
513        let mut engine = create_engine(&config)?;
514        #[cfg(test)]
515        crate::services::restart_diag::phase("engine.create.done");
516        #[cfg(test)]
517        crate::services::restart_diag::phase("engine.load_model.enter");
518        if let Some(ref artifact) = pinned_artifact {
519            artifact.ensure_current_process()?;
520        }
521        RuntimeEngine::load_model(&mut engine, &model_path).await?;
522        #[cfg(test)]
523        crate::services::restart_diag::phase("engine.load_model.done");
524
525        // Initialize KV cache registry
526        let model_info = RuntimeEngine::model_info(&engine);
527        let num_layers = model_info.num_hidden_layers.unwrap_or(32) as usize;
528        let max_seq_len = config.max_context.unwrap_or(model_info.context_length) as usize;
529        let kv_budget = engine.compute_kv_budget();
530        engine.initialize_kv_registry(num_layers, max_seq_len, config.kv_quant_type, kv_budget);
531
532        info!(
533            "KV cache registry initialized: {} layers, max_seq_len={}",
534            num_layers, max_seq_len
535        );
536
537        // Check for training mode in config.json
538        let training_config = ModelConfig::load_training_config(&model_path);
539
540        // Only initialize TTT trainer and tokenizer if TTT is enabled
541        // This ensures zero memory/compute overhead when not using TTT
542        let (ttt_trainer, tokenizer): (Option<Arc<TestTimeTrainer>>, Option<Arc<Tokenizer>>) =
543            if let Some(ref tc) = training_config {
544                if tc.is_enabled() && tc.mode == TrainingMode::TestTimeTraining {
545                    info!(
546                        "Test-Time Training enabled: lr={}, steps={}",
547                        tc.ttt.learning_rate, tc.ttt.gradient_steps
548                    );
549
550                    let ttt_config = TTTConfig {
551                        learning_rate: tc.ttt.learning_rate,
552                        gradient_steps: tc.ttt.gradient_steps,
553                        max_grad_norm: tc.ttt.max_grad_norm,
554                        min_input_length: tc.ttt.min_input_length,
555                        max_ttt_context: tc.ttt.max_ttt_context,
556                        enabled: true,
557                        ..TTTConfig::default()
558                    };
559
560                    let device = engine.device();
561                    let mut trainer = TestTimeTrainer::new(ttt_config, device);
562                    // Wire gradient gating config if provided
563                    if let Some(ref gating) = tc.ttt.gradient_gating {
564                        trainer = trainer.with_gradient_gating(gating.clone());
565                    }
566
567                    // Get tokenizer for input tokenization
568                    let tokenizer = engine.get_tokenizer().ok().map(Arc::new);
569                    (Some(Arc::new(trainer)), tokenizer)
570                } else {
571                    (None, None)
572                }
573            } else {
574                (None, None)
575            };
576
577        // Initialize delta pool if TTT is enabled
578        let delta_pool = if ttt_trainer.is_some() {
579            let module_dims = engine.get_lora_module_dims().unwrap_or_default();
580            let per_layer_dims = engine.get_per_layer_lora_dims();
581            let device = engine.device();
582
583            let base_delta_config = if let Some(ref tc) = training_config {
584                let alpha = tc.lora_alpha.unwrap_or(tc.lora_rank as f32);
585                TenantDeltaConfig {
586                    rank: tc.lora_rank,
587                    alpha,
588                    target_modules: tc.target_modules.clone(),
589                    learning_rate: tc.ttt.learning_rate,
590                    ..TenantDeltaConfig::default()
591                }
592            } else {
593                TenantDeltaConfig::default()
594            };
595
596            // Try to load TTN layer profile for non-uniform rank allocation
597            let model_config_result = crate::runtime::model_config::ModelConfig::load(
598                &model_path,
599                &std::collections::HashMap::new(),
600            );
601            let delta_config = if let Ok(ref mc) = model_config_result {
602                match crate::runtime::ttn_profile::get_layer_profile(&model_path, mc, None) {
603                    Ok(profile) => {
604                        let cfg = TenantDeltaConfig::from_profile(&base_delta_config, &profile);
605                        info!(
606                            "Delta pool: using TTN profile ({} layer overrides)",
607                            cfg.layer_overrides
608                                .as_ref()
609                                .map_or(0, std::collections::HashMap::len)
610                        );
611                        cfg
612                    }
613                    Err(e) => {
614                        info!("TTN profile unavailable ({}), using uniform config", e);
615                        base_delta_config
616                    }
617                }
618            } else {
619                base_delta_config
620            };
621
622            let kv_reg = engine.kv_registry();
623            let snapshots_dir = state_path.join("adapters").join(".snapshots");
624            let num_layers = engine.get_num_layers().unwrap_or(32);
625
626            info!(
627                "Delta pool initialized: rank={}, alpha={:.1}, modules={:?}, lr={:.1e}",
628                delta_config.rank,
629                delta_config.alpha,
630                delta_config.target_modules,
631                delta_config.learning_rate
632            );
633            let mut pool = DeltaPool::new(
634                delta_config,
635                module_dims,
636                device,
637                kv_reg,
638                snapshots_dir,
639                fs.clone(),
640                num_layers,
641            );
642            if let Some(pld) = per_layer_dims {
643                pool = pool.with_per_layer_dims(pld);
644            }
645            // Wire rank oracle config if provided in training config
646            if let Some(ref tc) = training_config {
647                if let Some(ref oracle_config) = tc.ttt.rank_oracle {
648                    pool = pool.with_rank_oracle(oracle_config.clone());
649                    info!(
650                        "Rank oracle enabled: interval={}, auto_adapt={}",
651                        oracle_config.adaptation_interval, oracle_config.auto_adapt
652                    );
653                }
654            }
655
656            Some(Arc::new(pool))
657        } else {
658            None
659        };
660
661        // Create StreamChannel upfront.
662        let stream_channel = StreamChannel::new(signing_key.clone())
663            .with_reach_config_handle(producer_reach_config)
664            .with_moq_origin_handle(moq_origin);
665        let object_label = inference_object_label();
666
667        Ok(InferenceService {
668            inner: Arc::new(InferenceServiceInner {
669                engine: parking_lot::RwLock::new(engine),
670                model_path,
671                state_path,
672                _pinned_artifact: pinned_artifact,
673                session_id: parking_lot::RwLock::new(None),
674                runtime_handle,
675                stream_channel: Some(stream_channel),
676                server_pubkey,
677                signing_key,
678                nonce_cache,
679                policy_client,
680                ttt_trainer,
681                tokenizer,
682                delta_pool,
683                base_delta: Mutex::new(None),
684                fs,
685                transport: hyprstream_rpc::transport::TransportConfig::inproc("inference-unset"),
686                lora_generation: Arc::new(AtomicU64::new(0)),
687                tenant_domain,
688                controller_pubkey,
689                network_ready,
690                draining,
691                shutdown,
692                #[cfg(feature = "ledger")]
693                ledger: parking_lot::RwLock::new(None),
694                object_label,
695            }),
696        })
697    }
698
699    /// Resolve the authorization domain for this tenant-bound engine.
700    ///
701    /// A direct caller must carry the same authority-verified tenant. The only
702    /// exception is the local ModelService bridge, whose verified signer key is
703    /// pinned when the instance is created; that bridge inherits this
704    /// instance's tenant instead of the generic `local` domain.
705    fn authorization_domain(&self, ctx: &EnvelopeContext) -> Result<String> {
706        resolve_instance_domain(&self.tenant_domain, &self.controller_pubkey, ctx)
707    }
708
709    /// Resolve the effective delta for a subject: compose base_delta + tenant delta if both exist.
710    ///
711    /// Returns None if no deltas exist (base model only), which is the common case
712    /// and incurs zero overhead.
713    ///
714    /// Returns `(delta, tenant_id, weight_epoch)` where `tenant_id` is
715    /// `Some(subject)` only when a per-tenant delta contributed to the effective
716    /// weights, and `weight_epoch` is that tenant delta's current weight epoch
717    /// (0 when no tenant delta is in play). The engine registers the pair
718    /// against the session KV cache so a later tenant-delta eviction/reset/removal
719    /// invalidates it, and so an epoch advance (in-place re-adaptation under the
720    /// same tenant) marks the cached KV weight-stale; base-delta-only and
721    /// bare-base requests carry no tenant dependency (`None`) and are never
722    /// cleared by a tenant eviction.
723    fn resolve_delta(
724        &self,
725        subject: &hyprstream_rpc::Subject,
726    ) -> (
727        Option<Arc<Mutex<crate::training::TenantDelta>>>,
728        Option<String>,
729        u64,
730    ) {
731        let base = self.base_delta.lock().clone();
732        let tenant = self.delta_pool.as_ref().and_then(|pool| pool.get(subject));
733
734        // The id is the verified request subject — the existing RPC tenant
735        // identity — not a new identity source.
736        let tenant_id = tenant.as_ref().map(|_| subject.to_string());
737        // Read the epoch AFTER any adaptation for this request has completed
738        // (resolve_delta runs after apply_ttt_adaptation on the serving path),
739        // so a re-adapted tenant's new epoch is what gets registered.
740        let weight_epoch = tenant
741            .as_ref()
742            .map(|d| d.lock().weight_epoch())
743            .unwrap_or(0);
744
745        let delta = match (base, tenant) {
746            (Some(base), Some(tenant)) => {
747                // Compose: base + tenant corrections
748                Some(crate::training::TenantDelta::compose(&base, &tenant))
749            }
750            (Some(base), None) => Some(base),
751            (None, Some(tenant)) => Some(tenant),
752            (None, None) => None,
753        };
754
755        (delta, tenant_id, weight_epoch)
756    }
757
758    /// Apply TTT adaptation if enabled (adapts model to input BEFORE generation)
759    ///
760    /// Returns:
761    /// - Ok(Some(result)) if TTT was configured and ran (or was skipped)
762    /// - Ok(None) if TTT is not configured
763    /// - Err(e) if TTT failed unexpectedly
764    ///
765    /// Apply TTT adaptation with per-request overrides.
766    ///
767    /// Uses subject-specific delta pool for isolated per-session adaptation.
768    async fn apply_ttt_adaptation_with_overrides(
769        &self,
770        prompt: &str,
771        subject: &Subject,
772        overrides: &crate::training::ttt::TTTOverrides,
773    ) -> Result<Option<crate::training::ttt::TTTResult>> {
774        use anyhow::anyhow;
775
776        let ttt_trainer = match self.ttt_trainer.as_ref() {
777            Some(t) => t,
778            None => return Ok(None), // TTT not configured
779        };
780
781        let pool = match self.delta_pool.as_ref() {
782            Some(p) => p,
783            None => return Ok(None), // No delta pool
784        };
785
786        let tokenizer = match self.tokenizer.as_ref() {
787            Some(t) => t,
788            None => return Ok(None), // No tokenizer available
789        };
790
791        let encoding = tokenizer
792            .encode(prompt, false)
793            .map_err(|e| anyhow!("Tokenization failed: {}", e))?;
794        let input_tokens: Vec<u32> = encoding.get_ids().to_vec();
795
796        // Ensure pool has capacity before creating/accessing delta
797        // (must happen before acquiring engine read lock to avoid holding lock across await)
798        pool.ensure_capacity().await?;
799
800        // Rehydrate from a persisted snapshot on cold miss (#869) before taking the
801        // engine read lock (snapshot I/O is async and must not straddle the lock).
802        let delta_arc = pool.get_or_hydrate(subject).await?;
803        let engine = self.engine.read();
804
805        // Lock the delta and run adaptation
806        let mut delta = delta_arc.lock();
807
808        // Note: capacity check is now handled by guard.at_capacity inside resolve().
809        // The guard returns ResolveOutcome::Skipped if the delta is at capacity.
810
811        // Snapshot Muon momentum and effective_ranks for rollback (before adapt_tenant
812        // mutates them). adapt_tenant already returns the weight pre_snapshot, but Muon
813        // and effective_ranks are mutated in-place during the gradient loop.
814        let pre_muon = crate::training::muon::snapshot_muon_states(&delta.muon_states);
815        let pre_eff_ranks = delta.effective_ranks.clone();
816
817        let adapt_result = ttt_trainer.adapt_tenant(&engine, &mut delta, &input_tokens, overrides);
818
819        // Early drop: release engine read lock before bookkeeping.
820        // Delta has its own mutex; the LoRA generation counter detects reconfiguration.
821        drop(engine);
822
823        match adapt_result {
824            Ok((mut result, pre_snapshot)) => {
825                if !result.skipped {
826                    debug!(
827                        "TTT (subject {}): steps={}, improvement={:.4}, time={}ms, ppl={:.1}->{:.1}, rec={}",
828                        subject,
829                        result.steps_performed,
830                        result.loss_improvement,
831                        result.adaptation_time_ms,
832                        result.initial_perplexity,
833                        result.final_perplexity,
834                        result.recommendation,
835                    );
836                }
837
838                // Map overrides to AdaptationStrategy and drive the state machine.
839                let strategy = overrides.adaptation_strategy.clone();
840
841                // Build guard status for state machine.
842                let guard = crate::training::GuardStatus {
843                    expired: delta.adaptation_state.is_expired(),
844                    at_capacity: delta.is_at_capacity(),
845                    lora_generation: self
846                        .lora_generation
847                        .load(std::sync::atomic::Ordering::Acquire),
848                };
849
850                let outcome = delta.adaptation_state.resolve(
851                    strategy,
852                    &guard,
853                    &result,
854                    pre_snapshot,
855                    pre_muon,
856                    pre_eff_ranks,
857                );
858
859                match outcome {
860                    crate::training::ResolveOutcome::WrittenBack => {
861                        delta.accumulated_steps += result.steps_performed as u64;
862                        delta.request_count += 1;
863                        let n = delta.request_count as f64;
864                        delta.avg_loss_improvement = delta.avg_loss_improvement * ((n - 1.0) / n)
865                            + result.loss_improvement as f64 / n;
866                        result.pending = false;
867                        debug!("TTT: auto-committed adaptation for subject {}", subject);
868                    }
869                    crate::training::ResolveOutcome::Evicted {
870                        snapshot,
871                        muon,
872                        eff_ranks,
873                    } => {
874                        // resolve() transitioned state to Idle and returned the baseline
875                        // snapshot (which may be the promoted old-pending baseline in the
876                        // stacked case, or the new adaptation's pre-state otherwise).
877                        // Restore delta weights, Muon momentum, and effective ranks.
878                        let _ = delta.load_state_dict(&snapshot);
879                        crate::training::muon::restore_muon_states(&mut delta.muon_states, &muon);
880                        delta.effective_ranks = eff_ranks;
881                        result.pending = false;
882                        debug!("TTT: auto-rolled back adaptation for subject {}", subject);
883                    }
884                    crate::training::ResolveOutcome::StoredPending => {
885                        // Pending state is stored inside delta.adaptation_state.
886                        result.pending = true;
887                        debug!("TTT: stored pending adaptation for subject {}", subject);
888                    }
889                    crate::training::ResolveOutcome::Skipped { reason } => {
890                        debug!("TTT: skipped for subject {}: {}", subject, reason);
891                    }
892                }
893
894                Ok(Some(result))
895            }
896            Err(e) => {
897                warn!("TTT adaptation failed for subject {}: {}", subject, e);
898                Err(e)
899            }
900        }
901    }
902
903    /// Prepare for streaming generation with DH-based key derivation.
904    ///
905    /// This is the first phase of streaming that runs BEFORE the REP response is sent.
906    /// The actual streaming happens in `execute_stream` which runs AFTER the response.
907    ///
908    /// Uses the explicitly named third-party interoperability DH edge from
909    /// hyprstream-rpc when no accepted identified-stream binding is available:
910    /// 1. Server generates ephemeral Ristretto255 keypair
911    /// 2. Server computes shared secret: DH(server_secret, client_ephemeral_pubkey)
912    /// 3. Both parties derive topic and mac_key from shared secret using HKDF
913    ///
914    /// # Returns
915    ///
916    /// (stream_id, server_pubkey, pending) where:
917    /// - stream_id: For client display/logging (not used for routing)
918    /// - server_pubkey: 32-byte Ristretto255 public key for client to derive same keys
919    /// - pending: Stream state including StreamContext with DH-derived keys
920    async fn prepare_stream(
921        &self,
922        request: GenerationRequest,
923        client_ephemeral_pubkey: Option<&[u8]>,
924        claims: Option<hyprstream_rpc::auth::Claims>,
925        expiry_secs: i64,
926        subject: &Subject,
927        ttt_overrides: crate::training::ttt::TTTOverrides,
928        owner_did: Option<String>,
929    ) -> Result<(
930        String,
931        [u8; 32],
932        String,
933        Vec<hyprstream_rpc::stream_info::Destination>,
934        PendingWork,
935    )> {
936        anyhow::ensure!(
937            !self.draining.load(Ordering::Acquire),
938            "inference service is draining and refuses new streams"
939        );
940        // TTT adaptation is deferred to execute_stream (runs in continuation after REP
941        // is sent) to avoid blocking the ZMQ REQ/REP handler with GPU-intensive work.
942
943        // DH key derivation is required - no legacy fallback
944        let client_pub_bytes = client_ephemeral_pubkey.ok_or_else(|| {
945            anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
946        })?;
947
948        // Use StreamChannel for DH key exchange and pre-authorization
949        let stream_channel = self
950            .stream_channel
951            .as_ref()
952            .ok_or_else(|| anyhow!("StreamChannel not initialized"))?;
953
954        let stream_ctx = stream_channel
955            .prepare_third_party_interop_stream_with_claims(client_pub_bytes, expiry_secs, claims)
956            .await?
957            .with_qos_preset::<hyprstream_rpc::stream_info::Job>();
958
959        debug!(
960            stream_id = %stream_ctx.stream_id(),
961            topic = %stream_ctx.topic(),
962            "Stream prepared via StreamChannel (DH + pre-authorization)"
963        );
964
965        let stream_id = stream_ctx.stream_id().to_owned();
966        let server_pubkey = *stream_ctx.server_pubkey();
967        let broadcast_path = stream_channel.broadcast_path(stream_ctx.topic())?;
968        // #384: per-stream reach (server-authored RelayChoice on the ctx);
969        // ServerDefault unless set to Only/Override for anonymized/per-tenant.
970        let reach = stream_ctx.reach();
971
972        // Delta lookup deferred to execute_stream (after TTT may modify it)
973        let pending = PendingWork::Generation {
974            stream_ctx,
975            request,
976            subject: subject.clone(),
977            ttt_overrides,
978            owner_did,
979        };
980
981        Ok((stream_id, server_pubkey, broadcast_path, reach, pending))
982    }
983
984    /// Execute streaming generation - called AFTER REP response is sent.
985    ///
986    /// Uses StreamChannel for publishing with DH-derived topic.
987    /// The topic is derived from DH shared secret, not guessable from stream_id.
988    ///
989    /// # Protocol (E2E Authenticated via DH)
990    ///
991    /// 1. Client calls generate_stream with ephemeral pubkey in envelope
992    /// 2. Service generates ephemeral keypair, computes DH shared secret
993    /// 3. Both derive topic and mac_key from DH using HKDF
994    /// 4. Service returns server_pubkey in response (client derives same keys)
995    /// 5. Client subscribes to DH-derived topic (unpredictable, non-colliding)
996    /// 6. Service publishes chunks with HMAC chain (verified by client, not StreamService)
997    /// 7. StreamService is blind forwarder (no HMAC verification)
998    ///
999    /// Note: The read lock must be held across await because TextStream<'_> borrows from the engine.
1000    /// tokio::sync::RwLock allows concurrent readers, but write-lock requests (createLora) will
1001    /// block until all active streams complete. GPU-intensive TTT is deferred to this continuation
1002    /// to avoid blocking the ZMQ REQ/REP handler.
1003    #[allow(clippy::await_holding_lock)] // engine read-lock must be held while stream borrows it
1004    async fn execute_stream(&self, pending: PendingWork) {
1005        // SAFETY: parking_lot::Mutex is safe here because InferenceService runs on
1006        // current_thread runtime — no task migration, no .await while lock held.
1007        debug_assert!(
1008            tokio::runtime::Handle::current().runtime_flavor()
1009                == tokio::runtime::RuntimeFlavor::CurrentThread,
1010            "InferenceService must run on current_thread runtime for parking_lot safety"
1011        );
1012
1013        // #1265: `owner_did` is read only on the ledger spend path (itself
1014        // `#[cfg(ledger)]`); in the default-off build it is an unused binding,
1015        // so scope an `allow(unused_variables)` to THIS statement (not the crate)
1016        // to keep the `-D warnings` clippy gate green without masking anything else.
1017        #[cfg_attr(not(feature = "ledger"), allow(unused_variables))]
1018        let PendingWork::Generation {
1019            stream_ctx,
1020            request,
1021            subject,
1022            ttt_overrides,
1023            owner_did,
1024        } = pending
1025        else {
1026            error!("execute_stream called with non-Generation PendingWork");
1027            return;
1028        };
1029        let stream_ctx = &stream_ctx;
1030        // #1264/#1265: the spend's correlation entropy + completion log key.
1031        // Only needed on the ledger path.
1032        #[cfg(feature = "ledger")]
1033        let stream_id_str = stream_ctx.stream_id().to_owned();
1034
1035        // Get StreamChannel
1036        let stream_channel = match &self.stream_channel {
1037            Some(sc) => sc,
1038            None => {
1039                error!("StreamChannel not initialized for streaming");
1040                return;
1041            }
1042        };
1043
1044        // #1265: fail CLOSED before generation begins when a #1264 completion-
1045        // spend emitter is attached but the caller has no self-certifying
1046        // pairwise DID — that DID is the account owner the spend debits, so an
1047        // authenticated-but-DID-less (anonymous) caller must not receive a
1048        // completed zero-cost generation. Deny with a content-free error frame
1049        // and NO model output (the engine is not even asked to generate). If the
1050        // owner later wants an explicit fail-open / debt policy for anonymous
1051        // callers, that is a separate policy change — do not implement it here.
1052        #[cfg(feature = "ledger")]
1053        {
1054            if ledger_requires_identity(self.ledger.read().is_some(), owner_did.as_deref()) {
1055                warn!(
1056                    stream_id = %stream_id_str,
1057                    "ledger: spend emitter attached but caller has no verified pairwise DID \
1058                     — denying generation before it starts (fail-closed, #1265)"
1059                );
1060                if let Err(e) = stream_channel
1061                    .run_stream(stream_ctx, |publisher| async move {
1062                        (publisher, Err::<(), _>(anyhow!(LEDGER_DENY_NO_IDENTITY)))
1063                    })
1064                    .await
1065                {
1066                    error!(
1067                        stream_id = %stream_id_str,
1068                        error = %e,
1069                        "failed to publish ledger deny frame"
1070                    );
1071                }
1072                return;
1073            }
1074        }
1075
1076        // Snapshot LoRA generation before TTT — used to detect reconfiguration mid-stream
1077        let lora_gen_before = self.lora_generation.load(Ordering::Acquire);
1078
1079        // TTT runs HERE (in continuation, after REP sent — ZMQ loop unblocked).
1080        // If the user explicitly enabled TTT (ttt_enabled=true), failures are stream errors.
1081        // If TTT is only from server config, failures are non-fatal (warn + continue).
1082        let ttt_result = match self
1083            .apply_ttt_adaptation_with_overrides(request.prompt.as_str(), &subject, &ttt_overrides)
1084            .await
1085        {
1086            Ok(Some(result)) => Some(result),
1087            Ok(None) => {
1088                // TTT not available on this server (no trainer/pool/tokenizer configured)
1089                if ttt_overrides.enabled == Some(true) {
1090                    // User explicitly requested TTT — this is an error
1091                    error!(stream_id = %stream_ctx.stream_id(), subject = %subject, "TTT explicitly enabled but not available on server");
1092                    let sc = stream_channel;
1093                    if let Err(send_err) = sc
1094                        .run_stream(stream_ctx, |publisher| async move {
1095                            (
1096                                publisher,
1097                                Err::<(), _>(anyhow::anyhow!(
1098                                    "TTT explicitly enabled but not configured on this server"
1099                                )),
1100                            )
1101                        })
1102                        .await
1103                    {
1104                        error!(stream_id = %stream_ctx.stream_id(), error = %send_err,
1105                            "Failed to publish TTT error to stream");
1106                    }
1107                    return;
1108                }
1109                None // TTT not requested or not configured — proceed without
1110            }
1111            Err(e) => {
1112                if ttt_overrides.enabled == Some(true) {
1113                    // User explicitly requested TTT — adaptation failure is a stream error
1114                    error!(stream_id = %stream_ctx.stream_id(), subject = %subject, error = %e, "TTT adaptation failed");
1115                    let sc = stream_channel;
1116                    if let Err(send_err) = sc
1117                        .run_stream(stream_ctx, |publisher| async move {
1118                            (
1119                                publisher,
1120                                Err::<(), _>(anyhow::anyhow!("TTT adaptation failed: {}", e)),
1121                            )
1122                        })
1123                        .await
1124                    {
1125                        error!(stream_id = %stream_ctx.stream_id(), error = %send_err,
1126                            "Failed to publish TTT error to stream");
1127                    }
1128                    return;
1129                }
1130                // TTT running from server config only — warn and continue
1131                warn!(
1132                    "TTT adaptation failed (server-config), continuing without: {}",
1133                    e
1134                );
1135                None
1136            }
1137        };
1138
1139        // Re-resolve delta after TTT (may have been updated by adaptation)
1140        let (delta, tenant_id, weight_epoch) = self.resolve_delta(&subject);
1141
1142        trace!(
1143            stream_id = %stream_ctx.stream_id(),
1144            topic = %stream_ctx.topic(),
1145            has_delta = delta.is_some(),
1146            has_ttt = ttt_result.is_some(),
1147            "Starting E2E authenticated stream via StreamChannel"
1148        );
1149
1150        // Check LoRA generation hasn't changed since we started (detects reconfiguration mid-stream)
1151        let lora_gen_now = self.lora_generation.load(Ordering::Acquire);
1152        if lora_gen_now != lora_gen_before {
1153            warn!(
1154                stream_id = %stream_ctx.stream_id(),
1155                "LoRA reconfigured during TTT adaptation (gen {} -> {}), aborting stream",
1156                lora_gen_before, lora_gen_now
1157            );
1158            let sc = stream_channel;
1159            if let Err(e) = sc
1160                .run_stream(stream_ctx, |publisher| async move {
1161                    (
1162                        publisher,
1163                        Err::<(), _>(anyhow::anyhow!(
1164                            "LoRA adapter was reconfigured during adaptation — retry request"
1165                        )),
1166                    )
1167                })
1168                .await
1169            {
1170                error!(stream_id = %stream_ctx.stream_id(), error = %e, "Failed to publish LoRA error");
1171            }
1172            return;
1173        }
1174
1175        // Run the stream with StreamChannel's async publisher callback.
1176        // Engine read-lock held across await: generate_with_delta returns a stream borrowing engine.
1177        let engine = self.engine.read();
1178        // Pass the live LoRA generation so the KV-compat expected identity reflects
1179        // the effective adapter actually applied (#1277): a re-adapted model is
1180        // detected as incompatible and its stale KV is rejected at the cache boundary.
1181        let stream_result =
1182            engine.generate_with_delta(request, delta, tenant_id, weight_epoch, lora_gen_now);
1183
1184        // #1264/#1265: capture the completion stats out of the stream closure so
1185        // the token-spend can be posted after the client-visible completion frame
1186        // — on EVERY terminal path (normal end, cancel, stream error, publish
1187        // failure), not only normal exhaustion.
1188        #[cfg(feature = "ledger")]
1189        let completion_stats: Arc<
1190            parking_lot::Mutex<Option<crate::runtime::GenerationStats>>,
1191        > = Arc::new(parking_lot::Mutex::new(None));
1192
1193        let result = stream_channel
1194            .run_stream(stream_ctx, |mut publisher| {
1195                #[cfg(feature = "ledger")]
1196                let completion_stats = completion_stats.clone();
1197                async move {
1198                    let result = match stream_result {
1199                        Ok(mut stream) => {
1200                            let cancel = stream_ctx.cancel_token();
1201                            // Drive the generation loop to a terminal outcome. The
1202                            // loop publishes each chunk; cancel / stream-error /
1203                            // publish-failure are all terminal. Completion stats are
1204                            // captured on the SINGLE common exit below, so no
1205                            // terminal path can skip accounting for the work actually
1206                            // done (#1265).
1207                            let outcome =
1208                                drive_generation_loop(&mut stream, cancel, &mut publisher).await;
1209
1210                            // Common exit: ALWAYS snapshot the stats once generation
1211                            // has begun, regardless of how it ended — so a cancelled,
1212                            // errored, or publish-failed generation still accounts.
1213                            let stats = stream.completion_stats();
1214                            #[cfg(feature = "ledger")]
1215                            {
1216                                *completion_stats.lock() = Some(stats.clone());
1217                            }
1218
1219                            match outcome {
1220                                GenLoopOutcome::Exhausted => {
1221                                    let mut complete =
1222                                        crate::services::rpc_types::InferenceComplete::from(&stats);
1223                                    // Attach TTT metrics to completion (deferred TTT).
1224                                    complete.ttt_metrics = ttt_result.map(std::convert::Into::into);
1225                                    publisher.complete_ref(&complete.to_bytes()).await
1226                                }
1227                                GenLoopOutcome::Cancelled => {
1228                                    let _ = publisher.publish_error("cancelled").await;
1229                                    Ok(())
1230                                }
1231                                GenLoopOutcome::StreamError(e) => {
1232                                    let _ = publisher.publish_error(&e.to_string()).await;
1233                                    Ok(())
1234                                }
1235                                GenLoopOutcome::PublishFailed(e) => Err(e),
1236                            }
1237                        }
1238                        Err(e) => Err(e), // framework sends Error frame automatically
1239                    };
1240                    (publisher, result)
1241                }
1242            })
1243            .await;
1244
1245        if let Err(e) = result {
1246            error!(
1247                stream_id = %stream_ctx.stream_id(),
1248                error = %e,
1249                "Stream execution failed"
1250            );
1251        }
1252
1253        // #1264/#1265: post the token-spend for the work actually done (quota
1254        // burn). Fail-safe: a ledger/accounting failure MUST NOT break a
1255        // generation that already produced output — the spend is best-effort and
1256        // every non-posted outcome is a content-free signal, never a silent drop.
1257        // Posted *after* the client-visible completion frame. (The engine
1258        // read-lock is held across this best-effort await; covered by the
1259        // `await_holding_lock` allow above and consistent with the stream itself.)
1260        #[cfg(feature = "ledger")]
1261        {
1262            if let Some(emitter) = self.ledger.read().clone() {
1263                let stats_opt = completion_stats.lock().clone();
1264                match post_completion_spend(
1265                    &emitter,
1266                    stats_opt.as_ref(),
1267                    owner_did.as_deref(),
1268                    &stream_id_str,
1269                )
1270                .await
1271                {
1272                    Some(res) => {
1273                        observe_spend_result(&res, &stream_id_str, emitter.unit());
1274                    }
1275                    None => {
1276                        // Nothing to account for: generation never began
1277                        // (`engine.generate_with_delta` returned Err before the
1278                        // loop ran). A DID-less caller cannot reach the stats arm
1279                        // — the gate above denied it before generation started
1280                        // (#1265).
1281                    }
1282                }
1283            }
1284        }
1285    }
1286
1287    /// Execute streaming training step - called AFTER REP response is sent.
1288    ///
1289    /// Runs the training step in the background and publishes results via StreamChannel.
1290    /// This avoids REQ/REP timeout on long-running training (e.g., backward pass compilation).
1291    async fn execute_training_stream(&self, pending: PendingWork) {
1292        let PendingWork::Training {
1293            stream_ctx,
1294            subject,
1295            input,
1296            gradient_steps,
1297            learning_rate,
1298            adaptation_strategy,
1299        } = pending
1300        else {
1301            error!("execute_training_stream called with non-Training PendingWork");
1302            return;
1303        };
1304        let stream_ctx = &stream_ctx;
1305
1306        let stream_channel = match &self.stream_channel {
1307            Some(sc) => sc,
1308            None => {
1309                error!("StreamChannel not initialized for training stream");
1310                return;
1311            }
1312        };
1313
1314        trace!(
1315            stream_id = %stream_ctx.stream_id(),
1316            "Starting training stream via StreamChannel"
1317        );
1318
1319        let result = stream_channel
1320            .run_stream(stream_ctx, |mut publisher| async move {
1321                if stream_ctx.cancel_token().is_cancelled() {
1322                    let _ = publisher.publish_error("cancelled").await;
1323                    return (publisher, Ok(()));
1324                }
1325                let result = match self
1326                    .handle_train_step(
1327                        &subject,
1328                        &input,
1329                        gradient_steps,
1330                        learning_rate,
1331                        adaptation_strategy,
1332                    )
1333                    .await
1334                {
1335                    Ok(result) => {
1336                        // Serialize training result as JSON for the completion payload
1337                        let payload = serde_json::to_vec(&result).unwrap_or_else(|e| {
1338                            format!("{{\"error\":\"serialize failed: {e}\"}}").into_bytes()
1339                        });
1340                        publisher.complete_ref(&payload).await
1341                    }
1342                    Err(e) => Err(e), // framework sends Error frame automatically
1343                };
1344                (publisher, result)
1345            })
1346            .await;
1347
1348        if let Err(e) = result {
1349            error!(
1350                stream_id = %stream_ctx.stream_id(),
1351                error = %e,
1352                "Training stream execution failed"
1353            );
1354        }
1355    }
1356
1357    /// Handle model info request
1358    async fn handle_model_info(&self) -> ModelInfo {
1359        RuntimeEngine::model_info(&*self.engine.read())
1360    }
1361
1362    /// Handle is ready request
1363    async fn handle_is_ready(&self) -> bool {
1364        self.engine.read().is_loaded()
1365            && self.network_ready.load(Ordering::Acquire)
1366            && !self.draining.load(Ordering::Acquire)
1367    }
1368
1369    /// Handle apply chat template
1370    async fn handle_apply_chat_template(
1371        &self,
1372        messages: Vec<crate::runtime::template_engine::ChatMessage>,
1373        add_generation_prompt: bool,
1374        tools: Option<&serde_json::Value>,
1375        enable_thinking: Option<bool>,
1376        template_vars_json: Option<&str>,
1377    ) -> Result<String> {
1378        self.engine.read().apply_chat_template_with_vars(
1379            &messages,
1380            add_generation_prompt,
1381            tools,
1382            enable_thinking,
1383            template_vars_json,
1384        )
1385    }
1386
1387    /// Truncate messages to fit within the context budget.
1388    ///
1389    /// Groups messages into atomic truncation units (assistant+tool_call groups
1390    /// are indivisible), estimates token cost per group, and drops oldest
1391    /// non-system groups until the total fits in `max_seq_len - reserve_tokens`.
1392    ///
1393    /// Returns an error if even the system prompt + the last message group
1394    /// exceeds the budget — the caller should surface this to the client.
1395    fn truncate_messages_to_budget(
1396        &self,
1397        messages: Vec<crate::runtime::template_engine::ChatMessage>,
1398        reserve_tokens: usize,
1399    ) -> Result<Vec<crate::runtime::template_engine::ChatMessage>> {
1400        let info = self.engine.read().model_info();
1401        let max_seq_len = info.context_length as usize;
1402        let budget = max_seq_len.saturating_sub(reserve_tokens);
1403        if budget == 0 {
1404            return Ok(messages);
1405        }
1406
1407        // Estimate per-message token cost
1408        const FRAMING_OVERHEAD: usize = 6; // <|im_start|>role\n...<|im_end|>\n
1409        let tokenizer = match self.engine.read().get_tokenizer() {
1410            Ok(t) => t,
1411            Err(_) => return Ok(messages), // Can't tokenize — skip truncation
1412        };
1413
1414        let msg_costs: Vec<usize> = messages
1415            .iter()
1416            .map(|m| {
1417                // Count content tokens
1418                let content_tokens = m
1419                    .content
1420                    .as_deref()
1421                    .and_then(|c| tokenizer.encode(c, false).ok())
1422                    .map(|e| e.get_ids().len())
1423                    .unwrap_or(0);
1424
1425                // Count tool_calls tokens (function name + serialized arguments)
1426                let tool_call_tokens = m.tool_calls.as_ref().map_or(0, |tcs| {
1427                    tcs.iter()
1428                        .map(|tc| {
1429                            let call_str = format!(
1430                                "{}({})",
1431                                tc.function.name,
1432                                serde_json::to_string(&tc.function.arguments).unwrap_or_default()
1433                            );
1434                            tokenizer
1435                                .encode(call_str.as_str(), false)
1436                                .ok()
1437                                .map(|e| e.get_ids().len())
1438                                .unwrap_or(10) // conservative fallback
1439                        })
1440                        .sum::<usize>()
1441                });
1442
1443                content_tokens + tool_call_tokens + FRAMING_OVERHEAD
1444            })
1445            .collect();
1446
1447        let total: usize = msg_costs.iter().sum();
1448        if total <= budget {
1449            return Ok(messages); // Fits — no truncation needed
1450        }
1451
1452        // Group messages into atomic truncation units.
1453        // An assistant with tool_calls + its matching tool responses = one group.
1454        let mut groups: Vec<(usize, Vec<usize>)> = Vec::new(); // (group_cost, message_indices)
1455        let mut i = 0;
1456        while i < messages.len() {
1457            let m = &messages[i];
1458            if m.role == "assistant" && m.tool_calls.as_ref().is_some_and(|tc| !tc.is_empty()) {
1459                let tool_ids: std::collections::HashSet<&str> = m
1460                    .tool_calls
1461                    .as_deref()
1462                    .unwrap_or_default()
1463                    .iter()
1464                    .map(|tc| tc.id.as_str())
1465                    .collect();
1466                let mut indices = vec![i];
1467                let mut cost = msg_costs[i];
1468                let mut j = i + 1;
1469                while j < messages.len()
1470                    && messages[j].role == "tool"
1471                    && messages[j]
1472                        .tool_call_id
1473                        .as_deref()
1474                        .is_some_and(|id| tool_ids.contains(id))
1475                {
1476                    indices.push(j);
1477                    cost += msg_costs[j];
1478                    j += 1;
1479                }
1480                groups.push((cost, indices));
1481                i = j;
1482            } else {
1483                groups.push((msg_costs[i], vec![i]));
1484                i += 1;
1485            }
1486        }
1487
1488        // Find cutoff: keep system groups + as many recent groups as fit
1489        let has_system = messages
1490            .first()
1491            .map(|m| m.role == "system")
1492            .unwrap_or(false);
1493        let system_cost = if has_system { groups[0].0 } else { 0 };
1494        let start = if has_system { 1 } else { 0 };
1495
1496        // Check if system + last group exceeds budget
1497        if groups.len() > start {
1498            let last_group_cost = groups.last().map(|(c, _)| *c).unwrap_or(0);
1499            if system_cost + last_group_cost > budget {
1500                return Err(anyhow::anyhow!(
1501                    "Input exceeds context window: system prompt (~{} tokens) + last message (~{} tokens) = ~{} tokens, \
1502                     but only {} tokens available (max_seq_len={}, reserved for generation={}). \
1503                     Reduce message size or use a model with a larger context window.",
1504                    system_cost, last_group_cost, system_cost + last_group_cost,
1505                    budget, max_seq_len, reserve_tokens
1506                ));
1507            }
1508        }
1509
1510        let mut kept_cost = system_cost;
1511        let mut keep_from = groups.len();
1512        for i in (start..groups.len()).rev() {
1513            if kept_cost + groups[i].0 <= budget {
1514                kept_cost += groups[i].0;
1515                keep_from = i;
1516            } else {
1517                break;
1518            }
1519        }
1520
1521        // Build truncated message list
1522        let mut kept_indices: Vec<usize> = Vec::new();
1523        if has_system {
1524            kept_indices.extend(&groups[0].1);
1525        }
1526        for g in &groups[keep_from..] {
1527            kept_indices.extend(&g.1);
1528        }
1529        kept_indices.sort();
1530
1531        let dropped = messages.len() - kept_indices.len();
1532        if dropped > 0 {
1533            tracing::info!(
1534                "Context truncation: dropped {} messages ({} → ~{} tokens, budget: {}, max_seq_len: {}, reserve: {})",
1535                dropped, total, kept_cost, budget, max_seq_len, reserve_tokens
1536            );
1537        }
1538
1539        Ok(kept_indices
1540            .into_iter()
1541            .map(|i| messages[i].clone())
1542            .collect())
1543    }
1544
1545    /// Handle create LoRA
1546    async fn handle_create_lora(&self, config: TenantDeltaConfig) -> Result<()> {
1547        // Propagate target modules to the delta pool so new deltas
1548        // create A/B matrices for ALL configured modules, not just the default q_proj/v_proj
1549        if let Some(pool) = &self.delta_pool {
1550            tracing::info!(
1551                "[TTT] Updating delta pool config: target_modules={:?}, rank={}, alpha={:.1}, lr={:.1e}",
1552                config.target_modules, config.rank, config.alpha, config.learning_rate
1553            );
1554            pool.update_config(config.clone());
1555        }
1556        let result = self.engine.write().create_lora(config);
1557        if result.is_ok() {
1558            self.lora_generation.fetch_add(1, Ordering::Release);
1559        }
1560        result
1561    }
1562
1563    // =========================================================================
1564    // Generic streaming setup (used by streaming handler variants)
1565    // =========================================================================
1566
1567    /// Set up a streaming context for a handler that needs to run work in a continuation.
1568    ///
1569    /// Returns `(StreamInfo, StreamContext)` — the caller builds the continuation
1570    /// and returns `Ok((stream_info, continuation))`.
1571    async fn setup_stream(
1572        &self,
1573        ctx: &EnvelopeContext,
1574    ) -> Result<(
1575        hyprstream_rpc_std::inference_client::StreamInfo,
1576        hyprstream_rpc::streaming::StreamContext,
1577    )> {
1578        let client_pub_bytes = ctx.ephemeral_pubkey().ok_or_else(|| {
1579            anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
1580        })?;
1581        let client_pub_ref: &[u8] = &client_pub_bytes;
1582        let stream_channel = self
1583            .stream_channel
1584            .as_ref()
1585            .ok_or_else(|| anyhow!("StreamChannel not initialized"))?;
1586
1587        let expiry_secs = ctx
1588            .claims()
1589            .map(|c| c.exp - chrono::Utc::now().timestamp())
1590            .unwrap_or(600)
1591            .max(600);
1592        let claims = ctx.claims().cloned();
1593
1594        let stream_ctx = stream_channel
1595            .prepare_third_party_interop_stream_with_claims(client_pub_ref, expiry_secs, claims)
1596            .await?
1597            .with_qos_preset::<hyprstream_rpc::stream_info::Job>();
1598
1599        let stream_id = stream_ctx.stream_id().to_owned();
1600        let server_pubkey = *stream_ctx.server_pubkey();
1601        let broadcast_path = hyprstream_rpc::moq_stream::global_moq_origin()
1602            .map(|o| o.broadcast_path(stream_ctx.topic()))
1603            .unwrap_or_default();
1604
1605        let stream_info = hyprstream_rpc_std::inference_client::StreamInfo {
1606            stream_id,
1607            dh_public: server_pubkey,
1608            qos: stream_ctx.qos().clone(),
1609            broadcast_path,
1610            announced_at: stream_ctx.reach(), // #384: per-stream reach via ctx
1611            moql_server_identity: stream_ctx.moql_server_identity(),
1612            kem_ciphertexts: Vec::new(), // #554: classical stream (dh_public path), no hybrid KEM
1613        };
1614
1615        Ok((stream_info, stream_ctx))
1616    }
1617
1618    // =========================================================================
1619    // Streaming execution helpers (called from continuations)
1620    // =========================================================================
1621
1622    async fn execute_create_lora_stream(
1623        &self,
1624        stream_ctx: hyprstream_rpc::streaming::StreamContext,
1625        config: crate::training::TenantDeltaConfig,
1626    ) {
1627        let sc = match &self.stream_channel {
1628            Some(sc) => sc,
1629            None => {
1630                error!("StreamChannel not initialized");
1631                return;
1632            }
1633        };
1634        let _ = sc
1635            .run_stream(&stream_ctx, |mut publisher| async {
1636                let result = InferenceService::handle_create_lora(self, config).await;
1637                match &result {
1638                    Ok(()) => {
1639                        let _ = publisher.complete_ref(b"{}").await;
1640                    }
1641                    Err(e) => {
1642                        let _ = publisher.publish_error(&format!("{:#}", e)).await;
1643                    }
1644                }
1645                (publisher, result.map(|_| ()))
1646            })
1647            .await;
1648    }
1649
1650    async fn execute_load_lora_stream(
1651        &self,
1652        stream_ctx: hyprstream_rpc::streaming::StreamContext,
1653        path: String,
1654    ) {
1655        let sc = match &self.stream_channel {
1656            Some(sc) => sc,
1657            None => {
1658                error!("StreamChannel not initialized");
1659                return;
1660            }
1661        };
1662        let _ = sc
1663            .run_stream(&stream_ctx, |mut publisher| async {
1664                let result = self.load_lora(Path::new(&path)).await;
1665                match &result {
1666                    Ok(()) => {
1667                        let _ = publisher.complete_ref(b"{}").await;
1668                    }
1669                    Err(e) => {
1670                        let _ = publisher.publish_error(&format!("{:#}", e)).await;
1671                    }
1672                }
1673                (publisher, result)
1674            })
1675            .await;
1676    }
1677
1678    async fn execute_save_lora_stream(
1679        &self,
1680        stream_ctx: hyprstream_rpc::streaming::StreamContext,
1681        name: String,
1682    ) {
1683        let sc = match &self.stream_channel {
1684            Some(sc) => sc,
1685            None => {
1686                error!("StreamChannel not initialized");
1687                return;
1688            }
1689        };
1690        let _ = sc
1691            .run_stream(&stream_ctx, |mut publisher| async {
1692                let result = self.save_lora(&name).await;
1693                match &result {
1694                    Ok(()) => {
1695                        let _ = publisher.complete_ref(b"{}").await;
1696                    }
1697                    Err(e) => {
1698                        let _ = publisher.publish_error(&format!("{:#}", e)).await;
1699                    }
1700                }
1701                (publisher, result)
1702            })
1703            .await;
1704    }
1705
1706    async fn execute_save_adaptation_stream(
1707        &self,
1708        stream_ctx: hyprstream_rpc::streaming::StreamContext,
1709        subject: Subject,
1710        name: String,
1711        merge_strategy: String,
1712        merge_weight: f32,
1713    ) {
1714        let sc = match &self.stream_channel {
1715            Some(sc) => sc,
1716            None => {
1717                error!("StreamChannel not initialized");
1718                return;
1719            }
1720        };
1721        let _ = sc
1722            .run_stream(&stream_ctx, |mut publisher| async {
1723                let result = InferenceService::handle_save_adaptation(
1724                    self,
1725                    &subject,
1726                    &name,
1727                    &merge_strategy,
1728                    merge_weight,
1729                )
1730                .await;
1731                match &result {
1732                    Ok(info) => {
1733                        let payload = serde_json::to_vec(&serde_json::json!({
1734                            "adapter_name": info.adapter_name,
1735                            "adapter_path": info.adapter_path,
1736                            "content_hash": info.content_hash,
1737                            "merge_strategy": info.merge_strategy,
1738                        }))
1739                        .unwrap_or_default();
1740                        let _ = publisher.complete_ref(&payload).await;
1741                    }
1742                    Err(e) => {
1743                        let _ = publisher.publish_error(&format!("{:#}", e)).await;
1744                    }
1745                }
1746                (publisher, result.map(|_| ()))
1747            })
1748            .await;
1749    }
1750
1751    async fn execute_snapshot_delta_stream(
1752        &self,
1753        stream_ctx: hyprstream_rpc::streaming::StreamContext,
1754        subject: Subject,
1755    ) {
1756        let sc = match &self.stream_channel {
1757            Some(sc) => sc,
1758            None => {
1759                error!("StreamChannel not initialized");
1760                return;
1761            }
1762        };
1763        let _ = sc
1764            .run_stream(&stream_ctx, |mut publisher| async {
1765                let result = InferenceService::handle_snapshot_delta(self, &subject).await;
1766                match &result {
1767                    Ok(info) => {
1768                        let payload = serde_json::to_vec(&serde_json::json!({
1769                            "content_hash": info.content_hash,
1770                            "size_bytes": info.size_bytes,
1771                            "accumulated_steps": info.accumulated_steps,
1772                            "request_count": info.request_count,
1773                        }))
1774                        .unwrap_or_default();
1775                        let _ = publisher.complete_ref(&payload).await;
1776                    }
1777                    Err(e) => {
1778                        let _ = publisher.publish_error(&format!("{:#}", e)).await;
1779                    }
1780                }
1781                (publisher, result.map(|_| ()))
1782            })
1783            .await;
1784    }
1785
1786    async fn execute_export_peft_adapter_stream(
1787        &self,
1788        stream_ctx: hyprstream_rpc::streaming::StreamContext,
1789        subject: Subject,
1790        name: String,
1791        commit_message: String,
1792    ) {
1793        let sc = match &self.stream_channel {
1794            Some(sc) => sc,
1795            None => {
1796                error!("StreamChannel not initialized");
1797                return;
1798            }
1799        };
1800        let _ = sc
1801            .run_stream(&stream_ctx, |mut publisher| async {
1802                let result = InferenceService::handle_export_peft_adapter(
1803                    self,
1804                    &subject,
1805                    &name,
1806                    &commit_message,
1807                )
1808                .await;
1809                match &result {
1810                    Ok(info) => {
1811                        let payload = serde_json::to_vec(&serde_json::json!({
1812                            "adapter_path": info.adapter_path,
1813                            "content_hash": info.content_hash,
1814                        }))
1815                        .unwrap_or_default();
1816                        let _ = publisher.complete_ref(&payload).await;
1817                    }
1818                    Err(e) => {
1819                        let _ = publisher.publish_error(&format!("{:#}", e)).await;
1820                    }
1821                }
1822                (publisher, result.map(|_| ()))
1823            })
1824            .await;
1825    }
1826
1827    async fn execute_merge_lora_stream(
1828        &self,
1829        stream_ctx: hyprstream_rpc::streaming::StreamContext,
1830        adapter_path: String,
1831        weight: f32,
1832        strategy: String,
1833    ) {
1834        let sc = match &self.stream_channel {
1835            Some(sc) => sc,
1836            None => {
1837                error!("StreamChannel not initialized");
1838                return;
1839            }
1840        };
1841        let _ = sc
1842            .run_stream(&stream_ctx, |mut publisher| async {
1843                let result = self
1844                    .merge_lora(Path::new(&adapter_path), weight, &strategy)
1845                    .await;
1846                match &result {
1847                    Ok(()) => {
1848                        let _ = publisher.complete_ref(b"{}").await;
1849                    }
1850                    Err(e) => {
1851                        let _ = publisher.publish_error(&format!("{:#}", e)).await;
1852                    }
1853                }
1854                (publisher, result)
1855            })
1856            .await;
1857    }
1858
1859    // =========================================================================
1860    // Training loop control handlers (tenant-aware TTT)
1861    // =========================================================================
1862
1863    /// Write back a pending TTT adaptation (commit path)
1864    fn handle_writeback(&self, subject: &Subject) -> Result<()> {
1865        if let Some(pool) = &self.delta_pool {
1866            if let Some(delta_arc) = pool.get(subject) {
1867                let mut delta = delta_arc.lock();
1868                let info = delta.adaptation_state.writeback_stats()?;
1869                delta.accumulated_steps += info.steps_performed as u64;
1870                delta.request_count += 1;
1871                let n = delta.request_count as f64;
1872                delta.avg_loss_improvement =
1873                    delta.avg_loss_improvement * ((n - 1.0) / n) + info.loss_improvement as f64 / n;
1874                debug!(
1875                    "Wrote back adaptation for '{}': steps={}, improvement={:.4}",
1876                    subject, info.steps_performed, info.loss_improvement
1877                );
1878                Ok(())
1879            } else {
1880                Err(anyhow::anyhow!("No delta found for subject '{}'", subject))
1881            }
1882        } else {
1883            Err(anyhow::anyhow!("No delta pool configured"))
1884        }
1885    }
1886
1887    /// Evict a pending TTT adaptation (rollback path)
1888    fn handle_evict(&self, subject: &Subject) -> Result<()> {
1889        if let Some(pool) = &self.delta_pool {
1890            if let Some(delta_arc) = pool.get(subject) {
1891                let mut delta = delta_arc.lock();
1892                let snapshot = delta.adaptation_state.evict()?;
1893                let _ = delta.load_state_dict(&snapshot.pre_snapshot);
1894                crate::training::muon::restore_muon_states(
1895                    &mut delta.muon_states,
1896                    &snapshot.pre_muon,
1897                );
1898                delta.effective_ranks = snapshot.pre_eff_ranks;
1899                debug!("Evicted adaptation for subject '{}'", subject);
1900                Ok(())
1901            } else {
1902                Err(anyhow::anyhow!("No delta found for subject '{}'", subject))
1903            }
1904        } else {
1905            Err(anyhow::anyhow!("No delta pool configured"))
1906        }
1907    }
1908
1909    /// Run pure training steps without generation
1910    async fn handle_train_step(
1911        &self,
1912        subject: &Subject,
1913        input: &str,
1914        gradient_steps: Option<u32>,
1915        learning_rate: Option<f32>,
1916        strategy: crate::training::adaptation_state::AdaptationStrategy,
1917    ) -> Result<crate::training::ttt::TTTResult> {
1918        let ttt_trainer = self
1919            .ttt_trainer
1920            .as_ref()
1921            .ok_or_else(|| anyhow!("TTT not configured"))?;
1922        let tokenizer = self
1923            .tokenizer
1924            .as_ref()
1925            .ok_or_else(|| anyhow!("No tokenizer available"))?;
1926        let pool = self
1927            .delta_pool
1928            .as_ref()
1929            .ok_or_else(|| anyhow!("Delta pool not initialized"))?;
1930
1931        // Ensure pool has capacity before creating/accessing delta
1932        pool.ensure_capacity().await?;
1933        // Rehydrate from a persisted snapshot on cold miss (#869): a replica that lost
1934        // this subject's warm delta (crash / reschedule / scale-in) reloads its
1935        // accumulated adaptation instead of silently starting from zero.
1936        let delta_arc = pool.get_or_hydrate(subject).await?;
1937
1938        // Phase 1: Sync work under scoped lock — tokenize and snapshot.
1939        // Always snapshot (even for auto_commit) so auto-rollback can restore state.
1940        let (input_tokens, pre_snapshot, pre_muon, pre_eff_ranks) = {
1941            let delta = delta_arc.lock();
1942            let encoding = tokenizer
1943                .encode(input, false)
1944                .map_err(|e| anyhow!("Tokenization failed: {}", e))?;
1945            let input_tokens: Vec<u32> = encoding.get_ids().to_vec();
1946            let pre_snapshot = delta.extract_state_dict();
1947            let pre_muon = crate::training::muon::snapshot_muon_states(&delta.muon_states);
1948            let pre_eff_ranks = delta.effective_ranks.clone();
1949            (input_tokens, pre_snapshot, pre_muon, pre_eff_ranks)
1950        }; // guard dropped
1951
1952        let steps = gradient_steps
1953            .map(|g| g as usize)
1954            .unwrap_or(ttt_trainer.config.gradient_steps as usize);
1955        let lr = learning_rate.map(|l| l as f64);
1956
1957        // Phase 2: Await engine (no lock held — safe for spawn_local concurrency)
1958        let engine = self.engine.read();
1959
1960        // Phase 3: Reacquire delta lock for train_step
1961        let mut delta = delta_arc.lock();
1962        let mut result =
1963            ttt_trainer.train_step(&engine, &mut delta, &input_tokens, steps, lr, None)?;
1964
1965        // strategy is passed in directly from the caller
1966
1967        // Build guard status
1968        let guard = crate::training::GuardStatus {
1969            expired: delta.adaptation_state.is_expired(),
1970            at_capacity: delta.is_at_capacity(),
1971            lora_generation: self
1972                .lora_generation
1973                .load(std::sync::atomic::Ordering::Acquire),
1974        };
1975
1976        let outcome = delta.adaptation_state.resolve(
1977            strategy,
1978            &guard,
1979            &result,
1980            pre_snapshot,
1981            pre_muon,
1982            pre_eff_ranks,
1983        );
1984
1985        match outcome {
1986            crate::training::ResolveOutcome::WrittenBack => {
1987                delta.accumulated_steps += result.steps_performed as u64;
1988                delta.request_count += 1;
1989                let n = delta.request_count as f64;
1990                delta.avg_loss_improvement = delta.avg_loss_improvement * ((n - 1.0) / n)
1991                    + result.loss_improvement as f64 / n;
1992                result.pending = false;
1993                debug!(
1994                    "TTT: auto-committed train_step adaptation for subject {}",
1995                    subject
1996                );
1997            }
1998            crate::training::ResolveOutcome::Evicted {
1999                snapshot,
2000                muon,
2001                eff_ranks,
2002            } => {
2003                let _ = delta.load_state_dict(&snapshot);
2004                crate::training::muon::restore_muon_states(&mut delta.muon_states, &muon);
2005                delta.effective_ranks = eff_ranks;
2006                result.pending = false;
2007                debug!("TTT: auto-rolled back train_step adaptation for subject {} (negative recommendation)", subject);
2008            }
2009            crate::training::ResolveOutcome::StoredPending => {
2010                result.pending = true;
2011                debug!(
2012                    "TTT: stored pending train_step adaptation for subject {}",
2013                    subject
2014                );
2015            }
2016            crate::training::ResolveOutcome::Skipped { reason } => {
2017                debug!(
2018                    "TTT: train_step skipped for subject {}: {}",
2019                    subject, reason
2020                );
2021            }
2022        }
2023
2024        Ok(result)
2025    }
2026
2027    /// Reset a tenant's delta to zeros
2028    fn handle_reset_delta(&self, subject: &Subject) -> Result<()> {
2029        if let Some(pool) = &self.delta_pool {
2030            if let Some(delta_arc) = pool.get(subject) {
2031                let mut delta = delta_arc.lock();
2032                delta.reset();
2033                debug!("Reset delta for subject '{}'", subject);
2034            } else {
2035                debug!("No delta to reset for subject '{}'", subject);
2036            }
2037        }
2038
2039        Ok(())
2040    }
2041
2042    /// Get status of a tenant's delta — returns generated DeltaStatusResult directly.
2043    fn handle_get_delta_status(&self, subject: &Subject) -> Result<DeltaStatusResult> {
2044        if let Some(pool) = &self.delta_pool {
2045            if let Some(delta_arc) = pool.get(subject) {
2046                let delta = delta_arc.lock();
2047                let norm_ratios = delta.delta_norm_ratio(pool.base_weight_norms());
2048                let has_pending = delta.adaptation_state.is_pending();
2049                return Ok(DeltaStatusResult {
2050                    exists: true,
2051                    accumulated_steps: delta.accumulated_steps,
2052                    max_accumulated_steps: delta.max_accumulated_steps,
2053                    request_count: delta.request_count,
2054                    avg_loss_improvement: delta.avg_loss_improvement as f32,
2055                    memory_bytes: delta.memory_bytes() as u64,
2056                    last_snapshot_hash: delta.last_snapshot_hash.clone().unwrap_or_default(),
2057                    delta_norm_ratios: norm_ratios
2058                        .into_iter()
2059                        .map(|(name, ratio)| ModuleNormRatio {
2060                            module_name: name,
2061                            ratio: ratio as f32,
2062                        })
2063                        .collect(),
2064                    has_pending,
2065                });
2066            }
2067        }
2068
2069        Ok(DeltaStatusResult {
2070            exists: false,
2071            accumulated_steps: 0,
2072            max_accumulated_steps: 0,
2073            request_count: 0,
2074            avg_loss_improvement: 0.0,
2075            memory_bytes: 0,
2076            last_snapshot_hash: String::new(),
2077            delta_norm_ratios: vec![],
2078            has_pending: false,
2079        })
2080    }
2081
2082    /// Save a tenant's delta as a permanent LoRA adapter
2083    ///
2084    /// Supports merge strategies: "replace", "additive", "do_merge" (default).
2085    /// When an existing adapter exists, the delta is merged using the specified strategy.
2086    async fn handle_save_adaptation(
2087        &self,
2088        subject: &Subject,
2089        name: &str,
2090        merge_strategy_name: &str,
2091        merge_weight: f32,
2092    ) -> Result<SaveAdaptationResult> {
2093        use crate::training::{merge_state_dicts, MergeStrategy};
2094
2095        let pool = self
2096            .delta_pool
2097            .as_ref()
2098            .ok_or_else(|| anyhow!("Delta pool not initialized"))?;
2099
2100        let delta_arc = pool
2101            .get(subject)
2102            .ok_or_else(|| anyhow!("No delta for subject '{}'", subject))?;
2103        let new_state_dict = {
2104            let delta = delta_arc.lock();
2105            delta.extract_state_dict()
2106        };
2107
2108        // Parse merge strategy (default to DO-Merge)
2109        let strategy_name = if merge_strategy_name.is_empty() {
2110            "do_merge"
2111        } else {
2112            merge_strategy_name
2113        };
2114        let weight = if merge_weight <= 0.0 || merge_weight > 1.0 {
2115            0.3
2116        } else {
2117            merge_weight as f64
2118        };
2119        let strategy = MergeStrategy::from_name(strategy_name, weight)?;
2120
2121        // Save as adapter file
2122        let adapter_mgr = crate::storage::AdapterManager::new(&self.state_path);
2123        let adapter_name = if name.is_empty() {
2124            format!("ttt_{}", subject)
2125        } else {
2126            name.to_owned()
2127        };
2128
2129        // Check for existing adapter to merge with (loads via FsOps)
2130        let existing_adapters = adapter_mgr.list_adapters().unwrap_or_default();
2131        let existing_state =
2132            if let Some(existing) = existing_adapters.iter().find(|a| a.name == adapter_name) {
2133                let rel_path = format!(
2134                    "adapters/{}",
2135                    existing
2136                        .path
2137                        .file_name()
2138                        .and_then(|f| f.to_str())
2139                        .unwrap_or("")
2140                );
2141                if let Some(ref fs) = self.fs {
2142                    match fs.read_file_chunked(&rel_path).await {
2143                        Ok(bytes) => crate::training::load_state_dict_from_bytes(&bytes)
2144                            .map_err(|e| {
2145                                tracing::debug!(
2146                                    "Could not load existing adapter '{}': {}",
2147                                    existing.name,
2148                                    e
2149                                );
2150                                e
2151                            })
2152                            .ok(),
2153                        Err(e) => {
2154                            tracing::debug!(
2155                                "Could not read existing adapter '{}': {}",
2156                                existing.name,
2157                                e
2158                            );
2159                            None
2160                        }
2161                    }
2162                } else {
2163                    None
2164                }
2165            } else {
2166                None
2167            };
2168
2169        // Determine adapter filename
2170        let adapter_filename =
2171            if let Some(existing) = existing_adapters.iter().find(|a| a.name == adapter_name) {
2172                existing
2173                    .path
2174                    .file_name()
2175                    .and_then(|f| f.to_str())
2176                    .unwrap_or("adapter.safetensors")
2177                    .to_owned()
2178            } else {
2179                let next_index = adapter_mgr.get_next_index().unwrap_or(0);
2180                format!("{:02}_{}.safetensors", next_index, adapter_name)
2181            };
2182
2183        // Apply merge strategy
2184        let final_state = if let Some(existing) = existing_state {
2185            merge_state_dicts(&existing, &new_state_dict, &strategy)?
2186        } else {
2187            new_state_dict
2188        };
2189
2190        // Write adapter file through FsOps (path-contained)
2191        let fs = self.fs.as_ref().ok_or_else(|| {
2192            anyhow!("FsOps not available — cannot write without path containment")
2193        })?;
2194        let rel_path = format!("adapters/{}", adapter_filename);
2195        fs.mkdir_p("adapters")
2196            .await
2197            .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
2198        let bytes = serialize_state_dict_to_bytes(&final_state)?;
2199        fs.write_file_chunked(&rel_path, &bytes)
2200            .await
2201            .map_err(|e| anyhow!("FsOps write_file failed: {}", e))?;
2202        let result_path = rel_path;
2203
2204        let actual_strategy = format!("{:?}", strategy).to_lowercase();
2205
2206        info!(
2207            "Saved adaptation for subject '{}' as adapter '{}' at {} (strategy: {})",
2208            subject, adapter_name, result_path, actual_strategy
2209        );
2210
2211        Ok(SaveAdaptationResult {
2212            adapter_name: adapter_name.clone(),
2213            adapter_path: result_path,
2214            content_hash: String::new(),
2215            merge_strategy: strategy_name.to_owned(),
2216        })
2217    }
2218
2219    /// Snapshot a tenant's delta to a file
2220    async fn handle_snapshot_delta(&self, subject: &Subject) -> Result<SnapshotDeltaResult> {
2221        let pool = self
2222            .delta_pool
2223            .as_ref()
2224            .ok_or_else(|| anyhow!("Delta pool not initialized"))?;
2225
2226        let delta_arc = pool
2227            .get(subject)
2228            .ok_or_else(|| anyhow!("No delta for subject '{}'", subject))?;
2229
2230        let filename = subject.to_string();
2231        let state_dict = {
2232            let delta = delta_arc.lock();
2233            delta.extract_state_dict()
2234        };
2235
2236        // Write snapshot through FsOps (path-contained)
2237        let fs = self.fs.as_ref().ok_or_else(|| {
2238            anyhow!("FsOps not available — cannot write without path containment")
2239        })?;
2240        let rel_snapshot = format!("adapters/.snapshots/{}.safetensors", filename);
2241        fs.mkdir_p("adapters/.snapshots")
2242            .await
2243            .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
2244        let bytes = serialize_state_dict_to_bytes(&state_dict)?;
2245        let size_bytes = bytes.len() as u64;
2246        fs.write_file_chunked(&rel_snapshot, &bytes)
2247            .await
2248            .map_err(|e| anyhow!("FsOps write_file failed: {}", e))?;
2249        let path_str = rel_snapshot;
2250
2251        // Re-acquire lock to update delta state
2252        let mut delta = delta_arc.lock();
2253        delta.last_snapshot_hash = Some(path_str.clone());
2254
2255        Ok(SnapshotDeltaResult {
2256            content_hash: path_str,
2257            size_bytes,
2258            accumulated_steps: delta.accumulated_steps,
2259            request_count: delta.request_count,
2260        })
2261    }
2262
2263    /// Export a tenant's delta as a PEFT-compatible adapter directory.
2264    ///
2265    /// Creates `adapters/{name}/adapter_model.safetensors` with HuggingFace PEFT naming
2266    /// and `adapters/{name}/adapter_config.json` with PEFT metadata.
2267    async fn handle_export_peft_adapter(
2268        &self,
2269        subject: &Subject,
2270        name: &str,
2271        _commit_message: &str,
2272    ) -> Result<ExportPeftResult> {
2273        let pool = self
2274            .delta_pool
2275            .as_ref()
2276            .ok_or_else(|| anyhow!("Delta pool not initialized"))?;
2277
2278        let delta_arc = pool
2279            .get(subject)
2280            .ok_or_else(|| anyhow!("No delta for subject '{}'", subject))?;
2281
2282        let adapter_name = if name.is_empty() {
2283            format!("ttt_{}", subject)
2284        } else {
2285            name.to_owned()
2286        };
2287
2288        // Extract all data from delta under lock, then drop lock before await
2289        let (safetensors_bytes, config_bytes) = {
2290            let delta = delta_arc.lock();
2291
2292            // Serialize as PEFT-compatible safetensors (with HuggingFace key naming)
2293            let safetensors_bytes = delta.serialize_to_safetensors_bytes()?;
2294
2295            // Generate PEFT adapter_config.json
2296            let adapter_config = serde_json::json!({
2297                "peft_type": "LORA",
2298                "auto_mapping": null,
2299                "base_model_name_or_path": "",
2300                "bias": "none",
2301                "fan_in_fan_out": false,
2302                "inference_mode": true,
2303                "init_lora_weights": true,
2304                "layers_to_transform": null,
2305                "layers_pattern": null,
2306                "lora_alpha": delta.scaling * delta.rank as f64,
2307                "lora_dropout": 0.0,
2308                "modules_to_save": null,
2309                "r": delta.rank,
2310                "rank_pattern": {},
2311                "alpha_pattern": {},
2312                "revision": null,
2313                "target_modules": delta.target_modules.clone(),
2314                "task_type": "CAUSAL_LM"
2315            });
2316            let config_bytes = serde_json::to_vec_pretty(&adapter_config)?;
2317            (safetensors_bytes, config_bytes)
2318        };
2319
2320        // Write through FsOps (path-contained)
2321        let fs = self.fs.as_ref().ok_or_else(|| {
2322            anyhow!("FsOps not available — cannot write without path containment")
2323        })?;
2324
2325        let dir_path = format!("adapters/{}", adapter_name);
2326        fs.mkdir_p(&dir_path)
2327            .await
2328            .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
2329
2330        let safetensors_path = format!("{}/adapter_model.safetensors", dir_path);
2331        fs.write_file_chunked(&safetensors_path, &safetensors_bytes)
2332            .await
2333            .map_err(|e| anyhow!("FsOps write adapter_model.safetensors failed: {}", e))?;
2334
2335        let config_path = format!("{}/adapter_config.json", dir_path);
2336        fs.write_file_chunked(&config_path, &config_bytes)
2337            .await
2338            .map_err(|e| anyhow!("FsOps write adapter_config.json failed: {}", e))?;
2339
2340        info!(
2341            "Exported PEFT adapter for subject '{}' as '{}' ({} bytes safetensors)",
2342            subject,
2343            adapter_name,
2344            safetensors_bytes.len()
2345        );
2346
2347        Ok(ExportPeftResult {
2348            adapter_path: dir_path,
2349            content_hash: String::new(),
2350        })
2351    }
2352
2353    /// Get TTT configuration (for status queries)
2354    #[allow(dead_code)]
2355    fn get_ttt_config(&self) -> Option<crate::training::ttt::TTTConfig> {
2356        self.ttt_trainer
2357            .as_ref()
2358            .map(|trainer| trainer.config.clone())
2359    }
2360
2361    /// Handle set session
2362    async fn handle_set_session(&self, session_id: String) -> Result<()> {
2363        // Track session ID for events
2364        *self.session_id.write() = Some(session_id.clone());
2365        self.engine
2366            .write()
2367            .set_session(CacheOwner::Session(session_id))
2368    }
2369
2370    /// Handle clear session
2371    async fn handle_clear_session(&self) {
2372        *self.session_id.write() = None;
2373        self.engine.write().clear_kv_cache();
2374    }
2375
2376    /// Handle release session
2377    async fn handle_release_session(&self, session_id: &str) -> Result<()> {
2378        self.engine
2379            .write()
2380            .release_session(&CacheOwner::Session(session_id.to_owned()))
2381    }
2382
2383    /// Load a PEFT adapter directory as the base delta.
2384    ///
2385    /// `path` is a PEFT adapter directory (e.g. `adapters/my-adapter`) containing
2386    /// `adapter_model.safetensors`. The loaded adapter is stored as `base_delta`
2387    /// and applied to all inference requests. If a per-tenant TTT delta also exists,
2388    /// the two are composed (corrections summed) during inference via `resolve_delta()`.
2389    pub async fn load_lora(&self, path: &Path) -> Result<()> {
2390        let device = self.engine.read().device();
2391        // Read via FsOps (path-contained)
2392        let fs = self
2393            .fs
2394            .as_ref()
2395            .ok_or_else(|| anyhow!("FsOps not available — cannot read without path containment"))?;
2396        let safetensors_path = format!("{}/adapter_model.safetensors", path.to_string_lossy());
2397        let bytes = fs.read_file_chunked(&safetensors_path).await.map_err(|e| {
2398            anyhow!(
2399                "Failed to read LoRA adapter from {}: {}",
2400                safetensors_path,
2401                e
2402            )
2403        })?;
2404        let delta = crate::training::TenantDelta::load_from_safetensors_bytes(&bytes, device)?;
2405
2406        // Warn if the adapter rank differs from the delta pool rank —
2407        // compose() now handles this via effective-weight decomposition,
2408        // but it's worth logging for visibility.
2409        if let Some(pool) = self.delta_pool.as_ref() {
2410            let pool_rank = pool.rank();
2411            if delta.rank != pool_rank {
2412                tracing::warn!(
2413                    "LoRA adapter rank ({}) differs from delta pool rank ({}). \
2414                     Composition will use effective-weight decomposition (slower but correct).",
2415                    delta.rank,
2416                    pool_rank
2417                );
2418            }
2419        }
2420
2421        *self.base_delta.lock() = Some(Arc::new(Mutex::new(delta)));
2422        self.lora_generation.fetch_add(1, Ordering::Release);
2423        tracing::info!("Loaded LoRA adapter as base delta from {}", path.display());
2424        Ok(())
2425    }
2426
2427    /// Save the current base delta to a safetensors file.
2428    pub async fn save_lora(&self, path: &str) -> Result<()> {
2429        let base = self.base_delta.lock().clone();
2430        if let Some(delta_arc) = base {
2431            let bytes = {
2432                let delta = delta_arc.lock();
2433                delta.serialize_to_safetensors_bytes()?
2434            };
2435            // Sanitize name and write via FsOps (path-contained)
2436            let fs = self.fs.as_ref().ok_or_else(|| {
2437                anyhow!("FsOps not available — cannot write without path containment")
2438            })?;
2439            let safe_name = sanitize_adapter_name(path)?;
2440            let rel_path = format!("adapters/{}.safetensors", safe_name);
2441            fs.mkdir_p("adapters")
2442                .await
2443                .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
2444            fs.write_file_chunked(&rel_path, &bytes)
2445                .await
2446                .map_err(|e| anyhow!("FsOps write_file failed: {}", e))?;
2447            Ok(())
2448        } else {
2449            Err(anyhow::anyhow!("No LoRA adapter loaded to save"))
2450        }
2451    }
2452
2453    /// Unload the current base LoRA adapter.
2454    pub async fn unload_lora(&self) -> Result<()> {
2455        let mut base = self.base_delta.lock();
2456        if base.is_some() {
2457            *base = None;
2458            self.lora_generation.fetch_add(1, Ordering::Release);
2459            tracing::info!("Unloaded base LoRA delta");
2460            Ok(())
2461        } else {
2462            Err(anyhow::anyhow!("No LoRA adapter loaded to unload"))
2463        }
2464    }
2465
2466    /// Check if a LoRA adapter (base delta) is loaded.
2467    pub async fn has_lora(&self) -> Result<bool> {
2468        Ok(self.base_delta.lock().is_some())
2469    }
2470
2471    /// Merge an on-disk PEFT adapter into the currently loaded base_delta.
2472    ///
2473    /// Reads the adapter from disk via WorktreeClient, loads it as a TenantDelta,
2474    /// then merges its corrections into the existing base_delta using the specified
2475    /// merge strategy. Requires a base_delta to already be loaded.
2476    pub async fn merge_lora(&self, path: &Path, weight: f32, strategy_name: &str) -> Result<()> {
2477        let device = self.engine.read().device();
2478
2479        // Read the adapter from disk via FsOps
2480        let fs = self
2481            .fs
2482            .as_ref()
2483            .ok_or_else(|| anyhow!("FsOps not available — cannot read without path containment"))?;
2484        let safetensors_path = format!("{}/adapter_model.safetensors", path.to_string_lossy());
2485        let bytes = fs
2486            .read_file_chunked(&safetensors_path)
2487            .await
2488            .map_err(|e| anyhow!("Failed to read adapter for merge: {}", e))?;
2489        let incoming = crate::training::TenantDelta::load_from_safetensors_bytes(&bytes, device)?;
2490
2491        // Get existing base_delta
2492        let base_arc = self.base_delta.lock().clone().ok_or_else(|| {
2493            anyhow!("No adapter loaded in base_delta register. Load one first with adapter.load.")
2494        })?;
2495
2496        // Parse merge strategy
2497        let strategy =
2498            crate::training::merge::MergeStrategy::from_name(strategy_name, weight as f64)?;
2499
2500        // Extract state dicts, merge, and update
2501        let merged_state = {
2502            let base = base_arc.lock();
2503            let existing = base.extract_state_dict();
2504            let new = incoming.extract_state_dict();
2505            crate::training::merge::merge_state_dicts(&existing, &new, &strategy)?
2506        };
2507
2508        // Load merged weights back into the base_delta
2509        {
2510            let mut base = base_arc.lock();
2511            base.load_state_dict(&merged_state)?;
2512        }
2513
2514        tracing::info!(
2515            "Merged adapter from {} into base_delta (strategy={}, weight={})",
2516            path.display(),
2517            strategy_name,
2518            weight
2519        );
2520        Ok(())
2521    }
2522}
2523
2524/// Sanitize an adapter name to prevent path traversal.
2525///
2526/// Strips path separators, `..`, and file extensions. Returns a safe filename stem.
2527/// Only allows alphanumeric characters, underscores, and hyphens.
2528fn sanitize_adapter_name(name: &str) -> Result<String> {
2529    let stem = std::path::Path::new(name)
2530        .file_stem()
2531        .and_then(|s| s.to_str())
2532        .unwrap_or(name);
2533
2534    // Reject path traversal
2535    if stem.contains("..") || stem.contains('/') || stem.contains('\\') || stem.is_empty() {
2536        return Err(anyhow!("Invalid adapter name: '{}'", name));
2537    }
2538
2539    // Only allow alphanumeric, underscore, hyphen
2540    let safe: String = stem
2541        .chars()
2542        .filter(|c| c.is_alphanumeric() || *c == '_' || *c == '-')
2543        .collect();
2544
2545    if safe.is_empty() {
2546        return Err(anyhow!(
2547            "Adapter name '{}' contains no valid characters",
2548            name
2549        ));
2550    }
2551
2552    Ok(safe)
2553}
2554
2555// ═══════════════════════════════════════════════════════════════════════════════
2556// InferenceHandler Implementation — generated dispatch for typed handler trait
2557// ═══════════════════════════════════════════════════════════════════════════════
2558
2559use crate::services::generated::inference_client::{
2560    dispatch_inference, serialize_response, InferenceHandler,
2561};
2562use hyprstream_rpc_std::inference_client::{
2563    AdaptationStrategy as AdaptationStrategyEnum, ChatTemplateRequest, DeltaStatusResult,
2564    EmbedImagesRequest, EmbedImagesResponse, ErrorInfo, ExportPeftRequest, ExportPeftResult,
2565    HealthStatus, InferenceResponseVariant, LoraConfig, MergeLoraRequest, ModuleNormRatio,
2566    SaveAdaptationRequest, SaveAdaptationResult, SnapshotDeltaResult, TrainStepRequest,
2567};
2568use hyprstream_rpc_std::policy_client::PolicyCheck;
2569// Conflicting names — use canonical path at usage sites:
2570//   inference_client::GenerationResult, inference_client::ModelInfo, inference_client::StreamInfo
2571
2572/// Map a capnp AdaptationStrategyEnum + optional threshold to the internal AdaptationStrategy.
2573fn map_adaptation_strategy(
2574    capnp_strategy: AdaptationStrategyEnum,
2575    writeback_threshold: Option<f32>,
2576) -> crate::training::adaptation_state::AdaptationStrategy {
2577    match capnp_strategy {
2578        AdaptationStrategyEnum::AutoWriteback => {
2579            crate::training::adaptation_state::AdaptationStrategy::AutoWriteback
2580        }
2581        AdaptationStrategyEnum::AutoEvict => {
2582            crate::training::adaptation_state::AdaptationStrategy::AutoEvict
2583        }
2584        AdaptationStrategyEnum::Speculative => {
2585            crate::training::adaptation_state::AdaptationStrategy::Speculative
2586        }
2587        AdaptationStrategyEnum::WritebackIfAbove => {
2588            crate::training::adaptation_state::AdaptationStrategy::WritebackIfAbove {
2589                threshold: writeback_threshold.unwrap_or(0.0),
2590            }
2591        }
2592    }
2593}
2594
2595#[async_trait::async_trait(?Send)]
2596impl InferenceHandler for InferenceService {
2597    async fn authorize(
2598        &self,
2599        ctx: &EnvelopeContext,
2600        resource: &str,
2601        operation: &str,
2602    ) -> Result<()> {
2603        // Single-service boundary (user-approved D5): controller internal work
2604        // was authorized at Model ingress — Model is the authority for this
2605        // leg. The worker re-checks the mandatory MAC floor against the
2606        // caller-claims snapshot carried in the verified work order and
2607        // requires the exact bound operation coordinate, but makes NO Policy
2608        // call: it is a subprocessor, not an independent authority domain.
2609        if let Some(work) = ctx.internal_work() {
2610            anyhow::ensure!(
2611                work.resource == resource && work.operation == operation,
2612                "internal work order does not authorize {operation} on {resource}"
2613            );
2614            enforce_inference_mac(ctx, &self.object_label)?;
2615            tracing::info!(
2616                target: "hyprstream.mac.audit",
2617                decision = "allow",
2618                subject = %work.subject,
2619                resource = %resource,
2620                action = %operation,
2621                request_id = ctx.request_id,
2622                plane = "rpc",
2623                via = "model-internal-work",
2624                controller = hex::encode(self.controller_pubkey.as_bytes()),
2625                "authorization decision"
2626            );
2627            return Ok(());
2628        }
2629        let subject = ctx.subject();
2630        let domain = self.authorization_domain(ctx)?;
2631        enforce_inference_mac(ctx, &self.object_label)?;
2632        let request = PolicyCheck {
2633            subject: subject.to_string(),
2634            domain,
2635            resource: resource.to_owned(),
2636            operation: operation.to_owned(),
2637        };
2638        let allowed = crate::services::policy::check_with_holder_evidence(
2639            &self.policy_client,
2640            &request,
2641            ctx.jwt_token(),
2642            &ctx.subject(),
2643            ctx.original_holder_evidence(),
2644        )
2645        .await
2646        .unwrap_or_else(|e| {
2647            warn!(
2648                "Policy check failed for {} on {}: {} - denying access",
2649                subject, resource, e
2650            );
2651            false
2652        });
2653        if allowed {
2654            Ok(())
2655        } else {
2656            anyhow::bail!(
2657                "Unauthorized: {} cannot {} on {}",
2658                subject,
2659                operation,
2660                resource
2661            )
2662        }
2663    }
2664
2665    async fn handle_generate_stream(
2666        &self,
2667        ctx: &EnvelopeContext,
2668        _request_id: u64,
2669        data: &hyprstream_rpc_std::inference_client::GenerationRequest,
2670    ) -> Result<(
2671        hyprstream_rpc_std::inference_client::StreamInfo,
2672        hyprstream_rpc::service::Continuation,
2673    )> {
2674        let subject = ctx.subject();
2675        let request = data.clone();
2676
2677        // Build per-request TTT overrides from RPC fields
2678        let ttt_overrides = crate::training::ttt::TTTOverrides {
2679            enabled: if data.ttt_enabled { Some(true) } else { None },
2680            gradient_steps: data.ttt_gradient_steps,
2681            learning_rate: data.ttt_learning_rate,
2682            adaptation_strategy: map_adaptation_strategy(
2683                data.adaptation_strategy,
2684                data.writeback_threshold,
2685            ),
2686            max_adaptation_ms: None,
2687        };
2688
2689        // Calculate expiry from claims
2690        let expiry_secs = ctx
2691            .claims()
2692            .map(|c| c.exp - chrono::Utc::now().timestamp())
2693            .unwrap_or(600)
2694            .max(600);
2695
2696        let client_ephemeral_pubkey = ctx.ephemeral_pubkey();
2697        let claims = ctx.claims().cloned();
2698        // #1264: the verified caller's self-certifying pairwise DID — the ledger
2699        // account owner spent at completion. `ctx.subject()` is the Casbin-facing
2700        // authorization label (e.g. "alice"), not a DID and never a ledger
2701        // principal; the self-certifying DID is derived from the same verified
2702        // envelope (svc.rs / the enforcer's S1 model). Anonymous ⇒ `None`.
2703        //
2704        // Single-service boundary: on controller internal work the envelope
2705        // signer is the PINNED CONTROLLER, so the spend owner comes from the
2706        // verified work order's original-caller DID; only direct callers fall
2707        // back to their own signer DID.
2708        #[cfg(feature = "ledger")]
2709        let owner_did = ctx
2710            .internal_work()
2711            .and_then(|work| work.owner_did.clone())
2712            .or_else(|| {
2713                ctx.authenticated_pairwise_did()
2714                    .map(|d| d.as_str().to_owned())
2715            });
2716        #[cfg(not(feature = "ledger"))]
2717        let owner_did: Option<String> = None;
2718        let (stream_id, server_pubkey, broadcast_path, reach, pending) = self
2719            .prepare_stream(
2720                request,
2721                client_ephemeral_pubkey.as_ref().map(<[u8; 32]>::as_slice),
2722                claims,
2723                expiry_secs,
2724                &subject,
2725                ttt_overrides,
2726                owner_did,
2727            )
2728            .await?;
2729
2730        let stream_info = hyprstream_rpc_std::inference_client::StreamInfo {
2731            stream_id,
2732            dh_public: server_pubkey,
2733            qos: <hyprstream_rpc::stream_info::Job as hyprstream_rpc::stream_info::StreamOptPreset>::stream_opt(),
2734            broadcast_path,
2735            announced_at: reach,
2736            moql_server_identity: Default::default(),
2737            kem_ciphertexts: Vec::new(), // #554: classical stream (dh_public path), no hybrid KEM
2738        };
2739
2740        // Build continuation that executes the stream after REP is sent.
2741        // With tokio::sync::RwLock, the read guard is Send-safe, so no
2742        // UnsafeSendFuture wrapper is needed.
2743        let service = self.clone();
2744        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
2745            service.execute_stream(pending).await;
2746        });
2747
2748        Ok((stream_info, continuation))
2749    }
2750
2751    async fn handle_model_info(
2752        &self,
2753        _ctx: &EnvelopeContext,
2754        _request_id: u64,
2755    ) -> Result<InferenceResponseVariant> {
2756        let mut info = InferenceService::handle_model_info(self).await;
2757        info.lora_loaded = self.base_delta.lock().is_some();
2758        Ok(InferenceResponseVariant::ModelInfoResult(info))
2759    }
2760
2761    async fn handle_is_ready(
2762        &self,
2763        _ctx: &EnvelopeContext,
2764        _request_id: u64,
2765    ) -> Result<InferenceResponseVariant> {
2766        let ready = InferenceService::handle_is_ready(self).await;
2767        Ok(InferenceResponseVariant::IsReadyResult(ready))
2768    }
2769
2770    async fn handle_get_layer_profile(
2771        &self,
2772        _ctx: &EnvelopeContext,
2773        _request_id: u64,
2774    ) -> Result<InferenceResponseVariant> {
2775        use crate::runtime::ttn_profile;
2776        use hyprstream_rpc_std::inference_client::LayerProfileResult;
2777
2778        let model_config = crate::runtime::model_config::ModelConfig::load(
2779            &self.model_path,
2780            &std::collections::HashMap::new(),
2781        )?;
2782        let profile = ttn_profile::get_layer_profile(&self.model_path, &model_config, None)?;
2783        let json = serde_json::to_string_pretty(&profile)?;
2784
2785        Ok(InferenceResponseVariant::GetLayerProfileResult(
2786            LayerProfileResult { json },
2787        ))
2788    }
2789
2790    async fn handle_apply_chat_template(
2791        &self,
2792        _ctx: &EnvelopeContext,
2793        _request_id: u64,
2794        data: &ChatTemplateRequest,
2795    ) -> Result<InferenceResponseVariant> {
2796        let chat_messages: Vec<crate::runtime::template_engine::ChatMessage> = data
2797            .messages
2798            .iter()
2799            .map(|m| {
2800                let tool_calls = if m.tool_calls.is_empty() {
2801                    None
2802                } else {
2803                    // Convert RPC ToolCall (arguments: String) to TemplateToolCall
2804                    // (arguments: Value) so templates can iterate with |items.
2805                    Some(
2806                        m.tool_calls
2807                            .iter()
2808                            .map(crate::runtime::template_engine::TemplatToolCall::from)
2809                            .collect(),
2810                    )
2811                };
2812                crate::runtime::template_engine::ChatMessage {
2813                    role: m.role.clone(),
2814                    content: if m.content.is_empty() {
2815                        None
2816                    } else {
2817                        Some(m.content.clone())
2818                    },
2819                    tool_calls,
2820                    tool_call_id: if m.tool_call_id.is_empty() {
2821                        None
2822                    } else {
2823                        Some(m.tool_call_id.clone())
2824                    },
2825                }
2826            })
2827            .collect();
2828        // tools_json is passed as a JSON string via the schema; parse it here
2829        let tools: Option<serde_json::Value> = data
2830            .tools_json
2831            .as_deref()
2832            .filter(|s| !s.is_empty())
2833            .and_then(|s| serde_json::from_str(s).ok());
2834
2835        // Context truncation: when maxTokens is set, drop oldest non-system messages
2836        // so the rendered prompt fits in max_seq_len - maxTokens.
2837        // Returns error if even system + last message exceeds the budget.
2838        let chat_messages = if let Some(reserve) = data.max_tokens {
2839            self.truncate_messages_to_budget(chat_messages, reserve as usize)?
2840        } else {
2841            chat_messages
2842        };
2843
2844        let result = InferenceService::handle_apply_chat_template(
2845            self,
2846            chat_messages,
2847            data.add_generation_prompt,
2848            tools.as_ref(),
2849            data.enable_thinking,
2850            data.template_vars_json.as_deref(),
2851        )
2852        .await?;
2853        Ok(InferenceResponseVariant::ApplyChatTemplateResult(result))
2854    }
2855
2856    async fn handle_create_lora(
2857        &self,
2858        _ctx: &EnvelopeContext,
2859        _request_id: u64,
2860        data: &LoraConfig,
2861    ) -> Result<InferenceResponseVariant> {
2862        let defaults = crate::training::TenantDeltaConfig::default();
2863        let config = crate::training::TenantDeltaConfig {
2864            rank: data.rank as usize,
2865            alpha: data.alpha.unwrap_or(defaults.alpha),
2866            dropout: data.dropout.unwrap_or(defaults.dropout),
2867            target_modules: data.target_modules.clone(),
2868            learning_rate: data
2869                .learning_rate
2870                .map(|v| v as f64)
2871                .unwrap_or(defaults.learning_rate),
2872            ..defaults
2873        };
2874        InferenceService::handle_create_lora(self, config).await?;
2875        Ok(InferenceResponseVariant::CreateLoraResult)
2876    }
2877
2878    async fn handle_load_lora(
2879        &self,
2880        _ctx: &EnvelopeContext,
2881        _request_id: u64,
2882        value: &str,
2883    ) -> Result<InferenceResponseVariant> {
2884        self.load_lora(Path::new(value)).await?;
2885        Ok(InferenceResponseVariant::LoadLoraResult)
2886    }
2887
2888    async fn handle_save_lora(
2889        &self,
2890        _ctx: &EnvelopeContext,
2891        _request_id: u64,
2892        value: &str,
2893    ) -> Result<InferenceResponseVariant> {
2894        self.save_lora(value).await?;
2895        Ok(InferenceResponseVariant::SaveLoraResult)
2896    }
2897
2898    async fn handle_unload_lora(
2899        &self,
2900        _ctx: &EnvelopeContext,
2901        _request_id: u64,
2902    ) -> Result<InferenceResponseVariant> {
2903        self.unload_lora().await?;
2904        Ok(InferenceResponseVariant::UnloadLoraResult)
2905    }
2906
2907    async fn handle_has_lora(
2908        &self,
2909        _ctx: &EnvelopeContext,
2910        _request_id: u64,
2911    ) -> Result<InferenceResponseVariant> {
2912        let has = self.has_lora().await?;
2913        Ok(InferenceResponseVariant::HasLoraResult(has))
2914    }
2915
2916    async fn handle_set_session(
2917        &self,
2918        _ctx: &EnvelopeContext,
2919        _request_id: u64,
2920        value: &str,
2921    ) -> Result<InferenceResponseVariant> {
2922        InferenceService::handle_set_session(self, value.to_owned()).await?;
2923        Ok(InferenceResponseVariant::SetSessionResult)
2924    }
2925
2926    async fn handle_clear_session(
2927        &self,
2928        _ctx: &EnvelopeContext,
2929        _request_id: u64,
2930    ) -> Result<InferenceResponseVariant> {
2931        InferenceService::handle_clear_session(self).await;
2932        Ok(InferenceResponseVariant::ClearSessionResult)
2933    }
2934
2935    async fn handle_release_session(
2936        &self,
2937        _ctx: &EnvelopeContext,
2938        _request_id: u64,
2939        value: &str,
2940    ) -> Result<InferenceResponseVariant> {
2941        InferenceService::handle_release_session(self, value).await?;
2942        Ok(InferenceResponseVariant::ReleaseSessionResult)
2943    }
2944
2945    async fn handle_health_check(
2946        &self,
2947        _ctx: &EnvelopeContext,
2948        _request_id: u64,
2949    ) -> Result<InferenceResponseVariant> {
2950        let model_loaded = self.engine.read().is_loaded();
2951        let network_ready = self.network_ready.load(Ordering::Acquire);
2952        let draining = self.draining.load(Ordering::Acquire);
2953        let status = if draining {
2954            "draining"
2955        } else if !model_loaded {
2956            "not_loaded"
2957        } else if !network_ready {
2958            "stream_plane_unavailable"
2959        } else {
2960            "ok"
2961        };
2962        Ok(InferenceResponseVariant::HealthCheckResult(HealthStatus {
2963            status: status.into(),
2964            model_loaded,
2965            kv_cache_usage_percent: 0.0,
2966            gpu_memory_used_mb: 0,
2967            gpu_memory_total_mb: 0,
2968        }))
2969    }
2970
2971    async fn handle_shutdown(
2972        &self,
2973        _ctx: &EnvelopeContext,
2974        _request_id: u64,
2975    ) -> Result<InferenceResponseVariant> {
2976        info!("Inference service shutdown requested");
2977        self.draining.store(true, Ordering::Release);
2978        self.network_ready.store(false, Ordering::Release);
2979        let shutdown = Arc::clone(&self.shutdown);
2980        tokio::spawn(async move {
2981            tokio::task::yield_now().await;
2982            shutdown.notify_waiters();
2983        });
2984        Ok(InferenceResponseVariant::Success)
2985    }
2986
2987    async fn handle_ttt_writeback(
2988        &self,
2989        ctx: &EnvelopeContext,
2990        _request_id: u64,
2991    ) -> Result<InferenceResponseVariant> {
2992        let subject = ctx.subject();
2993        InferenceService::handle_writeback(self, &subject)?;
2994        Ok(InferenceResponseVariant::TttWritebackResult)
2995    }
2996
2997    async fn handle_ttt_evict(
2998        &self,
2999        ctx: &EnvelopeContext,
3000        _request_id: u64,
3001    ) -> Result<InferenceResponseVariant> {
3002        let subject = ctx.subject();
3003        InferenceService::handle_evict(self, &subject)?;
3004        Ok(InferenceResponseVariant::TttEvictResult)
3005    }
3006
3007    async fn handle_train_step(
3008        &self,
3009        _ctx: &EnvelopeContext,
3010        _request_id: u64,
3011        _data: &TrainStepRequest,
3012    ) -> Result<InferenceResponseVariant> {
3013        // DEPRECATED: trainStep @17 blocks the REQ/REP loop during forward/backward pass.
3014        // Use trainStepStream instead, which returns StreamInfo immediately and runs
3015        // the training in a continuation after the REP is sent.
3016        Err(anyhow!(
3017            "trainStep is deprecated — use trainStepStream for non-blocking training"
3018        ))
3019    }
3020
3021    async fn handle_train_step_stream(
3022        &self,
3023        ctx: &EnvelopeContext,
3024        _request_id: u64,
3025        data: &TrainStepRequest,
3026    ) -> Result<(
3027        hyprstream_rpc_std::inference_client::StreamInfo,
3028        hyprstream_rpc::service::Continuation,
3029    )> {
3030        let subject = ctx.subject();
3031
3032        // DH key derivation
3033        let client_pub_bytes = ctx.ephemeral_pubkey().ok_or_else(|| {
3034            anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
3035        })?;
3036        let client_pub_ref: &[u8] = &client_pub_bytes;
3037        let stream_channel = self
3038            .stream_channel
3039            .as_ref()
3040            .ok_or_else(|| anyhow!("StreamChannel not initialized"))?;
3041
3042        let expiry_secs = ctx
3043            .claims()
3044            .map(|c| c.exp - chrono::Utc::now().timestamp())
3045            .unwrap_or(600)
3046            .max(600);
3047        let claims = ctx.claims().cloned();
3048
3049        let stream_ctx = stream_channel
3050            .prepare_third_party_interop_stream_with_claims(client_pub_ref, expiry_secs, claims)
3051            .await?
3052            .with_qos_preset::<hyprstream_rpc::stream_info::Job>();
3053
3054        let stream_id = stream_ctx.stream_id().to_owned();
3055        let server_pubkey = *stream_ctx.server_pubkey();
3056        let broadcast_path = hyprstream_rpc::moq_stream::global_moq_origin()
3057            .map(|o| o.broadcast_path(stream_ctx.topic()))
3058            .unwrap_or_default();
3059
3060        let stream_info = hyprstream_rpc_std::inference_client::StreamInfo {
3061            stream_id,
3062            dh_public: server_pubkey,
3063            qos: stream_ctx.qos().clone(),
3064            broadcast_path,
3065            announced_at: stream_ctx.reach(), // #384: per-stream reach via ctx
3066            moql_server_identity: stream_ctx.moql_server_identity(),
3067            kem_ciphertexts: Vec::new(), // #554: classical stream (dh_public path), no hybrid KEM
3068        };
3069
3070        let adaptation_strategy =
3071            map_adaptation_strategy(data.adaptation_strategy, data.writeback_threshold);
3072        let pending = PendingWork::Training {
3073            stream_ctx,
3074            subject,
3075            input: data.input.clone(),
3076            gradient_steps: data.gradient_steps,
3077            learning_rate: data.learning_rate,
3078            adaptation_strategy,
3079        };
3080
3081        // Build continuation
3082        let service = self.clone();
3083        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3084            service.execute_training_stream(pending).await;
3085        });
3086
3087        Ok((stream_info, continuation))
3088    }
3089
3090    async fn handle_ttt_zero(
3091        &self,
3092        ctx: &EnvelopeContext,
3093        _request_id: u64,
3094    ) -> Result<InferenceResponseVariant> {
3095        let subject = ctx.subject();
3096        InferenceService::handle_reset_delta(self, &subject)?;
3097        Ok(InferenceResponseVariant::TttZeroResult)
3098    }
3099
3100    async fn handle_get_delta_status(
3101        &self,
3102        ctx: &EnvelopeContext,
3103        _request_id: u64,
3104    ) -> Result<InferenceResponseVariant> {
3105        let subject = ctx.subject();
3106        let result = InferenceService::handle_get_delta_status(self, &subject)?;
3107        Ok(InferenceResponseVariant::GetDeltaStatusResult(result))
3108    }
3109
3110    async fn handle_save_adaptation(
3111        &self,
3112        ctx: &EnvelopeContext,
3113        _request_id: u64,
3114        data: &SaveAdaptationRequest,
3115    ) -> Result<InferenceResponseVariant> {
3116        let subject = ctx.subject();
3117        let result = InferenceService::handle_save_adaptation(
3118            self,
3119            &subject,
3120            &data.name,
3121            data.merge_strategy.as_deref().unwrap_or(""),
3122            data.merge_weight.unwrap_or(0.0),
3123        )
3124        .await?;
3125        Ok(InferenceResponseVariant::SaveAdaptationResult(result))
3126    }
3127
3128    async fn handle_snapshot_delta(
3129        &self,
3130        ctx: &EnvelopeContext,
3131        _request_id: u64,
3132    ) -> Result<InferenceResponseVariant> {
3133        let subject = ctx.subject();
3134        let result = InferenceService::handle_snapshot_delta(self, &subject).await?;
3135        Ok(InferenceResponseVariant::SnapshotDeltaResult(result))
3136    }
3137
3138    async fn handle_export_peft_adapter(
3139        &self,
3140        ctx: &EnvelopeContext,
3141        _request_id: u64,
3142        data: &ExportPeftRequest,
3143    ) -> Result<InferenceResponseVariant> {
3144        let subject = ctx.subject();
3145        let result = InferenceService::handle_export_peft_adapter(
3146            self,
3147            &subject,
3148            &data.name,
3149            data.commit_message.as_deref().unwrap_or(""),
3150        )
3151        .await?;
3152        Ok(InferenceResponseVariant::ExportPeftAdapterResult(result))
3153    }
3154
3155    async fn handle_merge_lora(
3156        &self,
3157        _ctx: &EnvelopeContext,
3158        _request_id: u64,
3159        data: &MergeLoraRequest,
3160    ) -> Result<InferenceResponseVariant> {
3161        let weight = data.weight.filter(|&w| w > 0.0).unwrap_or(1.0);
3162        let strategy = data
3163            .strategy
3164            .as_deref()
3165            .filter(|s| !s.is_empty())
3166            .unwrap_or("do_merge");
3167        self.merge_lora(Path::new(&data.adapter_path), weight, strategy)
3168            .await?;
3169        Ok(InferenceResponseVariant::MergeLoraResult)
3170    }
3171
3172    // =========================================================================
3173    // Streaming handler variants
3174    //
3175    // Each returns (StreamInfo, Continuation) immediately. The continuation
3176    // calls an execute_*_stream method on the cloned service, which accesses
3177    // self.stream_channel by reference (same pattern as execute_training_stream).
3178    // =========================================================================
3179
3180    async fn handle_create_lora_stream(
3181        &self,
3182        ctx: &EnvelopeContext,
3183        _request_id: u64,
3184        data: &LoraConfig,
3185    ) -> Result<(
3186        hyprstream_rpc_std::inference_client::StreamInfo,
3187        hyprstream_rpc::service::Continuation,
3188    )> {
3189        let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3190        let service = self.clone();
3191        let defaults = crate::training::TenantDeltaConfig::default();
3192        let config = crate::training::TenantDeltaConfig {
3193            rank: data.rank as usize,
3194            alpha: data.alpha.unwrap_or(defaults.alpha),
3195            dropout: data.dropout.unwrap_or(defaults.dropout),
3196            target_modules: data.target_modules.clone(),
3197            learning_rate: data
3198                .learning_rate
3199                .map(|v| v as f64)
3200                .unwrap_or(defaults.learning_rate),
3201            ..defaults
3202        };
3203        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3204            service.execute_create_lora_stream(stream_ctx, config).await;
3205        });
3206        Ok((stream_info, continuation))
3207    }
3208
3209    async fn handle_load_lora_stream(
3210        &self,
3211        ctx: &EnvelopeContext,
3212        _request_id: u64,
3213        value: &str,
3214    ) -> Result<(
3215        hyprstream_rpc_std::inference_client::StreamInfo,
3216        hyprstream_rpc::service::Continuation,
3217    )> {
3218        let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3219        let service = self.clone();
3220        let path = value.to_owned();
3221        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3222            service.execute_load_lora_stream(stream_ctx, path).await;
3223        });
3224        Ok((stream_info, continuation))
3225    }
3226
3227    async fn handle_save_lora_stream(
3228        &self,
3229        ctx: &EnvelopeContext,
3230        _request_id: u64,
3231        value: &str,
3232    ) -> Result<(
3233        hyprstream_rpc_std::inference_client::StreamInfo,
3234        hyprstream_rpc::service::Continuation,
3235    )> {
3236        let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3237        let service = self.clone();
3238        let name = value.to_owned();
3239        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3240            service.execute_save_lora_stream(stream_ctx, name).await;
3241        });
3242        Ok((stream_info, continuation))
3243    }
3244
3245    async fn handle_save_adaptation_stream(
3246        &self,
3247        ctx: &EnvelopeContext,
3248        _request_id: u64,
3249        data: &SaveAdaptationRequest,
3250    ) -> Result<(
3251        hyprstream_rpc_std::inference_client::StreamInfo,
3252        hyprstream_rpc::service::Continuation,
3253    )> {
3254        let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3255        let service = self.clone();
3256        let subject = ctx.subject();
3257        let name = data.name.clone();
3258        let merge_strategy = data.merge_strategy.clone();
3259        let merge_weight = data.merge_weight;
3260        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3261            service
3262                .execute_save_adaptation_stream(
3263                    stream_ctx,
3264                    subject,
3265                    name,
3266                    merge_strategy.unwrap_or_default(),
3267                    merge_weight.unwrap_or(0.0),
3268                )
3269                .await;
3270        });
3271        Ok((stream_info, continuation))
3272    }
3273
3274    async fn handle_snapshot_delta_stream(
3275        &self,
3276        ctx: &EnvelopeContext,
3277        _request_id: u64,
3278    ) -> Result<(
3279        hyprstream_rpc_std::inference_client::StreamInfo,
3280        hyprstream_rpc::service::Continuation,
3281    )> {
3282        let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3283        let service = self.clone();
3284        let subject = ctx.subject();
3285        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3286            service
3287                .execute_snapshot_delta_stream(stream_ctx, subject)
3288                .await;
3289        });
3290        Ok((stream_info, continuation))
3291    }
3292
3293    async fn handle_export_peft_adapter_stream(
3294        &self,
3295        ctx: &EnvelopeContext,
3296        _request_id: u64,
3297        data: &ExportPeftRequest,
3298    ) -> Result<(
3299        hyprstream_rpc_std::inference_client::StreamInfo,
3300        hyprstream_rpc::service::Continuation,
3301    )> {
3302        let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3303        let service = self.clone();
3304        let subject = ctx.subject();
3305        let name = data.name.clone();
3306        let commit_message = data.commit_message.clone();
3307        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3308            service
3309                .execute_export_peft_adapter_stream(
3310                    stream_ctx,
3311                    subject,
3312                    name,
3313                    commit_message.unwrap_or_default(),
3314                )
3315                .await;
3316        });
3317        Ok((stream_info, continuation))
3318    }
3319
3320    async fn handle_merge_lora_stream(
3321        &self,
3322        ctx: &EnvelopeContext,
3323        _request_id: u64,
3324        data: &MergeLoraRequest,
3325    ) -> Result<(
3326        hyprstream_rpc_std::inference_client::StreamInfo,
3327        hyprstream_rpc::service::Continuation,
3328    )> {
3329        let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3330        let service = self.clone();
3331        let adapter_path = data.adapter_path.clone();
3332        let weight = data.weight.filter(|&w| w > 0.0).unwrap_or(1.0);
3333        let strategy = data
3334            .strategy
3335            .as_deref()
3336            .filter(|s| !s.is_empty())
3337            .unwrap_or("do_merge")
3338            .to_owned();
3339        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3340            service
3341                .execute_merge_lora_stream(stream_ctx, adapter_path, weight, strategy)
3342                .await;
3343        });
3344        Ok((stream_info, continuation))
3345    }
3346
3347    async fn handle_embed(
3348        &self,
3349        _ctx: &EnvelopeContext,
3350        _request_id: u64,
3351        data: &EmbedImagesRequest,
3352    ) -> Result<InferenceResponseVariant> {
3353        let engine = self.engine.read();
3354        let embeddings = engine.embed_images(&data.images)?;
3355        let dimensions = embeddings.first().map(|v| v.len() as u32).unwrap_or(0);
3356        Ok(InferenceResponseVariant::EmbedResult(EmbedImagesResponse {
3357            embeddings,
3358            dimensions,
3359        }))
3360    }
3361}
3362
3363// ═══════════════════════════════════════════════════════════════════════════════
3364// RequestService adapter and Spawnable implementation
3365// ═══════════════════════════════════════════════════════════════════════════════
3366
3367// verify_claims is now handled by the default RequestService::verify_claims() implementation
3368// in hyprstream-rpc (E2E JWT verification for all non-local identities).
3369
3370/// RequestService adapter for InferenceService.
3371///
3372/// Wraps `InferenceService` to implement `RequestService` for use with `RequestLoop`.
3373/// This adapter is created inside `Spawnable::run()` on the service thread —
3374/// it never crosses thread boundaries.
3375struct InferenceZmqAdapter {
3376    service: InferenceService,
3377    transport: hyprstream_rpc::transport::TransportConfig,
3378    signing_key: SigningKey,
3379    expected_audience: Option<String>,
3380    jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
3381    /// Model reference binding; `None` ⇒ internal work orders are never
3382    /// admitted (standalone direct-caller service, fail-closed).
3383    instance_model_ref: Option<String>,
3384    /// EXACT versioned internal-work audience of THIS worker incarnation
3385    /// (`iw1/{service name}/{worker-generated incarnation}`). Work orders
3386    /// naming anything else are denied.
3387    instance_audience: String,
3388}
3389
3390#[async_trait::async_trait(?Send)]
3391impl hyprstream_rpc::service::RequestService for InferenceZmqAdapter {
3392    fn decode_request_body(
3393        &self,
3394        signed_body: &[u8],
3395    ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
3396        // The ONE bounded decode (v16 §5.2): the generated decoder derives
3397        // the full method leaf and returns the decoded message that policy,
3398        // MAC, and dispatch below all consume.
3399        crate::services::generated::inference_client::decode_inference_request_body(signed_body)
3400    }
3401
3402    async fn handle_request(
3403        &self,
3404        ctx: &EnvelopeContext,
3405        body: &hyprstream_rpc::service::DecodedRequestBody,
3406    ) -> anyhow::Result<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
3407        dispatch_inference(&self.service, ctx, body).await
3408    }
3409
3410    fn name(&self) -> &str {
3411        // Keep the canonical MAC object label even though the Spawnable
3412        // registration is instance-specific.
3413        "inference"
3414    }
3415
3416    fn transport(&self) -> &hyprstream_rpc::transport::TransportConfig {
3417        &self.transport
3418    }
3419
3420    fn signing_key(&self) -> SigningKey {
3421        self.signing_key.clone()
3422    }
3423
3424    fn expected_audience(&self) -> Option<&str> {
3425        self.expected_audience.as_deref()
3426    }
3427
3428    fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
3429        self.jwt_key_source.clone()
3430    }
3431    fn accept_delegated_bearer(&self, signer_pubkey: &[u8; 32]) -> bool {
3432        signer_pubkey == &self.service.controller_pubkey.to_bytes()
3433    }
3434
3435    /// Single-service boundary admission (user-approved D5): the pinned
3436    /// allocated instance executes only its controller's internal work orders,
3437    /// bound to THIS instance/tenant/model and one exact operation.
3438    ///
3439    /// Fail-closed on: missing instance binding (standalone service), a foreign
3440    /// envelope signer, wrong audience/tenant/model, expired/oversized/replayed
3441    /// tokens, and missing caller snapshots. Verified against the PINNED
3442    /// controller key — never against the CA key source.
3443    async fn verify_internal_work_bearer(
3444        &self,
3445        ctx: &EnvelopeContext,
3446        token: &str,
3447        _protected: &hyprstream_rpc::auth::ProtectedHeader,
3448    ) -> anyhow::Result<hyprstream_rpc::auth::internal_work::InternalWorkClaims> {
3449        let instance_model_ref = self.instance_model_ref.as_ref().ok_or_else(|| {
3450            anyhow::anyhow!("standalone inference accepts no internal work bearer")
3451        })?;
3452        anyhow::ensure!(
3453            ctx.cnf == self.service.controller_pubkey.to_bytes(),
3454            "internal work bearer envelope was not signed by the pinned controller"
3455        );
3456        let now = chrono::Utc::now().timestamp();
3457        let claims = hyprstream_rpc::auth::internal_work::verify_internal_work(
3458            token,
3459            &self.service.controller_pubkey,
3460            &self.instance_audience,
3461            now,
3462        )?;
3463        anyhow::ensure!(
3464            claims.tenant == self.service.tenant_domain,
3465            "internal work token tenant does not match this instance binding"
3466        );
3467        anyhow::ensure!(
3468            claims.model == *instance_model_ref,
3469            "internal work token model does not match this instance binding"
3470        );
3471        anyhow::ensure!(
3472            hyprstream_rpc::auth::internal_work::admit_internal_work_jti_once(
3473                &claims.jti,
3474                claims.exp,
3475                now,
3476            ),
3477            "internal work token jti was already admitted (replay)"
3478        );
3479        Ok(claims)
3480    }
3481
3482    /// Resolve a verified mesh-peer signer key to its per-host subject (#328).
3483    ///
3484    /// Routes through the global trust store, which is populated at startup from
3485    /// the admin-anchored `mesh_peers` roster (see
3486    /// `hyprstream::auth::mesh_trust::build_mesh_identity_roster`). A networked
3487    /// peer whose key is enrolled resolves to `service:inference:host-<label>`;
3488    /// an unenrolled peer resolves to `None` → anonymous (fail-closed,
3489    /// deny-by-default — never the `"system"` god principal).
3490    fn resolve_key_subject(
3491        &self,
3492        signer_pubkey: &[u8; 32],
3493    ) -> Option<hyprstream_rpc::envelope::Subject> {
3494        hyprstream_service::global_trust_store().resolve_subject(signer_pubkey)
3495    }
3496
3497    fn build_error_payload(&self, request_id: u64, error: &str) -> Vec<u8> {
3498        let variant = InferenceResponseVariant::Error(ErrorInfo {
3499            message: error.to_owned(),
3500            code: "INTERNAL".to_owned(),
3501            details: String::new(),
3502        });
3503        serialize_response(request_id, &variant).unwrap_or_else(|e| {
3504            error!("Failed to serialize error payload: {}", e);
3505            vec![]
3506        })
3507    }
3508}
3509
3510/// Private per-attempt readiness handoff from the worker to its owning
3511/// ModelService (Sol bounded incarnation plan). The worker generates its own
3512/// 256-bit incarnation inside its service thread and reports the exact
3513/// versioned audience plus the spawn-attempt identity through this channel —
3514/// never via a request, resolver record, or cached deterministic name.
3515#[derive(Clone)]
3516pub struct IncarnationHandoff {
3517    /// The deterministic instance name this worker was spawned for.
3518    pub instance_service_name: String,
3519    /// The pinned controller (Model) key this worker was configured with.
3520    pub controller_pubkey: [u8; 32],
3521    /// The spawn-attempt generation the ModelService allocated.
3522    pub generation: u64,
3523    /// Worker-generated 256-bit incarnation (hex-encoded).
3524    pub incarnation: String,
3525    /// Exact versioned internal-work audience:
3526    /// `iw1/{instance_service_name}/{incarnation}`.
3527    pub audience: String,
3528}
3529
3530/// Seconds a ModelService waits for the worker's incarnation handoff after
3531/// the spawner reports readiness. Missing readiness fails the load closed.
3532pub const INCARNATION_HANDOFF_TIMEOUT_SECS: u64 = 30;
3533
3534/// The versioned internal-work audience for one worker incarnation.
3535/// Exact-comparison string: no fallback to the deterministic name alone.
3536pub(crate) fn internal_work_audience(service_name: &str, incarnation: &str) -> String {
3537    format!("iw1/{service_name}/{incarnation}")
3538}
3539
3540/// Configuration for spawning an InferenceService.
3541///
3542/// This struct holds all parameters needed to initialize an InferenceService.
3543/// It is `Send` and can be moved to a dedicated thread via `Spawnable::run()`.
3544/// The actual GPU initialization happens on the service thread.
3545pub struct InferenceServiceConfig {
3546    service_name: String,
3547    model_path: PathBuf,
3548    state_path: PathBuf,
3549    pinned_artifact: Option<Arc<crate::storage::pinned_model::PinnedModelArtifact>>,
3550    config: RuntimeConfig,
3551    server_pubkey: VerifyingKey,
3552    signing_key: SigningKey,
3553    /// PolicyClient is created lazily inside `run()` on the service thread's runtime.
3554    /// ZMQ sockets must be registered with the correct runtime's reactor — a PolicyClient
3555    /// created in the caller's runtime will fail with "Tokio context being shutdown" when
3556    /// used on a different thread's runtime.
3557    policy_signing_key: SigningKey,
3558    policy_transport: hyprstream_rpc::transport::TransportConfig,
3559    transport: hyprstream_rpc::transport::TransportConfig,
3560    fs: Option<WorktreeClient>,
3561    /// Expected audience for JWT validation (resource URL)
3562    expected_audience: Option<String>,
3563    /// JWT key source for verifying JWTs (local and federated).
3564    jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
3565    /// Tenant binding inherited from the authority-verified ModelService call.
3566    tenant_domain: String,
3567    /// Model reference this instance serves, when spawned as a pinned
3568    /// Model-subprocessor instance. Present ⇒ the instance accepts controller
3569    /// internal work orders (`iw+jwt`); absent (standalone service) ⇒ it never
3570    /// accepts them, fail-closed.
3571    model_ref: Option<String>,
3572    /// Private per-attempt readiness handoff: the worker reports its
3573    /// worker-generated incarnation here after its engine is initialized.
3574    /// `None` (or a dropped sender) ⇒ the owning load attempt fails closed.
3575    incarnation_handoff: Option<tokio::sync::oneshot::Sender<IncarnationHandoff>>,
3576    /// Spawn-attempt generation echoed in the handoff for staleness checks.
3577    attempt_generation: u64,
3578    /// ModelService key allowed to bridge local calls into this tenant.
3579    controller_pubkey: VerifyingKey,
3580    /// Network reach published after the engine and Iroh endpoint are ready.
3581    network_reach: Arc<parking_lot::RwLock<Option<hyprstream_rpc::transport::TransportConfig>>>,
3582    /// Reach and origin shared with the browser-addressable QUIC/MoQ relay.
3583    producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
3584    moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
3585    /// Standalone services own and bind this QUIC/WebTransport endpoint.
3586    quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>,
3587    /// Explicit externally routable reach advertised for this QUIC bind.
3588    quic_advertise_addr: Option<std::net::SocketAddr>,
3589    /// False when an outer ModelService owns the advertised stream plane.
3590    owns_stream_plane: bool,
3591    network_ready: Arc<AtomicBool>,
3592    draining: Arc<AtomicBool>,
3593}
3594
3595impl InferenceServiceConfig {
3596    /// Create a new InferenceServiceConfig.
3597    ///
3598    /// No GPU work is done here — all heavy initialization is deferred
3599    /// to the service thread via `Spawnable::run()`.
3600    ///
3601    /// **Important**: Does NOT take a `PolicyClient`. The policy client is created
3602    /// lazily inside `run()` on the service thread's own Tokio runtime, because
3603    /// ZMQ sockets must be registered with the correct runtime's reactor.
3604    pub fn new(
3605        model_path: impl AsRef<Path>,
3606        config: RuntimeConfig,
3607        server_pubkey: VerifyingKey,
3608        signing_key: SigningKey,
3609        transport: hyprstream_rpc::transport::TransportConfig,
3610        policy_transport: hyprstream_rpc::transport::TransportConfig,
3611        fs: Option<WorktreeClient>,
3612    ) -> Self {
3613        let policy_signing_key = signing_key.clone();
3614        Self {
3615            service_name: "inference".to_owned(),
3616            model_path: model_path.as_ref().to_path_buf(),
3617            state_path: model_path.as_ref().to_path_buf(),
3618            pinned_artifact: None,
3619            config,
3620            server_pubkey,
3621            signing_key,
3622            policy_signing_key,
3623            policy_transport,
3624            transport,
3625            fs,
3626            expected_audience: None,
3627            jwt_key_source: None,
3628            tenant_domain: "local".to_owned(),
3629            model_ref: None,
3630            incarnation_handoff: None,
3631            attempt_generation: 0,
3632            controller_pubkey: server_pubkey,
3633            network_reach: Arc::new(parking_lot::RwLock::new(None)),
3634            producer_reach_config: Arc::new(parking_lot::RwLock::new(
3635                hyprstream_rpc::moq_stream::ProducerReachConfig::default(),
3636            )),
3637            moq_origin: Arc::new(parking_lot::RwLock::new(None)),
3638            quic_config: None,
3639            quic_advertise_addr: None,
3640            owns_stream_plane: true,
3641            // Legacy in-process/training services do not attach a browser
3642            // stream plane; preserve their model-based readiness until a
3643            // network plane is explicitly configured below.
3644            network_ready: Arc::new(AtomicBool::new(true)),
3645            draining: Arc::new(AtomicBool::new(false)),
3646        }
3647    }
3648
3649    /// Bind all base-model path reads to the owned exact-tree projection.
3650    /// Adapter and training writes continue to use the original checkout.
3651    #[must_use]
3652    pub fn with_pinned_artifact(
3653        mut self,
3654        artifact: Arc<crate::storage::pinned_model::PinnedModelArtifact>,
3655    ) -> Self {
3656        self.model_path = artifact.root().to_path_buf();
3657        self.pinned_artifact = Some(artifact);
3658        self
3659    }
3660
3661    /// Bind this engine to one verified tenant and one controller identity.
3662    #[must_use]
3663    pub fn with_instance_identity(
3664        mut self,
3665        service_name: String,
3666        tenant_domain: String,
3667        controller_pubkey: VerifyingKey,
3668    ) -> Self {
3669        self.service_name = service_name;
3670        self.tenant_domain = tenant_domain;
3671        self.controller_pubkey = controller_pubkey;
3672        self
3673    }
3674
3675    /// Bind the model reference this pinned instance serves. Present ⇒
3676    /// controller internal work orders (`iw+jwt`) are admissible; absent ⇒
3677    /// they are never admitted (standalone direct-caller service).
3678    #[must_use]
3679    pub fn with_model_ref(mut self, model_ref: impl Into<String>) -> Self {
3680        self.model_ref = Some(model_ref.into());
3681        self
3682    }
3683
3684    /// Attach this spawn attempt's private incarnation handoff. The worker
3685    /// sends its worker-generated incarnation + audience + attempt identity
3686    /// after its engine is initialized; the owner awaits it after readiness.
3687    #[must_use]
3688    pub fn with_incarnation_handoff(
3689        mut self,
3690        sender: tokio::sync::oneshot::Sender<IncarnationHandoff>,
3691        attempt_generation: u64,
3692    ) -> Self {
3693        self.incarnation_handoff = Some(sender);
3694        self.attempt_generation = attempt_generation;
3695        self
3696    }
3697
3698    /// Publish generated streams through an outer service's QUIC/MoQ relay.
3699    #[must_use]
3700    pub fn with_stream_plane(
3701        mut self,
3702        producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
3703        moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
3704    ) -> Self {
3705        self.producer_reach_config = producer_reach_config;
3706        self.moq_origin = moq_origin;
3707        self.owns_stream_plane = false;
3708        self.network_ready.store(
3709            self.producer_reach_config.read().quic_reach.is_some(),
3710            Ordering::Release,
3711        );
3712        self
3713    }
3714
3715    /// Bind a browser-addressable RPC/MoQ endpoint owned by this process.
3716    #[must_use]
3717    pub fn with_quic_config(
3718        mut self,
3719        quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>,
3720    ) -> Self {
3721        if quic_config.is_some() {
3722            self.network_ready.store(false, Ordering::Release);
3723        }
3724        self.quic_config = quic_config;
3725        self
3726    }
3727
3728    /// Set the externally routable address signed into `StreamInfo`.
3729    #[must_use]
3730    pub fn with_quic_advertise_addr(
3731        mut self,
3732        advertise_addr: Option<std::net::SocketAddr>,
3733    ) -> Self {
3734        self.quic_advertise_addr = advertise_addr;
3735        self
3736    }
3737
3738    /// Set the expected audience for JWT validation.
3739    pub fn with_expected_audience(mut self, audience: String) -> Self {
3740        self.expected_audience = Some(audience);
3741        self
3742    }
3743
3744    /// Set the JWT key source for verifying JWTs (local and federated).
3745    pub fn with_jwt_key_source(
3746        mut self,
3747        src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
3748    ) -> Self {
3749        self.jwt_key_source = Some(src);
3750        self
3751    }
3752    #[must_use]
3753    pub fn network_reach_handle(
3754        &self,
3755    ) -> Arc<parking_lot::RwLock<Option<hyprstream_rpc::transport::TransportConfig>>> {
3756        Arc::clone(&self.network_reach)
3757    }
3758}
3759
3760#[allow(clippy::too_many_arguments)]
3761async fn serve_inference_bridged(
3762    service_name: &str,
3763    transport: &hyprstream_rpc::transport::TransportConfig,
3764    processor: Arc<dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor>,
3765    signing_key: SigningKey,
3766    shutdown: Arc<tokio::sync::Notify>,
3767    on_ready: Option<tokio::sync::oneshot::Sender<()>>,
3768    quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>,
3769    quic_advertise_addr: Option<std::net::SocketAddr>,
3770    producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
3771    moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
3772    network_ready: Arc<AtomicBool>,
3773    draining: Arc<AtomicBool>,
3774) -> hyprstream_rpc::error::Result<()> {
3775    let Some(mut qc) = quic_config else {
3776        return hyprstream_rpc::service::serve::serve_bridged(
3777            transport,
3778            processor,
3779            signing_key,
3780            shutdown,
3781            on_ready,
3782        )
3783        .await
3784        .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(e.to_string()));
3785    };
3786
3787    hyprstream_rpc::transport::install_pq_crypto_provider().map_err(|error| {
3788        hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC crypto provider: {error}"))
3789    })?;
3790    let chain: Vec<rustls::pki_types::CertificateDer<'static>> = qc
3791        .cert_chain
3792        .iter()
3793        .map(|der| rustls::pki_types::CertificateDer::from(der.clone()))
3794        .collect();
3795    let key = rustls::pki_types::PrivateKeyDer::try_from((*qc.key_der).clone())
3796        .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC key: {e}")))?;
3797    let wt_server = web_transport_quinn::ServerBuilder::new()
3798        .with_addr(qc.bind_addr)
3799        .with_certificate(chain, key)
3800        .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC bind: {e}")))?;
3801    let actual_addr = wt_server.local_addr().map_err(|e| {
3802        hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC local_addr: {e}"))
3803    })?;
3804    let advertise_addr = quic_advertise_addr.ok_or_else(|| {
3805        hyprstream_rpc::error::RpcError::SpawnFailed(
3806            "standalone inference requires an explicit routable QUIC advertise address".to_owned(),
3807        )
3808    })?;
3809    if advertise_addr.ip().is_unspecified()
3810        || advertise_addr.ip().is_loopback()
3811        || advertise_addr.port() == 0
3812        || advertise_addr.port() != actual_addr.port()
3813    {
3814        return Err(hyprstream_rpc::error::RpcError::SpawnFailed(format!(
3815            "standalone inference advertise address {advertise_addr} is not routable or does not match bound QUIC port {}",
3816            actual_addr.port()
3817        )));
3818    }
3819    let pin = hyprstream_rpc::transport::quinn_transport::cert_sha256(&qc.cert_chain[0]);
3820    let origin = moq_origin
3821        .read()
3822        .clone()
3823        .or_else(|| hyprstream_rpc::moq_stream::global_moq_origin().cloned())
3824        .ok_or_else(|| {
3825            hyprstream_rpc::error::RpcError::SpawnFailed(
3826                "standalone inference has no MoQ origin".to_owned(),
3827            )
3828        })?;
3829
3830    let rpc_server = hyprstream_rpc::transport::quinn_transport::QuinnRpcServer::with_capacity(
3831        wt_server,
3832        Arc::clone(&processor),
3833        signing_key.clone(),
3834        hyprstream_rpc::transport::rpc_session::DEFAULT_STREAM_LIMIT,
3835    )
3836    .with_moq_consumer(origin.consumer().clone());
3837
3838    if let Some(registry) = hyprstream_rpc::registry::try_global() {
3839        registry.register(
3840            service_name,
3841            hyprstream_rpc::registry::SocketKind::Quic,
3842            hyprstream_rpc::transport::TransportConfig::quic_pinned(
3843                advertise_addr,
3844                &qc.server_name,
3845                pin,
3846            ),
3847            None,
3848        );
3849    }
3850    {
3851        let mut reach = producer_reach_config.write();
3852        reach.quic_reach = Some(hyprstream_rpc::moq_stream::NodeStreamReach {
3853            addr: advertise_addr,
3854            server_name: qc.server_name.clone(),
3855            cert_hashes: vec![pin],
3856        });
3857        reach.relay = qc.moq_relay.clone();
3858    }
3859    network_ready.store(true, Ordering::Release);
3860    if let Some(callback) = qc.on_quic_bound.take() {
3861        callback(
3862            service_name.to_owned(),
3863            advertise_addr,
3864            qc.server_name.clone(),
3865        );
3866    }
3867    if let Some(relay) = qc.moq_relay.take() {
3868        hyprstream_rpc::moq_stream::serve_origin_to_relay_background(
3869            origin.producer().clone(),
3870            relay,
3871            qc.moq_admission_proof.clone(),
3872            qc.moq_relay_server_identity.take(),
3873        );
3874    }
3875
3876    // Start REP before arming the QUIC drain waiter. The bridged serve helper
3877    // acknowledges only after its own shutdown waiter is registered, before
3878    // it signals socket readiness, so the owner can safely arm both waiters
3879    // before exposing external readiness.
3880    let (rep_armed_tx, rep_armed_rx) = tokio::sync::oneshot::channel();
3881    let rep_transport = transport.clone();
3882    let rep_processor = Arc::clone(&processor);
3883    let rep_shutdown = Arc::clone(&shutdown);
3884    let rep_task = tokio::spawn(async move {
3885        hyprstream_rpc::service::serve::serve_bridged_with_shutdown_armed_silent(
3886            &rep_transport,
3887            rep_processor,
3888            signing_key,
3889            rep_shutdown,
3890            None,
3891            Some(rep_armed_tx),
3892        )
3893        .await
3894    });
3895    let rep_owner = RepTaskOwner::new(rep_task);
3896    if rep_armed_rx.await.is_err() {
3897        network_ready.store(false, Ordering::Release);
3898        let rep_error = rep_owner.join().await;
3899        processor.close_admission();
3900        return Err(match rep_error {
3901            Err(error) => hyprstream_rpc::error::RpcError::SpawnFailed(error.to_string()),
3902            Ok(()) => {
3903                hyprstream_rpc::error::RpcError::SpawnFailed("REP waiter failed to arm".to_owned())
3904            }
3905        });
3906    }
3907    let rep_admission = Arc::clone(&processor);
3908    let rep = async move {
3909        let result = rep_owner.join().await;
3910        if result.is_err() {
3911            // A bind/setup or serve failure can happen after QUIC/Iroh was
3912            // advertised. Close admission before the QUIC drain waits so no
3913            // new work can enter while the owner tears down.
3914            rep_admission.close_admission();
3915        }
3916        result
3917    };
3918
3919    let drain_limit = rpc_server.stream_limit();
3920    let drain_capacity = rpc_server.capacity();
3921    let drain_token = rpc_server.shutdown_token();
3922    let drain_shutdown = Arc::clone(&shutdown);
3923    let drain_state = Arc::clone(&draining);
3924    let drain_ready = Arc::clone(&network_ready);
3925    let (drain_armed_tx, drain_armed_rx) = tokio::sync::oneshot::channel();
3926    let drain_task = tokio::spawn(async move {
3927        // Arm before notifying the owner can become possible. `Notify`'s
3928        // notify_waiters() does not retain a notification for a waiter that
3929        // has not registered yet.
3930        let notified = drain_shutdown.notified();
3931        tokio::pin!(notified);
3932        notified.as_mut().enable();
3933        let _ = drain_armed_tx.send(());
3934        notified.await;
3935        drain_state.store(true, Ordering::Release);
3936        drain_ready.store(false, Ordering::Release);
3937        hyprstream_rpc::transport::quinn_transport::QuinnRpcServer::shutdown(
3938            &drain_limit,
3939            drain_capacity,
3940            &drain_token,
3941        )
3942        .await;
3943    });
3944    let drain_owner = QuinnDrainOwner::new(drain_task);
3945    // Do not enter the select until both the REP and QUIC waiters have
3946    // registered. This closes the notification-lost race on an immediately
3947    // failing peer.
3948    let _ = drain_armed_rx.await;
3949    // The nested REP helper suppresses its own systemd readiness signal. The
3950    // standalone inference owner emits the single outer READY=1 only after
3951    // both REP and QUIC shutdown waiters are armed.
3952    let _ = hyprstream_rpc::notify::ready();
3953    if let Some(on_ready) = on_ready {
3954        let _ = on_ready.send(());
3955    }
3956    let quic = rpc_server.run();
3957    run_quinn_lifecycle(rep, quic, shutdown, network_ready, draining, drain_owner).await
3958}
3959
3960async fn run_quinn_lifecycle<Rep, Quic>(
3961    rep: Rep,
3962    quic: Quic,
3963    shutdown: Arc<tokio::sync::Notify>,
3964    network_ready: Arc<AtomicBool>,
3965    draining: Arc<AtomicBool>,
3966    drain_owner: QuinnDrainOwner,
3967) -> hyprstream_rpc::error::Result<()>
3968where
3969    Rep: std::future::Future<Output = anyhow::Result<()>>,
3970    Quic: std::future::Future<Output = anyhow::Result<()>>,
3971{
3972    tokio::pin!(rep);
3973    tokio::pin!(quic);
3974    tokio::select! {
3975        // Poll REP first so `serve_bridged` registers its Notify waiter before
3976        // the QUIC-first arm can notify it on an unexpected accept-loop exit.
3977        biased;
3978        rep_result = &mut rep => {
3979            shutdown.notify_waiters();
3980            let quic_result = quic.await;
3981            drain_owner.join().await;
3982            network_ready.store(false, Ordering::Release);
3983            if let Err(error) = quic_result {
3984                tracing::warn!(%error, "standalone inference QUIC loop ended during drain");
3985            }
3986            rep_result.map_err(|e| {
3987                hyprstream_rpc::error::RpcError::SpawnFailed(e.to_string())
3988            })
3989        }
3990        quic_result = &mut quic => {
3991            network_ready.store(false, Ordering::Release);
3992            if draining.load(Ordering::Acquire) {
3993                // The drain waiter and REP shutdown must run together.  The
3994                // shutdown notification has already closed admission; polling
3995                // both futures here makes the grace interval shared instead
3996                // of delaying REP teardown until QUIC drain completes.
3997                // Re-notify to cover a REP waiter that was registered after
3998                // the first notification but before this branch was polled.
3999                shutdown.notify_waiters();
4000                let (rep_result, _) = tokio::join!(rep, drain_owner.join());
4001                let rep_result = rep_result.map_err(|error| {
4002                    hyprstream_rpc::error::RpcError::SpawnFailed(error.to_string())
4003                });
4004                return rep_result;
4005            }
4006            shutdown.notify_waiters();
4007            let detail = match quic_result {
4008                Ok(()) => "QUIC server exited unexpectedly".to_owned(),
4009                Err(error) => format!("QUIC server failed: {error}"),
4010            };
4011            // Preserve the unexpected QUIC exit as the primary service error;
4012            // REP teardown and the QUIC drain share the same grace interval,
4013            // so a wedged request cannot hold admission open for a second
4014            // full drain period.
4015            let (_rep_result, _) = tokio::join!(rep, drain_owner.join());
4016            Err(hyprstream_rpc::error::RpcError::SpawnFailed(detail))
4017        }
4018    }
4019}
4020
4021/// Owns the QUIC drain waiter for the lifetime of the bridged service.
4022///
4023/// Normal service exits consume and join this handle. If the owning future is
4024/// cancelled before one of those exits, abort the child instead of detaching
4025/// it into the runtime.
4026struct QuinnDrainOwner {
4027    task: Option<tokio::task::JoinHandle<()>>,
4028}
4029
4030/// Owns the REP task for the lifetime of the bridged service. Cancellation of
4031/// the lifecycle must abort the child instead of detaching it while retaining
4032/// the processor and its admission handles.
4033struct RepTaskOwner {
4034    task: Option<tokio::task::JoinHandle<anyhow::Result<()>>>,
4035}
4036
4037impl RepTaskOwner {
4038    fn new(task: tokio::task::JoinHandle<anyhow::Result<()>>) -> Self {
4039        Self { task: Some(task) }
4040    }
4041
4042    async fn join(mut self) -> anyhow::Result<()> {
4043        let Some(task) = self.task.as_mut() else {
4044            return Err(anyhow::anyhow!("REP task missing"));
4045        };
4046        let result = task
4047            .await
4048            .map_err(|error| anyhow::anyhow!("REP task failed: {error}"))?;
4049        self.task.take();
4050        result
4051    }
4052}
4053
4054impl Drop for RepTaskOwner {
4055    fn drop(&mut self) {
4056        if let Some(task) = self.task.take() {
4057            task.abort();
4058        }
4059    }
4060}
4061
4062impl QuinnDrainOwner {
4063    fn new(task: tokio::task::JoinHandle<()>) -> Self {
4064        Self { task: Some(task) }
4065    }
4066
4067    async fn join(mut self) {
4068        if let Some(task) = self.task.as_mut() {
4069            if let Err(error) = task.await {
4070                tracing::warn!(%error, "standalone inference QUIC drain task ended unexpectedly");
4071            }
4072        }
4073        self.task.take();
4074    }
4075}
4076
4077impl Drop for QuinnDrainOwner {
4078    fn drop(&mut self) {
4079        if let Some(task) = self.task.take() {
4080            task.abort();
4081        }
4082    }
4083}
4084
4085async fn observe_inference_shutdown(
4086    shutdown: Arc<tokio::sync::Notify>,
4087    network_ready: Arc<AtomicBool>,
4088    draining: Arc<AtomicBool>,
4089    armed: tokio::sync::oneshot::Sender<()>,
4090) {
4091    let notified = shutdown.notified();
4092    tokio::pin!(notified);
4093    notified.as_mut().enable();
4094    let _ = armed.send(());
4095    notified.await;
4096    draining.store(true, Ordering::Release);
4097    network_ready.store(false, Ordering::Release);
4098}
4099
4100impl hyprstream_service::Spawnable for InferenceServiceConfig {
4101    fn name(&self) -> &str {
4102        &self.service_name
4103    }
4104
4105    fn registrations(
4106        &self,
4107    ) -> Vec<(
4108        hyprstream_rpc::registry::SocketKind,
4109        hyprstream_rpc::transport::TransportConfig,
4110    )> {
4111        vec![(
4112            hyprstream_rpc::registry::SocketKind::Rep,
4113            self.transport.clone(),
4114        )]
4115    }
4116
4117    fn run(
4118        self: Box<Self>,
4119        shutdown: Arc<tokio::sync::Notify>,
4120        on_ready: Option<tokio::sync::oneshot::Sender<()>>,
4121    ) -> hyprstream_rpc::error::Result<()> {
4122        let transport = self.transport.clone();
4123        let server_signing_key = self.signing_key.clone();
4124
4125        // Post-ZMQ serve (#136): the GPU `InferenceService` is `!Send`, so build it
4126        // on a dedicated LocalServiceBridge thread via `spawn_with`, then serve the
4127        // resulting processor over the registered transport with `serve_bridged`.
4128        // No ZMQ ROUTER. (QUIC is not enabled on this service.)
4129        let rt = tokio::runtime::Builder::new_multi_thread()
4130            .enable_all()
4131            .build()
4132            .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("runtime: {e}")))?;
4133
4134        rt.block_on(async move {
4135            // Shared between the bridge dispatch and the InferenceService.
4136            let nonce_cache = Arc::new(InMemoryNonceCache::new());
4137            let bridge_nonce = Arc::clone(&nonce_cache);
4138
4139            // Destructure so the (Send) config moves into the on-thread builder.
4140            let InferenceServiceConfig {
4141                service_name,
4142                model_path,
4143                state_path,
4144                pinned_artifact,
4145                config,
4146                server_pubkey,
4147                signing_key: svc_signing_key,
4148                policy_signing_key,
4149                policy_transport,
4150                transport: _transport,
4151                fs,
4152                expected_audience,
4153                jwt_key_source,
4154                tenant_domain,
4155                model_ref: model_ref_binding,
4156                incarnation_handoff,
4157                attempt_generation,
4158                controller_pubkey,
4159                network_reach,
4160                producer_reach_config,
4161                moq_origin,
4162                quic_config,
4163                quic_advertise_addr,
4164                owns_stream_plane,
4165                network_ready,
4166                draining,
4167            } = *self;
4168            let adapter_transport = transport.clone();
4169            // Instance audience for controller internal work orders (`iw+jwt`).
4170            let instance_service_name = service_name.clone();
4171
4172            let lifecycle_ready = Arc::clone(&network_ready);
4173            let lifecycle_draining = Arc::clone(&draining);
4174            let lifecycle_shutdown = Arc::clone(&shutdown);
4175            let (lifecycle_armed_tx, lifecycle_armed_rx) = tokio::sync::oneshot::channel();
4176            tokio::spawn(async move {
4177                observe_inference_shutdown(
4178                    lifecycle_shutdown,
4179                    lifecycle_ready,
4180                    lifecycle_draining,
4181                    lifecycle_armed_tx,
4182                )
4183                .await;
4184            });
4185            lifecycle_armed_rx.await.map_err(|_| {
4186                hyprstream_rpc::error::RpcError::SpawnFailed(
4187                    "inference lifecycle shutdown observer exited before arming".to_owned(),
4188                )
4189            })?;
4190
4191            // Standalone instances use an isolated origin so two CPU replicas
4192            // in one deployment cannot expose each other's broadcast namespace.
4193            if owns_stream_plane && quic_config.is_some() {
4194                *moq_origin.write() = Some(
4195                    hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone()
4196                        .with_prefix(hyprstream_rpc::moq_stream::DEFAULT_PREFIX)
4197                        .build(),
4198                );
4199            }
4200            let bridge_reach_config = Arc::clone(&producer_reach_config);
4201            let bridge_moq_origin = Arc::clone(&moq_origin);
4202            let bridge_network_ready = Arc::clone(&network_ready);
4203            let bridge_draining = Arc::clone(&draining);
4204            let bridge_shutdown = Arc::clone(&shutdown);
4205
4206            // Build PolicyClient + GPU service + adapter ON the bridge thread.
4207            #[cfg(test)]
4208            crate::services::restart_diag::phase("bridge.build.enter");
4209            let (bridge, ready) = hyprstream_rpc::transport::iroh_rpc::LocalServiceBridge::spawn_with(
4210                "inference",
4211                move || async move {
4212                    let policy_vk = hyprstream_service::global_trust_store()
4213                        .resolve_one("policy")
4214                        .ok_or_else(|| {
4215                            anyhow::anyhow!("trust store has no policy key — startup must populate it")
4216                        })?;
4217                    // Required profile resolves through the checkpoint-backed
4218                    // discovery resolver; compatibility dials the factory-
4219                    // resolved deterministic IPC transport.
4220                    let policy_client = if hyprstream_discovery::native_network_required() {
4221                        PolicyClient::from_provider(&hyprstream_discovery::ProductionRpcClientProvider, policy_signing_key, None)?
4222                    } else {
4223                        PolicyClient::for_local_transport_bootstrap(
4224                            &policy_transport,
4225                            policy_signing_key,
4226                            policy_vk,
4227                            None,
4228                        )?
4229                    };
4230                    let service = InferenceService::initialize(
4231                        model_path,
4232                        state_path,
4233                        pinned_artifact,
4234                        config,
4235                        server_pubkey,
4236                        svc_signing_key.clone(),
4237                        Arc::clone(&bridge_nonce),
4238                        policy_client,
4239                        fs,
4240                        tenant_domain,
4241                        controller_pubkey,
4242                        bridge_reach_config,
4243                        bridge_moq_origin,
4244                        bridge_network_ready,
4245                        bridge_draining,
4246                        bridge_shutdown,
4247                    )
4248                    .await
4249                    .map_err(|e| anyhow::anyhow!("inference init: {e}"))?;
4250
4251                    // Sol bounded incarnation plan: the WORKER generates its
4252                    // own fresh 256-bit incarnation inside this service
4253                    // thread, after the engine is initialized and before the
4254                    // request adapter exists. The versioned audience is stored
4255                    // on the adapter for its entire run and reported to the
4256                    // owner through the private per-attempt handoff. Never
4257                    // accepted from a request, resolver record, or cached name.
4258                    let incarnation_bytes = [
4259                        hyprstream_rpc::envelope::generate_nonce(),
4260                        hyprstream_rpc::envelope::generate_nonce(),
4261                    ]
4262                    .concat();
4263                    let incarnation = hex::encode(incarnation_bytes);
4264                    let instance_audience =
4265                        internal_work_audience(&instance_service_name, &incarnation);
4266                    match incarnation_handoff {
4267                        Some(sender) => sender
4268                            .send(IncarnationHandoff {
4269                                instance_service_name: instance_service_name.clone(),
4270                                controller_pubkey: controller_pubkey.to_bytes(),
4271                                generation: attempt_generation,
4272                                incarnation,
4273                                audience: instance_audience.clone(),
4274                            })
4275                            .map_err(|_| {
4276                                anyhow::anyhow!(
4277                                    "incarnation handoff receiver dropped by the owner;                                      refusing to serve internal work from an unbound worker"
4278                                )
4279                            })?,
4280                        // A pinned instance without a handoff receiver has no
4281                        // owner awaiting its binding — fail closed.
4282                        None if model_ref_binding.is_some() => {
4283                            anyhow::bail!(
4284                                "pinned instance has no incarnation handoff receiver; refusing to serve"
4285                            )
4286                        }
4287                        None => {}
4288                    }
4289                    #[cfg(test)]
4290                    crate::services::restart_diag::phase("incarnation.sent");
4291
4292                    Ok(InferenceZmqAdapter {
4293                        service,
4294                        transport: adapter_transport,
4295                        signing_key: svc_signing_key,
4296                        expected_audience,
4297                        jwt_key_source,
4298                        instance_model_ref: model_ref_binding,
4299                        instance_audience,
4300                    })
4301                },
4302                nonce_cache,
4303                0,
4304            )
4305            .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("bridge: {e}")))?;
4306
4307            // Surface GPU-init failure before advertising readiness.
4308            #[cfg(test)]
4309            crate::services::restart_diag::phase("bridge.ready.await.enter");
4310            ready
4311                .await
4312                .map_err(|_| {
4313                    hyprstream_rpc::error::RpcError::SpawnFailed(
4314                        "inference bridge readiness dropped".to_owned(),
4315                    )
4316                })?
4317                .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("init: {e}")))?;
4318            #[cfg(test)]
4319            crate::services::restart_diag::phase("bridge.ready.await.done");
4320
4321            let processor: Arc<dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor> =
4322                Arc::new(bridge);
4323            let purpose =
4324                crate::runtime::inference_profile::inference_iroh_key_purpose(&service_name);
4325            let iroh_key =
4326                hyprstream_rpc::node_identity::derive_purpose_key(&server_signing_key, &purpose);
4327            let rpc_handler =
4328                hyprstream_rpc::transport::iroh_rpc::IrohRpcProtocolHandler::with_stream_limit(
4329                    Arc::clone(&processor),
4330                    server_signing_key.clone(),
4331                    hyprstream_rpc::transport::rpc_session::DEFAULT_STREAM_LIMIT,
4332                );
4333            #[cfg(test)]
4334            crate::services::restart_diag::phase("iroh.bind.enter");
4335            let substrate =
4336                hyprstream_rpc::transport::iroh_substrate::IrohSubstrate::new(
4337                    iroh_key.to_bytes(),
4338                    hyprstream_rpc::transport::iroh_substrate::RefuseHandler::new(
4339                        "inference streaming uses the existing StreamService",
4340                    ),
4341                    rpc_handler,
4342                )
4343                .await
4344                .map_err(|e| {
4345                    hyprstream_rpc::error::RpcError::SpawnFailed(format!(
4346                        "inference iroh bind: {e}"
4347                    ))
4348                })?;
4349            #[cfg(test)]
4350            crate::services::restart_diag::phase("iroh.bind.done");
4351            let node_id = *substrate.endpoint_id().as_bytes();
4352            let direct_addrs = substrate.endpoint().bound_sockets().into_iter().collect();
4353            *network_reach.write() = Some(hyprstream_rpc::transport::TransportConfig::iroh(
4354                node_id,
4355                direct_addrs,
4356                None,
4357            ));
4358            #[cfg(test)]
4359            crate::services::restart_diag::phase("inproc.serve.enter");
4360            let result = serve_inference_bridged(
4361                &service_name,
4362                &transport,
4363                processor,
4364                server_signing_key,
4365                shutdown,
4366                on_ready,
4367                quic_config,
4368                quic_advertise_addr,
4369                producer_reach_config,
4370                moq_origin,
4371                network_ready,
4372                draining,
4373            )
4374            .await;
4375            if let Err(error) = substrate.shutdown().await {
4376                tracing::warn!(%error, "inference iroh shutdown failed");
4377            }
4378            result
4379        })
4380    }
4381}
4382
4383// InferenceZmqClient removed — use generated InferenceClient + InferenceRpc trait directly.
4384
4385// ============================================================================
4386// StreamChunkMessage - Client-side stream message handling
4387// ============================================================================
4388
4389/// Message received from StreamService during streaming generation.
4390///
4391/// Represents one of three possible message types:
4392/// - Chunk: Text chunk from the model
4393/// - Complete: Stream finished with generation stats
4394/// - Error: Generation failed
4395///
4396/// Note: Ordering is handled by prevMac in the outer streaming_capnp::StreamBlock wrapper.
4397/// Authentication via HMAC chain happens at the StreamBlock level.
4398pub enum StreamChunkMessage {
4399    Chunk {
4400        text: String,
4401    },
4402    Complete {
4403        stats: crate::runtime::GenerationStats,
4404    },
4405    Error {
4406        error: String,
4407    },
4408}
4409
4410impl StreamChunkMessage {
4411    /// Convert a generic StreamPayload to an inference-specific StreamChunkMessage.
4412    ///
4413    /// This bridges the generic streaming layer (StreamPayload) to the inference-specific
4414    /// message types used by consumers.
4415    pub fn from_stream_payload(payload: crate::services::rpc_types::StreamPayload) -> Self {
4416        use crate::services::rpc_types::{InferenceStreamPayload, StreamPayloadExt};
4417
4418        match payload.to_inference() {
4419            Ok(InferenceStreamPayload::Token(text)) => StreamChunkMessage::Chunk { text },
4420            Ok(InferenceStreamPayload::Error(message)) => {
4421                StreamChunkMessage::Error { error: message }
4422            }
4423            Ok(InferenceStreamPayload::Complete(stats)) => {
4424                let quality_metrics = if stats.perplexity.is_some() || stats.avg_entropy.is_some() {
4425                    Some(
4426                        crate::runtime::generation_metrics::GenerationQualityMetrics {
4427                            perplexity: stats.perplexity.unwrap_or(0.0),
4428                            avg_entropy: stats.avg_entropy.unwrap_or(0.0),
4429                            ..Default::default()
4430                        },
4431                    )
4432                } else {
4433                    None
4434                };
4435
4436                let gen_stats = crate::runtime::GenerationStats {
4437                    tokens_generated: stats.tokens_generated,
4438                    generation_time_ms: stats.generation_time_ms,
4439                    tokens_per_second: stats.tokens_per_second,
4440                    finish_reason: Some(match stats.finish_reason.as_str() {
4441                        "length" => crate::config::FinishReason::MaxTokens,
4442                        "eos" => crate::config::FinishReason::EndOfSequence,
4443                        "error" => crate::config::FinishReason::Error(String::new()),
4444                        _ => crate::config::FinishReason::Stop,
4445                    }),
4446                    quality_metrics,
4447                    prefill_tokens: stats.prefill_tokens,
4448                    prefill_time_ms: stats.prefill_time_ms,
4449                    prefill_tokens_per_sec: stats.prefill_tokens_per_sec,
4450                    inference_tokens: stats.inference_tokens,
4451                    inference_time_ms: stats.inference_time_ms,
4452                    inference_tokens_per_sec: stats.inference_tokens_per_sec,
4453                    inference_tokens_per_sec_ema: stats.inference_tokens_per_sec_ema,
4454                    // The streaming wire protocol has no speculative counters yet.
4455                    speculative_accepted: 0,
4456                    speculative_rejected: 0,
4457                };
4458                StreamChunkMessage::Complete { stats: gen_stats }
4459            }
4460            Err(e) => StreamChunkMessage::Error {
4461                error: format!("Failed to parse payload: {e}"),
4462            },
4463        }
4464    }
4465
4466    /// Returns true if this is the last message (Complete or Error)
4467    pub fn is_last(&self) -> bool {
4468        matches!(
4469            self,
4470            StreamChunkMessage::Complete { .. } | StreamChunkMessage::Error { .. }
4471        )
4472    }
4473}
4474
4475// StreamHandle consolidated: uses hyprstream_rpc::streaming::StreamHandle (re-exported via rpc_types)
4476// Use StreamChunkMessage::from_stream_payload() to convert StreamPayload → StreamChunkMessage
4477
4478#[cfg(test)]
4479mod tenant_binding_tests {
4480    use super::*;
4481    use hyprstream_rpc::auth::mac::{
4482        Assurance, CompartmentSet, Level, SecurityContext, SecurityLabel, VerifiedKeyMaterial,
4483    };
4484    use hyprstream_rpc::envelope::Subject;
4485
4486    fn key(seed: u8) -> SigningKey {
4487        SigningKey::from_bytes(&[seed; 32])
4488    }
4489
4490    #[tokio::test]
4491    async fn pinned_config_retains_base_inputs_but_not_writable_state() -> Result<()> {
4492        let dir = tempfile::tempdir()?;
4493        let repo = git2db::Repository::init(dir.path())?;
4494        std::fs::write(dir.path().join("config.json"), b"reviewed")?;
4495        let mut index = repo.index()?;
4496        index.add_path(Path::new("config.json"))?;
4497        index.write()?;
4498        let tree = repo.find_tree(index.write_tree()?)?;
4499        let sig = git2::Signature::now("test", "test@example.invalid")?;
4500        let oid = repo.commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])?;
4501        drop(tree);
4502        drop(repo);
4503        let artifact = crate::storage::pinned_model::acquire_pinned_model(dir.path(), oid).await?;
4504        let signing_key = key(3);
4505        let config = InferenceServiceConfig::new(
4506            dir.path(),
4507            RuntimeConfig::default(),
4508            signing_key.verifying_key(),
4509            signing_key,
4510            hyprstream_rpc::transport::TransportConfig::inproc("pinned-test"),
4511            hyprstream_rpc::transport::TransportConfig::inproc("policy"),
4512            None,
4513        )
4514        .with_pinned_artifact(artifact);
4515        std::fs::write(dir.path().join("config.json"), b"replaced")?;
4516        assert_eq!(
4517            std::fs::read(config.model_path.join("config.json"))?,
4518            b"reviewed"
4519        );
4520        assert_eq!(config.state_path, dir.path());
4521        assert_eq!(
4522            config
4523                .pinned_artifact
4524                .as_ref()
4525                .ok_or_else(|| anyhow!("missing pin"))?
4526                .commit(),
4527            oid
4528        );
4529        Ok(())
4530    }
4531
4532    #[test]
4533    fn inference_config_shares_outer_moq_plane_by_identity() {
4534        let reach = Arc::new(parking_lot::RwLock::new(
4535            hyprstream_rpc::moq_stream::ProducerReachConfig::default(),
4536        ));
4537        let origin = Arc::new(parking_lot::RwLock::new(None));
4538        let signing_key = key(4);
4539        let config = InferenceServiceConfig::new(
4540            "/model",
4541            RuntimeConfig::default(),
4542            signing_key.verifying_key(),
4543            signing_key.clone(),
4544            hyprstream_rpc::transport::TransportConfig::inproc("inference-stream-plane-test"),
4545            hyprstream_rpc::transport::TransportConfig::inproc("policy"),
4546            None,
4547        )
4548        .with_stream_plane(Arc::clone(&reach), Arc::clone(&origin));
4549
4550        assert!(Arc::ptr_eq(&config.producer_reach_config, &reach));
4551        assert!(Arc::ptr_eq(&config.moq_origin, &origin));
4552        assert!(!config.owns_stream_plane);
4553        assert!(!config.network_ready.load(Ordering::Acquire));
4554    }
4555
4556    #[test]
4557    fn legacy_in_process_config_preserves_model_based_readiness() {
4558        let signing_key = key(5);
4559        let config = InferenceServiceConfig::new(
4560            "/model",
4561            RuntimeConfig::default(),
4562            signing_key.verifying_key(),
4563            signing_key,
4564            hyprstream_rpc::transport::TransportConfig::inproc("inference-legacy-readiness-test"),
4565            hyprstream_rpc::transport::TransportConfig::inproc("policy"),
4566            None,
4567        );
4568
4569        assert!(config.network_ready.load(Ordering::Acquire));
4570    }
4571
4572    #[test]
4573    fn direct_request_must_match_instance_tenant() {
4574        let controller = key(7).verifying_key();
4575        let caller = key(8).verifying_key();
4576        let same = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4577            Subject::new("alice"),
4578            "tenant-a",
4579            caller,
4580        );
4581        assert_eq!(
4582            resolve_instance_domain("tenant-a", &controller, &same)
4583                .unwrap_or_else(|e| panic!("matching tenant must pass: {e}")),
4584            "tenant-a"
4585        );
4586
4587        let other = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4588            Subject::new("mallory"),
4589            "tenant-b",
4590            caller,
4591        );
4592        assert!(resolve_instance_domain("tenant-a", &controller, &other).is_err());
4593    }
4594
4595    #[test]
4596    fn only_pinned_controller_can_bridge_local_domain() {
4597        let controller = key(7).verifying_key();
4598        let local_controller = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4599            Subject::new("system"),
4600            "local",
4601            controller,
4602        );
4603        assert_eq!(
4604            resolve_instance_domain("tenant-a", &controller, &local_controller)
4605                .unwrap_or_else(|e| panic!("pinned controller must pass: {e}")),
4606            "tenant-a"
4607        );
4608
4609        let impostor = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4610            Subject::new("system"),
4611            "local",
4612            key(9).verifying_key(),
4613        );
4614        assert!(resolve_instance_domain("tenant-a", &controller, &impostor).is_err());
4615    }
4616
4617    #[test]
4618    fn mac_denies_missing_clearance() {
4619        let object =
4620            SecurityLabel::new(Level::Internal, Assurance::Classical, CompartmentSet::EMPTY);
4621        assert!(enforce_inference_security_context(None, &object).is_err());
4622    }
4623
4624    #[test]
4625    fn mac_denies_insufficient_clearance() {
4626        let subject = SecurityContext::new(
4627            Level::Public,
4628            CompartmentSet::EMPTY,
4629            VerifiedKeyMaterial::Classical,
4630        );
4631        let object =
4632            SecurityLabel::new(Level::Internal, Assurance::Classical, CompartmentSet::EMPTY);
4633        assert!(enforce_inference_security_context(Some(subject), &object).is_err());
4634    }
4635
4636    #[test]
4637    fn mac_allows_dominating_clearance() {
4638        let subject = SecurityContext::new(
4639            Level::Secret,
4640            CompartmentSet::EMPTY,
4641            VerifiedKeyMaterial::Classical,
4642        );
4643        let object =
4644            SecurityLabel::new(Level::Internal, Assurance::Classical, CompartmentSet::EMPTY);
4645        assert!(enforce_inference_security_context(Some(subject), &object).is_ok());
4646    }
4647}
4648
4649#[cfg(test)]
4650#[allow(clippy::unwrap_used, clippy::expect_used)]
4651mod quinn_drain_tests {
4652    use super::*;
4653    use anyhow::Context as _;
4654    use std::future::Future as _;
4655    use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
4656    use std::process::Command;
4657    use std::time::{Duration, Instant};
4658
4659    struct JoinGuard<T> {
4660        task: Option<tokio::task::JoinHandle<T>>,
4661    }
4662
4663    impl<T> JoinGuard<T> {
4664        fn new(task: tokio::task::JoinHandle<T>) -> Self {
4665            Self { task: Some(task) }
4666        }
4667
4668        fn handle_mut(&mut self) -> &mut tokio::task::JoinHandle<T> {
4669            self.task.as_mut().expect("join guard task present")
4670        }
4671
4672        async fn join(mut self) -> std::result::Result<T, tokio::task::JoinError> {
4673            let result = self.handle_mut().await;
4674            self.task.take();
4675            result
4676        }
4677    }
4678
4679    impl<T> Drop for JoinGuard<T> {
4680        fn drop(&mut self) {
4681            if let Some(task) = self.task.take() {
4682                task.abort();
4683            }
4684        }
4685    }
4686
4687    struct ReleaseOnDrop(Arc<tokio::sync::Semaphore>);
4688
4689    impl Drop for ReleaseOnDrop {
4690        fn drop(&mut self) {
4691            self.0.add_permits(1);
4692        }
4693    }
4694
4695    struct PermitInferenceDispatch;
4696
4697    impl hyprstream_rpc::auth::mac::MacDispatchPep for PermitInferenceDispatch {
4698        fn check(
4699            &self,
4700            _ctx: &EnvelopeContext,
4701            service_domain: &str,
4702            _method: Option<&[u16]>,
4703        ) -> hyprstream_rpc::auth::mac::MacDecision {
4704            if service_domain == "inference" {
4705                hyprstream_rpc::auth::mac::MacDecision::Permit
4706            } else {
4707                hyprstream_rpc::auth::mac::MacDecision::Deny(
4708                    hyprstream_rpc::auth::mac::MacDenyReason::UnlabeledObject,
4709                )
4710            }
4711        }
4712    }
4713
4714    struct DrainGateService {
4715        transport: hyprstream_rpc::transport::TransportConfig,
4716        signing_key: SigningKey,
4717        handler_entered: Arc<tokio::sync::Semaphore>,
4718        handler_completed: Arc<tokio::sync::Semaphore>,
4719        release: Arc<tokio::sync::Semaphore>,
4720    }
4721
4722    #[async_trait::async_trait(?Send)]
4723    impl hyprstream_rpc::service::RequestService for DrainGateService {
4724        async fn handle_request(
4725            &self,
4726            _ctx: &EnvelopeContext,
4727            _body: &hyprstream_rpc::service::DecodedRequestBody,
4728        ) -> Result<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
4729            self.handler_entered.add_permits(1);
4730            self.release.acquire().await?.forget();
4731            self.handler_completed.add_permits(1);
4732            Ok((b"held-response".to_vec(), None))
4733        }
4734
4735        fn decode_request_body(
4736            &self,
4737            signed_body: &[u8],
4738        ) -> Result<hyprstream_rpc::service::DecodedRequestBody> {
4739            crate::services::generated::inference_client::decode_inference_request_body(signed_body)
4740        }
4741
4742        async fn verify_claims(&self, _ctx: &mut EnvelopeContext) -> Result<()> {
4743            Ok(())
4744        }
4745
4746        fn build_error_payload(&self, _request_id: u64, _error: &str) -> Vec<u8> {
4747            b"error-response".to_vec()
4748        }
4749
4750        fn name(&self) -> &str {
4751            "inference"
4752        }
4753
4754        fn transport(&self) -> &hyprstream_rpc::transport::TransportConfig {
4755            &self.transport
4756        }
4757
4758        fn signing_key(&self) -> SigningKey {
4759            self.signing_key.clone()
4760        }
4761    }
4762
4763    fn test_quic_config(
4764        port: u16,
4765        cert_der: Vec<u8>,
4766        key_der: Vec<u8>,
4767    ) -> hyprstream_rpc::service::QuicLoopConfig {
4768        hyprstream_rpc::service::QuicLoopConfig {
4769            announcement_cancellation: tokio_util::sync::CancellationToken::new(),
4770            cert_chain: vec![cert_der],
4771            key_der: zeroize::Zeroizing::new(key_der),
4772            bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port),
4773            server_name: "localhost".to_owned(),
4774            protected_resource_json: None,
4775            on_quic_bound: None,
4776            iroh_enabled: false,
4777            iroh_required: false,
4778            on_iroh_bound: None,
4779            moq_relay: None,
4780            moq_relay_server_identity: None,
4781            moq_admission: None,
4782            moq_ingress_authorizer: None,
4783            moq_admission_proof: None,
4784        }
4785    }
4786
4787    fn valid_request_body() -> Result<Vec<u8>> {
4788        let mut message = capnp::message::Builder::new_default();
4789        {
4790            let mut request = message
4791                .init_root::<hyprstream_rpc_std::inference_capnp::inference_request::Builder>(
4792            );
4793            request.set_id(7);
4794            request.set_is_ready(());
4795        }
4796        let mut bytes = Vec::new();
4797        capnp::serialize::write_message(&mut bytes, &message)?;
4798        Ok(bytes)
4799    }
4800
4801    /// Install the process-global verification config in a child test process;
4802    /// its OnceLock must not contaminate the rest of the lib test binary.
4803    #[test]
4804    fn inference_quic_owner_joins_drain_before_returning() -> Result<()> {
4805        let executable = std::env::current_exe()?;
4806        let mut child = Command::new(executable)
4807            .args([
4808                "--exact",
4809                "services::inference::quinn_drain_tests::inference_quic_owner_joins_drain_before_returning_child",
4810                "--nocapture",
4811            ])
4812            .env("HYPRSTREAM_INFERENCE_DRAIN_CHILD", "1")
4813            .spawn()?;
4814        let deadline = Instant::now() + Duration::from_secs(30);
4815        let status = loop {
4816            if let Some(status) = child.try_wait()? {
4817                break status;
4818            }
4819            if Instant::now() >= deadline {
4820                child.kill()?;
4821                let _ = child.wait();
4822                anyhow::bail!("isolated causal child exceeded 30 second watchdog");
4823            }
4824            std::thread::sleep(Duration::from_millis(20));
4825        };
4826        anyhow::ensure!(
4827            status.success(),
4828            "isolated causal child exited with {status}"
4829        );
4830        Ok(())
4831    }
4832
4833    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4834    async fn inference_quic_owner_joins_drain_before_returning_child() -> Result<()> {
4835        if std::env::var_os("HYPRSTREAM_INFERENCE_DRAIN_CHILD").is_none() {
4836            return Ok(());
4837        }
4838        hyprstream_rpc::transport::install_pq_crypto_provider()?;
4839        let cert_key = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()])?;
4840        let cert_der = cert_key.cert.der().to_vec();
4841        let key_der = cert_key.key_pair.serialize_der();
4842        let probe = UdpSocket::bind("127.0.0.1:0")?;
4843        let port = probe.local_addr()?.port();
4844        drop(probe);
4845
4846        let handler_entered = Arc::new(tokio::sync::Semaphore::new(0));
4847        let handler_completed = Arc::new(tokio::sync::Semaphore::new(0));
4848        let release = Arc::new(tokio::sync::Semaphore::new(0));
4849        let _release_cleanup = ReleaseOnDrop(Arc::clone(&release));
4850        let service_key = SigningKey::from_bytes(&[0x31; 32]);
4851        let client_key = SigningKey::from_bytes(&[0x32; 32]);
4852        let mut verify_store = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
4853        let client_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&client_key);
4854        let client_pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(
4855            &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(&client_pq),
4856        )?;
4857        verify_store.bind(client_key.verifying_key().to_bytes(), &client_pq_vk);
4858        verify_store.bind(
4859            service_key.verifying_key().to_bytes(),
4860            &hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(
4861                &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(
4862                    &hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_key),
4863                ),
4864            )?,
4865        );
4866        hyprstream_rpc::envelope::install_verify_config(
4867            hyprstream_rpc::envelope::EnvelopeVerifyConfig {
4868                policy: hyprstream_rpc::crypto::CryptoPolicy::Hybrid,
4869                pq_store: Some(Arc::new(verify_store)),
4870            },
4871        )?;
4872        hyprstream_rpc::auth::mac::install_mac_dispatch_pep(Arc::new(PermitInferenceDispatch));
4873        let transport = hyprstream_rpc::transport::TransportConfig::inproc(format!(
4874            "inference-drain-test-{port}"
4875        ));
4876        let bridge = hyprstream_rpc::transport::iroh_rpc::LocalServiceBridge::spawn(
4877            DrainGateService {
4878                transport: transport.clone(),
4879                signing_key: service_key.clone(),
4880                handler_entered: Arc::clone(&handler_entered),
4881                handler_completed: Arc::clone(&handler_completed),
4882                release: Arc::clone(&release),
4883            },
4884            Arc::new(hyprstream_rpc::envelope::InMemoryNonceCache::new()),
4885            0,
4886        )?;
4887        let processor: Arc<dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor> =
4888            Arc::new(bridge);
4889        let shutdown = Arc::new(tokio::sync::Notify::new());
4890        let network_ready = Arc::new(AtomicBool::new(false));
4891        let draining = Arc::new(AtomicBool::new(false));
4892        let producer_reach_config = Arc::new(parking_lot::RwLock::new(
4893            hyprstream_rpc::moq_stream::ProducerReachConfig::default(),
4894        ));
4895        let moq_origin = Arc::new(parking_lot::RwLock::new(Some(
4896            hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone().build(),
4897        )));
4898        let advertised_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)), port);
4899        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
4900        let owner_transport = transport.clone();
4901        let owner_cert_der = cert_der.clone();
4902        let owner_shutdown = Arc::clone(&shutdown);
4903        let owner_draining = Arc::clone(&draining);
4904        let owner_service_key = service_key.clone();
4905        let mut owner = JoinGuard::new(tokio::spawn(async move {
4906            serve_inference_bridged(
4907                "inference",
4908                &owner_transport,
4909                Arc::clone(&processor),
4910                owner_service_key,
4911                owner_shutdown,
4912                Some(ready_tx),
4913                Some(test_quic_config(port, owner_cert_der, key_der)),
4914                Some(advertised_addr),
4915                Arc::clone(&producer_reach_config),
4916                Arc::clone(&moq_origin),
4917                Arc::clone(&network_ready),
4918                owner_draining,
4919            )
4920            .await
4921        }));
4922
4923        tokio::time::timeout(Duration::from_secs(5), ready_rx)
4924            .await
4925            .context("inference owner readiness timeout")??;
4926        let session = tokio::time::timeout(
4927            Duration::from_secs(5),
4928            hyprstream_rpc::transport::quinn_transport::connect_pinned(
4929                SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port),
4930                &cert_der,
4931            ),
4932        )
4933        .await
4934        .context("QUIC connect timed out")??;
4935        let service_vk = service_key.verifying_key();
4936        let mut kem_store = hyprstream_rpc::crypto::hybrid_kem::KeyedKemTrustStore::new();
4937        kem_store.bind(
4938            service_vk.to_bytes(),
4939            hyprstream_rpc::node_identity::derive_mesh_kem_recipient(&service_key)?.public(),
4940        );
4941        let service_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_key);
4942        let service_pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(
4943            &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(&service_pq),
4944        )?;
4945        let mut pq_store = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
4946        pq_store.bind(service_vk.to_bytes(), &service_pq_vk);
4947        let client = hyprstream_rpc::rpc_client::RpcClientImpl::new(
4948            hyprstream_rpc::signer::LocalSigner::new(client_key),
4949            hyprstream_rpc::transport::quinn_transport::QuinnTransport::new(session),
4950            Some(service_vk),
4951        )
4952        .with_request_kem_store(Arc::new(kem_store))
4953        .with_response_pq_store(Arc::new(pq_store));
4954        let response = JoinGuard::new(tokio::spawn(async move {
4955            client
4956                .call_for_service("inference", valid_request_body()?)
4957                .await
4958        }));
4959        let entered_permit =
4960            match tokio::time::timeout(Duration::from_secs(5), handler_entered.acquire()).await {
4961                Ok(Ok(permit)) => permit,
4962                Ok(Err(error)) => return Err(error.into()),
4963                Err(_) => {
4964                    let response_result =
4965                        tokio::time::timeout(Duration::from_secs(5), response.join())
4966                            .await
4967                            .context("held request task cleanup timed out")?
4968                            .context("held request task panicked")?;
4969                    anyhow::bail!(
4970                        "QUIC request did not enter service; response result: {response_result:?}"
4971                    );
4972                }
4973            };
4974        entered_permit.forget();
4975
4976        shutdown.notify_waiters();
4977        tokio::time::timeout(Duration::from_secs(5), async {
4978            while !draining.load(Ordering::Acquire) {
4979                tokio::task::yield_now().await;
4980            }
4981        })
4982        .await
4983        .context("inference drain did not start")?;
4984        let owner_done = tokio::time::timeout(Duration::from_millis(100), owner.handle_mut()).await;
4985        assert!(
4986            owner_done.is_err(),
4987            "inference owner returned before QUIC drain"
4988        );
4989
4990        release.add_permits(1);
4991        let response_result = tokio::time::timeout(Duration::from_secs(5), response.join())
4992            .await
4993            .context("held QUIC response did not complete")??;
4994        assert_eq!(response_result?, b"held-response");
4995        tokio::time::timeout(Duration::from_secs(5), handler_completed.acquire())
4996            .await
4997            .context("handler completion marker timed out")??
4998            .forget();
4999        let owner_result = tokio::time::timeout(Duration::from_secs(5), owner.join())
5000            .await
5001            .context("inference owner did not join QUIC drain")??;
5002        owner_result?;
5003        Ok(())
5004    }
5005
5006    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5007    async fn unexpected_quic_exit_notifies_unpolled_rep_and_joins_drain() -> Result<()> {
5008        let shutdown = Arc::new(tokio::sync::Notify::new());
5009        let drain_shutdown = Arc::clone(&shutdown);
5010        let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>();
5011        let (drain_started_tx, drain_started_rx) = tokio::sync::oneshot::channel();
5012        let (drain_release_tx, drain_release_rx) = tokio::sync::oneshot::channel();
5013        let drain_finished = Arc::new(AtomicBool::new(false));
5014        let drain_finished_task = Arc::clone(&drain_finished);
5015        let drain_task = tokio::spawn(async move {
5016            let notified = drain_shutdown.notified();
5017            tokio::pin!(notified);
5018            notified.as_mut().enable();
5019            let _ = armed_tx.send(());
5020            notified.await;
5021            let _ = drain_started_tx.send(());
5022            let _ = drain_release_rx.await;
5023            drain_finished_task.store(true, Ordering::Release);
5024        });
5025        tokio::time::timeout(Duration::from_secs(1), armed_rx)
5026            .await
5027            .context("test drain waiter did not arm")??;
5028
5029        let rep_shutdown_observed = Arc::new(AtomicBool::new(false));
5030        let rep_shutdown_observed_task = Arc::clone(&rep_shutdown_observed);
5031        let rep_shutdown = Arc::clone(&shutdown);
5032        let rep = async move {
5033            rep_shutdown.notified().await;
5034            rep_shutdown_observed_task.store(true, Ordering::Release);
5035            Ok::<(), anyhow::Error>(())
5036        };
5037        let quic = async { Err::<(), _>(anyhow::anyhow!("accept-loop sentinel")) };
5038        let lifecycle = tokio::spawn(run_quinn_lifecycle(
5039            rep,
5040            quic,
5041            shutdown,
5042            Arc::new(AtomicBool::new(true)),
5043            Arc::new(AtomicBool::new(false)),
5044            QuinnDrainOwner::new(drain_task),
5045        ));
5046        tokio::time::timeout(Duration::from_secs(1), drain_started_rx)
5047            .await
5048            .context("test drain did not start")??;
5049        tokio::time::timeout(Duration::from_millis(200), async {
5050            while !rep_shutdown_observed.load(Ordering::Acquire) {
5051                tokio::task::yield_now().await;
5052            }
5053        })
5054        .await
5055        .context("REP shutdown was not polled while QUIC drain was blocked")?;
5056        drain_release_tx
5057            .send(())
5058            .expect("drain waiter remains live");
5059        let result = tokio::time::timeout(Duration::from_secs(1), lifecycle)
5060            .await
5061            .context("unexpected QUIC lifecycle cleanup timed out")??;
5062        let error = result.expect_err("unexpected QUIC exit must remain the primary error");
5063        assert!(error.to_string().contains("accept-loop sentinel"));
5064        assert!(rep_shutdown_observed.load(Ordering::Acquire));
5065        assert!(drain_finished.load(Ordering::Acquire));
5066        Ok(())
5067    }
5068
5069    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5070    async fn draining_quic_exit_renotifies_rep_before_join() -> Result<()> {
5071        let shutdown = Arc::new(tokio::sync::Notify::new());
5072        let drain_shutdown = Arc::clone(&shutdown);
5073        let (drain_armed_tx, drain_armed_rx) = tokio::sync::oneshot::channel();
5074        let (drain_started_tx, drain_started_rx) = tokio::sync::oneshot::channel();
5075        let (drain_release_tx, drain_release_rx) = tokio::sync::oneshot::channel();
5076        let drain_task = tokio::spawn(async move {
5077            let notified = drain_shutdown.notified();
5078            tokio::pin!(notified);
5079            notified.as_mut().enable();
5080            let _ = drain_armed_tx.send(());
5081            notified.await;
5082            let _ = drain_started_tx.send(());
5083            let _ = drain_release_rx.await;
5084        });
5085
5086        let rep_shutdown = Arc::clone(&shutdown);
5087        let rep_observed = Arc::new(AtomicBool::new(false));
5088        let rep_observed_task = Arc::clone(&rep_observed);
5089        let rep = async move {
5090            // Register the REP waiter, then let the QUIC branch win. The
5091            // lifecycle's re-notify keeps this waiter live through the join.
5092            let notified = rep_shutdown.notified();
5093            tokio::pin!(notified);
5094            notified.as_mut().enable();
5095            tokio::task::yield_now().await;
5096            notified.await;
5097            rep_observed_task.store(true, Ordering::Release);
5098            Ok::<(), anyhow::Error>(())
5099        };
5100        tokio::time::timeout(Duration::from_secs(1), drain_armed_rx)
5101            .await
5102            .context("drain waiter did not arm")??;
5103        let quic = async { Err::<(), _>(anyhow::anyhow!("draining accept-loop sentinel")) };
5104        let lifecycle = tokio::spawn(run_quinn_lifecycle(
5105            rep,
5106            quic,
5107            Arc::clone(&shutdown),
5108            Arc::new(AtomicBool::new(true)),
5109            Arc::new(AtomicBool::new(true)),
5110            QuinnDrainOwner::new(drain_task),
5111        ));
5112
5113        tokio::time::timeout(Duration::from_secs(1), drain_started_rx)
5114            .await
5115            .context("drain did not start")??;
5116        tokio::time::timeout(Duration::from_secs(1), async {
5117            while !rep_observed.load(Ordering::Acquire) {
5118                tokio::task::yield_now().await;
5119            }
5120        })
5121        .await
5122        .context("draining QUIC exit did not re-notify REP")?;
5123        drain_release_tx
5124            .send(())
5125            .expect("drain waiter remains live");
5126        let result = tokio::time::timeout(Duration::from_secs(1), lifecycle)
5127            .await
5128            .context("draining lifecycle cleanup timed out")??;
5129        result?;
5130        Ok(())
5131    }
5132
5133    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5134    async fn cancelling_drain_join_aborts_owned_waiter() -> Result<()> {
5135        struct DropMarker(Arc<AtomicBool>);
5136
5137        impl Drop for DropMarker {
5138            fn drop(&mut self) {
5139                self.0.store(true, Ordering::Release);
5140            }
5141        }
5142
5143        let dropped = Arc::new(AtomicBool::new(false));
5144        let marker = Arc::clone(&dropped);
5145        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
5146        let task = tokio::spawn(async move {
5147            let _marker = DropMarker(marker);
5148            let _ = started_tx.send(());
5149            std::future::pending::<()>().await;
5150        });
5151        let owner = QuinnDrainOwner::new(task);
5152        tokio::time::timeout(Duration::from_secs(1), started_rx)
5153            .await
5154            .context("drain child did not start")??;
5155        let mut join = Box::pin(owner.join());
5156        let mut join_polled = false;
5157        let join_was_pending = std::future::poll_fn(|cx| {
5158            join_polled = true;
5159            match join.as_mut().poll(cx) {
5160                std::task::Poll::Pending => std::task::Poll::Ready(true),
5161                std::task::Poll::Ready(()) => std::task::Poll::Ready(false),
5162            }
5163        })
5164        .await;
5165        assert!(join_polled, "owner join was not polled");
5166        assert!(join_was_pending, "owner join unexpectedly completed");
5167        drop(join);
5168        tokio::time::timeout(Duration::from_secs(1), async {
5169            while !dropped.load(Ordering::Acquire) {
5170                tokio::task::yield_now().await;
5171            }
5172        })
5173        .await
5174        .context("cancellation did not abort the owned drain waiter")?;
5175        Ok(())
5176    }
5177
5178    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5179    async fn cancelling_rep_join_aborts_owned_task() -> Result<()> {
5180        struct DropMarker(Arc<AtomicBool>);
5181
5182        impl Drop for DropMarker {
5183            fn drop(&mut self) {
5184                self.0.store(true, Ordering::Release);
5185            }
5186        }
5187
5188        let dropped = Arc::new(AtomicBool::new(false));
5189        let marker = Arc::clone(&dropped);
5190        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
5191        let task = tokio::spawn(async move {
5192            let _marker = DropMarker(marker);
5193            let _ = started_tx.send(());
5194            std::future::pending::<()>().await;
5195            Ok::<(), anyhow::Error>(())
5196        });
5197        let owner = RepTaskOwner::new(task);
5198        tokio::time::timeout(Duration::from_secs(1), started_rx)
5199            .await
5200            .context("REP child did not start")??;
5201        let mut join = Box::pin(owner.join());
5202        let pending = std::future::poll_fn(|cx| match join.as_mut().poll(cx) {
5203            std::task::Poll::Pending => std::task::Poll::Ready(true),
5204            std::task::Poll::Ready(_) => std::task::Poll::Ready(false),
5205        })
5206        .await;
5207        assert!(pending, "REP owner join unexpectedly completed");
5208        drop(join);
5209        tokio::time::timeout(Duration::from_secs(1), async {
5210            while !dropped.load(Ordering::Acquire) {
5211                tokio::task::yield_now().await;
5212            }
5213        })
5214        .await
5215        .context("cancellation did not abort owned REP task")?;
5216        Ok(())
5217    }
5218
5219    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5220    async fn rep_startup_ack_failure_preserves_task_error() -> Result<()> {
5221        let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>();
5222        let task =
5223            tokio::spawn(async { Err::<(), anyhow::Error>(anyhow::anyhow!("REP bind sentinel")) });
5224        let owner = RepTaskOwner::new(task);
5225        drop(armed_tx);
5226        assert!(armed_rx.await.is_err(), "startup arm channel must close");
5227        let error = owner
5228            .join()
5229            .await
5230            .expect_err("REP startup failure must remain visible");
5231        assert!(error.to_string().contains("REP bind sentinel"));
5232        Ok(())
5233    }
5234}
5235
5236#[cfg(test)]
5237#[allow(clippy::unwrap_used, clippy::expect_used)]
5238mod tests {
5239    //! #1265 regression tests for the generation-loop accounting seam.
5240    //!
5241    //! These prove the three fixes without a live PyTorch engine:
5242    //! - **#1 (blocker):** cancel / publish-failure / stream-error all capture
5243    //!   the completion stats on the single common exit and a spend is posted
5244    //!   for the work actually done.
5245    //! - **#2 (major):** a missing verified billing DID fails closed before
5246    //!   generation when an emitter is attached.
5247    //!
5248    //! `drive_generation_loop` is driven by a `FakeStream` (GenOutput) + a
5249    //! recording/cancelling/failing `FakeSink` (GenSink); the spend is posted
5250    //! through the real `InferenceSpendEmitter` against a `MemLedger` fixture.
5251
5252    use super::*;
5253    use crate::runtime::GenerationStats;
5254    use std::future::Future;
5255    use std::pin::Pin;
5256    use std::task::{Context, Poll};
5257
5258    #[test]
5259    fn standalone_ready_follows_quic_drain_arm() {
5260        let source = include_str!("inference.rs");
5261        let drain_arm = source
5262            .find("let _ = drain_armed_rx.await;")
5263            .expect("standalone inference must await the drain arm barrier");
5264        let outer = &source[drain_arm..];
5265        let ready = outer
5266            .find("hyprstream_rpc::notify::ready()")
5267            .expect("outer inference must emit systemd readiness");
5268        let oneshot = outer
5269            .find("if let Some(on_ready) = on_ready")
5270            .expect("outer readiness oneshot must remain explicit");
5271        assert!(ready < oneshot, "systemd READY must follow drain arming");
5272        assert!(
5273            source.contains("serve_bridged_with_shutdown_armed_silent"),
5274            "nested REP helper must suppress its own readiness"
5275        );
5276    }
5277
5278    #[tokio::test]
5279    async fn lifecycle_shutdown_observer_arms_before_owner_readiness() {
5280        let source = include_str!("inference.rs");
5281        assert!(
5282            source.contains("lifecycle_armed_rx.await"),
5283            "inference startup must wait for the lifecycle shutdown observer to arm"
5284        );
5285
5286        let shutdown = Arc::new(tokio::sync::Notify::new());
5287        let network_ready = Arc::new(AtomicBool::new(true));
5288        let draining = Arc::new(AtomicBool::new(false));
5289        let (armed_tx, armed_rx) = tokio::sync::oneshot::channel();
5290        let observer = tokio::spawn(observe_inference_shutdown(
5291            Arc::clone(&shutdown),
5292            Arc::clone(&network_ready),
5293            Arc::clone(&draining),
5294            armed_tx,
5295        ));
5296
5297        tokio::time::timeout(std::time::Duration::from_secs(5), armed_rx)
5298            .await
5299            .expect("lifecycle observer did not arm")
5300            .expect("lifecycle observer dropped before arming");
5301        shutdown.notify_waiters();
5302        tokio::time::timeout(std::time::Duration::from_secs(5), observer)
5303            .await
5304            .expect("lifecycle observer did not finish")
5305            .expect("lifecycle observer panicked");
5306        assert!(draining.load(Ordering::Acquire));
5307        assert!(!network_ready.load(Ordering::Acquire));
5308    }
5309
5310    /// Build a `GenerationStats` with only the fields the spend reads.
5311    fn stats(prefill: usize, generated: usize) -> GenerationStats {
5312        GenerationStats {
5313            tokens_generated: generated,
5314            generation_time_ms: 0,
5315            tokens_per_second: 0.0,
5316            finish_reason: None,
5317            quality_metrics: None,
5318            prefill_tokens: prefill,
5319            prefill_time_ms: 0,
5320            prefill_tokens_per_sec: 0.0,
5321            inference_tokens: generated,
5322            inference_time_ms: 0,
5323            inference_tokens_per_sec: 0.0,
5324            inference_tokens_per_sec_ema: 0.0,
5325            speculative_accepted: 0,
5326            speculative_rejected: 0,
5327        }
5328    }
5329
5330    /// A generation output stream that yields a fixed sequence of chunks. Counts
5331    /// Ok chunks consumed so far as the generated-token count.
5332    struct FakeStream {
5333        // `anyhow::Error` is not `Clone`, so store the error as a `String` and
5334        // wrap it at yield time.
5335        chunks: Vec<Result<String, String>>,
5336        pos: usize,
5337        prefill: usize,
5338    }
5339
5340    impl FakeStream {
5341        fn new(chunks: Vec<Result<String, String>>, prefill: usize) -> Self {
5342            FakeStream {
5343                chunks,
5344                pos: 0,
5345                prefill,
5346            }
5347        }
5348        fn generated(&self) -> usize {
5349            self.chunks[..self.pos].iter().filter(|c| c.is_ok()).count()
5350        }
5351    }
5352
5353    impl futures::Stream for FakeStream {
5354        type Item = anyhow::Result<String>;
5355        fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
5356            let this = self.get_mut();
5357            if this.pos < this.chunks.len() {
5358                let c = this.chunks[this.pos].clone();
5359                this.pos += 1;
5360                Poll::Ready(Some(c.map_err(anyhow::Error::msg)))
5361            } else {
5362                Poll::Ready(None)
5363            }
5364        }
5365    }
5366
5367    impl GenOutput for FakeStream {
5368        fn ema_rate(&self) -> f32 {
5369            1.0
5370        }
5371        fn completion_stats(&self) -> GenerationStats {
5372            stats(self.prefill, self.generated())
5373        }
5374    }
5375
5376    /// A sink that records every chunk it publishes.
5377    #[derive(Default)]
5378    struct RecordingSink {
5379        published: Vec<String>,
5380    }
5381
5382    impl GenSink for RecordingSink {
5383        fn publish_chunk(&mut self, data: &str, _rate: f32) -> impl Future<Output = Result<()>> {
5384            self.published.push(data.to_owned());
5385            std::future::ready(Ok(()))
5386        }
5387    }
5388
5389    /// A sink that cancels the generation after the Nth published chunk, so the
5390    /// loop's cancel branch is exercised after partial output.
5391    struct CancelAfterSink {
5392        published: Vec<String>,
5393        cancel: tokio_util::sync::CancellationToken,
5394        cancel_after: usize,
5395    }
5396
5397    impl CancelAfterSink {
5398        fn new(cancel: tokio_util::sync::CancellationToken, cancel_after: usize) -> Self {
5399            CancelAfterSink {
5400                published: vec![],
5401                cancel,
5402                cancel_after,
5403            }
5404        }
5405    }
5406
5407    impl GenSink for CancelAfterSink {
5408        fn publish_chunk(&mut self, data: &str, _rate: f32) -> impl Future<Output = Result<()>> {
5409            self.published.push(data.to_owned());
5410            if self.published.len() >= self.cancel_after {
5411                self.cancel.cancel();
5412            }
5413            std::future::ready(Ok(()))
5414        }
5415    }
5416
5417    /// A sink that fails the Nth publish, exercising the publish-failure path.
5418    struct FailOnSink {
5419        published: Vec<String>,
5420        fail_on: usize,
5421    }
5422
5423    impl FailOnSink {
5424        fn new(fail_on: usize) -> Self {
5425            FailOnSink {
5426                published: vec![],
5427                fail_on,
5428            }
5429        }
5430    }
5431
5432    impl GenSink for FailOnSink {
5433        fn publish_chunk(&mut self, data: &str, _rate: f32) -> impl Future<Output = Result<()>> {
5434            let will_fail = self.published.len() + 1 == self.fail_on;
5435            self.published.push(data.to_owned());
5436            std::future::ready(if will_fail {
5437                Err(anyhow!("publish failed"))
5438            } else {
5439                Ok(())
5440            })
5441        }
5442    }
5443
5444    // ── #1: stats captured on EVERY terminal path (default build) ─────────────
5445
5446    #[tokio::test]
5447    async fn cancel_after_one_token_captures_partial_stats() {
5448        let cancel = tokio_util::sync::CancellationToken::new();
5449        let mut stream = FakeStream::new(vec![Ok("a".into()), Ok("b".into()), Ok("c".into())], 5);
5450        let mut sink = CancelAfterSink::new(cancel.clone(), 1);
5451        let outcome = drive_generation_loop(&mut stream, &cancel, &mut sink).await;
5452        assert!(matches!(outcome, GenLoopOutcome::Cancelled), "{outcome:?}");
5453        // One token made it to the subscriber before cancel.
5454        assert_eq!(sink.published.len(), 1);
5455        // Stats captured on the cancel exit reflect prefill + the one token.
5456        let s = stream.completion_stats();
5457        assert_eq!(s.prefill_tokens, 5);
5458        assert_eq!(s.tokens_generated, 1);
5459    }
5460
5461    #[tokio::test]
5462    async fn publish_failure_after_partial_captures_stats() {
5463        let cancel = tokio_util::sync::CancellationToken::new();
5464        let mut stream = FakeStream::new(vec![Ok("a".into()), Ok("b".into()), Ok("c".into())], 5);
5465        let mut sink = FailOnSink::new(2);
5466        let outcome = drive_generation_loop(&mut stream, &cancel, &mut sink).await;
5467        assert!(
5468            matches!(outcome, GenLoopOutcome::PublishFailed(_)),
5469            "{outcome:?}"
5470        );
5471        // "a" published; "b" generated then its publish failed.
5472        let s = stream.completion_stats();
5473        assert_eq!(s.prefill_tokens, 5);
5474        assert_eq!(s.tokens_generated, 2);
5475    }
5476
5477    #[tokio::test]
5478    async fn stream_error_after_partial_captures_stats() {
5479        let cancel = tokio_util::sync::CancellationToken::new();
5480        let mut stream = FakeStream::new(vec![Ok("a".into()), Err("decode boom".to_owned())], 5);
5481        let mut sink = RecordingSink::default();
5482        let outcome = drive_generation_loop(&mut stream, &cancel, &mut sink).await;
5483        assert!(
5484            matches!(outcome, GenLoopOutcome::StreamError(_)),
5485            "{outcome:?}"
5486        );
5487        // Only the one Ok token counts; the error chunk does not.
5488        let s = stream.completion_stats();
5489        assert_eq!(s.prefill_tokens, 5);
5490        assert_eq!(s.tokens_generated, 1);
5491    }
5492
5493    #[tokio::test]
5494    async fn normal_completion_captures_full_stats() {
5495        let cancel = tokio_util::sync::CancellationToken::new();
5496        let mut stream = FakeStream::new(vec![Ok("a".into()), Ok("b".into())], 5);
5497        let mut sink = RecordingSink::default();
5498        let outcome = drive_generation_loop(&mut stream, &cancel, &mut sink).await;
5499        assert!(matches!(outcome, GenLoopOutcome::Exhausted), "{outcome:?}");
5500        let s = stream.completion_stats();
5501        assert_eq!(s.prefill_tokens, 5);
5502        assert_eq!(s.tokens_generated, 2);
5503    }
5504
5505    // ── #2: missing billing identity fails closed (default build) ─────────────
5506
5507    #[test]
5508    fn missing_identity_denies_only_when_emitter_attached() {
5509        // Emitter attached + no DID ⇒ deny (fail-closed before generation).
5510        assert!(ledger_requires_identity(true, None));
5511        // Authenticated caller with a DID ⇒ never deny.
5512        assert!(!ledger_requires_identity(true, Some("did:web:alice")));
5513        // Subsystem inert (no emitter) ⇒ anonymous callers unaffected.
5514        assert!(!ledger_requires_identity(false, None));
5515        assert!(!ledger_requires_identity(false, Some("did:web:alice")));
5516    }
5517
5518    #[test]
5519    fn deny_frame_is_content_free() {
5520        // The fail-closed frame must carry no subject DID and no prompt material.
5521        assert!(LEDGER_DENY_NO_IDENTITY.contains("billing identity"));
5522        assert!(!LEDGER_DENY_NO_IDENTITY.contains("did:"));
5523    }
5524
5525    // ── #1 + #2 spend posting (ledger build) ─────────────────────────────────
5526    #[cfg(feature = "ledger")]
5527    mod spend {
5528        use super::*;
5529        use crate::services::ledger::inference_spend::tests::{available, fixture};
5530
5531        // Drive a generation to a terminal outcome, then post the spend for the
5532        // stats captured on that exit. Asserts the ledger is debited for the
5533        // work actually done (prefill + generated).
5534        async fn drive_and_post(
5535            stream: &mut FakeStream,
5536            cancel: &tokio_util::sync::CancellationToken,
5537            sink: &mut impl GenSink,
5538            emitter: &InferenceSpendEmitter,
5539            stream_id: &str,
5540        ) -> Option<SpendResult> {
5541            drive_generation_loop(stream, cancel, sink).await;
5542            post_completion_spend(
5543                emitter,
5544                Some(&stream.completion_stats()),
5545                Some("did:web:alice"),
5546                stream_id,
5547            )
5548            .await
5549        }
5550
5551        #[tokio::test]
5552        async fn cancelled_after_one_token_posts_spend() {
5553            let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5554            let cancel = tokio_util::sync::CancellationToken::new();
5555            let mut stream =
5556                FakeStream::new(vec![Ok("a".into()), Ok("b".into()), Ok("c".into())], 5);
5557            let mut sink = CancelAfterSink::new(cancel.clone(), 1);
5558            let res =
5559                drive_and_post(&mut stream, &cancel, &mut sink, &emitter, "stream-cancel").await;
5560            assert!(
5561                matches!(res, Some(SpendResult::Posted { amount: 6, .. })),
5562                "{res:?}"
5563            );
5564            // prefill(5) + 1 generated token debited; the rest never ran.
5565            assert_eq!(available(&handle, "did:web:alice").await, 994);
5566        }
5567
5568        #[tokio::test]
5569        async fn publish_failure_after_partial_posts_spend() {
5570            let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5571            let cancel = tokio_util::sync::CancellationToken::new();
5572            let mut stream =
5573                FakeStream::new(vec![Ok("a".into()), Ok("b".into()), Ok("c".into())], 5);
5574            let mut sink = FailOnSink::new(2);
5575            let res =
5576                drive_and_post(&mut stream, &cancel, &mut sink, &emitter, "stream-pubfail").await;
5577            assert!(
5578                matches!(res, Some(SpendResult::Posted { amount: 7, .. })),
5579                "{res:?}"
5580            );
5581            // prefill(5) + 2 generated tokens (a published, b's publish failed).
5582            assert_eq!(available(&handle, "did:web:alice").await, 993);
5583        }
5584
5585        #[tokio::test]
5586        async fn stream_error_after_partial_posts_spend() {
5587            let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5588            let cancel = tokio_util::sync::CancellationToken::new();
5589            let mut stream = FakeStream::new(vec![Ok("a".into()), Err("boom".to_owned())], 5);
5590            let mut sink = RecordingSink::default();
5591            let res = drive_and_post(&mut stream, &cancel, &mut sink, &emitter, "stream-err").await;
5592            assert!(
5593                matches!(res, Some(SpendResult::Posted { amount: 6, .. })),
5594                "{res:?}"
5595            );
5596            // prefill(5) + 1 generated token before the error.
5597            assert_eq!(available(&handle, "did:web:alice").await, 994);
5598        }
5599
5600        #[tokio::test]
5601        async fn normal_completion_posts_spend() {
5602            let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5603            let cancel = tokio_util::sync::CancellationToken::new();
5604            let mut stream = FakeStream::new(vec![Ok("a".into()), Ok("b".into())], 5);
5605            let mut sink = RecordingSink::default();
5606            let res = drive_and_post(&mut stream, &cancel, &mut sink, &emitter, "stream-ok").await;
5607            assert!(
5608                matches!(res, Some(SpendResult::Posted { amount: 7, .. })),
5609                "{res:?}"
5610            );
5611            assert_eq!(available(&handle, "did:web:alice").await, 993);
5612        }
5613
5614        #[tokio::test]
5615        async fn missing_identity_posts_no_spend() {
5616            // A DID-less caller (which the execute_stream gate denies
5617            // pre-generation) posts no spend and touches no account.
5618            let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5619            let s = stats(5, 1);
5620            let res = post_completion_spend(&emitter, Some(&s), None, "stream-anon").await;
5621            assert!(res.is_none(), "{res:?}");
5622            assert_eq!(available(&handle, "did:web:alice").await, 1000);
5623        }
5624
5625        #[tokio::test]
5626        async fn no_stats_means_nothing_to_account() {
5627            // Generation never began (engine returned Err) ⇒ no spend attempted.
5628            let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5629            let res =
5630                post_completion_spend(&emitter, None, Some("did:web:alice"), "stream-nostart")
5631                    .await;
5632            assert!(res.is_none(), "{res:?}");
5633            assert_eq!(available(&handle, "did:web:alice").await, 1000);
5634        }
5635    }
5636}
5637
5638/// User-approved single-service Model→Inference boundary (D5) — causal tests.
5639///
5640/// These exercise the REAL changed boundaries without a live engine:
5641/// - the adapter's `verify_internal_work_bearer` hook through the REAL generic
5642///   `verify_claims` pipeline (typ routing, pinned-controller signature,
5643///   instance/tenant/model binding, freshness, replay);
5644/// - the REAL `InferenceHandler::authorize` internal-work branch (exact
5645///   operation binding, MAC floor on the carried claims, and — via a dead
5646///   Policy transport — proof that the worker consults NO Policy on the
5647///   internal leg).
5648///
5649/// The spawned-engine transport leg remains covered only by the GPU-gated
5650/// `e2e_inference` suite; that facet is reported as untested in the receipt.
5651#[cfg(test)]
5652#[allow(clippy::unwrap_used, clippy::expect_used)]
5653mod single_service_boundary_tests {
5654    use super::*;
5655
5656    use crate::runtime::inference_profile::InferenceInstanceId;
5657    use hyprstream_rpc::auth::internal_work::{
5658        encode_internal_work, InternalWorkClaims, INTERNAL_WORK_ISSUER,
5659    };
5660    use hyprstream_rpc::envelope::{RequestEnvelope, SignedEnvelope};
5661
5662    const CONTROLLER_KEY: [u8; 32] = [0xC7; 32];
5663    const STRANGER_KEY: [u8; 32] = [0xC9; 32];
5664    const TENANT: &str = "tenant-boundary";
5665    const MODEL: &str = "boundary-model:main";
5666
5667    /// A policy transport that fails loudly if dialed. The internal leg must
5668    /// never consult Policy — an `Ok` from `authorize` IS the causal proof.
5669    struct DeadPolicyRpc;
5670
5671    #[async_trait::async_trait]
5672    impl hyprstream_rpc::RpcClient for DeadPolicyRpc {
5673        async fn call(&self, _payload: Vec<u8>) -> Result<Vec<u8>> {
5674            anyhow::bail!("policy must not be dialed on the internal work leg")
5675        }
5676        async fn call_for_service(
5677            &self,
5678            _service_domain: &str,
5679            _payload: Vec<u8>,
5680        ) -> Result<Vec<u8>> {
5681            anyhow::bail!("policy must not be dialed on the internal work leg")
5682        }
5683        async fn call_for_service_with_method(
5684            &self,
5685            _service_domain: &str,
5686            _method_discriminator: u16,
5687            _payload: Vec<u8>,
5688        ) -> Result<Vec<u8>> {
5689            anyhow::bail!("policy must not be dialed on the internal work leg")
5690        }
5691        async fn call_with_options(
5692            &self,
5693            _payload: Vec<u8>,
5694            _options: hyprstream_rpc::CallOptions,
5695        ) -> Result<Vec<u8>> {
5696            anyhow::bail!("policy must not be dialed on the internal work leg")
5697        }
5698        async fn call_with_options_for_service(
5699            &self,
5700            _service_domain: &str,
5701            _payload: Vec<u8>,
5702            _options: hyprstream_rpc::CallOptions,
5703        ) -> Result<Vec<u8>> {
5704            anyhow::bail!("policy must not be dialed on the internal work leg")
5705        }
5706        async fn call_streaming(
5707            &self,
5708            _payload: Vec<u8>,
5709            _ephemeral_pubkey: [u8; 32],
5710        ) -> Result<Vec<u8>> {
5711            anyhow::bail!("policy must not be dialed on the internal work leg")
5712        }
5713        async fn call_streaming_for_service(
5714            &self,
5715            _service_domain: &str,
5716            _payload: Vec<u8>,
5717            _ephemeral_pubkey: [u8; 32],
5718        ) -> Result<Vec<u8>> {
5719            anyhow::bail!("policy must not be dialed on the internal work leg")
5720        }
5721        async fn call_streaming_for_service_with_method(
5722            &self,
5723            _service_domain: &str,
5724            _method_discriminator: u16,
5725            _payload: Vec<u8>,
5726            _ephemeral_pubkey: [u8; 32],
5727        ) -> Result<Vec<u8>> {
5728            anyhow::bail!("policy must not be dialed on the internal work leg")
5729        }
5730        async fn open_stream(
5731            &self,
5732            _payload: Vec<u8>,
5733        ) -> Result<Box<dyn hyprstream_rpc::stream_consumer::StreamHandle>> {
5734            anyhow::bail!("policy must not be dialed on the internal work leg")
5735        }
5736        async fn open_stream_from_info(
5737            &self,
5738            _stream_info: hyprstream_rpc::stream_info::StreamInfo,
5739            _client_secret: [u8; 32],
5740            _client_pubkey: [u8; 32],
5741        ) -> Result<Box<dyn hyprstream_rpc::stream_consumer::StreamHandle>> {
5742            anyhow::bail!("policy must not be dialed on the internal work leg")
5743        }
5744        fn next_id(&self) -> u64 {
5745            1
5746        }
5747    }
5748
5749    /// Build a real adapter over an unloaded engine (no weights needed: these
5750    /// tests never execute generation) with a DEAD policy transport.
5751    /// The test incarnation bound into every default fixture adapter.
5752    const TEST_INCARNATION: &str = "test-incarnation-0000";
5753
5754    fn boundary_adapter(
5755        controller: &SigningKey,
5756        instance_service_name: &str,
5757        tenant: &str,
5758        model: &str,
5759        with_model_binding: bool,
5760    ) -> InferenceZmqAdapter {
5761        boundary_adapter_with_pool(
5762            controller,
5763            instance_service_name,
5764            tenant,
5765            model,
5766            with_model_binding,
5767            None,
5768            TEST_INCARNATION,
5769        )
5770    }
5771
5772    fn boundary_adapter_with_pool(
5773        controller: &SigningKey,
5774        instance_service_name: &str,
5775        tenant: &str,
5776        model: &str,
5777        with_model_binding: bool,
5778        delta_pool: Option<Arc<DeltaPool>>,
5779        incarnation: &str,
5780    ) -> InferenceZmqAdapter {
5781        let service = InferenceService {
5782            inner: Arc::new(InferenceServiceInner {
5783                engine: parking_lot::RwLock::new(
5784                    create_engine(&RuntimeConfig::default())
5785                        .expect("engine construction (no weights)"),
5786                ),
5787                model_path: PathBuf::from("/nonexistent"),
5788                state_path: PathBuf::from("/nonexistent"),
5789                _pinned_artifact: None,
5790                session_id: parking_lot::RwLock::new(None),
5791                runtime_handle: Handle::current(),
5792                stream_channel: None,
5793                server_pubkey: controller.verifying_key(),
5794                signing_key: controller.clone(),
5795                nonce_cache: Arc::new(InMemoryNonceCache::new()),
5796                policy_client: PolicyClient::new(Arc::new(DeadPolicyRpc)),
5797                ttt_trainer: None,
5798                tokenizer: None,
5799                delta_pool,
5800                base_delta: Mutex::new(None),
5801                fs: None,
5802                transport: hyprstream_rpc::transport::TransportConfig::inproc("boundary-test"),
5803                lora_generation: Arc::new(AtomicU64::new(0)),
5804                tenant_domain: tenant.to_owned(),
5805                controller_pubkey: controller.verifying_key(),
5806                network_ready: Arc::new(AtomicBool::new(true)),
5807                draining: Arc::new(AtomicBool::new(false)),
5808                shutdown: Arc::new(tokio::sync::Notify::new()),
5809                #[cfg(feature = "ledger")]
5810                ledger: parking_lot::RwLock::new(None),
5811                object_label: inference_object_label(),
5812            }),
5813        };
5814        InferenceZmqAdapter {
5815            service,
5816            transport: hyprstream_rpc::transport::TransportConfig::inproc("boundary-test"),
5817            signing_key: controller.clone(),
5818            expected_audience: None,
5819            jwt_key_source: None,
5820            instance_model_ref: with_model_binding.then(|| model.to_owned()),
5821            instance_audience: internal_work_audience(instance_service_name, incarnation),
5822        }
5823    }
5824
5825    fn fixture_work_claims(
5826        controller: &SigningKey,
5827        instance_service_name: &str,
5828        now: i64,
5829    ) -> InternalWorkClaims {
5830        fixture_work_claims_for_subject(controller, instance_service_name, now, "alice")
5831    }
5832
5833    fn fixture_work_claims_for_subject(
5834        controller: &SigningKey,
5835        instance_service_name: &str,
5836        now: i64,
5837        subject_string: &str,
5838    ) -> InternalWorkClaims {
5839        InternalWorkClaims {
5840            iss: INTERNAL_WORK_ISSUER.to_owned(),
5841            sub: subject_string.to_owned(),
5842            aud: internal_work_audience(instance_service_name, TEST_INCARNATION),
5843            tenant: TENANT.to_owned(),
5844            model: MODEL.to_owned(),
5845            resource: "inference:GenerateStream".to_owned(),
5846            operation: "infer".to_owned(),
5847            iat: now,
5848            exp: now + 60,
5849            jti: format!(
5850                "jti-{}-{}",
5851                hex::encode(&controller.verifying_key().as_bytes()[..4]),
5852                std::time::SystemTime::now()
5853                    .duration_since(std::time::UNIX_EPOCH)
5854                    .expect("clock")
5855                    .as_nanos()
5856            ),
5857            cnf: InternalWorkClaims::controller_cnf(&controller.verifying_key()),
5858            owner_did: Some("did:key:z6Mkboundary".to_owned()),
5859            caller: hyprstream_rpc::auth::Claims::new(subject_string.to_owned(), now, now + 3600)
5860                .with_tenant(TENANT.to_owned())
5861                // The MAC floor the worker enforces on the internal leg — the
5862                // same clearance Model's ingress gate required of the caller.
5863                .with_clearance(inference_object_label()),
5864        }
5865    }
5866
5867    /// Context fixture shaped like the production AnySigner bridge: the
5868    /// envelope signer (`cnf`) is the controller, the bearer is the minted
5869    /// work order.
5870    fn ctx_with_work(token: String, controller: &SigningKey) -> EnvelopeContext {
5871        let envelope = SignedEnvelope::new_signed(
5872            RequestEnvelope::anonymous(Vec::new()).with_jwt_token(token),
5873            controller,
5874        );
5875        EnvelopeContext::from_verified_as_system(&envelope)
5876    }
5877
5878    #[tokio::test]
5879    async fn internal_work_admits_through_real_verify_claims_and_authorize() {
5880        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
5881        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
5882        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5883        let now = chrono::Utc::now().timestamp();
5884        let token = encode_internal_work(
5885            &fixture_work_claims(&controller, &instance.service_name(), now),
5886            &controller,
5887        );
5888        let mut ctx = ctx_with_work(token, &controller);
5889
5890        // The REAL generic pipeline routes `iw+jwt` into the adapter hook.
5891        adapter
5892            .verify_claims(&mut ctx)
5893            .await
5894            .expect("controller work order must pass the real verify_claims routing");
5895        let work = ctx.internal_work().expect("work order installed");
5896        assert_eq!(work.resource, "inference:GenerateStream");
5897        assert_eq!(work.operation, "infer");
5898        assert_eq!(ctx.verified_tenant(), Some(TENANT));
5899        assert_eq!(ctx.claims().map(|c| c.sub.as_str()), Some("alice"));
5900
5901        // The REAL per-method PEP accepts the exact bound coordinate with the
5902        // policy transport DEAD — causal proof of non-consultation.
5903        InferenceHandler::authorize(&adapter.service, &ctx, "inference:GenerateStream", "infer")
5904            .await
5905            .expect("internal work authorizes exactly its bound operation");
5906        // The original caller subject the work order carries (on the
5907        // production AnySigner bridge — the only serving plane — ctx.subject()
5908        // resolves to exactly this; proven at the svc.rs routing test).
5909        assert_eq!(
5910            ctx.internal_work().expect("work order").subject.name(),
5911            Some("alice")
5912        );
5913    }
5914
5915    #[tokio::test]
5916    async fn internal_work_rejects_wrong_instance_tenant_model_and_standalone() {
5917        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
5918        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
5919        let now = chrono::Utc::now().timestamp();
5920
5921        // Wrong audience (a DIFFERENT allocated instance).
5922        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5923        let mut ctx = ctx_with_work(
5924            encode_internal_work(
5925                &fixture_work_claims(&controller, "inference-other-instance", now),
5926                &controller,
5927            ),
5928            &controller,
5929        );
5930        let err = adapter
5931            .verify_claims(&mut ctx)
5932            .await
5933            .expect_err("a work order for another instance must deny");
5934        assert!(err.to_string().contains("audience"), "{err:#}");
5935
5936        // Wrong tenant binding: the instance is bound to TENANT while the work
5937        // order claims a different tenant.
5938        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5939        let mut claims = fixture_work_claims(&controller, &instance.service_name(), now);
5940        claims.tenant = "tenant-other".to_owned();
5941        let mut ctx = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
5942        let err = adapter
5943            .verify_claims(&mut ctx)
5944            .await
5945            .expect_err("a work order for another tenant must deny");
5946        assert!(err.to_string().contains("tenant"), "{err:#}");
5947
5948        // Wrong model binding: the instance serves MODEL while the work order
5949        // claims a different model.
5950        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5951        let mut claims = fixture_work_claims(&controller, &instance.service_name(), now);
5952        claims.model = "model-other:main".to_owned();
5953        let mut ctx = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
5954        let err = adapter
5955            .verify_claims(&mut ctx)
5956            .await
5957            .expect_err("a work order for another model must deny");
5958        assert!(err.to_string().contains("model"), "{err:#}");
5959
5960        // Standalone instance (no model binding): internal work never admitted.
5961        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, false);
5962        let claims = fixture_work_claims(&controller, &instance.service_name(), now);
5963        let mut ctx = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
5964        let err = adapter
5965            .verify_claims(&mut ctx)
5966            .await
5967            .expect_err("a standalone instance accepts no internal work");
5968        assert!(err.to_string().contains("standalone"), "{err:#}");
5969    }
5970
5971    #[tokio::test]
5972    async fn internal_work_rejects_foreign_controller_expired_and_replayed() {
5973        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
5974        let stranger = SigningKey::from_bytes(&STRANGER_KEY);
5975        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
5976        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5977        let now = chrono::Utc::now().timestamp();
5978
5979        // A foreign controller minting for our instance: signature fails
5980        // against the PINNED key.
5981        let mut ctx = ctx_with_work(
5982            encode_internal_work(
5983                &fixture_work_claims(&stranger, &instance.service_name(), now),
5984                &stranger,
5985            ),
5986            &stranger,
5987        );
5988        let err = adapter
5989            .verify_claims(&mut ctx)
5990            .await
5991            .expect_err("a foreign controller cannot mint for a pinned instance");
5992        assert!(err.to_string().contains("pinned controller"), "{err:#}");
5993
5994        // Expired work order.
5995        let mut expired = fixture_work_claims(&controller, &instance.service_name(), now);
5996        expired.exp = now - 1;
5997        let mut ctx = ctx_with_work(encode_internal_work(&expired, &controller), &controller);
5998        let err = adapter
5999            .verify_claims(&mut ctx)
6000            .await
6001            .expect_err("expired work orders deny");
6002        assert!(err.to_string().contains("expired"), "{err:#}");
6003
6004        // Oversized lifetime (forged exp beyond the hard bound).
6005        let mut long = fixture_work_claims(&controller, &instance.service_name(), now);
6006        long.exp = now + 60 * 60;
6007        let mut ctx = ctx_with_work(encode_internal_work(&long, &controller), &controller);
6008        let err = adapter
6009            .verify_claims(&mut ctx)
6010            .await
6011            .expect_err("oversized lifetimes deny");
6012        assert!(
6013            err.to_string().contains("maximum internal lifetime"),
6014            "{err:#}"
6015        );
6016
6017        // Replay: the same jti admitted twice.
6018        let claims = fixture_work_claims(&controller, &instance.service_name(), now);
6019        let token = encode_internal_work(&claims, &controller);
6020        let mut first = ctx_with_work(token.clone(), &controller);
6021        adapter
6022            .verify_claims(&mut first)
6023            .await
6024            .expect("first admission succeeds");
6025        let mut second = ctx_with_work(token, &controller);
6026        let err = adapter
6027            .verify_claims(&mut second)
6028            .await
6029            .expect_err("the same work order cannot be admitted twice");
6030        assert!(err.to_string().contains("replay"), "{err:#}");
6031    }
6032
6033    #[tokio::test]
6034    async fn internal_work_authorize_denies_unbound_operations() {
6035        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6036        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6037        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6038        let now = chrono::Utc::now().timestamp();
6039        let token = encode_internal_work(
6040            &fixture_work_claims(&controller, &instance.service_name(), now),
6041            &controller,
6042        );
6043        let mut ctx = ctx_with_work(token, &controller);
6044        adapter.verify_claims(&mut ctx).await.expect("admitted");
6045
6046        // A different operation on the same resource is NOT authorized by this
6047        // work order — Model authorized exactly one operation.
6048        let err =
6049            InferenceHandler::authorize(&adapter.service, &ctx, "inference:tttZero", "manage")
6050                .await
6051                .expect_err("work order does not authorize other operations");
6052        assert!(err.to_string().contains("does not authorize"), "{err:#}");
6053
6054        // The same operation on a different resource is NOT authorized either.
6055        let err = InferenceHandler::authorize(&adapter.service, &ctx, "inference:embed", "infer")
6056            .await
6057            .expect_err("work order does not authorize other resources");
6058        assert!(err.to_string().contains("does not authorize"), "{err:#}");
6059    }
6060
6061    #[tokio::test]
6062    async fn direct_caller_path_still_requires_policy_on_dead_transport() {
6063        // Control: a NON-internal caller on the same service still goes through
6064        // Policy — the dead transport must deny, proving the internal branch
6065        // (not a blanket bypass) is what skipped Policy above.
6066        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6067        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6068        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6069        // NOT the internal-work path (no internal_work on ctx). A
6070        // clearance-bearing fixture reaches the Policy dial.
6071        let mut label_claims = hyprstream_rpc::auth::Claims::new(
6072            "alice".to_owned(),
6073            chrono::Utc::now().timestamp(),
6074            chrono::Utc::now().timestamp() + 3600,
6075        )
6076        .with_tenant(TENANT.to_owned());
6077        label_claims.clearance = Some(
6078            crate::mac::genesis::SitePolicy::conservative()
6079                .label_for("/srv/inference")
6080                .into(),
6081        );
6082        let ctx = EnvelopeContext::for_test_authenticated_subject_with_claims(
6083            Subject::new("alice"),
6084            TENANT,
6085            controller.verifying_key(),
6086            label_claims,
6087        );
6088        let err = InferenceHandler::authorize(
6089            &adapter.service,
6090            &ctx,
6091            "inference:generateStream",
6092            "infer",
6093        )
6094        .await
6095        .expect_err("direct callers still require the Policy decision");
6096        assert!(
6097            err.to_string().contains("policy must not be dialed")
6098                || err.to_string().contains("Unauthorized")
6099                || err.to_string().contains("denying"),
6100            "expected the dead-policy denial, got: {err:#}"
6101        );
6102    }
6103
6104    // ── REAL generated-dispatch coverage (K3 findings B1/B2) ────────────────
6105    //
6106    // The earlier boundary tests called `authorize` directly with
6107    // hand-written coordinates, so mint and PEP agreed with each other while
6108    // both disagreed with the generated dispatch grammar (camelCase mint vs
6109    // PascalCase dispatch; Text variants embedding the payload). These tests
6110    // close that gap: each mints through the Model grammar helper, admits
6111    // through the REAL `verify_claims` + adapter hook, and then drives the
6112    // REAL generated `dispatch_inference` over a real capnp request body —
6113    // one forwarded operation per dispatch variant class (Void, Text,
6114    // struct).
6115
6116    use crate::services::generated::inference_client::dispatch_inference;
6117
6118    /// Serialize a real `InferenceRequest` wire body for the given variant.
6119    fn encoded_request(
6120        build: impl FnOnce(&mut hyprstream_rpc_std::inference_capnp::inference_request::Builder<'_>),
6121    ) -> Result<Vec<u8>> {
6122        let mut message = capnp::message::Builder::new_default();
6123        {
6124            let mut request = message
6125                .init_root::<hyprstream_rpc_std::inference_capnp::inference_request::Builder>(
6126            );
6127            request.set_id(42);
6128            build(&mut request);
6129        }
6130        let mut bytes = Vec::new();
6131        capnp::serialize::write_message(&mut bytes, &message)?;
6132        Ok(bytes)
6133    }
6134
6135    /// Mint a work order with the PRODUCTION grammar coordinate and admit it
6136    /// through the real pipeline, returning the dispatch-ready context.
6137    async fn admitted_ctx(
6138        adapter: &InferenceZmqAdapter,
6139        controller: &SigningKey,
6140        instance_service_name: &str,
6141        resource: String,
6142        operation: &str,
6143    ) -> Result<EnvelopeContext> {
6144        let now = chrono::Utc::now().timestamp();
6145        let mut claims = fixture_work_claims(controller, instance_service_name, now);
6146        claims.resource = resource;
6147        claims.operation = operation.to_owned();
6148        let mut ctx = ctx_with_work(encode_internal_work(&claims, controller), controller);
6149        adapter.verify_claims(&mut ctx).await?;
6150        Ok(ctx)
6151    }
6152
6153    #[tokio::test]
6154    async fn real_dispatch_void_variant_has_lora_succeeds_through_full_chain() {
6155        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6156        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6157        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6158        // The EXACT coordinate the production mint emits (Model grammar
6159        // helper), which must equal what generated dispatch enforces.
6160        let resource = crate::services::model::ModelService::inference_dispatch_resource("hasLora");
6161        assert_eq!(resource, "inference:HasLora");
6162        let ctx = admitted_ctx(
6163            &adapter,
6164            &controller,
6165            &instance.service_name(),
6166            resource,
6167            "query",
6168        )
6169        .await
6170        .expect("work order must pass the real verify_claims routing");
6171
6172        let payload = encoded_request(|request| {
6173            request.set_has_lora(());
6174        })
6175        .expect("encode hasLora");
6176        let decoded =
6177            crate::services::generated::inference_client::decode_inference_request_body(&payload)
6178                .expect("decode hasLora");
6179        // The REAL generated dispatch: PEP + handler + serialization.
6180        let (response, continuation) =
6181            match dispatch_inference(&adapter.service, &ctx, &decoded).await {
6182                Ok(pair) => pair,
6183                Err(error) => panic!(
6184                "real dispatch of a correctly-scoped work order must reach the handler: {error:#}"
6185            ),
6186            };
6187        assert!(continuation.is_none());
6188        let parsed =
6189            hyprstream_rpc_std::inference_client::InferenceClient::parse_response(&response)
6190                .expect("parse hasLora response");
6191        match parsed {
6192            InferenceResponseVariant::HasLoraResult(has) => {
6193                assert!(!has, "unloaded engine reports no adapter");
6194            }
6195            other => panic!("expected HasLoraResult, got {other:?}"),
6196        }
6197    }
6198
6199    #[tokio::test]
6200    async fn real_dispatch_text_variant_load_lora_binds_exact_payload_coordinate() {
6201        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6202        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6203        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6204        // K3 finding B2: Text-payload variants dispatch as
6205        // `inference:{payload}` — the production mint binds the exact
6206        // adapter path, so the payload below must match.
6207        let path = "probe/adapters/k3";
6208        let ctx = admitted_ctx(
6209            &adapter,
6210            &controller,
6211            &instance.service_name(),
6212            format!("inference:{path}"),
6213            "write",
6214        )
6215        .await
6216        .expect("payload-bound work order must pass the real verify_claims routing");
6217
6218        let payload = encoded_request(|request| {
6219            request.set_load_lora(path);
6220        })
6221        .expect("encode loadLora");
6222        let decoded =
6223            crate::services::generated::inference_client::decode_inference_request_body(&payload)
6224                .expect("decode loadLora");
6225        let (response, _continuation) =
6226            match dispatch_inference(&adapter.service, &ctx, &decoded).await {
6227                Ok(pair) => pair,
6228                Err(error) => panic!(
6229                    "real dispatch must pass the PEP for the payload-bound coordinate: {error:#}"
6230                ),
6231            };
6232        // The handler runs and fails on the FIXTURE's missing path containment
6233        // (post-PEP), never on authorization.
6234        let parsed =
6235            hyprstream_rpc_std::inference_client::InferenceClient::parse_response(&response)
6236                .expect("parse loadLora response");
6237        match parsed {
6238            InferenceResponseVariant::Error(info) => {
6239                assert!(
6240                    !info.message.contains("does not authorize")
6241                        && !info.message.contains("UNAUTHORIZED"),
6242                    "handler error must be post-PEP, got: {}",
6243                    info.message
6244                );
6245                assert!(
6246                    info.message.contains("FsOps not available"),
6247                    "expected the fixture's path-containment handler error, got: {}",
6248                    info.message
6249                );
6250            }
6251            other => panic!("expected the handler Error variant, got {other:?}"),
6252        }
6253    }
6254
6255    #[tokio::test]
6256    async fn real_dispatch_struct_variant_generate_stream_reaches_handler_after_pep() {
6257        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6258        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6259        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6260        let ctx = admitted_ctx(
6261            &adapter,
6262            &controller,
6263            &instance.service_name(),
6264            crate::services::model::ModelService::inference_dispatch_resource("generateStream"),
6265            "infer",
6266        )
6267        .await
6268        .expect("work order must pass the real verify_claims routing");
6269
6270        let payload = encoded_request(|request| {
6271            request.reborrow().init_generate_stream();
6272        })
6273        .expect("encode generateStream");
6274        let decoded =
6275            crate::services::generated::inference_client::decode_inference_request_body(&payload)
6276                .expect("decode generateStream");
6277        // Streaming arms propagate handler errors: the PEP must pass and the
6278        // handler must fail on the missing caller ephemeral pubkey (this
6279        // fixture carries none) — K3's own control observation. The Ok
6280        // payload is not Debug, so match instead of expect_err.
6281        let error = match dispatch_inference(&adapter.service, &ctx, &decoded).await {
6282            Ok(_) => panic!("struct variant must reach the handler after the PEP"),
6283            Err(error) => error,
6284        };
6285        assert!(
6286            error.to_string().contains("ephemeral pubkey"),
6287            "expected the post-PEP handler error, got: {error:#}"
6288        );
6289        assert!(
6290            !error.to_string().contains("does not authorize"),
6291            "the PEP must not deny a correctly-scoped work order"
6292        );
6293    }
6294
6295    #[tokio::test]
6296    async fn real_dispatch_denies_when_mint_grammar_and_dispatch_disagree() {
6297        // Guard the B1 fix itself: a camelCase coordinate (the old, wrong
6298        // grammar) must still deny at the REAL dispatch.
6299        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6300        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6301        let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6302        let ctx = admitted_ctx(
6303            &adapter,
6304            &controller,
6305            &instance.service_name(),
6306            "inference:hasLora".to_owned(),
6307            "query",
6308        )
6309        .await
6310        .expect("admission is coordinate-agnostic; the PEP compares");
6311        let payload = encoded_request(|request| {
6312            request.set_has_lora(());
6313        })
6314        .expect("encode hasLora");
6315        let decoded =
6316            crate::services::generated::inference_client::decode_inference_request_body(&payload)
6317                .expect("decode hasLora");
6318        let error = match dispatch_inference(&adapter.service, &ctx, &decoded).await {
6319            Ok(_) => panic!("the stale camelCase grammar must keep denying"),
6320            Err(error) => error,
6321        };
6322        assert!(
6323            error.to_string().contains("does not authorize"),
6324            "expected the exact-match PEP denial, got: {error:#}"
6325        );
6326    }
6327
6328    // ── Sol bounded incarnation plan: worker incarnation binding ────────────
6329    //
6330    // The work-order audience is now `iw1/{instance}/{worker incarnation}`.
6331    // These tests prove: a token minted for incarnation A is DENIED at a
6332    // worker incarnation B of the SAME deterministic instance/tenant/model;
6333    // a token for B is admitted at B; a second submission of the same receipt
6334    // is denied through the JTI gate; and standalone instances still deny.
6335
6336    #[tokio::test]
6337    async fn incarnation_a_token_denied_at_worker_incarnation_b() {
6338        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6339        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6340        let adapter_a = boundary_adapter_with_pool(
6341            &controller,
6342            &instance.service_name(),
6343            TENANT,
6344            MODEL,
6345            true,
6346            None,
6347            "incarnation-aaaa",
6348        );
6349        let adapter_b = boundary_adapter_with_pool(
6350            &controller,
6351            &instance.service_name(),
6352            TENANT,
6353            MODEL,
6354            true,
6355            None,
6356            "incarnation-bbbb",
6357        );
6358        let now = chrono::Utc::now().timestamp();
6359        let mut claims =
6360            fixture_work_claims_for_subject(&controller, &instance.service_name(), now, "alice");
6361        // A's token binds EXACTLY A's versioned audience.
6362        claims.aud = internal_work_audience(&instance.service_name(), "incarnation-aaaa");
6363        let mut ctx = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
6364        adapter_a
6365            .verify_claims(&mut ctx)
6366            .await
6367            .expect("A token admits at A");
6368        // The same receipt is DENIED at incarnation B — pre-restart receipts
6369        // do not survive a worker replacement.
6370        let mut ctx_b = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
6371        let error = adapter_b
6372            .verify_claims(&mut ctx_b)
6373            .await
6374            .expect_err("A token must fail at worker incarnation B");
6375        assert!(error.to_string().contains("audience"), "{error:#}");
6376        assert!(ctx_b.internal_work().is_none());
6377
6378        // B's own receipt admits at B through the same real path.
6379        let mut b_claims =
6380            fixture_work_claims_for_subject(&controller, &instance.service_name(), now, "alice");
6381        b_claims.aud = internal_work_audience(&instance.service_name(), "incarnation-bbbb");
6382        b_claims.jti = format!("{}-b", b_claims.jti);
6383        let mut ctx_b2 = ctx_with_work(encode_internal_work(&b_claims, &controller), &controller);
6384        adapter_b
6385            .verify_claims(&mut ctx_b2)
6386            .await
6387            .expect("B token admits at B");
6388    }
6389
6390    #[tokio::test]
6391    async fn duplicate_incarnation_receipt_denies_through_jti_gate() {
6392        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6393        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6394        let adapter = boundary_adapter_with_pool(
6395            &controller,
6396            &instance.service_name(),
6397            TENANT,
6398            MODEL,
6399            true,
6400            None,
6401            "incarnation-cccc",
6402        );
6403        let now = chrono::Utc::now().timestamp();
6404        let mut claims =
6405            fixture_work_claims_for_subject(&controller, &instance.service_name(), now, "alice");
6406        claims.aud = internal_work_audience(&instance.service_name(), "incarnation-cccc");
6407        let token = encode_internal_work(&claims, &controller);
6408        // Same jti, same aud, same everything: first submission admitted,
6409        // second submission denied through the JTI gate INSIDE one incarnation.
6410        let mut first = ctx_with_work(token.clone(), &controller);
6411        adapter
6412            .verify_claims(&mut first)
6413            .await
6414            .expect("first submission admitted");
6415        let mut second = ctx_with_work(token, &controller);
6416        let error = adapter
6417            .verify_claims(&mut second)
6418            .await
6419            .expect_err("second submission of one receipt must deny");
6420        assert!(error.to_string().contains("replay"), "{error:#}");
6421    }
6422
6423    // ── Sol S1: distinct callers keep subject-keyed TTT state separate ─────
6424    //
6425    // Admission installs the EXACT resolved subject (proven at the real
6426    // verify_claims seam by the svc.rs Sol S1 regressions; the production
6427    // LocalServiceBridge serves AnySigner, where ctx.subject() IS the
6428    // installed jwt_subject). This test proves the DOWNSTREAM half: the
6429    // subject-keyed TTT operations key on those exact typed identities at the
6430    // real DeltaPool and real handler code, so `user:alice` and `alice` never
6431    // share adaptation state.
6432    #[tokio::test]
6433    async fn sol_s1_distinct_callers_keep_subject_keyed_ttt_state_separate() {
6434        use hyprstream_rpc::envelope::Subject;
6435
6436        let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6437        let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6438        // Real subject-keyed pool (CPU device reached through the engine trait
6439        // without naming the tch type).
6440        let device = create_engine(&RuntimeConfig::default())
6441            .expect("engine construction (no weights)")
6442            .device();
6443        let snapshots_dir = std::env::temp_dir().join(format!(
6444            "sol-s1-delta-pool-{}",
6445            std::time::SystemTime::now()
6446                .duration_since(std::time::UNIX_EPOCH)
6447                .expect("clock")
6448                .as_nanos()
6449        ));
6450        // Module dims for the default target modules — blank delta creation
6451        // allocates LoRA matrices per (layer, module) against these dims.
6452        let module_dims = std::collections::HashMap::from([
6453            ("q_proj".to_owned(), (16usize, 16usize)),
6454            ("v_proj".to_owned(), (16usize, 16usize)),
6455        ]);
6456        let pool = Arc::new(DeltaPool::new(
6457            crate::training::TenantDeltaConfig::default(),
6458            module_dims,
6459            device,
6460            None,
6461            snapshots_dir,
6462            None,
6463            2,
6464        ));
6465        let adapter = boundary_adapter_with_pool(
6466            &controller,
6467            &instance.service_name(),
6468            TENANT,
6469            MODEL,
6470            true,
6471            Some(Arc::clone(&pool)),
6472            "test-incarnation-0000",
6473        );
6474
6475        let caller_a = "user:alice";
6476        let caller_b = "alice";
6477
6478        // A's TTT state is seeded exactly like adaptation state would be after
6479        // A's own training steps: subject-keyed at the real pool.
6480        let subject_a = Subject::new(caller_a);
6481        let delta_a = pool.get_or_create(&subject_a).expect("seed caller A delta");
6482        delta_a.lock().accumulated_steps = 7;
6483
6484        // Admit BOTH callers' work orders through the real adapter hook, then
6485        // check the typed identities the admission installed.
6486        async fn admit(
6487            adapter: &InferenceZmqAdapter,
6488            controller: &SigningKey,
6489            instance_service_name: &str,
6490            subject_string: &str,
6491        ) -> EnvelopeContext {
6492            let now = chrono::Utc::now().timestamp();
6493            let mut claims = fixture_work_claims_for_subject(
6494                controller,
6495                instance_service_name,
6496                now,
6497                subject_string,
6498            );
6499            claims.resource = "inference:GetDeltaStatus".to_owned();
6500            claims.operation = "query".to_owned();
6501            let mut ctx = ctx_with_work(encode_internal_work(&claims, controller), controller);
6502            adapter
6503                .verify_claims(&mut ctx)
6504                .await
6505                .unwrap_or_else(|error| panic!("{subject_string} must be admitted: {error:#}"));
6506            ctx
6507        }
6508        let ctx_a = admit(&adapter, &controller, &instance.service_name(), caller_a).await;
6509        let ctx_b = admit(&adapter, &controller, &instance.service_name(), caller_b).await;
6510        assert_eq!(
6511            ctx_a.internal_work().expect("A work order").subject,
6512            Subject::new(caller_a),
6513            "caller A identity must install verbatim"
6514        );
6515        assert_eq!(
6516            ctx_b.internal_work().expect("B work order").subject,
6517            Subject::new(caller_b),
6518            "caller B identity must install verbatim"
6519        );
6520
6521        // Handler contexts carrying the EXACT typed identities admission just
6522        // installed (the svc.rs Sol S1 regressions prove real `verify_claims`
6523        // installs these on the AnySigner serving plane, where ctx.subject()
6524        // is the installed identity; this fixture's FixedSigner constructor
6525        // cannot express that shape, so the identity enters here explicitly
6526        // instead of being shadowed by `system`).
6527        let handler_ctx = |subject_string: &str| {
6528            let now = chrono::Utc::now().timestamp();
6529            let caller_claims =
6530                hyprstream_rpc::auth::Claims::new(subject_string.to_owned(), now, now + 3600)
6531                    .with_tenant(TENANT.to_owned());
6532            EnvelopeContext::for_test_authenticated_subject_with_claims(
6533                Subject::new(subject_string),
6534                TENANT,
6535                controller.verifying_key(),
6536                caller_claims,
6537            )
6538        };
6539        let handler_ctx_a = handler_ctx(caller_a);
6540        let handler_ctx_b = handler_ctx(caller_b);
6541
6542        // B's REAL generated handler reads ctx.subject() and must NOT observe
6543        // A's subject-keyed adaptation state.
6544        let status_b =
6545            InferenceHandler::handle_get_delta_status(&adapter.service, &handler_ctx_b, 1)
6546                .await
6547                .unwrap_or_else(|error| panic!("B delta status failed: {error:#}"));
6548        match status_b {
6549            InferenceResponseVariant::GetDeltaStatusResult(status) => {
6550                assert!(
6551                    !status.exists,
6552                    "caller B must not see caller A's subject-keyed delta"
6553                );
6554            }
6555            other => panic!("expected GetDeltaStatusResult, got {other:?}"),
6556        }
6557
6558        // B's reset operation (real handler) must not touch A's keyed state.
6559        InferenceHandler::handle_ttt_zero(&adapter.service, &handler_ctx_b, 2)
6560            .await
6561            .unwrap_or_else(|error| panic!("B tttZero failed: {error:#}"));
6562        let delta_a_after = pool.get(&subject_a).expect("A delta survives B's reset");
6563        assert_eq!(
6564            delta_a_after.lock().accumulated_steps,
6565            7,
6566            "A's accumulated TTT steps must be untouched by caller B's operations"
6567        );
6568
6569        // And A's own status view observes exactly A's state.
6570        let status_a =
6571            InferenceHandler::handle_get_delta_status(&adapter.service, &handler_ctx_a, 3)
6572                .await
6573                .unwrap_or_else(|error| panic!("A delta status failed: {error:#}"));
6574        match status_a {
6575            InferenceResponseVariant::GetDeltaStatusResult(status) => {
6576                assert!(status.exists, "A observes its own delta");
6577                assert_eq!(status.accumulated_steps, 7);
6578            }
6579            other => panic!("expected GetDeltaStatusResult, got {other:?}"),
6580        }
6581    }
6582}