Skip to main content

hyprstream_inference/runtime/
inference_profile.rs

1//! Deployment and scheduling contract for inference-engine instances.
2//!
3//! Engine isolation is deliberately separate from model architecture. A
4//! `TorchEngine` keeps one contract whether an operator places it in the host
5//! process, a tenant subprocess, or an isolated task/microVM.
6
7use anyhow::{ensure, Result};
8use serde::{Deserialize, Serialize};
9use sha2::{Digest, Sha256};
10use std::path::{Path, PathBuf};
11
12/// Address-space isolation selected for an inference engine.
13#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
14#[serde(rename_all = "kebab-case")]
15pub enum InferenceIsolationProfile {
16    /// Dedicated engine thread in the model-service process.
17    ///
18    /// Valid only for a single-tenant development deployment.
19    InProcess,
20    /// One operating-system process per authority-verified tenant.
21    PerTenantSubprocess,
22    /// One sandbox task or microVM per authority-verified tenant.
23    PerTenantMicroVmTask {
24        /// Explicit worker backend (for example `kata`). An unavailable backend
25        /// must fail rather than downgrade.
26        backend: String,
27    },
28}
29
30/// Tenancy posture of the deployment hosting inference.
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
32#[serde(rename_all = "kebab-case")]
33pub enum InferenceTenancy {
34    /// One tenant, intended for development and trusted appliances.
35    SingleTenantDevelopment,
36    /// More than one mutually untrusted tenant may reach the deployment.
37    SharedMultiTenant,
38}
39
40/// Accelerator class reserved for each engine instance.
41#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
42#[serde(rename_all = "kebab-case")]
43pub enum InferenceCompute {
44    /// CPU-only execution. This is the initial demo profile.
45    Cpu,
46    /// A specific GPU is assigned by the scheduler.
47    Gpu,
48    /// Preserve legacy auto-detection in single-tenant development.
49    Auto,
50}
51
52/// Per-instance resource and lifecycle limits.
53#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
54pub struct InferenceResourceBudget {
55    /// CPU reservation in millicores.
56    pub cpu_millis: u32,
57    /// Resident-memory ceiling.
58    pub memory_bytes: u64,
59    /// VRAM ceiling. Must be zero for [`InferenceCompute::Cpu`].
60    pub gpu_vram_bytes: u64,
61    /// Maximum time allowed for model preparation.
62    pub startup_timeout_ms: u64,
63    /// Maximum time a health probe may take.
64    pub health_check_timeout_ms: u64,
65    /// Maximum time allowed for drain and teardown.
66    pub teardown_timeout_ms: u64,
67}
68
69impl Default for InferenceResourceBudget {
70    fn default() -> Self {
71        Self {
72            cpu_millis: 1_000,
73            memory_bytes: 8 * 1024 * 1024 * 1024,
74            gpu_vram_bytes: 0,
75            startup_timeout_ms: 300_000,
76            health_check_timeout_ms: 5_000,
77            teardown_timeout_ms: 30_000,
78        }
79    }
80}
81
82/// Explicit inference deployment profile validated before scheduling.
83///
84/// The current in-process path consumes the compute selector. Isolation and
85/// resource budgets are the contract for an external launcher; selecting one
86/// of those profiles fails closed until that launcher is present.
87#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88pub struct InferenceDeploymentProfile {
89    pub isolation: InferenceIsolationProfile,
90    pub tenancy: InferenceTenancy,
91    pub compute: InferenceCompute,
92    pub resources: InferenceResourceBudget,
93}
94
95impl Default for InferenceDeploymentProfile {
96    fn default() -> Self {
97        Self {
98            isolation: InferenceIsolationProfile::InProcess,
99            tenancy: InferenceTenancy::SingleTenantDevelopment,
100            compute: InferenceCompute::Auto,
101            resources: InferenceResourceBudget::default(),
102        }
103    }
104}
105
106impl InferenceDeploymentProfile {
107    /// CPU-only profile used by the demo.
108    #[must_use]
109    pub fn cpu_demo(isolation: InferenceIsolationProfile, tenancy: InferenceTenancy) -> Self {
110        Self {
111            isolation,
112            tenancy,
113            compute: InferenceCompute::Cpu,
114            resources: InferenceResourceBudget::default(),
115        }
116    }
117
118    /// Load the production scheduling profile from explicit environment
119    /// signals. Unknown values fail startup; they never fall back to weaker
120    /// isolation or automatic accelerator selection.
121    pub fn from_env() -> Result<Self> {
122        let mut profile = Self::default();
123        if let Ok(value) = std::env::var("HYPRSTREAM_INFERENCE_ISOLATION") {
124            profile.isolation = match value.as_str() {
125                "in-process" => InferenceIsolationProfile::InProcess,
126                "per-tenant-subprocess" => InferenceIsolationProfile::PerTenantSubprocess,
127                "per-tenant-microvm-task" => {
128                    let backend =
129                        std::env::var("HYPRSTREAM_INFERENCE_SANDBOX_BACKEND").map_err(|_| {
130                            anyhow::anyhow!(
131                                "HYPRSTREAM_INFERENCE_SANDBOX_BACKEND is required for \
132                                 per-tenant-microvm-task"
133                            )
134                        })?;
135                    InferenceIsolationProfile::PerTenantMicroVmTask { backend }
136                }
137                other => anyhow::bail!(
138                    "HYPRSTREAM_INFERENCE_ISOLATION must be 'in-process', \
139                     'per-tenant-subprocess', or 'per-tenant-microvm-task', got {other:?}"
140                ),
141            };
142        }
143        if let Ok(value) = std::env::var("HYPRSTREAM_INFERENCE_TENANCY") {
144            profile.tenancy = match value.as_str() {
145                "single-tenant-development" => InferenceTenancy::SingleTenantDevelopment,
146                "shared-multi-tenant" => InferenceTenancy::SharedMultiTenant,
147                other => anyhow::bail!(
148                    "HYPRSTREAM_INFERENCE_TENANCY must be 'single-tenant-development' \
149                     or 'shared-multi-tenant', got {other:?}"
150                ),
151            };
152        }
153        if let Ok(value) = std::env::var("HYPRSTREAM_INFERENCE_COMPUTE") {
154            profile.compute = match value.as_str() {
155                "cpu" => InferenceCompute::Cpu,
156                "gpu" => InferenceCompute::Gpu,
157                "auto" => InferenceCompute::Auto,
158                other => anyhow::bail!(
159                    "HYPRSTREAM_INFERENCE_COMPUTE must be 'cpu', 'gpu', or 'auto', got {other:?}"
160                ),
161            };
162        }
163        apply_u32_env(
164            "HYPRSTREAM_INFERENCE_CPU_MILLIS",
165            &mut profile.resources.cpu_millis,
166        )?;
167        apply_u64_env(
168            "HYPRSTREAM_INFERENCE_MEMORY_BYTES",
169            &mut profile.resources.memory_bytes,
170        )?;
171        apply_u64_env(
172            "HYPRSTREAM_INFERENCE_GPU_VRAM_BYTES",
173            &mut profile.resources.gpu_vram_bytes,
174        )?;
175        apply_u64_env(
176            "HYPRSTREAM_INFERENCE_STARTUP_TIMEOUT_MS",
177            &mut profile.resources.startup_timeout_ms,
178        )?;
179        apply_u64_env(
180            "HYPRSTREAM_INFERENCE_HEALTH_TIMEOUT_MS",
181            &mut profile.resources.health_check_timeout_ms,
182        )?;
183        apply_u64_env(
184            "HYPRSTREAM_INFERENCE_TEARDOWN_TIMEOUT_MS",
185            &mut profile.resources.teardown_timeout_ms,
186        )?;
187        profile.validate()?;
188        Ok(profile)
189    }
190
191    /// Validate the deployment contract before an engine is loaded.
192    pub fn validate(&self) -> Result<()> {
193        ensure!(
194            self.resources.cpu_millis > 0,
195            "inference CPU budget must be non-zero"
196        );
197        ensure!(
198            self.resources.memory_bytes > 0,
199            "inference memory budget must be non-zero"
200        );
201        ensure!(
202            self.resources.startup_timeout_ms > 0
203                && self.resources.health_check_timeout_ms > 0
204                && self.resources.teardown_timeout_ms > 0,
205            "inference lifecycle timeouts must be non-zero"
206        );
207        if self.compute == InferenceCompute::Cpu {
208            ensure!(
209                self.resources.gpu_vram_bytes == 0,
210                "CPU inference profile cannot reserve GPU VRAM"
211            );
212        }
213        if self.compute == InferenceCompute::Gpu {
214            ensure!(
215                self.resources.gpu_vram_bytes > 0,
216                "GPU inference profile requires an explicit VRAM budget"
217            );
218        }
219        ensure!(
220            !(self.tenancy == InferenceTenancy::SharedMultiTenant
221                && self.isolation == InferenceIsolationProfile::InProcess),
222            "shared multi-tenant inference requires per-tenant subprocess or microVM/task isolation"
223        );
224        if let InferenceIsolationProfile::PerTenantMicroVmTask { backend } = &self.isolation {
225            ensure!(
226                !backend.trim().is_empty() && backend != "auto",
227                "microVM/task inference requires an explicit sandbox backend"
228            );
229        }
230        Ok(())
231    }
232
233    /// Whether this profile needs an out-of-process launcher.
234    #[must_use]
235    pub fn requires_isolated_launcher(&self) -> bool {
236        !matches!(self.isolation, InferenceIsolationProfile::InProcess)
237    }
238}
239
240fn apply_u32_env(name: &str, target: &mut u32) -> Result<()> {
241    if let Ok(value) = std::env::var(name) {
242        *target = value.parse().map_err(|e| {
243            anyhow::anyhow!("{name} must be an unsigned integer, got {value:?}: {e}")
244        })?;
245    }
246    Ok(())
247}
248
249fn apply_u64_env(name: &str, target: &mut u64) -> Result<()> {
250    if let Ok(value) = std::env::var(name) {
251        *target = value.parse().map_err(|e| {
252            anyhow::anyhow!("{name} must be an unsigned integer, got {value:?}: {e}")
253        })?;
254    }
255    Ok(())
256}
257
258/// Tenant/model/replica identity used for placement and endpoint derivation.
259///
260/// The tenant is accepted only after it came from `EnvelopeContext::domain()`.
261/// It is never derived from subject text or caller-supplied model input.
262#[derive(Debug, Clone, PartialEq, Eq, Hash)]
263pub struct InferenceInstanceId {
264    tenant: String,
265    model_ref: String,
266    replica: u16,
267}
268
269/// Domain-separated carrier-key purpose for one opaque service instance.
270///
271/// Public: consumed by the main crate's inference service wiring across the
272/// `hyprstream-inference` boundary.
273pub fn inference_iroh_key_purpose(service_name: &str) -> String {
274    format!("hyprstream-inference-iroh-v1/{service_name}")
275}
276
277impl InferenceInstanceId {
278    pub fn new(verified_tenant: &str, model_ref: &str, replica: u16) -> Result<Self> {
279        let tenant = verified_tenant.trim();
280        ensure!(
281            !tenant.is_empty() && tenant != "*",
282            "inference instance requires a non-empty, non-wildcard verified tenant"
283        );
284        ensure!(
285            !model_ref.trim().is_empty(),
286            "inference instance requires a model ref"
287        );
288        Ok(Self {
289            tenant: tenant.to_owned(),
290            model_ref: model_ref.to_owned(),
291            replica,
292        })
293    }
294
295    #[must_use]
296    pub fn tenant(&self) -> &str {
297        &self.tenant
298    }
299
300    #[must_use]
301    pub fn model_ref(&self) -> &str {
302        &self.model_ref
303    }
304
305    #[must_use]
306    pub fn replica(&self) -> u16 {
307        self.replica
308    }
309
310    /// Opaque, deterministic name safe for registries and process managers.
311    #[must_use]
312    pub fn service_name(&self) -> String {
313        let mut digest = Sha256::new();
314        digest.update(b"hyprstream-inference-instance-v1\0");
315        digest.update(self.tenant.as_bytes());
316        digest.update(b"\0");
317        digest.update(self.model_ref.as_bytes());
318        digest.update(b"\0");
319        digest.update(self.replica.to_be_bytes());
320        let suffix = hex::encode(&digest.finalize()[..12]);
321        format!("inference-{suffix}")
322    }
323    #[must_use]
324    pub fn iroh_key_purpose(&self) -> String {
325        inference_iroh_key_purpose(&self.service_name())
326    }
327
328    /// Same-host cross-process dial target for a subprocess placement.
329    #[must_use]
330    pub fn ipc_transport(&self, runtime_dir: &Path) -> hyprstream_rpc::transport::TransportConfig {
331        let path: PathBuf = runtime_dir.join(format!("{}.sock", self.service_name()));
332        hyprstream_rpc::transport::TransportConfig::ipc(path)
333    }
334}
335
336#[cfg(test)]
337mod tests {
338    use super::*;
339
340    #[test]
341    fn shared_tenancy_rejects_in_process_profile() {
342        let profile = InferenceDeploymentProfile::cpu_demo(
343            InferenceIsolationProfile::InProcess,
344            InferenceTenancy::SharedMultiTenant,
345        );
346        assert!(profile.validate().is_err());
347    }
348
349    #[test]
350    fn shared_cpu_subprocess_profile_is_valid() {
351        let profile = InferenceDeploymentProfile::cpu_demo(
352            InferenceIsolationProfile::PerTenantSubprocess,
353            InferenceTenancy::SharedMultiTenant,
354        );
355        assert!(profile.validate().is_ok());
356        assert!(profile.requires_isolated_launcher());
357    }
358
359    #[test]
360    fn two_tenants_get_distinct_opaque_dial_targets() -> Result<()> {
361        let first = InferenceInstanceId::new("tenant-a", "tiny-llama:main", 0)?;
362        let second = InferenceInstanceId::new("tenant-b", "tiny-llama:main", 0)?;
363        assert_ne!(first.service_name(), second.service_name());
364        assert_ne!(
365            first.ipc_transport(Path::new("/run/hyprstream")),
366            second.ipc_transport(Path::new("/run/hyprstream"))
367        );
368        assert_ne!(first.iroh_key_purpose(), second.iroh_key_purpose());
369        assert!(!first.service_name().contains("tenant-a"));
370        Ok(())
371    }
372
373    #[test]
374    fn wildcard_or_blank_tenant_fails_closed() {
375        assert!(InferenceInstanceId::new("*", "model:main", 0).is_err());
376        assert!(InferenceInstanceId::new(" ", "model:main", 0).is_err());
377    }
378
379    #[test]
380    fn replica_identity_changes_dial_target() -> Result<()> {
381        let first = InferenceInstanceId::new("tenant-a", "tiny-llama:main", 0)?;
382        let second = InferenceInstanceId::new("tenant-a", "tiny-llama:main", 1)?;
383        assert_ne!(first.service_name(), second.service_name());
384        Ok(())
385    }
386
387    #[test]
388    fn two_cpu_models_get_distinct_service_addresses() -> Result<()> {
389        let first = InferenceInstanceId::new("tenant-a", "model-one:main", 0)?;
390        let second = InferenceInstanceId::new("tenant-a", "model-two:main", 0)?;
391        assert_ne!(first.service_name(), second.service_name());
392        assert_ne!(
393            first.ipc_transport(Path::new("/run/hyprstream")),
394            second.ipc_transport(Path::new("/run/hyprstream"))
395        );
396        let service_key = hyprstream_rpc::prelude::SigningKey::from_bytes(&[31u8; 32]);
397        let first_carrier = hyprstream_rpc::node_identity::derive_purpose_key(
398            &service_key,
399            &first.iroh_key_purpose(),
400        );
401        let second_carrier = hyprstream_rpc::node_identity::derive_purpose_key(
402            &service_key,
403            &second.iroh_key_purpose(),
404        );
405        assert_ne!(
406            first_carrier.verifying_key(),
407            service_key.verifying_key(),
408            "carrier address must not reuse application authority"
409        );
410        assert_ne!(
411            first_carrier.verifying_key(),
412            second_carrier.verifying_key()
413        );
414        Ok(())
415    }
416}