hyprstream_inference/runtime/
inference_profile.rs1use anyhow::{ensure, Result};
8use serde::{Deserialize, Serialize};
9use sha2::{Digest, Sha256};
10use std::path::{Path, PathBuf};
11
12#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
14#[serde(rename_all = "kebab-case")]
15pub enum InferenceIsolationProfile {
16 InProcess,
20 PerTenantSubprocess,
22 PerTenantMicroVmTask {
24 backend: String,
27 },
28}
29
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
32#[serde(rename_all = "kebab-case")]
33pub enum InferenceTenancy {
34 SingleTenantDevelopment,
36 SharedMultiTenant,
38}
39
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
42#[serde(rename_all = "kebab-case")]
43pub enum InferenceCompute {
44 Cpu,
46 Gpu,
48 Auto,
50}
51
52#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
54pub struct InferenceResourceBudget {
55 pub cpu_millis: u32,
57 pub memory_bytes: u64,
59 pub gpu_vram_bytes: u64,
61 pub startup_timeout_ms: u64,
63 pub health_check_timeout_ms: u64,
65 pub teardown_timeout_ms: u64,
67}
68
69impl Default for InferenceResourceBudget {
70 fn default() -> Self {
71 Self {
72 cpu_millis: 1_000,
73 memory_bytes: 8 * 1024 * 1024 * 1024,
74 gpu_vram_bytes: 0,
75 startup_timeout_ms: 300_000,
76 health_check_timeout_ms: 5_000,
77 teardown_timeout_ms: 30_000,
78 }
79 }
80}
81
82#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88pub struct InferenceDeploymentProfile {
89 pub isolation: InferenceIsolationProfile,
90 pub tenancy: InferenceTenancy,
91 pub compute: InferenceCompute,
92 pub resources: InferenceResourceBudget,
93}
94
95impl Default for InferenceDeploymentProfile {
96 fn default() -> Self {
97 Self {
98 isolation: InferenceIsolationProfile::InProcess,
99 tenancy: InferenceTenancy::SingleTenantDevelopment,
100 compute: InferenceCompute::Auto,
101 resources: InferenceResourceBudget::default(),
102 }
103 }
104}
105
106impl InferenceDeploymentProfile {
107 #[must_use]
109 pub fn cpu_demo(isolation: InferenceIsolationProfile, tenancy: InferenceTenancy) -> Self {
110 Self {
111 isolation,
112 tenancy,
113 compute: InferenceCompute::Cpu,
114 resources: InferenceResourceBudget::default(),
115 }
116 }
117
118 pub fn from_env() -> Result<Self> {
122 let mut profile = Self::default();
123 if let Ok(value) = std::env::var("HYPRSTREAM_INFERENCE_ISOLATION") {
124 profile.isolation = match value.as_str() {
125 "in-process" => InferenceIsolationProfile::InProcess,
126 "per-tenant-subprocess" => InferenceIsolationProfile::PerTenantSubprocess,
127 "per-tenant-microvm-task" => {
128 let backend =
129 std::env::var("HYPRSTREAM_INFERENCE_SANDBOX_BACKEND").map_err(|_| {
130 anyhow::anyhow!(
131 "HYPRSTREAM_INFERENCE_SANDBOX_BACKEND is required for \
132 per-tenant-microvm-task"
133 )
134 })?;
135 InferenceIsolationProfile::PerTenantMicroVmTask { backend }
136 }
137 other => anyhow::bail!(
138 "HYPRSTREAM_INFERENCE_ISOLATION must be 'in-process', \
139 'per-tenant-subprocess', or 'per-tenant-microvm-task', got {other:?}"
140 ),
141 };
142 }
143 if let Ok(value) = std::env::var("HYPRSTREAM_INFERENCE_TENANCY") {
144 profile.tenancy = match value.as_str() {
145 "single-tenant-development" => InferenceTenancy::SingleTenantDevelopment,
146 "shared-multi-tenant" => InferenceTenancy::SharedMultiTenant,
147 other => anyhow::bail!(
148 "HYPRSTREAM_INFERENCE_TENANCY must be 'single-tenant-development' \
149 or 'shared-multi-tenant', got {other:?}"
150 ),
151 };
152 }
153 if let Ok(value) = std::env::var("HYPRSTREAM_INFERENCE_COMPUTE") {
154 profile.compute = match value.as_str() {
155 "cpu" => InferenceCompute::Cpu,
156 "gpu" => InferenceCompute::Gpu,
157 "auto" => InferenceCompute::Auto,
158 other => anyhow::bail!(
159 "HYPRSTREAM_INFERENCE_COMPUTE must be 'cpu', 'gpu', or 'auto', got {other:?}"
160 ),
161 };
162 }
163 apply_u32_env(
164 "HYPRSTREAM_INFERENCE_CPU_MILLIS",
165 &mut profile.resources.cpu_millis,
166 )?;
167 apply_u64_env(
168 "HYPRSTREAM_INFERENCE_MEMORY_BYTES",
169 &mut profile.resources.memory_bytes,
170 )?;
171 apply_u64_env(
172 "HYPRSTREAM_INFERENCE_GPU_VRAM_BYTES",
173 &mut profile.resources.gpu_vram_bytes,
174 )?;
175 apply_u64_env(
176 "HYPRSTREAM_INFERENCE_STARTUP_TIMEOUT_MS",
177 &mut profile.resources.startup_timeout_ms,
178 )?;
179 apply_u64_env(
180 "HYPRSTREAM_INFERENCE_HEALTH_TIMEOUT_MS",
181 &mut profile.resources.health_check_timeout_ms,
182 )?;
183 apply_u64_env(
184 "HYPRSTREAM_INFERENCE_TEARDOWN_TIMEOUT_MS",
185 &mut profile.resources.teardown_timeout_ms,
186 )?;
187 profile.validate()?;
188 Ok(profile)
189 }
190
191 pub fn validate(&self) -> Result<()> {
193 ensure!(
194 self.resources.cpu_millis > 0,
195 "inference CPU budget must be non-zero"
196 );
197 ensure!(
198 self.resources.memory_bytes > 0,
199 "inference memory budget must be non-zero"
200 );
201 ensure!(
202 self.resources.startup_timeout_ms > 0
203 && self.resources.health_check_timeout_ms > 0
204 && self.resources.teardown_timeout_ms > 0,
205 "inference lifecycle timeouts must be non-zero"
206 );
207 if self.compute == InferenceCompute::Cpu {
208 ensure!(
209 self.resources.gpu_vram_bytes == 0,
210 "CPU inference profile cannot reserve GPU VRAM"
211 );
212 }
213 if self.compute == InferenceCompute::Gpu {
214 ensure!(
215 self.resources.gpu_vram_bytes > 0,
216 "GPU inference profile requires an explicit VRAM budget"
217 );
218 }
219 ensure!(
220 !(self.tenancy == InferenceTenancy::SharedMultiTenant
221 && self.isolation == InferenceIsolationProfile::InProcess),
222 "shared multi-tenant inference requires per-tenant subprocess or microVM/task isolation"
223 );
224 if let InferenceIsolationProfile::PerTenantMicroVmTask { backend } = &self.isolation {
225 ensure!(
226 !backend.trim().is_empty() && backend != "auto",
227 "microVM/task inference requires an explicit sandbox backend"
228 );
229 }
230 Ok(())
231 }
232
233 #[must_use]
235 pub fn requires_isolated_launcher(&self) -> bool {
236 !matches!(self.isolation, InferenceIsolationProfile::InProcess)
237 }
238}
239
240fn apply_u32_env(name: &str, target: &mut u32) -> Result<()> {
241 if let Ok(value) = std::env::var(name) {
242 *target = value.parse().map_err(|e| {
243 anyhow::anyhow!("{name} must be an unsigned integer, got {value:?}: {e}")
244 })?;
245 }
246 Ok(())
247}
248
249fn apply_u64_env(name: &str, target: &mut u64) -> Result<()> {
250 if let Ok(value) = std::env::var(name) {
251 *target = value.parse().map_err(|e| {
252 anyhow::anyhow!("{name} must be an unsigned integer, got {value:?}: {e}")
253 })?;
254 }
255 Ok(())
256}
257
258#[derive(Debug, Clone, PartialEq, Eq, Hash)]
263pub struct InferenceInstanceId {
264 tenant: String,
265 model_ref: String,
266 replica: u16,
267}
268
269pub fn inference_iroh_key_purpose(service_name: &str) -> String {
274 format!("hyprstream-inference-iroh-v1/{service_name}")
275}
276
277impl InferenceInstanceId {
278 pub fn new(verified_tenant: &str, model_ref: &str, replica: u16) -> Result<Self> {
279 let tenant = verified_tenant.trim();
280 ensure!(
281 !tenant.is_empty() && tenant != "*",
282 "inference instance requires a non-empty, non-wildcard verified tenant"
283 );
284 ensure!(
285 !model_ref.trim().is_empty(),
286 "inference instance requires a model ref"
287 );
288 Ok(Self {
289 tenant: tenant.to_owned(),
290 model_ref: model_ref.to_owned(),
291 replica,
292 })
293 }
294
295 #[must_use]
296 pub fn tenant(&self) -> &str {
297 &self.tenant
298 }
299
300 #[must_use]
301 pub fn model_ref(&self) -> &str {
302 &self.model_ref
303 }
304
305 #[must_use]
306 pub fn replica(&self) -> u16 {
307 self.replica
308 }
309
310 #[must_use]
312 pub fn service_name(&self) -> String {
313 let mut digest = Sha256::new();
314 digest.update(b"hyprstream-inference-instance-v1\0");
315 digest.update(self.tenant.as_bytes());
316 digest.update(b"\0");
317 digest.update(self.model_ref.as_bytes());
318 digest.update(b"\0");
319 digest.update(self.replica.to_be_bytes());
320 let suffix = hex::encode(&digest.finalize()[..12]);
321 format!("inference-{suffix}")
322 }
323 #[must_use]
324 pub fn iroh_key_purpose(&self) -> String {
325 inference_iroh_key_purpose(&self.service_name())
326 }
327
328 #[must_use]
330 pub fn ipc_transport(&self, runtime_dir: &Path) -> hyprstream_rpc::transport::TransportConfig {
331 let path: PathBuf = runtime_dir.join(format!("{}.sock", self.service_name()));
332 hyprstream_rpc::transport::TransportConfig::ipc(path)
333 }
334}
335
336#[cfg(test)]
337mod tests {
338 use super::*;
339
340 #[test]
341 fn shared_tenancy_rejects_in_process_profile() {
342 let profile = InferenceDeploymentProfile::cpu_demo(
343 InferenceIsolationProfile::InProcess,
344 InferenceTenancy::SharedMultiTenant,
345 );
346 assert!(profile.validate().is_err());
347 }
348
349 #[test]
350 fn shared_cpu_subprocess_profile_is_valid() {
351 let profile = InferenceDeploymentProfile::cpu_demo(
352 InferenceIsolationProfile::PerTenantSubprocess,
353 InferenceTenancy::SharedMultiTenant,
354 );
355 assert!(profile.validate().is_ok());
356 assert!(profile.requires_isolated_launcher());
357 }
358
359 #[test]
360 fn two_tenants_get_distinct_opaque_dial_targets() -> Result<()> {
361 let first = InferenceInstanceId::new("tenant-a", "tiny-llama:main", 0)?;
362 let second = InferenceInstanceId::new("tenant-b", "tiny-llama:main", 0)?;
363 assert_ne!(first.service_name(), second.service_name());
364 assert_ne!(
365 first.ipc_transport(Path::new("/run/hyprstream")),
366 second.ipc_transport(Path::new("/run/hyprstream"))
367 );
368 assert_ne!(first.iroh_key_purpose(), second.iroh_key_purpose());
369 assert!(!first.service_name().contains("tenant-a"));
370 Ok(())
371 }
372
373 #[test]
374 fn wildcard_or_blank_tenant_fails_closed() {
375 assert!(InferenceInstanceId::new("*", "model:main", 0).is_err());
376 assert!(InferenceInstanceId::new(" ", "model:main", 0).is_err());
377 }
378
379 #[test]
380 fn replica_identity_changes_dial_target() -> Result<()> {
381 let first = InferenceInstanceId::new("tenant-a", "tiny-llama:main", 0)?;
382 let second = InferenceInstanceId::new("tenant-a", "tiny-llama:main", 1)?;
383 assert_ne!(first.service_name(), second.service_name());
384 Ok(())
385 }
386
387 #[test]
388 fn two_cpu_models_get_distinct_service_addresses() -> Result<()> {
389 let first = InferenceInstanceId::new("tenant-a", "model-one:main", 0)?;
390 let second = InferenceInstanceId::new("tenant-a", "model-two:main", 0)?;
391 assert_ne!(first.service_name(), second.service_name());
392 assert_ne!(
393 first.ipc_transport(Path::new("/run/hyprstream")),
394 second.ipc_transport(Path::new("/run/hyprstream"))
395 );
396 let service_key = hyprstream_rpc::prelude::SigningKey::from_bytes(&[31u8; 32]);
397 let first_carrier = hyprstream_rpc::node_identity::derive_purpose_key(
398 &service_key,
399 &first.iroh_key_purpose(),
400 );
401 let second_carrier = hyprstream_rpc::node_identity::derive_purpose_key(
402 &service_key,
403 &second.iroh_key_purpose(),
404 );
405 assert_ne!(
406 first_carrier.verifying_key(),
407 service_key.verifying_key(),
408 "carrier address must not reuse application authority"
409 );
410 assert_ne!(
411 first_carrier.verifying_key(),
412 second_carrier.verifying_key()
413 );
414 Ok(())
415 }
416}