hyprstream_inference/runtime/device_pool.rs
1//! Multi-GPU device abstraction (`DevicePool`).
2//!
3//! This module introduces the device-strategy seam for the multi-GPU epic (#310,
4//! part of #313). It resolves a set of configured device indices
5//! (`RuntimeConfig.devices`) into concrete [`tch::Device`] values and exposes a
6//! small, stable API that the later phases build on:
7//!
8//! - **2a replication** (#313): N `TorchEngine`s, one per local GPU, requests
9//! routed across [`DevicePool::devices`].
10//! - **2b intra-host pipeline** (#314): a layer→device `device_map` layered on
11//! top of the same pool (see [`DevicePool`] note on extensibility).
12//!
13//! # Send / !Send boundary
14//!
15//! `tch-rs` tensors are `!Send`: a `Tensor` must never cross a thread boundary,
16//! and engines that hold tensors stay pinned to one thread (as
17//! `services/inference.rs` already does — one thread per engine). However, a
18//! *single* thread may legitimately own tensors on `Cuda(0)` **and** `Cuda(1)`
19//! at once: libtorch dispatches each op to the correct device via a
20//! `DeviceGuard`, so multi-device-on-one-thread is fine; only multi-*thread*
21//! tensor moves are forbidden.
22//!
23//! `DevicePool` deliberately holds **only** device indices / [`tch::Device`]
24//! values (which are `Copy` and contain no tensor state). It is therefore
25//! `Send + Sync` and can be shared across threads (e.g. handed to each
26//! engine-owning thread so each can construct its own thread-pinned engine).
27//! **Do not** add a `Tensor`, `VarStore`, or any `!Send` field to this type — it
28//! would silently make the pool `!Send` and break the replication design.
29
30use anyhow::{anyhow, Result};
31use tch::Device;
32
33/// A resolved, validated set of compute devices a single inference service owns.
34///
35/// Constructed from [`crate::config::RuntimeConfig`] via [`DevicePool::from_config`].
36/// The pool performs **fail-fast** device resolution: when devices are explicitly
37/// requested it never silently downgrades to CPU ("no-fragile-fallbacks", per the
38/// multi-GPU/multi-host inference design). A process told to use GPU 1 that would
39/// land on CPU is an error, not a warning.
40///
41/// # Send / Sync
42///
43/// `DevicePool` is `Send + Sync` because it holds only [`Device`] values (no
44/// tensors). See the module-level docs for the `!Send` boundary — keep this type
45/// tensor-free.
46///
47/// # Extensibility (#314)
48///
49/// The pipeline phase needs a layer→device map. That belongs *on* this type
50/// (e.g. a `fn device_for_layer(&self, layer_idx, num_layers) -> Device` or an
51/// explicit `device_map`), reusing the already-resolved [`Self::devices`]. New
52/// strategy methods can be added without changing the existing accessors below,
53/// so current callers (which only need [`Self::primary`]) keep working.
54#[derive(Debug, Clone)]
55pub struct DevicePool {
56 /// Resolved devices, in the order they were requested. Always non-empty.
57 devices: Vec<Device>,
58}
59
60impl DevicePool {
61 /// Build a pool for the given GPU indices, validating availability fail-fast.
62 ///
63 /// `indices` must be non-empty and free of duplicates (the caller —
64 /// [`crate::config::RuntimeConfig::resolve_device_indices`] — guarantees
65 /// this, but it is re-checked here so the type's invariant holds regardless
66 /// of construction path).
67 ///
68 /// Every index is validated against the live CUDA/HIP device count. If CUDA
69 /// is unavailable, or any requested index is out of range, this returns an
70 /// error rather than degrading to CPU — devices were *explicitly* requested.
71 ///
72 /// Note: ROCm presents as [`Device::Cuda`] via HIP, so this path is
73 /// vendor-agnostic and contains no vendor-special casing.
74 pub fn from_cuda_indices(indices: &[usize]) -> Result<Self> {
75 if indices.is_empty() {
76 return Err(anyhow!(
77 "DevicePool requires at least one device index (got none)"
78 ));
79 }
80
81 // Reject duplicates defensively (config layer also enforces this).
82 let mut seen = std::collections::HashSet::with_capacity(indices.len());
83 for &idx in indices {
84 if !seen.insert(idx) {
85 return Err(anyhow!(
86 "DevicePool: duplicate device index {idx} in requested set {indices:?}"
87 ));
88 }
89 }
90
91 // Fail-fast: devices were explicitly requested, so a CPU downgrade is an
92 // error, not a fallback (no `Device::cuda_if_available()` here). Surface
93 // the actual cause: a bare "not available" hid a libcuda.so.1 / wrong-
94 // backend problem on the live cuda130 node and made it undiagnosable.
95 if !tch::Cuda::is_available() {
96 let has_cuda = tch::utils::has_cuda();
97 let has_hip = tch::utils::has_hip();
98 let device_count = tch::Cuda::device_count();
99 let ld_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_else(|_| "<unset>".to_owned());
100 return Err(anyhow!(
101 "DevicePool: GPU devices {indices:?} requested but no CUDA/ROCm device is \
102 available (refusing to silently fall back to CPU). \
103 Diagnostics: has_cuda={has_cuda}, has_hip={has_hip}, \
104 device_count={device_count}, LD_LIBRARY_PATH={ld_path}. If has_cuda=true \
105 but device_count=0, the host driver lib (libcuda.so.1) is not loadable — \
106 add the NVIDIA driver lib dir to LD_LIBRARY_PATH. If has_cuda=false, a \
107 CPU-only libtorch/backend is installed."
108 ));
109 }
110
111 let available = tch::Cuda::device_count();
112 if available < 0 {
113 return Err(anyhow!(
114 "DevicePool: CUDA reported a negative device count ({available})"
115 ));
116 }
117 let available = available as usize;
118
119 let devices = indices
120 .iter()
121 .map(|&idx| {
122 if idx >= available {
123 Err(anyhow!(
124 "DevicePool: GPU device index {idx} is out of range \
125 (only {available} CUDA/ROCm device(s) present)"
126 ))
127 } else {
128 Ok(Device::Cuda(idx))
129 }
130 })
131 .collect::<Result<Vec<_>>>()?;
132
133 Ok(Self { devices })
134 }
135
136 /// Construct a pool directly from already-resolved [`Device`] values.
137 ///
138 /// Intended for tests and for callers that have an explicit device list
139 /// (e.g. CPU-only setups). Validates only the structural invariants
140 /// (non-empty, no duplicates); it does **not** probe CUDA, so prefer
141 /// [`Self::from_cuda_indices`] / [`Self::from_config`] for GPU resolution.
142 pub fn from_devices(devices: Vec<Device>) -> Result<Self> {
143 if devices.is_empty() {
144 return Err(anyhow!(
145 "DevicePool requires at least one device (got none)"
146 ));
147 }
148 let mut seen = std::collections::HashSet::with_capacity(devices.len());
149 for dev in &devices {
150 if !seen.insert(*dev) {
151 return Err(anyhow!(
152 "DevicePool: duplicate device {dev:?} in {devices:?}"
153 ));
154 }
155 }
156 Ok(Self { devices })
157 }
158
159 /// Resolve a [`DevicePool`] from runtime configuration.
160 ///
161 /// Behavior:
162 /// - `use_gpu == false` → a single-entry CPU pool ([`Device::Cpu`]).
163 /// - `use_gpu == true` → resolve the configured indices
164 /// ([`crate::config::RuntimeConfig::resolve_device_indices`], which honors
165 /// `devices` then falls back to the legacy single `gpu_device_id`) and
166 /// validate them fail-fast via [`Self::from_cuda_indices`].
167 ///
168 /// When `use_gpu` is true but no index is configured, this falls back to
169 /// `cuda_if_available()`-style auto-detection of device 0 (preserving the
170 /// pre-existing single-GPU default). That auto-detect path is the *only*
171 /// place a CPU result is allowed when GPU is on, and only because nothing
172 /// was explicitly requested.
173 pub fn from_config(config: &crate::config::RuntimeConfig) -> Result<Self> {
174 if !config.use_gpu {
175 return Self::from_devices(vec![Device::Cpu]);
176 }
177
178 match config.resolve_device_indices()? {
179 // Explicit device(s) requested → strict, fail-fast resolution.
180 Some(indices) => Self::from_cuda_indices(&indices),
181 // Nothing explicitly requested → preserve legacy auto-detect (may
182 // be CPU if no GPU is present; this is NOT a silent downgrade because
183 // the user requested no specific device).
184 None => Self::from_devices(vec![Device::cuda_if_available()]),
185 }
186 }
187
188 /// All resolved devices, in requested order. Always non-empty.
189 ///
190 /// The primary seam for #313/2a replication (one engine per device).
191 #[must_use]
192 pub fn devices(&self) -> &[Device] {
193 &self.devices
194 }
195
196 /// The primary device — the first configured device.
197 ///
198 /// Single-device callers (e.g. today's `TorchEngine`) use this to keep
199 /// existing behavior while the multi-engine routing lands in a later PR.
200 #[must_use]
201 pub fn primary(&self) -> Device {
202 self.devices[0]
203 }
204
205 /// Number of devices in the pool (always >= 1).
206 ///
207 /// No `is_empty()` companion exists: a `DevicePool` is never empty by
208 /// construction. Use [`Self::is_single`] for the meaningful predicate.
209 #[must_use]
210 #[allow(clippy::len_without_is_empty)]
211 pub fn len(&self) -> usize {
212 self.devices.len()
213 }
214
215 /// Whether the pool holds a single device. (`is_empty` is meaningless here —
216 /// the pool is never empty — so this single-device predicate is provided
217 /// instead; clippy's `len`/`is_empty` lint is allowed below.)
218 #[must_use]
219 pub fn is_single(&self) -> bool {
220 self.devices.len() == 1
221 }
222}
223
224/// A per-layer device assignment for **2b intra-host pipeline** (#314).
225///
226/// This is the layer→device `device_map` the [`DevicePool`] doc-comment
227/// anticipates. It is a thin, validated newtype over the per-global-layer
228/// `Vec<Device>` derived from a [`DevicePool`]: entry `g` is the device that
229/// owns global decoder layer `g`. A *stage* is a contiguous run of layers that
230/// map to the same device; the only cross-device copy in the forward pass is a
231/// single `hidden.to_device(next)` at a boundary where consecutive layers map to
232/// different devices (see [`LayerDeviceMap::is_boundary`]).
233///
234/// # Why a map over `num_hidden_layers`, not the stage-local count
235///
236/// The map is **global** — indexed by global layer index, length
237/// `num_hidden_layers` — so a single source of truth drives both shard-aware
238/// construction (per-layer placement + per-layer type/RoPE selection, which need
239/// the global index) and the forward loop. A stage that owns layers `[a..b)`
240/// queries the same global map; the architecture is responsible for the
241/// global↔local remap of its own `self.layers` / KV / SSM vectors via the
242/// `layer_offset = a` it was built with.
243///
244/// # Send / Sync
245///
246/// Holds only [`Device`] values (`Copy`, no tensors), so it is `Send + Sync` and
247/// may be handed to an engine-owning thread. Like [`DevicePool`], **never** add a
248/// `Tensor`/`VarStore` field — see the module-level `!Send` boundary note.
249#[derive(Debug, Clone)]
250pub struct LayerDeviceMap {
251 /// One device per global layer index. Always non-empty; `len()` equals the
252 /// model's `num_hidden_layers`.
253 per_layer: Vec<Device>,
254}
255
256impl LayerDeviceMap {
257 /// All layers on a single device — the unsplit fast path.
258 ///
259 /// Used both by single-GPU inference and by the CPU-only equivalence tests
260 /// (`forward_layers(0..N)` over an all-CPU map must equal the whole-model
261 /// forward). `num_layers` must be non-zero.
262 pub fn single(device: Device, num_layers: usize) -> Result<Self> {
263 if num_layers == 0 {
264 return Err(anyhow!("LayerDeviceMap requires num_layers >= 1 (got 0)"));
265 }
266 Ok(Self {
267 per_layer: vec![device; num_layers],
268 })
269 }
270
271 /// Build an explicit per-layer map, validating it is non-empty.
272 ///
273 /// `per_layer[g]` is the device that owns global layer `g`.
274 pub fn from_per_layer(per_layer: Vec<Device>) -> Result<Self> {
275 if per_layer.is_empty() {
276 return Err(anyhow!(
277 "LayerDeviceMap requires at least one layer (got none)"
278 ));
279 }
280 Ok(Self { per_layer })
281 }
282
283 /// Spread `num_layers` contiguously across a [`DevicePool`]'s devices in
284 /// requested order, balanced as evenly as possible (capacity-symmetric).
285 ///
286 /// Earlier stages get the extra layer when `num_layers` is not divisible by
287 /// the device count, matching the natural prefix-heavy split. This is the
288 /// parameter-balanced planner; the capacity-aware (asymmetric-VRAM) variant
289 /// is a later epic concern (plan §H) and would replace this constructor
290 /// without touching the forward path.
291 ///
292 /// Layers are assigned in **contiguous runs** so each device owns a single
293 /// pipeline stage `[a..b)` — never interleaved — which keeps boundary copies
294 /// to exactly `(num_stages - 1)` per forward.
295 pub fn even_split(pool: &DevicePool, num_layers: usize) -> Result<Self> {
296 if num_layers == 0 {
297 return Err(anyhow!("LayerDeviceMap requires num_layers >= 1 (got 0)"));
298 }
299 let devices = pool.devices();
300 let n_dev = devices.len();
301 let base = num_layers / n_dev;
302 let rem = num_layers % n_dev;
303
304 let mut per_layer = Vec::with_capacity(num_layers);
305 for (d, &dev) in devices.iter().enumerate() {
306 // First `rem` devices get one extra layer (prefix-heavy split).
307 let count = base + usize::from(d < rem);
308 for _ in 0..count {
309 per_layer.push(dev);
310 }
311 }
312 debug_assert_eq!(per_layer.len(), num_layers);
313 Ok(Self { per_layer })
314 }
315
316 /// Device owning global layer `global_layer_idx`.
317 ///
318 /// # Panics
319 /// Panics if `global_layer_idx >= len()` — a programming error (the caller
320 /// iterates a known global range). Use [`Self::try_device_for`] for a checked
321 /// variant.
322 #[must_use]
323 pub fn device_for(&self, global_layer_idx: usize) -> Device {
324 self.per_layer[global_layer_idx]
325 }
326
327 /// Checked [`Self::device_for`].
328 pub fn try_device_for(&self, global_layer_idx: usize) -> Result<Device> {
329 self.per_layer
330 .get(global_layer_idx)
331 .copied()
332 .ok_or_else(|| {
333 anyhow!(
334 "LayerDeviceMap: global layer index {global_layer_idx} out of range \
335 (map covers {} layers)",
336 self.per_layer.len()
337 )
338 })
339 }
340
341 /// Whether a cross-device boundary copy is needed *before* running layer
342 /// `global_layer_idx`, given the device the previous layer's output is on.
343 ///
344 /// Returns `false` for the first layer of a stage when its device equals
345 /// `prev_device` (zero-copy within a stage and across same-device stages), and
346 /// `true` only when the device actually changes. This is the single point
347 /// that gates the lone `hidden.to_device()` in the forward loop.
348 #[must_use]
349 pub fn is_boundary(&self, prev_device: Device, global_layer_idx: usize) -> bool {
350 self.device_for(global_layer_idx) != prev_device
351 }
352
353 /// Number of global layers covered (equals the model's `num_hidden_layers`).
354 #[must_use]
355 #[allow(clippy::len_without_is_empty)]
356 pub fn len(&self) -> usize {
357 self.per_layer.len()
358 }
359
360 /// Whether every layer maps to the same device (the unsplit fast path).
361 ///
362 /// When true, `forward_layers` performs zero cross-device copies and is
363 /// numerically identical to the single-device whole-model forward.
364 #[must_use]
365 pub fn is_single_device(&self) -> bool {
366 self.per_layer
367 .first()
368 .is_some_and(|&first| self.per_layer.iter().all(|&d| d == first))
369 }
370}
371
372#[cfg(test)]
373#[allow(clippy::unwrap_used, clippy::expect_used)]
374mod tests {
375 use super::*;
376
377 #[test]
378 fn from_devices_rejects_empty() {
379 let err = DevicePool::from_devices(vec![]).unwrap_err();
380 assert!(err.to_string().contains("at least one device"));
381 }
382
383 #[test]
384 fn from_devices_rejects_duplicates() {
385 let err = DevicePool::from_devices(vec![Device::Cpu, Device::Cpu]).unwrap_err();
386 assert!(err.to_string().contains("duplicate"));
387 }
388
389 #[test]
390 fn from_devices_preserves_order_and_primary() {
391 let pool = DevicePool::from_devices(vec![Device::Cuda(2), Device::Cuda(0)]).unwrap();
392 assert_eq!(pool.devices(), &[Device::Cuda(2), Device::Cuda(0)]);
393 assert_eq!(pool.primary(), Device::Cuda(2));
394 assert_eq!(pool.len(), 2);
395 assert!(!pool.is_single());
396 }
397
398 #[test]
399 fn from_devices_single() {
400 let pool = DevicePool::from_devices(vec![Device::Cpu]).unwrap();
401 assert_eq!(pool.primary(), Device::Cpu);
402 assert!(pool.is_single());
403 assert_eq!(pool.len(), 1);
404 }
405
406 #[test]
407 fn from_cuda_indices_rejects_empty() {
408 let err = DevicePool::from_cuda_indices(&[]).unwrap_err();
409 assert!(err.to_string().contains("at least one device index"));
410 }
411
412 #[test]
413 fn from_cuda_indices_rejects_duplicates_before_probing() {
414 // Duplicate detection must happen before any CUDA probe so it is
415 // deterministic on CPU-only CI.
416 let err = DevicePool::from_cuda_indices(&[1, 1]).unwrap_err();
417 assert!(err.to_string().contains("duplicate"));
418 }
419
420 #[test]
421 fn from_cuda_indices_no_gpu_is_fail_fast() {
422 // On CPU-only CI, requesting an explicit GPU index must error rather
423 // than degrade to CPU. (If CUDA happens to be present, this asserts the
424 // out-of-range guard instead via a deliberately huge index.)
425 if tch::Cuda::is_available() {
426 let err = DevicePool::from_cuda_indices(&[usize::MAX]).unwrap_err();
427 assert!(err.to_string().contains("out of range"));
428 } else {
429 let err = DevicePool::from_cuda_indices(&[0]).unwrap_err();
430 assert!(err.to_string().contains("no CUDA/ROCm device is available"));
431 }
432 }
433
434 #[test]
435 fn pool_is_send_sync() {
436 // Compile-time proof that the !Send boundary is upheld: DevicePool holds
437 // no tensors, so it must be Send + Sync for the replication design.
438 fn assert_send_sync<T: Send + Sync>() {}
439 assert_send_sync::<DevicePool>();
440 // The layer→device map crosses to the engine thread too, so it must also
441 // be Send + Sync (it holds only Device values).
442 assert_send_sync::<LayerDeviceMap>();
443 }
444
445 #[test]
446 fn layer_map_single_is_zero_copy() {
447 let map = LayerDeviceMap::single(Device::Cpu, 4).unwrap();
448 assert_eq!(map.len(), 4);
449 assert!(map.is_single_device());
450 for g in 0..4 {
451 assert_eq!(map.device_for(g), Device::Cpu);
452 }
453 // No boundary anywhere when every layer is the same device.
454 for g in 1..4 {
455 assert!(!map.is_boundary(Device::Cpu, g));
456 }
457 }
458
459 #[test]
460 fn layer_map_single_rejects_zero() {
461 assert!(LayerDeviceMap::single(Device::Cpu, 0).is_err());
462 assert!(LayerDeviceMap::from_per_layer(vec![]).is_err());
463 let pool = DevicePool::from_devices(vec![Device::Cpu]).unwrap();
464 assert!(LayerDeviceMap::even_split(&pool, 0).is_err());
465 }
466
467 #[test]
468 fn even_split_is_contiguous_and_prefix_heavy() {
469 // 5 layers over 2 devices → [d0,d0,d0, d1,d1] (prefix gets the extra).
470 let pool = DevicePool::from_devices(vec![Device::Cpu, Device::Cuda(0)]).unwrap();
471 let map = LayerDeviceMap::even_split(&pool, 5).unwrap();
472 assert_eq!(map.len(), 5);
473 assert_eq!(map.device_for(0), Device::Cpu);
474 assert_eq!(map.device_for(1), Device::Cpu);
475 assert_eq!(map.device_for(2), Device::Cpu);
476 assert_eq!(map.device_for(3), Device::Cuda(0));
477 assert_eq!(map.device_for(4), Device::Cuda(0));
478 assert!(!map.is_single_device());
479
480 // Exactly one boundary, at the device change (layer 3).
481 let mut boundaries = 0;
482 let mut prev = map.device_for(0);
483 for g in 1..map.len() {
484 if map.is_boundary(prev, g) {
485 boundaries += 1;
486 }
487 prev = map.device_for(g);
488 }
489 assert_eq!(
490 boundaries, 1,
491 "contiguous 2-device split has exactly one boundary"
492 );
493 }
494
495 #[test]
496 fn even_split_balanced_when_divisible() {
497 let pool =
498 DevicePool::from_devices(vec![Device::Cpu, Device::Cuda(0), Device::Cuda(1)]).unwrap();
499 let map = LayerDeviceMap::even_split(&pool, 6).unwrap();
500 assert_eq!(
501 (0..6).map(|g| map.device_for(g)).collect::<Vec<_>>(),
502 vec![
503 Device::Cpu,
504 Device::Cpu,
505 Device::Cuda(0),
506 Device::Cuda(0),
507 Device::Cuda(1),
508 Device::Cuda(1),
509 ]
510 );
511 }
512
513 #[test]
514 fn try_device_for_is_checked() {
515 let map = LayerDeviceMap::single(Device::Cpu, 2).unwrap();
516 assert_eq!(map.try_device_for(1).unwrap(), Device::Cpu);
517 assert!(map
518 .try_device_for(2)
519 .unwrap_err()
520 .to_string()
521 .contains("out of range"));
522 }
523}