Skip to main content

hyprstream_inference/runtime/
device_pool.rs

1//! Multi-GPU device abstraction (`DevicePool`).
2//!
3//! This module introduces the device-strategy seam for the multi-GPU epic (#310,
4//! part of #313). It resolves a set of configured device indices
5//! (`RuntimeConfig.devices`) into concrete [`tch::Device`] values and exposes a
6//! small, stable API that the later phases build on:
7//!
8//! - **2a replication** (#313): N `TorchEngine`s, one per local GPU, requests
9//!   routed across [`DevicePool::devices`].
10//! - **2b intra-host pipeline** (#314): a layer→device `device_map` layered on
11//!   top of the same pool (see [`DevicePool`] note on extensibility).
12//!
13//! # Send / !Send boundary
14//!
15//! `tch-rs` tensors are `!Send`: a `Tensor` must never cross a thread boundary,
16//! and engines that hold tensors stay pinned to one thread (as
17//! `services/inference.rs` already does — one thread per engine). However, a
18//! *single* thread may legitimately own tensors on `Cuda(0)` **and** `Cuda(1)`
19//! at once: libtorch dispatches each op to the correct device via a
20//! `DeviceGuard`, so multi-device-on-one-thread is fine; only multi-*thread*
21//! tensor moves are forbidden.
22//!
23//! `DevicePool` deliberately holds **only** device indices / [`tch::Device`]
24//! values (which are `Copy` and contain no tensor state). It is therefore
25//! `Send + Sync` and can be shared across threads (e.g. handed to each
26//! engine-owning thread so each can construct its own thread-pinned engine).
27//! **Do not** add a `Tensor`, `VarStore`, or any `!Send` field to this type — it
28//! would silently make the pool `!Send` and break the replication design.
29
30use anyhow::{anyhow, Result};
31use tch::Device;
32
33/// A resolved, validated set of compute devices a single inference service owns.
34///
35/// Constructed from [`crate::config::RuntimeConfig`] via [`DevicePool::from_config`].
36/// The pool performs **fail-fast** device resolution: when devices are explicitly
37/// requested it never silently downgrades to CPU ("no-fragile-fallbacks", per the
38/// multi-GPU/multi-host inference design). A process told to use GPU 1 that would
39/// land on CPU is an error, not a warning.
40///
41/// # Send / Sync
42///
43/// `DevicePool` is `Send + Sync` because it holds only [`Device`] values (no
44/// tensors). See the module-level docs for the `!Send` boundary — keep this type
45/// tensor-free.
46///
47/// # Extensibility (#314)
48///
49/// The pipeline phase needs a layer→device map. That belongs *on* this type
50/// (e.g. a `fn device_for_layer(&self, layer_idx, num_layers) -> Device` or an
51/// explicit `device_map`), reusing the already-resolved [`Self::devices`]. New
52/// strategy methods can be added without changing the existing accessors below,
53/// so current callers (which only need [`Self::primary`]) keep working.
54#[derive(Debug, Clone)]
55pub struct DevicePool {
56    /// Resolved devices, in the order they were requested. Always non-empty.
57    devices: Vec<Device>,
58}
59
60impl DevicePool {
61    /// Build a pool for the given GPU indices, validating availability fail-fast.
62    ///
63    /// `indices` must be non-empty and free of duplicates (the caller —
64    /// [`crate::config::RuntimeConfig::resolve_device_indices`] — guarantees
65    /// this, but it is re-checked here so the type's invariant holds regardless
66    /// of construction path).
67    ///
68    /// Every index is validated against the live CUDA/HIP device count. If CUDA
69    /// is unavailable, or any requested index is out of range, this returns an
70    /// error rather than degrading to CPU — devices were *explicitly* requested.
71    ///
72    /// Note: ROCm presents as [`Device::Cuda`] via HIP, so this path is
73    /// vendor-agnostic and contains no vendor-special casing.
74    pub fn from_cuda_indices(indices: &[usize]) -> Result<Self> {
75        if indices.is_empty() {
76            return Err(anyhow!(
77                "DevicePool requires at least one device index (got none)"
78            ));
79        }
80
81        // Reject duplicates defensively (config layer also enforces this).
82        let mut seen = std::collections::HashSet::with_capacity(indices.len());
83        for &idx in indices {
84            if !seen.insert(idx) {
85                return Err(anyhow!(
86                    "DevicePool: duplicate device index {idx} in requested set {indices:?}"
87                ));
88            }
89        }
90
91        // Fail-fast: devices were explicitly requested, so a CPU downgrade is an
92        // error, not a fallback (no `Device::cuda_if_available()` here). Surface
93        // the actual cause: a bare "not available" hid a libcuda.so.1 / wrong-
94        // backend problem on the live cuda130 node and made it undiagnosable.
95        if !tch::Cuda::is_available() {
96            let has_cuda = tch::utils::has_cuda();
97            let has_hip = tch::utils::has_hip();
98            let device_count = tch::Cuda::device_count();
99            let ld_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_else(|_| "<unset>".to_owned());
100            return Err(anyhow!(
101                "DevicePool: GPU devices {indices:?} requested but no CUDA/ROCm device is \
102                 available (refusing to silently fall back to CPU). \
103                 Diagnostics: has_cuda={has_cuda}, has_hip={has_hip}, \
104                 device_count={device_count}, LD_LIBRARY_PATH={ld_path}. If has_cuda=true \
105                 but device_count=0, the host driver lib (libcuda.so.1) is not loadable — \
106                 add the NVIDIA driver lib dir to LD_LIBRARY_PATH. If has_cuda=false, a \
107                 CPU-only libtorch/backend is installed."
108            ));
109        }
110
111        let available = tch::Cuda::device_count();
112        if available < 0 {
113            return Err(anyhow!(
114                "DevicePool: CUDA reported a negative device count ({available})"
115            ));
116        }
117        let available = available as usize;
118
119        let devices = indices
120            .iter()
121            .map(|&idx| {
122                if idx >= available {
123                    Err(anyhow!(
124                        "DevicePool: GPU device index {idx} is out of range \
125                         (only {available} CUDA/ROCm device(s) present)"
126                    ))
127                } else {
128                    Ok(Device::Cuda(idx))
129                }
130            })
131            .collect::<Result<Vec<_>>>()?;
132
133        Ok(Self { devices })
134    }
135
136    /// Construct a pool directly from already-resolved [`Device`] values.
137    ///
138    /// Intended for tests and for callers that have an explicit device list
139    /// (e.g. CPU-only setups). Validates only the structural invariants
140    /// (non-empty, no duplicates); it does **not** probe CUDA, so prefer
141    /// [`Self::from_cuda_indices`] / [`Self::from_config`] for GPU resolution.
142    pub fn from_devices(devices: Vec<Device>) -> Result<Self> {
143        if devices.is_empty() {
144            return Err(anyhow!(
145                "DevicePool requires at least one device (got none)"
146            ));
147        }
148        let mut seen = std::collections::HashSet::with_capacity(devices.len());
149        for dev in &devices {
150            if !seen.insert(*dev) {
151                return Err(anyhow!(
152                    "DevicePool: duplicate device {dev:?} in {devices:?}"
153                ));
154            }
155        }
156        Ok(Self { devices })
157    }
158
159    /// Resolve a [`DevicePool`] from runtime configuration.
160    ///
161    /// Behavior:
162    /// - `use_gpu == false` → a single-entry CPU pool ([`Device::Cpu`]).
163    /// - `use_gpu == true` → resolve the configured indices
164    ///   ([`crate::config::RuntimeConfig::resolve_device_indices`], which honors
165    ///   `devices` then falls back to the legacy single `gpu_device_id`) and
166    ///   validate them fail-fast via [`Self::from_cuda_indices`].
167    ///
168    /// When `use_gpu` is true but no index is configured, this falls back to
169    /// `cuda_if_available()`-style auto-detection of device 0 (preserving the
170    /// pre-existing single-GPU default). That auto-detect path is the *only*
171    /// place a CPU result is allowed when GPU is on, and only because nothing
172    /// was explicitly requested.
173    pub fn from_config(config: &crate::config::RuntimeConfig) -> Result<Self> {
174        if !config.use_gpu {
175            return Self::from_devices(vec![Device::Cpu]);
176        }
177
178        match config.resolve_device_indices()? {
179            // Explicit device(s) requested → strict, fail-fast resolution.
180            Some(indices) => Self::from_cuda_indices(&indices),
181            // Nothing explicitly requested → preserve legacy auto-detect (may
182            // be CPU if no GPU is present; this is NOT a silent downgrade because
183            // the user requested no specific device).
184            None => Self::from_devices(vec![Device::cuda_if_available()]),
185        }
186    }
187
188    /// All resolved devices, in requested order. Always non-empty.
189    ///
190    /// The primary seam for #313/2a replication (one engine per device).
191    #[must_use]
192    pub fn devices(&self) -> &[Device] {
193        &self.devices
194    }
195
196    /// The primary device — the first configured device.
197    ///
198    /// Single-device callers (e.g. today's `TorchEngine`) use this to keep
199    /// existing behavior while the multi-engine routing lands in a later PR.
200    #[must_use]
201    pub fn primary(&self) -> Device {
202        self.devices[0]
203    }
204
205    /// Number of devices in the pool (always >= 1).
206    ///
207    /// No `is_empty()` companion exists: a `DevicePool` is never empty by
208    /// construction. Use [`Self::is_single`] for the meaningful predicate.
209    #[must_use]
210    #[allow(clippy::len_without_is_empty)]
211    pub fn len(&self) -> usize {
212        self.devices.len()
213    }
214
215    /// Whether the pool holds a single device. (`is_empty` is meaningless here —
216    /// the pool is never empty — so this single-device predicate is provided
217    /// instead; clippy's `len`/`is_empty` lint is allowed below.)
218    #[must_use]
219    pub fn is_single(&self) -> bool {
220        self.devices.len() == 1
221    }
222}
223
224/// A per-layer device assignment for **2b intra-host pipeline** (#314).
225///
226/// This is the layer→device `device_map` the [`DevicePool`] doc-comment
227/// anticipates. It is a thin, validated newtype over the per-global-layer
228/// `Vec<Device>` derived from a [`DevicePool`]: entry `g` is the device that
229/// owns global decoder layer `g`. A *stage* is a contiguous run of layers that
230/// map to the same device; the only cross-device copy in the forward pass is a
231/// single `hidden.to_device(next)` at a boundary where consecutive layers map to
232/// different devices (see [`LayerDeviceMap::is_boundary`]).
233///
234/// # Why a map over `num_hidden_layers`, not the stage-local count
235///
236/// The map is **global** — indexed by global layer index, length
237/// `num_hidden_layers` — so a single source of truth drives both shard-aware
238/// construction (per-layer placement + per-layer type/RoPE selection, which need
239/// the global index) and the forward loop. A stage that owns layers `[a..b)`
240/// queries the same global map; the architecture is responsible for the
241/// global↔local remap of its own `self.layers` / KV / SSM vectors via the
242/// `layer_offset = a` it was built with.
243///
244/// # Send / Sync
245///
246/// Holds only [`Device`] values (`Copy`, no tensors), so it is `Send + Sync` and
247/// may be handed to an engine-owning thread. Like [`DevicePool`], **never** add a
248/// `Tensor`/`VarStore` field — see the module-level `!Send` boundary note.
249#[derive(Debug, Clone)]
250pub struct LayerDeviceMap {
251    /// One device per global layer index. Always non-empty; `len()` equals the
252    /// model's `num_hidden_layers`.
253    per_layer: Vec<Device>,
254}
255
256impl LayerDeviceMap {
257    /// All layers on a single device — the unsplit fast path.
258    ///
259    /// Used both by single-GPU inference and by the CPU-only equivalence tests
260    /// (`forward_layers(0..N)` over an all-CPU map must equal the whole-model
261    /// forward). `num_layers` must be non-zero.
262    pub fn single(device: Device, num_layers: usize) -> Result<Self> {
263        if num_layers == 0 {
264            return Err(anyhow!("LayerDeviceMap requires num_layers >= 1 (got 0)"));
265        }
266        Ok(Self {
267            per_layer: vec![device; num_layers],
268        })
269    }
270
271    /// Build an explicit per-layer map, validating it is non-empty.
272    ///
273    /// `per_layer[g]` is the device that owns global layer `g`.
274    pub fn from_per_layer(per_layer: Vec<Device>) -> Result<Self> {
275        if per_layer.is_empty() {
276            return Err(anyhow!(
277                "LayerDeviceMap requires at least one layer (got none)"
278            ));
279        }
280        Ok(Self { per_layer })
281    }
282
283    /// Spread `num_layers` contiguously across a [`DevicePool`]'s devices in
284    /// requested order, balanced as evenly as possible (capacity-symmetric).
285    ///
286    /// Earlier stages get the extra layer when `num_layers` is not divisible by
287    /// the device count, matching the natural prefix-heavy split. This is the
288    /// parameter-balanced planner; the capacity-aware (asymmetric-VRAM) variant
289    /// is a later epic concern (plan §H) and would replace this constructor
290    /// without touching the forward path.
291    ///
292    /// Layers are assigned in **contiguous runs** so each device owns a single
293    /// pipeline stage `[a..b)` — never interleaved — which keeps boundary copies
294    /// to exactly `(num_stages - 1)` per forward.
295    pub fn even_split(pool: &DevicePool, num_layers: usize) -> Result<Self> {
296        if num_layers == 0 {
297            return Err(anyhow!("LayerDeviceMap requires num_layers >= 1 (got 0)"));
298        }
299        let devices = pool.devices();
300        let n_dev = devices.len();
301        let base = num_layers / n_dev;
302        let rem = num_layers % n_dev;
303
304        let mut per_layer = Vec::with_capacity(num_layers);
305        for (d, &dev) in devices.iter().enumerate() {
306            // First `rem` devices get one extra layer (prefix-heavy split).
307            let count = base + usize::from(d < rem);
308            for _ in 0..count {
309                per_layer.push(dev);
310            }
311        }
312        debug_assert_eq!(per_layer.len(), num_layers);
313        Ok(Self { per_layer })
314    }
315
316    /// Device owning global layer `global_layer_idx`.
317    ///
318    /// # Panics
319    /// Panics if `global_layer_idx >= len()` — a programming error (the caller
320    /// iterates a known global range). Use [`Self::try_device_for`] for a checked
321    /// variant.
322    #[must_use]
323    pub fn device_for(&self, global_layer_idx: usize) -> Device {
324        self.per_layer[global_layer_idx]
325    }
326
327    /// Checked [`Self::device_for`].
328    pub fn try_device_for(&self, global_layer_idx: usize) -> Result<Device> {
329        self.per_layer
330            .get(global_layer_idx)
331            .copied()
332            .ok_or_else(|| {
333                anyhow!(
334                    "LayerDeviceMap: global layer index {global_layer_idx} out of range \
335                     (map covers {} layers)",
336                    self.per_layer.len()
337                )
338            })
339    }
340
341    /// Whether a cross-device boundary copy is needed *before* running layer
342    /// `global_layer_idx`, given the device the previous layer's output is on.
343    ///
344    /// Returns `false` for the first layer of a stage when its device equals
345    /// `prev_device` (zero-copy within a stage and across same-device stages), and
346    /// `true` only when the device actually changes. This is the single point
347    /// that gates the lone `hidden.to_device()` in the forward loop.
348    #[must_use]
349    pub fn is_boundary(&self, prev_device: Device, global_layer_idx: usize) -> bool {
350        self.device_for(global_layer_idx) != prev_device
351    }
352
353    /// Number of global layers covered (equals the model's `num_hidden_layers`).
354    #[must_use]
355    #[allow(clippy::len_without_is_empty)]
356    pub fn len(&self) -> usize {
357        self.per_layer.len()
358    }
359
360    /// Whether every layer maps to the same device (the unsplit fast path).
361    ///
362    /// When true, `forward_layers` performs zero cross-device copies and is
363    /// numerically identical to the single-device whole-model forward.
364    #[must_use]
365    pub fn is_single_device(&self) -> bool {
366        self.per_layer
367            .first()
368            .is_some_and(|&first| self.per_layer.iter().all(|&d| d == first))
369    }
370}
371
372#[cfg(test)]
373#[allow(clippy::unwrap_used, clippy::expect_used)]
374mod tests {
375    use super::*;
376
377    #[test]
378    fn from_devices_rejects_empty() {
379        let err = DevicePool::from_devices(vec![]).unwrap_err();
380        assert!(err.to_string().contains("at least one device"));
381    }
382
383    #[test]
384    fn from_devices_rejects_duplicates() {
385        let err = DevicePool::from_devices(vec![Device::Cpu, Device::Cpu]).unwrap_err();
386        assert!(err.to_string().contains("duplicate"));
387    }
388
389    #[test]
390    fn from_devices_preserves_order_and_primary() {
391        let pool = DevicePool::from_devices(vec![Device::Cuda(2), Device::Cuda(0)]).unwrap();
392        assert_eq!(pool.devices(), &[Device::Cuda(2), Device::Cuda(0)]);
393        assert_eq!(pool.primary(), Device::Cuda(2));
394        assert_eq!(pool.len(), 2);
395        assert!(!pool.is_single());
396    }
397
398    #[test]
399    fn from_devices_single() {
400        let pool = DevicePool::from_devices(vec![Device::Cpu]).unwrap();
401        assert_eq!(pool.primary(), Device::Cpu);
402        assert!(pool.is_single());
403        assert_eq!(pool.len(), 1);
404    }
405
406    #[test]
407    fn from_cuda_indices_rejects_empty() {
408        let err = DevicePool::from_cuda_indices(&[]).unwrap_err();
409        assert!(err.to_string().contains("at least one device index"));
410    }
411
412    #[test]
413    fn from_cuda_indices_rejects_duplicates_before_probing() {
414        // Duplicate detection must happen before any CUDA probe so it is
415        // deterministic on CPU-only CI.
416        let err = DevicePool::from_cuda_indices(&[1, 1]).unwrap_err();
417        assert!(err.to_string().contains("duplicate"));
418    }
419
420    #[test]
421    fn from_cuda_indices_no_gpu_is_fail_fast() {
422        // On CPU-only CI, requesting an explicit GPU index must error rather
423        // than degrade to CPU. (If CUDA happens to be present, this asserts the
424        // out-of-range guard instead via a deliberately huge index.)
425        if tch::Cuda::is_available() {
426            let err = DevicePool::from_cuda_indices(&[usize::MAX]).unwrap_err();
427            assert!(err.to_string().contains("out of range"));
428        } else {
429            let err = DevicePool::from_cuda_indices(&[0]).unwrap_err();
430            assert!(err.to_string().contains("no CUDA/ROCm device is available"));
431        }
432    }
433
434    #[test]
435    fn pool_is_send_sync() {
436        // Compile-time proof that the !Send boundary is upheld: DevicePool holds
437        // no tensors, so it must be Send + Sync for the replication design.
438        fn assert_send_sync<T: Send + Sync>() {}
439        assert_send_sync::<DevicePool>();
440        // The layer→device map crosses to the engine thread too, so it must also
441        // be Send + Sync (it holds only Device values).
442        assert_send_sync::<LayerDeviceMap>();
443    }
444
445    #[test]
446    fn layer_map_single_is_zero_copy() {
447        let map = LayerDeviceMap::single(Device::Cpu, 4).unwrap();
448        assert_eq!(map.len(), 4);
449        assert!(map.is_single_device());
450        for g in 0..4 {
451            assert_eq!(map.device_for(g), Device::Cpu);
452        }
453        // No boundary anywhere when every layer is the same device.
454        for g in 1..4 {
455            assert!(!map.is_boundary(Device::Cpu, g));
456        }
457    }
458
459    #[test]
460    fn layer_map_single_rejects_zero() {
461        assert!(LayerDeviceMap::single(Device::Cpu, 0).is_err());
462        assert!(LayerDeviceMap::from_per_layer(vec![]).is_err());
463        let pool = DevicePool::from_devices(vec![Device::Cpu]).unwrap();
464        assert!(LayerDeviceMap::even_split(&pool, 0).is_err());
465    }
466
467    #[test]
468    fn even_split_is_contiguous_and_prefix_heavy() {
469        // 5 layers over 2 devices → [d0,d0,d0, d1,d1] (prefix gets the extra).
470        let pool = DevicePool::from_devices(vec![Device::Cpu, Device::Cuda(0)]).unwrap();
471        let map = LayerDeviceMap::even_split(&pool, 5).unwrap();
472        assert_eq!(map.len(), 5);
473        assert_eq!(map.device_for(0), Device::Cpu);
474        assert_eq!(map.device_for(1), Device::Cpu);
475        assert_eq!(map.device_for(2), Device::Cpu);
476        assert_eq!(map.device_for(3), Device::Cuda(0));
477        assert_eq!(map.device_for(4), Device::Cuda(0));
478        assert!(!map.is_single_device());
479
480        // Exactly one boundary, at the device change (layer 3).
481        let mut boundaries = 0;
482        let mut prev = map.device_for(0);
483        for g in 1..map.len() {
484            if map.is_boundary(prev, g) {
485                boundaries += 1;
486            }
487            prev = map.device_for(g);
488        }
489        assert_eq!(
490            boundaries, 1,
491            "contiguous 2-device split has exactly one boundary"
492        );
493    }
494
495    #[test]
496    fn even_split_balanced_when_divisible() {
497        let pool =
498            DevicePool::from_devices(vec![Device::Cpu, Device::Cuda(0), Device::Cuda(1)]).unwrap();
499        let map = LayerDeviceMap::even_split(&pool, 6).unwrap();
500        assert_eq!(
501            (0..6).map(|g| map.device_for(g)).collect::<Vec<_>>(),
502            vec![
503                Device::Cpu,
504                Device::Cpu,
505                Device::Cuda(0),
506                Device::Cuda(0),
507                Device::Cuda(1),
508                Device::Cuda(1),
509            ]
510        );
511    }
512
513    #[test]
514    fn try_device_for_is_checked() {
515        let map = LayerDeviceMap::single(Device::Cpu, 2).unwrap();
516        assert_eq!(map.try_device_for(1).unwrap(), Device::Cpu);
517        assert!(map
518            .try_device_for(2)
519            .unwrap_err()
520            .to_string()
521            .contains("out of range"));
522    }
523}