Skip to main content

hyprstream_inference/runtime/
torch_utils.rs

1//! Utilities for safe PyTorch operations with proper error handling
2//!
3//! ## Panic Handling and FFI Boundaries
4//!
5//! PyTorch's C++ backend (libtorch) throws exceptions on OOM and other errors.
6//! The tch-rs bindings translate these C++ exceptions into Rust panics during
7//! operations like tensor allocation (`Tensor::zeros`) and device transfers
8//! (`tensor.to_device`).
9//!
10//! This module provides wrappers that use `catch_unwind` to catch these panics
11//! and convert them to proper `Result` errors with actionable error messages.
12//!
13//! **Important**: This only works because tch-rs performs the C++ exception → Rust
14//! panic translation. Raw libtorch panics from C++ cannot be caught by `catch_unwind`.
15//!
16//! ## Thread Safety
17//!
18//! These utilities are safe to call from multiple threads, but be aware that
19//! catching panics during tensor operations may leave PyTorch's CUDA/ROCm
20//! allocator in an inconsistent state. After catching an OOM panic, it's
21//! recommended to avoid further allocations on the same device, or restart
22//! the application to reset allocator state.
23
24use anyhow::{anyhow, Result};
25use std::panic::catch_unwind;
26use tch::{Device, Tensor};
27use tracing::{error, warn};
28
29/// Safely execute a tensor operation that might panic (e.g., OOM)
30///
31/// PyTorch operations like `Tensor::zeros()` and `to_device()` can panic
32/// when GPU memory is exhausted. This wrapper catches such panics and
33/// converts them to proper Result errors.
34///
35/// # Example
36/// ```no_run
37/// use tch::{Device, Tensor};
38/// use hyprstream_inference::runtime::torch_utils::safe_tensor_op;
39/// let result = safe_tensor_op(|| {
40///     Tensor::zeros([1024, 1024], (tch::Kind::BFloat16, Device::Cuda(0)))
41/// });
42/// ```
43pub fn safe_tensor_op<F, T>(operation: F) -> Result<T>
44where
45    F: FnOnce() -> T + std::panic::UnwindSafe,
46{
47    match catch_unwind(operation) {
48        Ok(result) => Ok(result),
49        Err(panic_info) => {
50            // Try to extract panic message
51            let panic_msg = if let Some(s) = panic_info.downcast_ref::<String>() {
52                s.clone()
53            } else if let Some(s) = panic_info.downcast_ref::<&str>() {
54                (*s).to_owned()
55            } else {
56                "Unknown panic during tensor operation".to_owned()
57            };
58
59            error!("PyTorch operation panicked: {}", panic_msg);
60
61            // Check if it's an OOM error
62            if panic_msg.contains("out of memory") || panic_msg.contains("OOM") {
63                Err(anyhow!("GPU OOM: {}", panic_msg))
64            } else {
65                Err(anyhow!("Tensor operation failed: {}", panic_msg))
66            }
67        }
68    }
69}
70
71/// Safely transfer a tensor to a device (CPU to GPU)
72///
73/// This is a common operation that can OOM, so we provide a dedicated wrapper
74/// with clear error messages.
75pub fn safe_to_device(tensor: &Tensor, device: Device) -> Result<Tensor> {
76    safe_tensor_op(|| tensor.to_device(device)).map_err(|e| {
77        anyhow!(
78            "Failed to_device({:?}): {} | shape: {:?}",
79            device,
80            e,
81            tensor.size()
82        )
83    })
84}
85
86/// Safely allocate a tensor (zeros, ones, randn, etc.)
87///
88/// Wraps tensor allocation operations that might OOM.
89pub fn safe_zeros(shape: &[i64], kind_device: (tch::Kind, Device)) -> Result<Tensor> {
90    let (kind, device) = kind_device;
91    let size_mb = estimate_tensor_size_mb(shape, kind);
92    safe_tensor_op(|| Tensor::zeros(shape, (kind, device))).map_err(|e| {
93        anyhow!(
94            "Failed to allocate tensor on {:?}: {} | shape: {:?} ({:.1} MB)",
95            device,
96            e,
97            shape,
98            size_mb
99        )
100    })
101}
102
103/// Estimate tensor size in megabytes
104pub fn estimate_tensor_size_mb(shape: &[i64], dtype: tch::Kind) -> f64 {
105    let num_elements: i64 = shape.iter().product();
106    let bytes_per_element = match dtype {
107        tch::Kind::Double | tch::Kind::Int64 => 8,
108        tch::Kind::Half | tch::Kind::BFloat16 | tch::Kind::Int16 => 2,
109        tch::Kind::Int8
110        | tch::Kind::Uint8
111        | tch::Kind::Float8e4m3fn
112        | tch::Kind::Float8e5m2
113        | tch::Kind::Float8e4m3fnuz
114        | tch::Kind::Float8e5m2fnuz => 1,
115        // Float, Int, and all other types default to 4 bytes
116        _ => 4,
117    };
118
119    (num_elements * bytes_per_element) as f64 / (1024.0 * 1024.0)
120}
121
122/// Check available GPU memory before attempting an operation
123///
124/// Returns (free_bytes, total_bytes) for the specified device.
125/// Only works for CUDA/ROCm devices.
126pub fn check_gpu_memory(device: Device) -> Result<(usize, usize)> {
127    match device {
128        Device::Cuda(device_idx) => {
129            // Try to get CUDA memory info
130            // Note: This requires tch-rs to expose CUDA memory APIs
131            // For now, we'll try to call it via catch_unwind in case it panics
132            match catch_unwind(|| {
133                // Try to allocate a small tensor to test memory
134                let test = Tensor::zeros([1], (tch::Kind::Float, device));
135                drop(test);
136            }) {
137                Ok(_) => {
138                    warn!("GPU memory check not fully implemented - using workaround");
139                    // Return dummy values - this API needs tch-rs support
140                    Ok((0, 0))
141                }
142                Err(_) => Err(anyhow!("GPU device {} is not accessible", device_idx)),
143            }
144        }
145        Device::Cpu => {
146            // CPU memory check is different and less critical
147            Ok((0, 0))
148        }
149        _ => Ok((0, 0)),
150    }
151}
152
153/// Pre-flight check before loading a model
154///
155/// Attempts to detect if there's enough GPU memory before starting
156/// to load a model. This is a best-effort check.
157pub fn preflight_gpu_check(device: Device, estimated_model_size_mb: f64) -> Result<()> {
158    if !device.is_cuda() {
159        return Ok(()); // Only check GPU devices
160    }
161
162    // Try to get memory info
163    match check_gpu_memory(device) {
164        Ok((free, total)) if total > 0 => {
165            let free_mb = free as f64 / (1024.0 * 1024.0);
166            if free_mb < estimated_model_size_mb {
167                return Err(anyhow!(
168                    "Insufficient GPU memory: {:.2} MB free, but model requires ~{:.2} MB\n\
169                     \n\
170                     Free GPU memory before retrying:\n\
171                     - Kill other GPU processes\n\
172                     - Restart hyprstream server\n\
173                     - Use a smaller model",
174                    free_mb,
175                    estimated_model_size_mb
176                ));
177            }
178        }
179        _ => {
180            // Memory check not available, proceed with caution
181            warn!(
182                "Could not check GPU memory before loading model. \
183                 Estimated size: {:.2} MB",
184                estimated_model_size_mb
185            );
186        }
187    }
188
189    Ok(())
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195
196    #[test]
197    fn test_estimate_tensor_size() {
198        // BFloat16 tensor: [1, 2048, 32, 128]
199        let shape = vec![1, 2048, 32, 128];
200        let size_mb = estimate_tensor_size_mb(&shape, tch::Kind::BFloat16);
201
202        // Expected: 1 * 2048 * 32 * 128 * 2 bytes = 16,777,216 bytes = 16 MB
203        assert!((size_mb - 16.0).abs() < 0.1);
204    }
205
206    #[test]
207    fn test_safe_tensor_op_cpu() {
208        let result = safe_tensor_op(|| Tensor::zeros([10, 10], (tch::Kind::Float, Device::Cpu)));
209        assert!(result.is_ok());
210    }
211}