hyprstream_inference/runtime/torch_utils.rs
1//! Utilities for safe PyTorch operations with proper error handling
2//!
3//! ## Panic Handling and FFI Boundaries
4//!
5//! PyTorch's C++ backend (libtorch) throws exceptions on OOM and other errors.
6//! The tch-rs bindings translate these C++ exceptions into Rust panics during
7//! operations like tensor allocation (`Tensor::zeros`) and device transfers
8//! (`tensor.to_device`).
9//!
10//! This module provides wrappers that use `catch_unwind` to catch these panics
11//! and convert them to proper `Result` errors with actionable error messages.
12//!
13//! **Important**: This only works because tch-rs performs the C++ exception → Rust
14//! panic translation. Raw libtorch panics from C++ cannot be caught by `catch_unwind`.
15//!
16//! ## Thread Safety
17//!
18//! These utilities are safe to call from multiple threads, but be aware that
19//! catching panics during tensor operations may leave PyTorch's CUDA/ROCm
20//! allocator in an inconsistent state. After catching an OOM panic, it's
21//! recommended to avoid further allocations on the same device, or restart
22//! the application to reset allocator state.
23
24use anyhow::{anyhow, Result};
25use std::panic::catch_unwind;
26use tch::{Device, Tensor};
27use tracing::{error, warn};
28
29/// Safely execute a tensor operation that might panic (e.g., OOM)
30///
31/// PyTorch operations like `Tensor::zeros()` and `to_device()` can panic
32/// when GPU memory is exhausted. This wrapper catches such panics and
33/// converts them to proper Result errors.
34///
35/// # Example
36/// ```no_run
37/// use tch::{Device, Tensor};
38/// use hyprstream_inference::runtime::torch_utils::safe_tensor_op;
39/// let result = safe_tensor_op(|| {
40/// Tensor::zeros([1024, 1024], (tch::Kind::BFloat16, Device::Cuda(0)))
41/// });
42/// ```
43pub fn safe_tensor_op<F, T>(operation: F) -> Result<T>
44where
45 F: FnOnce() -> T + std::panic::UnwindSafe,
46{
47 match catch_unwind(operation) {
48 Ok(result) => Ok(result),
49 Err(panic_info) => {
50 // Try to extract panic message
51 let panic_msg = if let Some(s) = panic_info.downcast_ref::<String>() {
52 s.clone()
53 } else if let Some(s) = panic_info.downcast_ref::<&str>() {
54 (*s).to_owned()
55 } else {
56 "Unknown panic during tensor operation".to_owned()
57 };
58
59 error!("PyTorch operation panicked: {}", panic_msg);
60
61 // Check if it's an OOM error
62 if panic_msg.contains("out of memory") || panic_msg.contains("OOM") {
63 Err(anyhow!("GPU OOM: {}", panic_msg))
64 } else {
65 Err(anyhow!("Tensor operation failed: {}", panic_msg))
66 }
67 }
68 }
69}
70
71/// Safely transfer a tensor to a device (CPU to GPU)
72///
73/// This is a common operation that can OOM, so we provide a dedicated wrapper
74/// with clear error messages.
75pub fn safe_to_device(tensor: &Tensor, device: Device) -> Result<Tensor> {
76 safe_tensor_op(|| tensor.to_device(device)).map_err(|e| {
77 anyhow!(
78 "Failed to_device({:?}): {} | shape: {:?}",
79 device,
80 e,
81 tensor.size()
82 )
83 })
84}
85
86/// Safely allocate a tensor (zeros, ones, randn, etc.)
87///
88/// Wraps tensor allocation operations that might OOM.
89pub fn safe_zeros(shape: &[i64], kind_device: (tch::Kind, Device)) -> Result<Tensor> {
90 let (kind, device) = kind_device;
91 let size_mb = estimate_tensor_size_mb(shape, kind);
92 safe_tensor_op(|| Tensor::zeros(shape, (kind, device))).map_err(|e| {
93 anyhow!(
94 "Failed to allocate tensor on {:?}: {} | shape: {:?} ({:.1} MB)",
95 device,
96 e,
97 shape,
98 size_mb
99 )
100 })
101}
102
103/// Estimate tensor size in megabytes
104pub fn estimate_tensor_size_mb(shape: &[i64], dtype: tch::Kind) -> f64 {
105 let num_elements: i64 = shape.iter().product();
106 let bytes_per_element = match dtype {
107 tch::Kind::Double | tch::Kind::Int64 => 8,
108 tch::Kind::Half | tch::Kind::BFloat16 | tch::Kind::Int16 => 2,
109 tch::Kind::Int8
110 | tch::Kind::Uint8
111 | tch::Kind::Float8e4m3fn
112 | tch::Kind::Float8e5m2
113 | tch::Kind::Float8e4m3fnuz
114 | tch::Kind::Float8e5m2fnuz => 1,
115 // Float, Int, and all other types default to 4 bytes
116 _ => 4,
117 };
118
119 (num_elements * bytes_per_element) as f64 / (1024.0 * 1024.0)
120}
121
122/// Check available GPU memory before attempting an operation
123///
124/// Returns (free_bytes, total_bytes) for the specified device.
125/// Only works for CUDA/ROCm devices.
126pub fn check_gpu_memory(device: Device) -> Result<(usize, usize)> {
127 match device {
128 Device::Cuda(device_idx) => {
129 // Try to get CUDA memory info
130 // Note: This requires tch-rs to expose CUDA memory APIs
131 // For now, we'll try to call it via catch_unwind in case it panics
132 match catch_unwind(|| {
133 // Try to allocate a small tensor to test memory
134 let test = Tensor::zeros([1], (tch::Kind::Float, device));
135 drop(test);
136 }) {
137 Ok(_) => {
138 warn!("GPU memory check not fully implemented - using workaround");
139 // Return dummy values - this API needs tch-rs support
140 Ok((0, 0))
141 }
142 Err(_) => Err(anyhow!("GPU device {} is not accessible", device_idx)),
143 }
144 }
145 Device::Cpu => {
146 // CPU memory check is different and less critical
147 Ok((0, 0))
148 }
149 _ => Ok((0, 0)),
150 }
151}
152
153/// Pre-flight check before loading a model
154///
155/// Attempts to detect if there's enough GPU memory before starting
156/// to load a model. This is a best-effort check.
157pub fn preflight_gpu_check(device: Device, estimated_model_size_mb: f64) -> Result<()> {
158 if !device.is_cuda() {
159 return Ok(()); // Only check GPU devices
160 }
161
162 // Try to get memory info
163 match check_gpu_memory(device) {
164 Ok((free, total)) if total > 0 => {
165 let free_mb = free as f64 / (1024.0 * 1024.0);
166 if free_mb < estimated_model_size_mb {
167 return Err(anyhow!(
168 "Insufficient GPU memory: {:.2} MB free, but model requires ~{:.2} MB\n\
169 \n\
170 Free GPU memory before retrying:\n\
171 - Kill other GPU processes\n\
172 - Restart hyprstream server\n\
173 - Use a smaller model",
174 free_mb,
175 estimated_model_size_mb
176 ));
177 }
178 }
179 _ => {
180 // Memory check not available, proceed with caution
181 warn!(
182 "Could not check GPU memory before loading model. \
183 Estimated size: {:.2} MB",
184 estimated_model_size_mb
185 );
186 }
187 }
188
189 Ok(())
190}
191
192#[cfg(test)]
193mod tests {
194 use super::*;
195
196 #[test]
197 fn test_estimate_tensor_size() {
198 // BFloat16 tensor: [1, 2048, 32, 128]
199 let shape = vec![1, 2048, 32, 128];
200 let size_mb = estimate_tensor_size_mb(&shape, tch::Kind::BFloat16);
201
202 // Expected: 1 * 2048 * 32 * 128 * 2 bytes = 16,777,216 bytes = 16 MB
203 assert!((size_mb - 16.0).abs() < 0.1);
204 }
205
206 #[test]
207 fn test_safe_tensor_op_cpu() {
208 let result = safe_tensor_op(|| Tensor::zeros([10, 10], (tch::Kind::Float, Device::Cpu)));
209 assert!(result.is_ok());
210 }
211}