Skip to main content

hyprstream_inference/runtime/
image_utils.rs

1//! Image loading and preprocessing utilities for multimodal models
2//!
3//! This module provides utilities for:
4//! - Loading images from disk (PNG, JPEG, etc.)
5//! - Preprocessing images for vision encoders
6//! - Creating fallback minimal images
7//! - Batch image processing
8
9use anyhow::{anyhow, Result};
10use std::path::Path;
11use tch::{Device, Kind, Tensor};
12use tracing::{debug, warn};
13
14/// Image preprocessing configuration
15#[derive(Debug, Clone)]
16pub struct ImagePreprocessConfig {
17    /// Target image size (width and height)
18    pub image_size: usize,
19    /// Mean values for normalization [R, G, B]
20    pub mean: [f32; 3],
21    /// Std values for normalization [R, G, B]
22    pub std: [f32; 3],
23    /// Whether to convert to RGB (from RGBA, grayscale, etc.)
24    pub convert_rgb: bool,
25}
26
27impl Default for ImagePreprocessConfig {
28    fn default() -> Self {
29        // SigLIP/CLIP default normalization
30        Self {
31            image_size: 384,
32            mean: [0.48145466, 0.4578275, 0.40821073],
33            std: [0.26862954, 0.261_302_6, 0.275_777_1],
34            convert_rgb: true,
35        }
36    }
37}
38
39impl ImagePreprocessConfig {
40    /// Configuration for SigLIP vision encoder
41    pub fn siglip() -> Self {
42        Self {
43            image_size: 384,
44            mean: [0.5, 0.5, 0.5],
45            std: [0.5, 0.5, 0.5],
46            convert_rgb: true,
47        }
48    }
49
50    /// Configuration for CLIP vision encoder
51    pub fn clip() -> Self {
52        Self::default()
53    }
54}
55
56/// Load an image from disk and preprocess it for vision encoder
57///
58/// # Arguments
59/// * `path` - Path to the image file
60/// * `config` - Preprocessing configuration
61/// * `device` - Device to load tensor onto
62///
63/// # Returns
64/// Tensor of shape [1, 3, H, W] ready for vision encoder
65pub fn load_image(path: &Path, config: &ImagePreprocessConfig, device: Device) -> Result<Tensor> {
66    debug!("Loading image from: {}", path.display());
67
68    // Load image using image crate
69    let img = image::open(path).map_err(|e| anyhow!("Failed to load image: {}", e))?;
70
71    // Convert to RGB (always needed for model input)
72    let img = img.to_rgb8();
73
74    // Resize to target size (keeping aspect ratio, then center crop)
75    let img = image::imageops::resize(
76        &img,
77        config.image_size as u32,
78        config.image_size as u32,
79        image::imageops::FilterType::Lanczos3,
80    );
81
82    // Convert to tensor [H, W, C]
83    let (width, height) = img.dimensions();
84    let img_vec: Vec<f32> = img.into_raw().iter().map(|&x| x as f32 / 255.0).collect();
85
86    // Reshape to [H, W, C]
87    let tensor = Tensor::from_slice(&img_vec).reshape([height as i64, width as i64, 3]);
88
89    // Transpose to [C, H, W]
90    let tensor = tensor.permute([2, 0, 1]);
91
92    // Normalize
93    let tensor = normalize_tensor(&tensor, &config.mean, &config.std)?;
94
95    // Add batch dimension [1, C, H, W]
96    let tensor = tensor.unsqueeze(0);
97
98    // Move to target device
99    Ok(tensor.to_device(device))
100}
101
102/// Load an image from raw bytes (PNG, JPEG, or other formats supported by `image` crate)
103pub fn load_image_from_bytes(
104    bytes: &[u8],
105    config: &ImagePreprocessConfig,
106    device: Device,
107) -> Result<Tensor> {
108    debug!("Loading image from {} bytes", bytes.len());
109
110    let img = image::load_from_memory(bytes)
111        .map_err(|e| anyhow!("Failed to decode image bytes: {}", e))?;
112
113    let img = img.to_rgb8();
114
115    let img = image::imageops::resize(
116        &img,
117        config.image_size as u32,
118        config.image_size as u32,
119        image::imageops::FilterType::Lanczos3,
120    );
121
122    let (width, height) = img.dimensions();
123    let img_vec: Vec<f32> = img.into_raw().iter().map(|&x| x as f32 / 255.0).collect();
124
125    let tensor = Tensor::from_slice(&img_vec).reshape([height as i64, width as i64, 3]);
126
127    let tensor = tensor.permute([2, 0, 1]);
128    let tensor = normalize_tensor(&tensor, &config.mean, &config.std)?;
129    let tensor = tensor.unsqueeze(0);
130
131    Ok(tensor.to_device(device))
132}
133
134/// Load multiple images and batch them
135pub fn load_images(
136    paths: &[&Path],
137    config: &ImagePreprocessConfig,
138    device: Device,
139) -> Result<Tensor> {
140    if paths.is_empty() {
141        return Err(anyhow!("No image paths provided"));
142    }
143
144    let images: Result<Vec<Tensor>> = paths
145        .iter()
146        .map(|path| load_image(path, config, device))
147        .collect();
148
149    let images = images?;
150
151    // Stack into batch [N, C, H, W]
152    Ok(Tensor::cat(&images, 0))
153}
154
155/// Create a minimal fallback image (1x1 random pixel expanded to full size)
156///
157/// This is used when no image is provided but the model requires vision input.
158/// The image is essentially a solid color with a random RGB value.
159///
160/// # Arguments
161/// * `config` - Preprocessing configuration (determines output size)
162/// * `device` - Device to create tensor on
163///
164/// # Returns
165/// Tensor of shape [1, 3, H, W] with random solid color
166pub fn create_fallback_image(config: &ImagePreprocessConfig, device: Device) -> Result<Tensor> {
167    warn!("Creating fallback minimal image (no image provided)");
168
169    // Generate random RGB pixel values [0, 1]
170    let r = fastrand::f32();
171    let g = fastrand::f32();
172    let b = fastrand::f32();
173
174    debug!("Fallback image color: R={:.3}, G={:.3}, B={:.3}", r, g, b);
175
176    // Create solid color image [C, H, W]
177    let r_channel = Tensor::full(
178        [1, config.image_size as i64, config.image_size as i64],
179        r as f64,
180        (Kind::Float, device),
181    );
182    let g_channel = Tensor::full(
183        [1, config.image_size as i64, config.image_size as i64],
184        g as f64,
185        (Kind::Float, device),
186    );
187    let b_channel = Tensor::full(
188        [1, config.image_size as i64, config.image_size as i64],
189        b as f64,
190        (Kind::Float, device),
191    );
192
193    // Stack channels [3, H, W]
194    let tensor = Tensor::cat(&[r_channel, g_channel, b_channel], 0);
195
196    // Normalize
197    let tensor = normalize_tensor(&tensor, &config.mean, &config.std)?;
198
199    // Add batch dimension [1, 3, H, W]
200    Ok(tensor.unsqueeze(0))
201}
202
203/// Normalize a tensor using mean and std
204///
205/// Formula: (x - mean) / std
206fn normalize_tensor(tensor: &Tensor, mean: &[f32; 3], std: &[f32; 3]) -> Result<Tensor> {
207    let mean_tensor = Tensor::from_slice(mean)
208        .to_device(tensor.device())
209        .reshape([3, 1, 1]);
210    let std_tensor = Tensor::from_slice(std)
211        .to_device(tensor.device())
212        .reshape([3, 1, 1]);
213
214    Ok((tensor - mean_tensor) / std_tensor)
215}
216
217/// Image data structure for passing around
218pub struct ImageInput {
219    /// Preprocessed image tensor [1, 3, H, W]
220    pub tensor: Tensor,
221    /// Original path (if loaded from file)
222    pub path: Option<String>,
223    /// Whether this is a fallback minimal image
224    pub is_fallback: bool,
225}
226
227impl ImageInput {
228    /// Create from a file path
229    pub fn from_path(path: &Path, config: &ImagePreprocessConfig, device: Device) -> Result<Self> {
230        let tensor = load_image(path, config, device)?;
231        Ok(Self {
232            tensor,
233            path: Some(path.to_string_lossy().to_string()),
234            is_fallback: false,
235        })
236    }
237
238    /// Create a fallback image
239    pub fn fallback(config: &ImagePreprocessConfig, device: Device) -> Result<Self> {
240        let tensor = create_fallback_image(config, device)?;
241        Ok(Self {
242            tensor,
243            path: None,
244            is_fallback: true,
245        })
246    }
247
248    /// Get the tensor
249    pub fn tensor(&self) -> &Tensor {
250        &self.tensor
251    }
252
253    /// Check if this is a fallback image
254    pub fn is_fallback(&self) -> bool {
255        self.is_fallback
256    }
257}
258
259/// Batch multiple images into a single tensor
260///
261/// Takes a vector of ImageInput instances and concatenates them along the batch dimension.
262///
263/// # Arguments
264/// * `images` - Vector of ImageInput instances, each with shape [1, C, H, W]
265///
266/// # Returns
267/// Batched tensor with shape [N, C, H, W] where N is the number of images
268///
269/// # Errors
270/// Returns an error if the images have inconsistent shapes or if concatenation fails
271pub fn batch_images(images: &[ImageInput]) -> Result<Tensor> {
272    if images.is_empty() {
273        return Err(anyhow::anyhow!("Cannot batch empty image list"));
274    }
275
276    if images.len() == 1 {
277        // Single image - return as-is (already has batch dimension)
278        return Ok(images[0].tensor.shallow_clone());
279    }
280
281    // Concatenate along batch dimension (dim 0)
282    let tensors: Vec<&Tensor> = images.iter().map(|img| &img.tensor).collect();
283    let batched = Tensor::cat(&tensors, 0);
284
285    Ok(batched)
286}
287
288#[cfg(test)]
289#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
290mod tests {
291    use super::*;
292
293    #[test]
294    fn test_fallback_image_creation() {
295        let config = ImagePreprocessConfig::default();
296        let device = Device::Cpu;
297
298        let image = create_fallback_image(&config, device).expect("test: create fallback image");
299
300        // Check shape
301        assert_eq!(image.size(), &[1, 3, 384, 384]);
302
303        // Check device
304        assert_eq!(image.device(), device);
305    }
306
307    #[test]
308    fn test_normalize_tensor() {
309        let device = Device::Cpu;
310        let tensor = Tensor::ones([3, 224, 224], (Kind::Float, device));
311
312        let mean = [0.5, 0.5, 0.5];
313        let std = [0.5, 0.5, 0.5];
314
315        let normalized = normalize_tensor(&tensor, &mean, &std).expect("test: normalize tensor");
316
317        // After normalization: (1.0 - 0.5) / 0.5 = 1.0
318        let expected = Tensor::ones([3, 224, 224], (Kind::Float, device));
319
320        assert!(
321            (normalized - expected)
322                .abs()
323                .mean(Kind::Float)
324                .double_value(&[])
325                < 1e-6
326        );
327    }
328
329    #[test]
330    fn test_image_input_fallback() {
331        let config = ImagePreprocessConfig::default();
332        let device = Device::Cpu;
333
334        let image = ImageInput::fallback(&config, device).expect("test: create fallback image");
335
336        assert!(image.is_fallback());
337        assert!(image.path.is_none());
338        assert_eq!(image.tensor().size(), &[1, 3, 384, 384]);
339    }
340}