hyprstream_inference/runtime/
image_utils.rs1use anyhow::{anyhow, Result};
10use std::path::Path;
11use tch::{Device, Kind, Tensor};
12use tracing::{debug, warn};
13
14#[derive(Debug, Clone)]
16pub struct ImagePreprocessConfig {
17 pub image_size: usize,
19 pub mean: [f32; 3],
21 pub std: [f32; 3],
23 pub convert_rgb: bool,
25}
26
27impl Default for ImagePreprocessConfig {
28 fn default() -> Self {
29 Self {
31 image_size: 384,
32 mean: [0.48145466, 0.4578275, 0.40821073],
33 std: [0.26862954, 0.261_302_6, 0.275_777_1],
34 convert_rgb: true,
35 }
36 }
37}
38
39impl ImagePreprocessConfig {
40 pub fn siglip() -> Self {
42 Self {
43 image_size: 384,
44 mean: [0.5, 0.5, 0.5],
45 std: [0.5, 0.5, 0.5],
46 convert_rgb: true,
47 }
48 }
49
50 pub fn clip() -> Self {
52 Self::default()
53 }
54}
55
56pub fn load_image(path: &Path, config: &ImagePreprocessConfig, device: Device) -> Result<Tensor> {
66 debug!("Loading image from: {}", path.display());
67
68 let img = image::open(path).map_err(|e| anyhow!("Failed to load image: {}", e))?;
70
71 let img = img.to_rgb8();
73
74 let img = image::imageops::resize(
76 &img,
77 config.image_size as u32,
78 config.image_size as u32,
79 image::imageops::FilterType::Lanczos3,
80 );
81
82 let (width, height) = img.dimensions();
84 let img_vec: Vec<f32> = img.into_raw().iter().map(|&x| x as f32 / 255.0).collect();
85
86 let tensor = Tensor::from_slice(&img_vec).reshape([height as i64, width as i64, 3]);
88
89 let tensor = tensor.permute([2, 0, 1]);
91
92 let tensor = normalize_tensor(&tensor, &config.mean, &config.std)?;
94
95 let tensor = tensor.unsqueeze(0);
97
98 Ok(tensor.to_device(device))
100}
101
102pub fn load_image_from_bytes(
104 bytes: &[u8],
105 config: &ImagePreprocessConfig,
106 device: Device,
107) -> Result<Tensor> {
108 debug!("Loading image from {} bytes", bytes.len());
109
110 let img = image::load_from_memory(bytes)
111 .map_err(|e| anyhow!("Failed to decode image bytes: {}", e))?;
112
113 let img = img.to_rgb8();
114
115 let img = image::imageops::resize(
116 &img,
117 config.image_size as u32,
118 config.image_size as u32,
119 image::imageops::FilterType::Lanczos3,
120 );
121
122 let (width, height) = img.dimensions();
123 let img_vec: Vec<f32> = img.into_raw().iter().map(|&x| x as f32 / 255.0).collect();
124
125 let tensor = Tensor::from_slice(&img_vec).reshape([height as i64, width as i64, 3]);
126
127 let tensor = tensor.permute([2, 0, 1]);
128 let tensor = normalize_tensor(&tensor, &config.mean, &config.std)?;
129 let tensor = tensor.unsqueeze(0);
130
131 Ok(tensor.to_device(device))
132}
133
134pub fn load_images(
136 paths: &[&Path],
137 config: &ImagePreprocessConfig,
138 device: Device,
139) -> Result<Tensor> {
140 if paths.is_empty() {
141 return Err(anyhow!("No image paths provided"));
142 }
143
144 let images: Result<Vec<Tensor>> = paths
145 .iter()
146 .map(|path| load_image(path, config, device))
147 .collect();
148
149 let images = images?;
150
151 Ok(Tensor::cat(&images, 0))
153}
154
155pub fn create_fallback_image(config: &ImagePreprocessConfig, device: Device) -> Result<Tensor> {
167 warn!("Creating fallback minimal image (no image provided)");
168
169 let r = fastrand::f32();
171 let g = fastrand::f32();
172 let b = fastrand::f32();
173
174 debug!("Fallback image color: R={:.3}, G={:.3}, B={:.3}", r, g, b);
175
176 let r_channel = Tensor::full(
178 [1, config.image_size as i64, config.image_size as i64],
179 r as f64,
180 (Kind::Float, device),
181 );
182 let g_channel = Tensor::full(
183 [1, config.image_size as i64, config.image_size as i64],
184 g as f64,
185 (Kind::Float, device),
186 );
187 let b_channel = Tensor::full(
188 [1, config.image_size as i64, config.image_size as i64],
189 b as f64,
190 (Kind::Float, device),
191 );
192
193 let tensor = Tensor::cat(&[r_channel, g_channel, b_channel], 0);
195
196 let tensor = normalize_tensor(&tensor, &config.mean, &config.std)?;
198
199 Ok(tensor.unsqueeze(0))
201}
202
203fn normalize_tensor(tensor: &Tensor, mean: &[f32; 3], std: &[f32; 3]) -> Result<Tensor> {
207 let mean_tensor = Tensor::from_slice(mean)
208 .to_device(tensor.device())
209 .reshape([3, 1, 1]);
210 let std_tensor = Tensor::from_slice(std)
211 .to_device(tensor.device())
212 .reshape([3, 1, 1]);
213
214 Ok((tensor - mean_tensor) / std_tensor)
215}
216
217pub struct ImageInput {
219 pub tensor: Tensor,
221 pub path: Option<String>,
223 pub is_fallback: bool,
225}
226
227impl ImageInput {
228 pub fn from_path(path: &Path, config: &ImagePreprocessConfig, device: Device) -> Result<Self> {
230 let tensor = load_image(path, config, device)?;
231 Ok(Self {
232 tensor,
233 path: Some(path.to_string_lossy().to_string()),
234 is_fallback: false,
235 })
236 }
237
238 pub fn fallback(config: &ImagePreprocessConfig, device: Device) -> Result<Self> {
240 let tensor = create_fallback_image(config, device)?;
241 Ok(Self {
242 tensor,
243 path: None,
244 is_fallback: true,
245 })
246 }
247
248 pub fn tensor(&self) -> &Tensor {
250 &self.tensor
251 }
252
253 pub fn is_fallback(&self) -> bool {
255 self.is_fallback
256 }
257}
258
259pub fn batch_images(images: &[ImageInput]) -> Result<Tensor> {
272 if images.is_empty() {
273 return Err(anyhow::anyhow!("Cannot batch empty image list"));
274 }
275
276 if images.len() == 1 {
277 return Ok(images[0].tensor.shallow_clone());
279 }
280
281 let tensors: Vec<&Tensor> = images.iter().map(|img| &img.tensor).collect();
283 let batched = Tensor::cat(&tensors, 0);
284
285 Ok(batched)
286}
287
288#[cfg(test)]
289#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
290mod tests {
291 use super::*;
292
293 #[test]
294 fn test_fallback_image_creation() {
295 let config = ImagePreprocessConfig::default();
296 let device = Device::Cpu;
297
298 let image = create_fallback_image(&config, device).expect("test: create fallback image");
299
300 assert_eq!(image.size(), &[1, 3, 384, 384]);
302
303 assert_eq!(image.device(), device);
305 }
306
307 #[test]
308 fn test_normalize_tensor() {
309 let device = Device::Cpu;
310 let tensor = Tensor::ones([3, 224, 224], (Kind::Float, device));
311
312 let mean = [0.5, 0.5, 0.5];
313 let std = [0.5, 0.5, 0.5];
314
315 let normalized = normalize_tensor(&tensor, &mean, &std).expect("test: normalize tensor");
316
317 let expected = Tensor::ones([3, 224, 224], (Kind::Float, device));
319
320 assert!(
321 (normalized - expected)
322 .abs()
323 .mean(Kind::Float)
324 .double_value(&[])
325 < 1e-6
326 );
327 }
328
329 #[test]
330 fn test_image_input_fallback() {
331 let config = ImagePreprocessConfig::default();
332 let device = Device::Cpu;
333
334 let image = ImageInput::fallback(&config, device).expect("test: create fallback image");
335
336 assert!(image.is_fallback());
337 assert!(image.path.is_none());
338 assert_eq!(image.tensor().size(), &[1, 3, 384, 384]);
339 }
340}