Skip to main content

hyprstream_inference/training/
data_loader.rs

1//! Training data loading and processing for LoRA fine-tuning
2
3use anyhow::{Context, Result};
4use serde::{Deserialize, Serialize};
5use std::path::Path;
6
7/// A single training sample
8#[derive(Debug, Clone, Serialize, Deserialize)]
9pub struct TrainingSample {
10    /// Input text (prompt)
11    pub input: String,
12    /// Expected output (completion)
13    pub output: String,
14    /// Optional system message
15    pub system: Option<String>,
16    /// Optional metadata
17    pub metadata: Option<serde_json::Value>,
18}
19
20/// Training dataset
21#[derive(Clone)]
22pub struct TrainingDataset {
23    pub samples: Vec<TrainingSample>,
24}
25
26impl TrainingDataset {
27    /// Load training data from JSONL file
28    pub fn from_jsonl(path: &Path) -> Result<Self> {
29        let content = std::fs::read_to_string(path)
30            .with_context(|| format!("Failed to read training data from {path:?}"))?;
31
32        let mut samples = Vec::new();
33        for (line_num, line) in content.lines().enumerate() {
34            if line.trim().is_empty() {
35                continue;
36            }
37
38            let sample: TrainingSample = serde_json::from_str(line)
39                .with_context(|| format!("Failed to parse line {} in {:?}", line_num + 1, path))?;
40
41            samples.push(sample);
42        }
43
44        Ok(Self { samples })
45    }
46
47    /// Create dataset from conversation format
48    /// Expects format: [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
49    pub fn from_conversations(path: &Path) -> Result<Self> {
50        let content = std::fs::read_to_string(path)
51            .with_context(|| format!("Failed to read conversation data from {path:?}"))?;
52
53        let mut samples = Vec::new();
54
55        for line in content.lines() {
56            if line.trim().is_empty() {
57                continue;
58            }
59
60            let messages: Vec<serde_json::Value> = serde_json::from_str(line)?;
61
62            // Convert conversation to training samples
63            let mut current_input = String::new();
64
65            for message in messages {
66                let role = message["role"].as_str().unwrap_or("");
67                let content = message["content"].as_str().unwrap_or("");
68
69                match role {
70                    "user" | "human" => {
71                        if !current_input.is_empty() {
72                            current_input.push('\n');
73                        }
74                        current_input.push_str(content);
75                    }
76                    "assistant" | "ai" => {
77                        if !current_input.is_empty() {
78                            samples.push(TrainingSample {
79                                input: current_input.clone(),
80                                output: content.to_owned(),
81                                system: None,
82                                metadata: None,
83                            });
84                        }
85                        current_input.clear();
86                    }
87                    _ => {} // ignore system messages for now
88                }
89            }
90        }
91
92        Ok(Self { samples })
93    }
94
95    /// Get number of samples
96    pub fn len(&self) -> usize {
97        self.samples.len()
98    }
99
100    /// Check if dataset is empty
101    pub fn is_empty(&self) -> bool {
102        self.samples.is_empty()
103    }
104
105    /// Get sample by index
106    pub fn get(&self, index: usize) -> Option<&TrainingSample> {
107        self.samples.get(index)
108    }
109
110    /// Iterator over samples
111    pub fn iter(&self) -> impl Iterator<Item = &TrainingSample> {
112        self.samples.iter()
113    }
114
115    /// Create batches of specified size
116    pub fn batches(&self, batch_size: usize) -> impl Iterator<Item = &[TrainingSample]> {
117        self.samples.chunks(batch_size)
118    }
119
120    /// Split dataset into train/validation
121    pub fn train_test_split(&self, train_ratio: f64) -> (TrainingDataset, TrainingDataset) {
122        let split_point = ((self.samples.len() as f64) * train_ratio) as usize;
123
124        let train_samples = self.samples[..split_point].to_vec();
125        let test_samples = self.samples[split_point..].to_vec();
126
127        (
128            TrainingDataset {
129                samples: train_samples,
130            },
131            TrainingDataset {
132                samples: test_samples,
133            },
134        )
135    }
136}
137
138/// Training data loader with chat template formatting
139pub struct ChatTemplateDataLoader {
140    dataset: TrainingDataset,
141    template_engine: Option<crate::runtime::template_engine::TemplateEngine>,
142}
143
144impl ChatTemplateDataLoader {
145    /// Create new data loader
146    pub fn new(dataset: TrainingDataset) -> Self {
147        Self {
148            dataset,
149            template_engine: None,
150        }
151    }
152
153    /// Set chat template engine
154    pub fn with_template(
155        mut self,
156        template_engine: crate::runtime::template_engine::TemplateEngine,
157    ) -> Self {
158        self.template_engine = Some(template_engine);
159        self
160    }
161
162    /// Format a sample using chat template
163    pub fn format_sample(&self, sample: &TrainingSample) -> Result<(String, String)> {
164        use crate::runtime::template_engine::ChatMessage;
165
166        if let Some(ref template) = self.template_engine {
167            // Create chat messages
168            let mut messages = Vec::new();
169
170            if let Some(ref system) = sample.system {
171                messages.push(ChatMessage {
172                    role: "system".into(),
173                    content: Some(system.clone()),
174                    ..Default::default()
175                });
176            }
177
178            messages.push(ChatMessage {
179                role: "user".into(),
180                content: Some(sample.input.clone()),
181                ..Default::default()
182            });
183
184            // Format input (without assistant response)
185            let formatted_input = template.apply_chat_template(&messages, Some(true), None)?;
186
187            // Add assistant response for target
188            messages.push(ChatMessage {
189                role: "assistant".into(),
190                content: Some(sample.output.clone()),
191                ..Default::default()
192            });
193
194            let full_formatted = template.apply_chat_template(&messages, Some(false), None)?;
195
196            Ok((formatted_input, full_formatted))
197        } else {
198            // Simple concatenation without template
199            let input = format!("Human: {}\n\nAssistant: ", sample.input);
200            let target = format!("{}{}", input, sample.output);
201            Ok((input, target))
202        }
203    }
204
205    /// Get formatted batches for training
206    pub fn formatted_batches(
207        &self,
208        batch_size: usize,
209    ) -> impl Iterator<Item = Result<Vec<(String, String)>>> + '_ {
210        self.dataset.batches(batch_size).map(|batch| {
211            batch
212                .iter()
213                .map(|sample| self.format_sample(sample))
214                .collect::<Result<Vec<_>>>()
215        })
216    }
217}
218
219#[cfg(test)]
220#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
221mod tests {
222    use super::*;
223    use tempfile::tempdir;
224
225    #[test]
226    fn test_jsonl_loading() {
227        let temp_dir = tempdir().expect("test: create temp dir");
228        let data_path = temp_dir.path().join("train.jsonl");
229
230        let content = r#"{"input": "What is 2+2?", "output": "4"}
231{"input": "Hello", "output": "Hi there!"}
232"#;
233        std::fs::write(&data_path, content).expect("test: write training data");
234
235        let dataset = TrainingDataset::from_jsonl(&data_path).expect("test: load dataset");
236        assert_eq!(dataset.len(), 2);
237        assert_eq!(
238            dataset.get(0).expect("test: get sample").input,
239            "What is 2+2?"
240        );
241        assert_eq!(dataset.get(0).expect("test: get sample").output, "4");
242    }
243
244    #[test]
245    fn test_train_test_split() {
246        let samples = vec![
247            TrainingSample {
248                input: "1".to_owned(),
249                output: "a".to_owned(),
250                system: None,
251                metadata: None,
252            },
253            TrainingSample {
254                input: "2".to_owned(),
255                output: "b".to_owned(),
256                system: None,
257                metadata: None,
258            },
259            TrainingSample {
260                input: "3".to_owned(),
261                output: "c".to_owned(),
262                system: None,
263                metadata: None,
264            },
265            TrainingSample {
266                input: "4".to_owned(),
267                output: "d".to_owned(),
268                system: None,
269                metadata: None,
270            },
271        ];
272
273        let dataset = TrainingDataset { samples };
274        let (train, test) = dataset.train_test_split(0.75);
275
276        assert_eq!(train.len(), 3);
277        assert_eq!(test.len(), 1);
278    }
279}