hyprstream_inference/training/
data_loader.rs1use anyhow::{Context, Result};
4use serde::{Deserialize, Serialize};
5use std::path::Path;
6
7#[derive(Debug, Clone, Serialize, Deserialize)]
9pub struct TrainingSample {
10 pub input: String,
12 pub output: String,
14 pub system: Option<String>,
16 pub metadata: Option<serde_json::Value>,
18}
19
20#[derive(Clone)]
22pub struct TrainingDataset {
23 pub samples: Vec<TrainingSample>,
24}
25
26impl TrainingDataset {
27 pub fn from_jsonl(path: &Path) -> Result<Self> {
29 let content = std::fs::read_to_string(path)
30 .with_context(|| format!("Failed to read training data from {path:?}"))?;
31
32 let mut samples = Vec::new();
33 for (line_num, line) in content.lines().enumerate() {
34 if line.trim().is_empty() {
35 continue;
36 }
37
38 let sample: TrainingSample = serde_json::from_str(line)
39 .with_context(|| format!("Failed to parse line {} in {:?}", line_num + 1, path))?;
40
41 samples.push(sample);
42 }
43
44 Ok(Self { samples })
45 }
46
47 pub fn from_conversations(path: &Path) -> Result<Self> {
50 let content = std::fs::read_to_string(path)
51 .with_context(|| format!("Failed to read conversation data from {path:?}"))?;
52
53 let mut samples = Vec::new();
54
55 for line in content.lines() {
56 if line.trim().is_empty() {
57 continue;
58 }
59
60 let messages: Vec<serde_json::Value> = serde_json::from_str(line)?;
61
62 let mut current_input = String::new();
64
65 for message in messages {
66 let role = message["role"].as_str().unwrap_or("");
67 let content = message["content"].as_str().unwrap_or("");
68
69 match role {
70 "user" | "human" => {
71 if !current_input.is_empty() {
72 current_input.push('\n');
73 }
74 current_input.push_str(content);
75 }
76 "assistant" | "ai" => {
77 if !current_input.is_empty() {
78 samples.push(TrainingSample {
79 input: current_input.clone(),
80 output: content.to_owned(),
81 system: None,
82 metadata: None,
83 });
84 }
85 current_input.clear();
86 }
87 _ => {} }
89 }
90 }
91
92 Ok(Self { samples })
93 }
94
95 pub fn len(&self) -> usize {
97 self.samples.len()
98 }
99
100 pub fn is_empty(&self) -> bool {
102 self.samples.is_empty()
103 }
104
105 pub fn get(&self, index: usize) -> Option<&TrainingSample> {
107 self.samples.get(index)
108 }
109
110 pub fn iter(&self) -> impl Iterator<Item = &TrainingSample> {
112 self.samples.iter()
113 }
114
115 pub fn batches(&self, batch_size: usize) -> impl Iterator<Item = &[TrainingSample]> {
117 self.samples.chunks(batch_size)
118 }
119
120 pub fn train_test_split(&self, train_ratio: f64) -> (TrainingDataset, TrainingDataset) {
122 let split_point = ((self.samples.len() as f64) * train_ratio) as usize;
123
124 let train_samples = self.samples[..split_point].to_vec();
125 let test_samples = self.samples[split_point..].to_vec();
126
127 (
128 TrainingDataset {
129 samples: train_samples,
130 },
131 TrainingDataset {
132 samples: test_samples,
133 },
134 )
135 }
136}
137
138pub struct ChatTemplateDataLoader {
140 dataset: TrainingDataset,
141 template_engine: Option<crate::runtime::template_engine::TemplateEngine>,
142}
143
144impl ChatTemplateDataLoader {
145 pub fn new(dataset: TrainingDataset) -> Self {
147 Self {
148 dataset,
149 template_engine: None,
150 }
151 }
152
153 pub fn with_template(
155 mut self,
156 template_engine: crate::runtime::template_engine::TemplateEngine,
157 ) -> Self {
158 self.template_engine = Some(template_engine);
159 self
160 }
161
162 pub fn format_sample(&self, sample: &TrainingSample) -> Result<(String, String)> {
164 use crate::runtime::template_engine::ChatMessage;
165
166 if let Some(ref template) = self.template_engine {
167 let mut messages = Vec::new();
169
170 if let Some(ref system) = sample.system {
171 messages.push(ChatMessage {
172 role: "system".into(),
173 content: Some(system.clone()),
174 ..Default::default()
175 });
176 }
177
178 messages.push(ChatMessage {
179 role: "user".into(),
180 content: Some(sample.input.clone()),
181 ..Default::default()
182 });
183
184 let formatted_input = template.apply_chat_template(&messages, Some(true), None)?;
186
187 messages.push(ChatMessage {
189 role: "assistant".into(),
190 content: Some(sample.output.clone()),
191 ..Default::default()
192 });
193
194 let full_formatted = template.apply_chat_template(&messages, Some(false), None)?;
195
196 Ok((formatted_input, full_formatted))
197 } else {
198 let input = format!("Human: {}\n\nAssistant: ", sample.input);
200 let target = format!("{}{}", input, sample.output);
201 Ok((input, target))
202 }
203 }
204
205 pub fn formatted_batches(
207 &self,
208 batch_size: usize,
209 ) -> impl Iterator<Item = Result<Vec<(String, String)>>> + '_ {
210 self.dataset.batches(batch_size).map(|batch| {
211 batch
212 .iter()
213 .map(|sample| self.format_sample(sample))
214 .collect::<Result<Vec<_>>>()
215 })
216 }
217}
218
219#[cfg(test)]
220#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
221mod tests {
222 use super::*;
223 use tempfile::tempdir;
224
225 #[test]
226 fn test_jsonl_loading() {
227 let temp_dir = tempdir().expect("test: create temp dir");
228 let data_path = temp_dir.path().join("train.jsonl");
229
230 let content = r#"{"input": "What is 2+2?", "output": "4"}
231{"input": "Hello", "output": "Hi there!"}
232"#;
233 std::fs::write(&data_path, content).expect("test: write training data");
234
235 let dataset = TrainingDataset::from_jsonl(&data_path).expect("test: load dataset");
236 assert_eq!(dataset.len(), 2);
237 assert_eq!(
238 dataset.get(0).expect("test: get sample").input,
239 "What is 2+2?"
240 );
241 assert_eq!(dataset.get(0).expect("test: get sample").output, "4");
242 }
243
244 #[test]
245 fn test_train_test_split() {
246 let samples = vec![
247 TrainingSample {
248 input: "1".to_owned(),
249 output: "a".to_owned(),
250 system: None,
251 metadata: None,
252 },
253 TrainingSample {
254 input: "2".to_owned(),
255 output: "b".to_owned(),
256 system: None,
257 metadata: None,
258 },
259 TrainingSample {
260 input: "3".to_owned(),
261 output: "c".to_owned(),
262 system: None,
263 metadata: None,
264 },
265 TrainingSample {
266 input: "4".to_owned(),
267 output: "d".to_owned(),
268 system: None,
269 metadata: None,
270 },
271 ];
272
273 let dataset = TrainingDataset { samples };
274 let (train, test) = dataset.train_test_split(0.75);
275
276 assert_eq!(train.len(), 3);
277 assert_eq!(test.len(), 1);
278 }
279}