1#![allow(clippy::print_stdout, clippy::print_stderr)]
10#![allow(deprecated)]
14
15use anyhow::{bail, Result};
16use tracing::{info, warn};
17
18use crate::cli::commands::KVQuantArg;
19use crate::config::{
20 default_lora_rank, default_target_modules, HyprstreamTrainingConfig, TTTTrainingConfig,
21 TrainingMode,
22};
23use crate::runtime::model_config::ModelConfig;
24use crate::services::{InferenceServiceConfig, WorktreeClientExt, INFERENCE_ENDPOINT};
25use crate::storage::ModelRef;
26use hyprstream_rpc::{SigningKey, VerifyingKey};
27use hyprstream_rpc_std::inference_client::InferenceClient;
28use hyprstream_rpc_std::inference_client::{ChatMessage, ChatTemplateRequest};
29use hyprstream_rpc_std::registry_client::{
30 BranchRequest, CommitRequest, CreateWorktreeRequest, PushRequest, StageFilesRequest,
31};
32use hyprstream_rpc_std::registry_client::{RegistryClient, WorktreeClient};
33use hyprstream_service::ServiceSpawner;
34use std::path::PathBuf;
35
36#[allow(clippy::too_many_arguments)]
40pub async fn handle_training_init(
41 registry: &RegistryClient,
42 model_ref_str: &str,
43 branch_name: Option<String>,
44 adapter_name: Option<String>,
45 index: Option<u32>,
46 rank: u32,
47 alpha: u32,
48 mode: &str,
49 learning_rate: f32,
50) -> Result<()> {
51 let model_ref = ModelRef::parse(model_ref_str)?;
52 let tracked = registry
53 .get_by_name(&model_ref.model)
54 .await
55 .map_err(|e| anyhow::anyhow!("Model '{}' not found: {}", model_ref.model, e))?;
56 let repo_client = registry.repo(&tracked.id);
57
58 if let Some(new_branch) = branch_name {
60 info!(
61 "Creating isolated training environment: branch {} from {}",
62 new_branch,
63 model_ref.git_ref.display_name()
64 );
65
66 let from_ref = model_ref.git_ref.to_ref_string();
68 repo_client
69 .create_branch(&BranchRequest {
70 branch_name: new_branch.clone(),
71 start_point: from_ref.as_deref().unwrap_or("").to_owned(),
72 })
73 .await
74 .map_err(|e| anyhow::anyhow!("Failed to create branch: {}", e))?;
75 println!(
76 "✓ Created branch {} from {}",
77 new_branch,
78 model_ref.git_ref.display_name()
79 );
80
81 repo_client
83 .create_worktree(&CreateWorktreeRequest {
84 branch: new_branch.clone(),
85 })
86 .await
87 .map_err(|e| anyhow::anyhow!("Failed to create worktree: {}", e))?;
88 println!("✓ Created worktree for branch {new_branch}");
89
90 let storage_paths = crate::storage::StoragePaths::new()?;
92 let worktree_path = storage_paths.worktree_path(&model_ref.model, &new_branch)?;
93
94 let fs = repo_client.worktree(&new_branch);
96 init_adapter_at_path(
97 &worktree_path,
98 adapter_name.as_deref(),
99 index,
100 rank,
101 alpha,
102 learning_rate,
103 mode,
104 &format!("{}:{}", model_ref.model, new_branch),
105 Some(&fs),
106 )
107 .await?;
108
109 println!("\n✓ Isolated training environment ready!");
110 println!("\n→ Next steps:");
111 println!(
112 " hyprstream training infer {}:{} --prompt \"...\"",
113 model_ref.model, new_branch
114 );
115 println!(
116 " hyprstream training checkpoint {}:{}",
117 model_ref.model, new_branch
118 );
119
120 return Ok(());
121 }
122
123 let branch_name = match &model_ref.git_ref {
125 git2db::GitRef::Branch(name) => name.clone(),
126 git2db::GitRef::DefaultBranch => repo_client
127 .get_head()
128 .await
129 .map_err(|e| anyhow::anyhow!("Failed to get default branch: {}", e))?,
130 _ => {
131 bail!(
132 "Training requires a branch reference. Use model:branch format (e.g., {}:main)",
133 model_ref.model
134 );
135 }
136 };
137
138 let worktrees = repo_client
140 .list_worktrees()
141 .await
142 .map_err(|e| anyhow::anyhow!("Failed to list worktrees: {}", e))?;
143 if !worktrees.iter().any(|wt| wt.branch_name == branch_name) {
144 anyhow::bail!(
145 "Worktree '{}' does not exist for model '{}'. Create with:\n \
146 hyprstream training init {} --branch {}",
147 branch_name,
148 model_ref.model,
149 model_ref.model,
150 branch_name
151 );
152 }
153
154 let storage_paths = crate::storage::StoragePaths::new()?;
156 let model_path = storage_paths.worktree_path(&model_ref.model, &branch_name)?;
157
158 let fs = repo_client.worktree(&branch_name);
159 init_adapter_at_path(
160 &model_path,
161 adapter_name.as_deref(),
162 index,
163 rank,
164 alpha,
165 learning_rate,
166 mode,
167 &model_ref.to_string(),
168 Some(&fs),
169 )
170 .await?;
171
172 println!("\n✓ Training initialization complete!");
173 println!("\n→ Next steps:");
174 println!(" hyprstream training infer {model_ref} --prompt \"...\"");
175 println!(" hyprstream training checkpoint {model_ref}");
176
177 Ok(())
178}
179
180#[allow(clippy::too_many_arguments)]
184async fn init_adapter_at_path(
185 model_path: &std::path::Path,
186 adapter_name: Option<&str>,
187 index: Option<u32>,
188 rank: u32,
189 alpha: u32,
190 learning_rate: f32,
191 mode: &str,
192 model_ref_str: &str,
193 fs: Option<&WorktreeClient>,
194) -> Result<()> {
195 use crate::runtime::{create_engine, RuntimeEngine};
196 use crate::storage::AdapterManager;
197
198 let adapter_manager = if let Some(fs_ref) = fs {
199 AdapterManager::with_fs(fs_ref.clone())
200 } else {
201 AdapterManager::new(model_path)
202 };
203
204 adapter_manager.ensure_adapters_dir_async().await?;
205
206 let adapter_base_name = adapter_name.unwrap_or("default");
207 let indexed_adapter_name = adapter_manager
208 .create_indexed_name_async(adapter_base_name, index)
209 .await?;
210
211 println!("\n→ Initializing adapter: {indexed_adapter_name}");
212
213 let adapter_config = crate::storage::AdapterConfig {
215 rank,
216 alpha: alpha as f32,
217 learning_rate: learning_rate as f64,
218 batch_size: 4,
219 epochs: 10,
220 model_ref: model_ref_str.to_owned(),
221 training_data: None,
222 ..Default::default()
223 };
224
225 info!("Loading model to determine LoRA structure");
227 let config = crate::config::RuntimeConfig::default();
228 let mut engine = create_engine(&config)?;
229 RuntimeEngine::load_model(&mut engine, model_path).await?;
230
231 let lora_config = crate::training::TenantDeltaConfig {
233 rank: rank as usize,
234 alpha: alpha as f32,
235 dropout: 0.0,
236 target_modules: vec![
237 "q_proj".to_owned(),
238 "k_proj".to_owned(),
239 "v_proj".to_owned(),
240 "o_proj".to_owned(),
241 "gate_proj".to_owned(),
242 "up_proj".to_owned(),
243 "down_proj".to_owned(),
244 ],
245 learning_rate: learning_rate as f64,
246 ..crate::training::TenantDeltaConfig::default()
247 };
248
249 engine.create_lora(lora_config.clone())?;
250
251 let adapter_path = adapter_manager
253 .adapters_dir
254 .join(format!("{indexed_adapter_name}.safetensors"));
255 let device = engine.device();
256 let module_dims = engine.get_lora_module_dims()?;
257 let delta_config = lora_config.clone();
258 let num_layers = engine.get_num_layers().unwrap_or(32);
259 let delta = crate::training::TenantDelta::new(&delta_config, &module_dims, device, num_layers)?;
260 let bytes = delta.serialize_to_safetensors_bytes()?;
261 std::fs::write(&adapter_path, &bytes)?;
262
263 adapter_manager
265 .write_config_async(&indexed_adapter_name, &adapter_config)
266 .await?;
267
268 println!("✓ Created adapter: adapters/{indexed_adapter_name}.safetensors");
269
270 save_training_config(model_path, mode, &indexed_adapter_name, learning_rate)?;
272
273 Ok(())
274}
275
276fn save_training_config(
278 model_path: &std::path::Path,
279 mode_str: &str,
280 target_adapter: &str,
281 learning_rate: f32,
282) -> Result<()> {
283 let mode = match mode_str.to_lowercase().as_str() {
284 "ttt" | "test_time_training" | "test-time-training" => TrainingMode::TestTimeTraining,
285 "supervised" => TrainingMode::Supervised,
286 "disabled" | "off" | "none" => TrainingMode::Disabled,
287 _ => {
288 warn!("Unknown training mode '{}', defaulting to ttt", mode_str);
289 TrainingMode::TestTimeTraining
290 }
291 };
292
293 let training_config = HyprstreamTrainingConfig {
294 mode: mode.clone(),
295 target_adapter: Some(target_adapter.to_owned()),
296 learning_rate: learning_rate as f64,
297 batch_size: 4,
298 steps_per_cycle: 10,
299 min_quality_threshold: 0.3,
300 train_base_model: false,
301 lora_rank: default_lora_rank(),
302 lora_alpha: None,
303 target_modules: default_target_modules(),
304 ttt: TTTTrainingConfig {
305 learning_rate: learning_rate as f64,
306 gradient_steps: 3,
307 max_grad_norm: 1.0,
308 min_input_length: 32,
309 max_ttt_context: 512,
310 rank_oracle: None,
311 gradient_gating: None,
312 },
313 };
314
315 ModelConfig::save_training_config(model_path, &training_config)?;
316
317 match mode {
318 TrainingMode::Disabled => println!("✓ Training mode: disabled"),
319 TrainingMode::TestTimeTraining => {
320 println!("✓ Training mode: TTT (test-time-training)");
321 println!(
322 " → {} gradient steps, lr={}",
323 training_config.ttt.gradient_steps, training_config.ttt.learning_rate
324 );
325 }
326 TrainingMode::Supervised => println!("✓ Training mode: supervised"),
327 }
328
329 Ok(())
330}
331
332#[allow(clippy::too_many_arguments)]
336pub async fn handle_training_infer(
337 registry: &RegistryClient,
338 policy_transport: hyprstream_rpc::transport::TransportConfig,
339 model_ref_str: &str,
340 prompt: &str,
341 image_path: Option<String>,
342 max_tokens: Option<usize>,
343 temperature: Option<f32>,
344 top_p: Option<f32>,
345 top_k: Option<usize>,
346 repeat_penalty: Option<f32>,
347 sync: bool,
348 max_context: Option<usize>,
349 kv_quant: KVQuantArg,
350 signing_key: SigningKey,
351 verifying_key: VerifyingKey,
352) -> Result<()> {
353 use crate::runtime::RuntimeConfig;
354
355 info!(
356 "Training inference: model={}, prompt_len={}",
357 model_ref_str,
358 prompt.len()
359 );
360
361 let model_ref = ModelRef::parse(model_ref_str)?;
363 let tracked = registry
364 .get_by_name(&model_ref.model)
365 .await
366 .map_err(|e| anyhow::anyhow!("Model '{}' not found: {}", model_ref, e))?;
367 let repo_client = registry.repo(&tracked.id);
368 let branch_name = match &model_ref.git_ref {
369 git2db::GitRef::Branch(name) => name.clone(),
370 _ => repo_client
371 .get_head()
372 .await
373 .unwrap_or_else(|_| "main".to_owned()),
374 };
375 let worktrees = repo_client.list_worktrees().await?;
376 if !worktrees.iter().any(|wt| wt.branch_name == branch_name) {
377 bail!("worktree for {}:{} not found", model_ref.model, branch_name);
378 }
379
380 let storage_paths = crate::storage::StoragePaths::new()?;
382 let model_path = storage_paths.worktree_path(&model_ref.model, &branch_name)?;
383
384 if !model_path.exists() {
385 bail!(
386 "Model '{}' not found. Use 'hyprstream list' to see available models",
387 model_ref.model
388 );
389 }
390
391 ensure_ttt_enabled(&model_path)?;
393
394 info!("Using model at: {} (TTT enabled)", model_path.display());
395
396 let runtime_config = RuntimeConfig {
398 max_context: max_context.map(|v| v as u32),
399 kv_quant_type: kv_quant.into(),
400 ..Default::default()
401 };
402
403 let transport = hyprstream_rpc::transport::TransportConfig::from_endpoint(INFERENCE_ENDPOINT);
405 let service_config = InferenceServiceConfig::new(
406 &model_path,
407 runtime_config,
408 verifying_key,
409 signing_key.clone(),
410 transport,
411 policy_transport,
412 None, );
414 let spawner = ServiceSpawner::threaded();
415 let mut service_handle = spawner
416 .spawn(service_config)
417 .await
418 .map_err(|e| anyhow::anyhow!("Failed to spawn inference service: {}", e))?;
419
420 let client = InferenceClient::for_local_bootstrap(
421 signing_key.clone(),
422 signing_key.verifying_key(),
423 None,
424 )?;
425
426 let messages = vec![ChatMessage {
428 role: "user".into(),
429 content: prompt.into(),
430 tool_calls: vec![],
431 tool_call_id: String::new(),
432 }];
433
434 let formatted_prompt = match client
435 .apply_chat_template(&ChatTemplateRequest {
436 messages: messages.clone(),
437 add_generation_prompt: true,
438 tools_json: Some(String::new()),
439 max_tokens: None,
440 enable_thinking: None,
441 template_vars_json: None,
442 })
443 .await
444 {
445 Ok(formatted) => formatted,
446 Err(e) => {
447 warn!("Could not apply chat template: {}. Using raw prompt.", e);
448 prompt.to_owned()
449 }
450 };
451
452 let mut params = crate::config::SamplingParams::from_model_path(&model_path)
454 .await
455 .unwrap_or_default();
456
457 if let Some(t) = temperature {
458 params.temperature = Some(t);
459 }
460 if let Some(p) = top_p {
461 params.top_p = Some(p);
462 }
463 if let Some(k) = top_k {
464 params.top_k = Some(k);
465 }
466 if let Some(r) = repeat_penalty {
467 params.repeat_penalty = Some(r);
468 }
469 if let Some(m) = max_tokens {
470 params.max_tokens = Some(m);
471 }
472
473 let mut request = params.into_generation_request(formatted_prompt);
474
475 if let Some(img_path) = image_path {
477 info!("Using image: {}", img_path);
478 let img_bytes = std::fs::read(&img_path)
479 .map_err(|e| anyhow::anyhow!("Failed to read image {}: {}", img_path, e))?;
480 request.images = Some(vec![img_bytes]);
481 }
482
483 if sync {
485 let response = collect_inference_stream(&client, &request).await?;
487 println!("{}", response.text);
488 } else {
489 use hyprstream_rpc::streaming::StreamPayload;
491 use std::io::Write;
492 let mut handle = {
493 use hyprstream_rpc_std::inference_client::InferenceRpc;
494 InferenceRpc::generate_stream(&client, &request).await?
495 };
496 println!();
497 loop {
498 match handle.recv_next().await? {
499 Some(StreamPayload::Data(data)) => {
500 if let Ok(text) = String::from_utf8(data) {
501 print!("{text}");
502 std::io::stdout().flush()?;
503 }
504 }
505 Some(StreamPayload::Complete(_)) | None => break,
506 Some(StreamPayload::Error(msg)) => bail!("Generation error: {msg}"),
507 Some(StreamPayload::Tagged { .. }) => continue,
508 }
509 }
510 println!();
511 }
512
513 println!("\n[TTT applied - adapter weights modified]");
514 println!("Run 'hyprstream training checkpoint {model_ref}' to commit changes");
515
516 let _ = service_handle.stop().await;
518
519 Ok(())
520}
521
522fn ensure_ttt_enabled(model_path: &std::path::Path) -> Result<()> {
524 let mut config =
525 ModelConfig::load_training_config(model_path).unwrap_or_else(|| HyprstreamTrainingConfig {
526 mode: TrainingMode::TestTimeTraining,
527 ..Default::default()
528 });
529
530 if config.mode != TrainingMode::TestTimeTraining {
531 config.mode = TrainingMode::TestTimeTraining;
532 ModelConfig::save_training_config(model_path, &config)?;
533 info!("Enabled TTT mode in config");
534 }
535
536 Ok(())
537}
538
539async fn collect_inference_stream(
543 client: &InferenceClient,
544 request: &hyprstream_rpc_std::inference_client::GenerationRequest,
545) -> Result<crate::config::GenerationResult> {
546 use hyprstream_rpc::streaming::StreamPayload;
547
548 use hyprstream_rpc_std::inference_client::InferenceRpc;
549 let mut handle = InferenceRpc::generate_stream(client, request).await?;
550 let mut text = String::new();
551
552 loop {
553 match handle.recv_next().await? {
554 Some(payload) => match payload {
555 StreamPayload::Data(data) => {
556 text.push_str(&String::from_utf8_lossy(&data));
557 }
558 StreamPayload::Complete(meta) => {
559 let stats: crate::services::rpc_types::InferenceComplete =
560 serde_json::from_slice(&meta).unwrap_or_else(|e| {
561 tracing::warn!("Failed to parse InferenceComplete: {e}");
562 crate::services::rpc_types::InferenceComplete::empty()
563 });
564 return Ok(crate::config::GenerationResult {
565 text,
566 tokens_generated: stats.tokens_generated,
567 finish_reason: crate::config::FinishReason::Stop,
568 generation_time_ms: stats.generation_time_ms,
569 tokens_per_second: stats.tokens_per_second,
570 quality_metrics: None,
571 prefill_tokens: stats.prefill_tokens,
572 prefill_time_ms: stats.prefill_time_ms,
573 prefill_tokens_per_sec: stats.prefill_tokens_per_sec,
574 inference_tokens: stats.inference_tokens,
575 inference_time_ms: stats.inference_time_ms,
576 inference_tokens_per_sec: stats.inference_tokens_per_sec,
577 ttt_metrics: stats.ttt_metrics,
578 });
579 }
580 StreamPayload::Error(msg) => {
581 bail!("Generation error: {msg}");
582 }
583 StreamPayload::Tagged { .. } => {
584 continue;
586 }
587 },
588 None => {
589 bail!("Stream ended without completion");
590 }
591 }
592 }
593}
594
595#[allow(clippy::too_many_arguments)]
600pub async fn handle_training_batch(
601 registry: &RegistryClient,
602 policy_transport: hyprstream_rpc::transport::TransportConfig,
603 model_ref_str: &str,
604 input_files: Vec<String>,
605 input_dir: Option<PathBuf>,
606 pattern: &str,
607 format: &str,
608 max_tokens: usize,
609 chunk_size: usize,
610 skip: usize,
611 limit: Option<usize>,
612 progress_interval: usize,
613 checkpoint_interval: usize,
614 test_set: Option<String>,
615 max_context: Option<usize>,
616 kv_quant: KVQuantArg,
617 signing_key: SigningKey,
618 verifying_key: VerifyingKey,
619) -> Result<()> {
620 use crate::runtime::RuntimeConfig;
621 use crate::storage::AdapterManager;
622 use glob::glob;
623
624 info!("Batch training: model={}, format={}", model_ref_str, format);
625
626 let model_ref = ModelRef::parse(model_ref_str)?;
627 let tracked = registry
628 .get_by_name(&model_ref.model)
629 .await
630 .map_err(|e| anyhow::anyhow!("Model '{}' not found: {}", model_ref.model, e))?;
631 let repo_client = registry.repo(&tracked.id);
632 let branch_name_resolved = match &model_ref.git_ref {
633 git2db::GitRef::Branch(name) => name.clone(),
634 _ => repo_client
635 .get_head()
636 .await
637 .unwrap_or_else(|_| "main".to_owned()),
638 };
639 let worktrees = repo_client.list_worktrees().await?;
640 if !worktrees
641 .iter()
642 .any(|wt| wt.branch_name == branch_name_resolved)
643 {
644 bail!(
645 "Worktree '{}' not found for model {}.\n\
646 Run: hyprstream training init {} --branch {}",
647 branch_name_resolved,
648 model_ref.model,
649 model_ref.model,
650 branch_name_resolved,
651 );
652 }
653
654 let storage_paths = crate::storage::StoragePaths::new()?;
656 let model_path = storage_paths.worktree_path(&model_ref.model, &branch_name_resolved)?;
657
658 let mut files: Vec<PathBuf> = Vec::new();
660
661 for file_pattern in &input_files {
662 for path in (glob(file_pattern)?).flatten() {
663 if path.is_file() {
664 files.push(path);
665 }
666 }
667 }
668
669 if let Some(dir) = input_dir {
670 if !dir.exists() {
671 bail!("Input directory does not exist: {}", dir.display());
672 }
673 let full_pattern = dir.join(pattern);
674 for path in (glob(&full_pattern.to_string_lossy())?).flatten() {
675 if path.is_file() {
676 files.push(path);
677 }
678 }
679 }
680
681 if files.is_empty() {
682 bail!("No files found matching input patterns");
683 }
684
685 files.sort();
686 let files: Vec<_> = files
687 .into_iter()
688 .skip(skip)
689 .take(limit.unwrap_or(usize::MAX))
690 .collect();
691
692 let total_files = files.len();
693 println!("Found {total_files} files to process");
694
695 ensure_ttt_enabled(&model_path)?;
697
698 let test_files: Vec<PathBuf> = if let Some(ref pattern) = test_set {
700 glob(pattern)?
701 .filter_map(std::result::Result::ok)
702 .filter(|p| p.is_file())
703 .collect()
704 } else {
705 vec![]
706 };
707
708 if !test_files.is_empty() {
709 println!("Loaded {} test files for validation", test_files.len());
710 }
711
712 println!("Starting InferenceService for {model_ref_str}...");
714 let runtime_config = RuntimeConfig {
715 max_context: max_context.map(|v| v as u32),
716 kv_quant_type: kv_quant.into(),
717 ..Default::default()
718 };
719
720 let transport = hyprstream_rpc::transport::TransportConfig::from_endpoint(INFERENCE_ENDPOINT);
722 let service_config = InferenceServiceConfig::new(
723 &model_path,
724 runtime_config,
725 verifying_key,
726 signing_key.clone(),
727 transport,
728 policy_transport,
729 None, );
731 let spawner = ServiceSpawner::threaded();
732 let mut service_handle = spawner
733 .spawn(service_config)
734 .await
735 .map_err(|e| anyhow::anyhow!("Failed to spawn inference service: {}", e))?;
736
737 let client = InferenceClient::for_local_bootstrap(
738 signing_key.clone(),
739 signing_key.verifying_key(),
740 None,
741 )?;
742
743 let adapter_manager = AdapterManager::new(&model_path);
745 let adapters = adapter_manager.list_adapters().unwrap_or_default();
746
747 println!("\nStarting batch training...\n");
748
749 let mut processed = 0;
750 let mut total_input_tokens = 0usize;
751 let mut total_output_tokens = 0usize;
752 let start_time = std::time::Instant::now();
753
754 for file_path in &files {
755 let content = std::fs::read_to_string(file_path)?;
757 let chunk: String = content.chars().take(chunk_size).collect();
758
759 if chunk.is_empty() {
760 continue;
761 }
762
763 let prompt = match format {
765 "jsonl" => {
766 if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(&chunk) {
768 parsed
769 .get("prompt")
770 .or_else(|| parsed.get("text"))
771 .or_else(|| parsed.get("content"))
772 .and_then(|v| v.as_str())
773 .unwrap_or(&chunk)
774 .to_owned()
775 } else {
776 chunk.clone()
777 }
778 }
779 _ => chunk.clone(), };
781
782 let request = hyprstream_rpc_std::inference_client::GenerationRequest {
783 prompt,
784 max_tokens: Some(max_tokens as u32),
785 temperature: Some(0.7),
786 ..Default::default()
787 };
788
789 match collect_inference_stream(&client, &request).await {
791 Ok(response) => {
792 total_input_tokens += response.prefill_tokens;
794 total_output_tokens += response.tokens_generated;
795 }
796 Err(e) => {
797 warn!("Generation failed for {}: {}", file_path.display(), e);
798 }
799 }
800
801 processed += 1;
802
803 if processed % progress_interval == 0 {
805 let elapsed = start_time.elapsed().as_secs_f32();
806 let rate = processed as f32 / elapsed;
807 let tokens_per_sec = (total_input_tokens + total_output_tokens) as f32 / elapsed;
808 println!(
809 "[{processed}/{total_files}] {rate:.1} files/sec, {total_input_tokens} input tokens, {total_output_tokens} output tokens ({tokens_per_sec:.0} tok/s)"
810 );
811 }
812
813 if processed % checkpoint_interval == 0 {
815 println!("\n→ Checkpoint at {processed} files...");
816 if let Some(adapter) = adapters.first() {
818 println!(" Adapter: {}", adapter.name);
819 }
820 println!(" ✓ Weights updated (will be saved on service stop)");
821 }
822 }
823
824 println!("\nStopping InferenceService and saving weights...");
826 let _ = service_handle.stop().await;
827
828 let elapsed = start_time.elapsed();
829 let total_tokens = total_input_tokens + total_output_tokens;
830 let tokens_per_sec = total_tokens as f32 / elapsed.as_secs_f32();
831 println!("\n✓ Batch training complete!");
832 println!(" Processed: {processed} files");
833 println!(" Input tokens: {total_input_tokens}");
834 println!(" Output tokens: {total_output_tokens}");
835 println!(" Total tokens: {total_tokens} ({tokens_per_sec:.0} tok/s)");
836 println!(" Time: {:.1}s", elapsed.as_secs_f32());
837 println!("\n→ Run 'hyprstream training checkpoint {model_ref}' to commit");
838
839 Ok(())
840}
841
842pub async fn handle_training_checkpoint(
846 registry: &RegistryClient,
847 model_ref_str: &str,
848 message: Option<String>,
849 push: bool,
850 remote: &str,
851) -> Result<()> {
852 use crate::storage::AdapterManager;
853
854 let model_ref = ModelRef::parse(model_ref_str)?;
855 let tracked = registry
856 .get_by_name(&model_ref.model)
857 .await
858 .map_err(|e| anyhow::anyhow!("Model '{}' not found: {}", model_ref, e))?;
859 let repo_client = registry.repo(&tracked.id);
860
861 let branch_for_fs = match &model_ref.git_ref {
862 git2db::GitRef::Branch(name) => name.clone(),
863 _ => repo_client
864 .get_head()
865 .await
866 .unwrap_or_else(|_| "main".to_owned()),
867 };
868 let fs = repo_client.worktree(&branch_for_fs);
869
870 let adapter_manager = AdapterManager::with_fs(fs.clone());
871 let adapters = adapter_manager.list_adapters_async().await?;
872
873 if adapters.is_empty() {
874 bail!("No adapters found for model {}", model_ref);
875 }
876
877 let status = repo_client
878 .status()
879 .await
880 .map_err(|e| anyhow::anyhow!("Failed to get repository status: {}", e))?;
881
882 let dirty_adapters: Vec<_> = status
884 .modified_files
885 .into_iter()
886 .filter(|p| p.contains("adapters/") && p.ends_with(".safetensors"))
887 .collect();
888
889 if dirty_adapters.is_empty() {
890 println!("No dirty adapter files to commit.");
891 return Ok(());
892 }
893
894 println!("Found {} modified adapter files:", dirty_adapters.len());
895 for path in &dirty_adapters {
896 println!(" {}", path);
897 }
898
899 let mut files_to_stage: Vec<String> = Vec::new();
901 for adapter in &adapters {
902 let rel_path = adapter.path.to_string_lossy().to_string();
904 files_to_stage.push(rel_path);
905
906 let base_name = adapter.filename.trim_end_matches(".safetensors");
908 let rel_config = format!("adapters/{base_name}.config.json");
909 if fs
910 .stat_path(&rel_config)
911 .await
912 .map(|s| s.exists)
913 .unwrap_or(false)
914 {
915 files_to_stage.push(rel_config);
916 }
917 }
918
919 let wt = repo_client.worktree(&branch_for_fs);
921 wt.stage_files(&StageFilesRequest {
922 files: files_to_stage.clone(),
923 })
924 .await
925 .map_err(|e| anyhow::anyhow!("Failed to stage files: {}", e))?;
926
927 let commit_message = message.unwrap_or_else(|| {
929 format!(
930 "Training checkpoint: {} adapters updated\n\n\
931 🤖 Generated with hyprstream training",
932 adapters.len()
933 )
934 });
935
936 let commit_id = wt
937 .commit(&CommitRequest {
938 message: commit_message,
939 author: String::new(),
940 email: String::new(),
941 })
942 .await
943 .map_err(|e| anyhow::anyhow!("Failed to create commit: {}", e))?;
944
945 println!(
946 "\n✓ Created commit: {}",
947 &commit_id[..8.min(commit_id.len())]
948 );
949
950 if push {
952 println!("Pushing to {remote}...");
953
954 let refspec = format!(
955 "refs/heads/{}:refs/heads/{}",
956 model_ref.git_ref, model_ref.git_ref
957 );
958 repo_client
959 .push(&PushRequest {
960 remote: remote.to_owned(),
961 refspec,
962 force: false,
963 })
964 .await
965 .map_err(|e| anyhow::anyhow!("Failed to push: {}", e))?;
966 println!("✓ Pushed to {remote}");
967 }
968
969 Ok(())
970}