Skip to main content

hyprstream_core/cli/
training_handlers.rs

1//! Training command handlers
2//!
3//! Handlers for the `training` subcommand:
4//! - `training init` - Initialize adapter for training
5//! - `training infer` - Inference with TTT (dirty writes)
6//! - `training batch` - Batch training with checkpoints
7//! - `training checkpoint` - Commit dirty adapter changes
8// CLI handlers intentionally print to stdout/stderr for user interaction
9#![allow(clippy::print_stdout, clippy::print_stderr)]
10// TODO(#395): migrate `model_ref.model` reads to `model_ref.name()` as part of the
11// name-layer retirement. Suppressed here so this prep PR stays focused on the CID
12// encoder + grammar; the deprecation is enforced at the ModelRef type definition.
13#![allow(deprecated)]
14
15use anyhow::{bail, Result};
16use tracing::{info, warn};
17
18use crate::cli::commands::KVQuantArg;
19use crate::config::{
20    default_lora_rank, default_target_modules, HyprstreamTrainingConfig, TTTTrainingConfig,
21    TrainingMode,
22};
23use crate::runtime::model_config::ModelConfig;
24use crate::services::{InferenceServiceConfig, WorktreeClientExt, INFERENCE_ENDPOINT};
25use crate::storage::ModelRef;
26use hyprstream_rpc::{SigningKey, VerifyingKey};
27use hyprstream_rpc_std::inference_client::InferenceClient;
28use hyprstream_rpc_std::inference_client::{ChatMessage, ChatTemplateRequest};
29use hyprstream_rpc_std::registry_client::{
30    BranchRequest, CommitRequest, CreateWorktreeRequest, PushRequest, StageFilesRequest,
31};
32use hyprstream_rpc_std::registry_client::{RegistryClient, WorktreeClient};
33use hyprstream_service::ServiceSpawner;
34use std::path::PathBuf;
35
36/// Handle `training init` command
37///
38/// Initializes a LoRA adapter for training. Optionally creates a new branch/worktree.
39#[allow(clippy::too_many_arguments)]
40pub async fn handle_training_init(
41    registry: &RegistryClient,
42    model_ref_str: &str,
43    branch_name: Option<String>,
44    adapter_name: Option<String>,
45    index: Option<u32>,
46    rank: u32,
47    alpha: u32,
48    mode: &str,
49    learning_rate: f32,
50) -> Result<()> {
51    let model_ref = ModelRef::parse(model_ref_str)?;
52    let tracked = registry
53        .get_by_name(&model_ref.model)
54        .await
55        .map_err(|e| anyhow::anyhow!("Model '{}' not found: {}", model_ref.model, e))?;
56    let repo_client = registry.repo(&tracked.id);
57
58    // If branch specified, create isolated training environment
59    if let Some(new_branch) = branch_name {
60        info!(
61            "Creating isolated training environment: branch {} from {}",
62            new_branch,
63            model_ref.git_ref.display_name()
64        );
65
66        // Create branch from model_ref's git_ref
67        let from_ref = model_ref.git_ref.to_ref_string();
68        repo_client
69            .create_branch(&BranchRequest {
70                branch_name: new_branch.clone(),
71                start_point: from_ref.as_deref().unwrap_or("").to_owned(),
72            })
73            .await
74            .map_err(|e| anyhow::anyhow!("Failed to create branch: {}", e))?;
75        println!(
76            "✓ Created branch {} from {}",
77            new_branch,
78            model_ref.git_ref.display_name()
79        );
80
81        // Create worktree for new branch
82        repo_client
83            .create_worktree(&CreateWorktreeRequest {
84                branch: new_branch.clone(),
85            })
86            .await
87            .map_err(|e| anyhow::anyhow!("Failed to create worktree: {}", e))?;
88        println!("✓ Created worktree for branch {new_branch}");
89
90        // Derive worktree path locally for adapter initialization
91        let storage_paths = crate::storage::StoragePaths::new()?;
92        let worktree_path = storage_paths.worktree_path(&model_ref.model, &new_branch)?;
93
94        // Initialize adapter in worktree (use FsOps from repo_client)
95        let fs = repo_client.worktree(&new_branch);
96        init_adapter_at_path(
97            &worktree_path,
98            adapter_name.as_deref(),
99            index,
100            rank,
101            alpha,
102            learning_rate,
103            mode,
104            &format!("{}:{}", model_ref.model, new_branch),
105            Some(&fs),
106        )
107        .await?;
108
109        println!("\n✓ Isolated training environment ready!");
110        println!("\n→ Next steps:");
111        println!(
112            "  hyprstream training infer {}:{} --prompt \"...\"",
113            model_ref.model, new_branch
114        );
115        println!(
116            "  hyprstream training checkpoint {}:{}",
117            model_ref.model, new_branch
118        );
119
120        return Ok(());
121    }
122
123    // Train on existing worktree
124    let branch_name = match &model_ref.git_ref {
125        git2db::GitRef::Branch(name) => name.clone(),
126        git2db::GitRef::DefaultBranch => repo_client
127            .get_head()
128            .await
129            .map_err(|e| anyhow::anyhow!("Failed to get default branch: {}", e))?,
130        _ => {
131            bail!(
132                "Training requires a branch reference. Use model:branch format (e.g., {}:main)",
133                model_ref.model
134            );
135        }
136    };
137
138    // Verify worktree exists
139    let worktrees = repo_client
140        .list_worktrees()
141        .await
142        .map_err(|e| anyhow::anyhow!("Failed to list worktrees: {}", e))?;
143    if !worktrees.iter().any(|wt| wt.branch_name == branch_name) {
144        anyhow::bail!(
145            "Worktree '{}' does not exist for model '{}'. Create with:\n  \
146             hyprstream training init {} --branch {}",
147            branch_name,
148            model_ref.model,
149            model_ref.model,
150            branch_name
151        );
152    }
153
154    // Derive worktree path locally
155    let storage_paths = crate::storage::StoragePaths::new()?;
156    let model_path = storage_paths.worktree_path(&model_ref.model, &branch_name)?;
157
158    let fs = repo_client.worktree(&branch_name);
159    init_adapter_at_path(
160        &model_path,
161        adapter_name.as_deref(),
162        index,
163        rank,
164        alpha,
165        learning_rate,
166        mode,
167        &model_ref.to_string(),
168        Some(&fs),
169    )
170    .await?;
171
172    println!("\n✓ Training initialization complete!");
173    println!("\n→ Next steps:");
174    println!("  hyprstream training infer {model_ref} --prompt \"...\"");
175    println!("  hyprstream training checkpoint {model_ref}");
176
177    Ok(())
178}
179
180/// Initialize adapter at a specific path
181///
182/// When `fs` is provided, uses worktree-scoped FsOps for config writes.
183#[allow(clippy::too_many_arguments)]
184async fn init_adapter_at_path(
185    model_path: &std::path::Path,
186    adapter_name: Option<&str>,
187    index: Option<u32>,
188    rank: u32,
189    alpha: u32,
190    learning_rate: f32,
191    mode: &str,
192    model_ref_str: &str,
193    fs: Option<&WorktreeClient>,
194) -> Result<()> {
195    use crate::runtime::{create_engine, RuntimeEngine};
196    use crate::storage::AdapterManager;
197
198    let adapter_manager = if let Some(fs_ref) = fs {
199        AdapterManager::with_fs(fs_ref.clone())
200    } else {
201        AdapterManager::new(model_path)
202    };
203
204    adapter_manager.ensure_adapters_dir_async().await?;
205
206    let adapter_base_name = adapter_name.unwrap_or("default");
207    let indexed_adapter_name = adapter_manager
208        .create_indexed_name_async(adapter_base_name, index)
209        .await?;
210
211    println!("\n→ Initializing adapter: {indexed_adapter_name}");
212
213    // Create adapter configuration
214    let adapter_config = crate::storage::AdapterConfig {
215        rank,
216        alpha: alpha as f32,
217        learning_rate: learning_rate as f64,
218        batch_size: 4,
219        epochs: 10,
220        model_ref: model_ref_str.to_owned(),
221        training_data: None,
222        ..Default::default()
223    };
224
225    // Load model to get proper dimensions
226    info!("Loading model to determine LoRA structure");
227    let config = crate::config::RuntimeConfig::default();
228    let mut engine = create_engine(&config)?;
229    RuntimeEngine::load_model(&mut engine, model_path).await?;
230
231    // Create LoRA configuration
232    let lora_config = crate::training::TenantDeltaConfig {
233        rank: rank as usize,
234        alpha: alpha as f32,
235        dropout: 0.0,
236        target_modules: vec![
237            "q_proj".to_owned(),
238            "k_proj".to_owned(),
239            "v_proj".to_owned(),
240            "o_proj".to_owned(),
241            "gate_proj".to_owned(),
242            "up_proj".to_owned(),
243            "down_proj".to_owned(),
244        ],
245        learning_rate: learning_rate as f64,
246        ..crate::training::TenantDeltaConfig::default()
247    };
248
249    engine.create_lora(lora_config.clone())?;
250
251    // Create initial adapter weights via TenantDelta and save as safetensors
252    let adapter_path = adapter_manager
253        .adapters_dir
254        .join(format!("{indexed_adapter_name}.safetensors"));
255    let device = engine.device();
256    let module_dims = engine.get_lora_module_dims()?;
257    let delta_config = lora_config.clone();
258    let num_layers = engine.get_num_layers().unwrap_or(32);
259    let delta = crate::training::TenantDelta::new(&delta_config, &module_dims, device, num_layers)?;
260    let bytes = delta.serialize_to_safetensors_bytes()?;
261    std::fs::write(&adapter_path, &bytes)?;
262
263    // Save adapter config via FsOps or direct
264    adapter_manager
265        .write_config_async(&indexed_adapter_name, &adapter_config)
266        .await?;
267
268    println!("✓ Created adapter: adapters/{indexed_adapter_name}.safetensors");
269
270    // Save training mode config
271    save_training_config(model_path, mode, &indexed_adapter_name, learning_rate)?;
272
273    Ok(())
274}
275
276/// Save training configuration to model's config.json
277fn save_training_config(
278    model_path: &std::path::Path,
279    mode_str: &str,
280    target_adapter: &str,
281    learning_rate: f32,
282) -> Result<()> {
283    let mode = match mode_str.to_lowercase().as_str() {
284        "ttt" | "test_time_training" | "test-time-training" => TrainingMode::TestTimeTraining,
285        "supervised" => TrainingMode::Supervised,
286        "disabled" | "off" | "none" => TrainingMode::Disabled,
287        _ => {
288            warn!("Unknown training mode '{}', defaulting to ttt", mode_str);
289            TrainingMode::TestTimeTraining
290        }
291    };
292
293    let training_config = HyprstreamTrainingConfig {
294        mode: mode.clone(),
295        target_adapter: Some(target_adapter.to_owned()),
296        learning_rate: learning_rate as f64,
297        batch_size: 4,
298        steps_per_cycle: 10,
299        min_quality_threshold: 0.3,
300        train_base_model: false,
301        lora_rank: default_lora_rank(),
302        lora_alpha: None,
303        target_modules: default_target_modules(),
304        ttt: TTTTrainingConfig {
305            learning_rate: learning_rate as f64,
306            gradient_steps: 3,
307            max_grad_norm: 1.0,
308            min_input_length: 32,
309            max_ttt_context: 512,
310            rank_oracle: None,
311            gradient_gating: None,
312        },
313    };
314
315    ModelConfig::save_training_config(model_path, &training_config)?;
316
317    match mode {
318        TrainingMode::Disabled => println!("✓ Training mode: disabled"),
319        TrainingMode::TestTimeTraining => {
320            println!("✓ Training mode: TTT (test-time-training)");
321            println!(
322                "  → {} gradient steps, lr={}",
323                training_config.ttt.gradient_steps, training_config.ttt.learning_rate
324            );
325        }
326        TrainingMode::Supervised => println!("✓ Training mode: supervised"),
327    }
328
329    Ok(())
330}
331
332/// Handle `training infer` command
333///
334/// Runs inference with TTT enabled, making dirty writes to adapter weights.
335#[allow(clippy::too_many_arguments)]
336pub async fn handle_training_infer(
337    registry: &RegistryClient,
338    policy_transport: hyprstream_rpc::transport::TransportConfig,
339    model_ref_str: &str,
340    prompt: &str,
341    image_path: Option<String>,
342    max_tokens: Option<usize>,
343    temperature: Option<f32>,
344    top_p: Option<f32>,
345    top_k: Option<usize>,
346    repeat_penalty: Option<f32>,
347    sync: bool,
348    max_context: Option<usize>,
349    kv_quant: KVQuantArg,
350    signing_key: SigningKey,
351    verifying_key: VerifyingKey,
352) -> Result<()> {
353    use crate::runtime::RuntimeConfig;
354
355    info!(
356        "Training inference: model={}, prompt_len={}",
357        model_ref_str,
358        prompt.len()
359    );
360
361    // Parse model reference and get path
362    let model_ref = ModelRef::parse(model_ref_str)?;
363    let tracked = registry
364        .get_by_name(&model_ref.model)
365        .await
366        .map_err(|e| anyhow::anyhow!("Model '{}' not found: {}", model_ref, e))?;
367    let repo_client = registry.repo(&tracked.id);
368    let branch_name = match &model_ref.git_ref {
369        git2db::GitRef::Branch(name) => name.clone(),
370        _ => repo_client
371            .get_head()
372            .await
373            .unwrap_or_else(|_| "main".to_owned()),
374    };
375    let worktrees = repo_client.list_worktrees().await?;
376    if !worktrees.iter().any(|wt| wt.branch_name == branch_name) {
377        bail!("worktree for {}:{} not found", model_ref.model, branch_name);
378    }
379
380    // Derive worktree path locally
381    let storage_paths = crate::storage::StoragePaths::new()?;
382    let model_path = storage_paths.worktree_path(&model_ref.model, &branch_name)?;
383
384    if !model_path.exists() {
385        bail!(
386            "Model '{}' not found. Use 'hyprstream list' to see available models",
387            model_ref.model
388        );
389    }
390
391    // Ensure TTT is enabled in config
392    ensure_ttt_enabled(&model_path)?;
393
394    info!("Using model at: {} (TTT enabled)", model_path.display());
395
396    // Configure runtime
397    let runtime_config = RuntimeConfig {
398        max_context: max_context.map(|v| v as u32),
399        kv_quant_type: kv_quant.into(),
400        ..Default::default()
401    };
402
403    // Start InferenceService with TTT enabled
404    let transport = hyprstream_rpc::transport::TransportConfig::from_endpoint(INFERENCE_ENDPOINT);
405    let service_config = InferenceServiceConfig::new(
406        &model_path,
407        runtime_config,
408        verifying_key,
409        signing_key.clone(),
410        transport,
411        policy_transport,
412        None, // CLI: no FsOps
413    );
414    let spawner = ServiceSpawner::threaded();
415    let mut service_handle = spawner
416        .spawn(service_config)
417        .await
418        .map_err(|e| anyhow::anyhow!("Failed to spawn inference service: {}", e))?;
419
420    let client = InferenceClient::for_local_bootstrap(
421        signing_key.clone(),
422        signing_key.verifying_key(),
423        None,
424    )?;
425
426    // Apply chat template
427    let messages = vec![ChatMessage {
428        role: "user".into(),
429        content: prompt.into(),
430        tool_calls: vec![],
431        tool_call_id: String::new(),
432    }];
433
434    let formatted_prompt = match client
435        .apply_chat_template(&ChatTemplateRequest {
436            messages: messages.clone(),
437            add_generation_prompt: true,
438            tools_json: Some(String::new()),
439            max_tokens: None,
440            enable_thinking: None,
441            template_vars_json: None,
442        })
443        .await
444    {
445        Ok(formatted) => formatted,
446        Err(e) => {
447            warn!("Could not apply chat template: {}. Using raw prompt.", e);
448            prompt.to_owned()
449        }
450    };
451
452    // Build generation request
453    let mut params = crate::config::SamplingParams::from_model_path(&model_path)
454        .await
455        .unwrap_or_default();
456
457    if let Some(t) = temperature {
458        params.temperature = Some(t);
459    }
460    if let Some(p) = top_p {
461        params.top_p = Some(p);
462    }
463    if let Some(k) = top_k {
464        params.top_k = Some(k);
465    }
466    if let Some(r) = repeat_penalty {
467        params.repeat_penalty = Some(r);
468    }
469    if let Some(m) = max_tokens {
470        params.max_tokens = Some(m);
471    }
472
473    let mut request = params.into_generation_request(formatted_prompt);
474
475    // Add image path if provided (for multimodal models)
476    if let Some(img_path) = image_path {
477        info!("Using image: {}", img_path);
478        let img_bytes = std::fs::read(&img_path)
479            .map_err(|e| anyhow::anyhow!("Failed to read image {}: {}", img_path, e))?;
480        request.images = Some(vec![img_bytes]);
481    }
482
483    // Generate with TTT
484    if sync {
485        // Collect full response before printing
486        let response = collect_inference_stream(&client, &request).await?;
487        println!("{}", response.text);
488    } else {
489        // Stream tokens live to stdout
490        use hyprstream_rpc::streaming::StreamPayload;
491        use std::io::Write;
492        let mut handle = {
493            use hyprstream_rpc_std::inference_client::InferenceRpc;
494            InferenceRpc::generate_stream(&client, &request).await?
495        };
496        println!();
497        loop {
498            match handle.recv_next().await? {
499                Some(StreamPayload::Data(data)) => {
500                    if let Ok(text) = String::from_utf8(data) {
501                        print!("{text}");
502                        std::io::stdout().flush()?;
503                    }
504                }
505                Some(StreamPayload::Complete(_)) | None => break,
506                Some(StreamPayload::Error(msg)) => bail!("Generation error: {msg}"),
507                Some(StreamPayload::Tagged { .. }) => continue,
508            }
509        }
510        println!();
511    }
512
513    println!("\n[TTT applied - adapter weights modified]");
514    println!("Run 'hyprstream training checkpoint {model_ref}' to commit changes");
515
516    // Stop service to properly release resources
517    let _ = service_handle.stop().await;
518
519    Ok(())
520}
521
522/// Ensure TTT is enabled in model config
523fn ensure_ttt_enabled(model_path: &std::path::Path) -> Result<()> {
524    let mut config =
525        ModelConfig::load_training_config(model_path).unwrap_or_else(|| HyprstreamTrainingConfig {
526            mode: TrainingMode::TestTimeTraining,
527            ..Default::default()
528        });
529
530    if config.mode != TrainingMode::TestTimeTraining {
531        config.mode = TrainingMode::TestTimeTraining;
532        ModelConfig::save_training_config(model_path, &config)?;
533        info!("Enabled TTT mode in config");
534    }
535
536    Ok(())
537}
538
539/// Collect an inference stream into a GenerationResult.
540///
541/// Replaces the removed `InferenceZmqClient::generate()` sync method.
542async fn collect_inference_stream(
543    client: &InferenceClient,
544    request: &hyprstream_rpc_std::inference_client::GenerationRequest,
545) -> Result<crate::config::GenerationResult> {
546    use hyprstream_rpc::streaming::StreamPayload;
547
548    use hyprstream_rpc_std::inference_client::InferenceRpc;
549    let mut handle = InferenceRpc::generate_stream(client, request).await?;
550    let mut text = String::new();
551
552    loop {
553        match handle.recv_next().await? {
554            Some(payload) => match payload {
555                StreamPayload::Data(data) => {
556                    text.push_str(&String::from_utf8_lossy(&data));
557                }
558                StreamPayload::Complete(meta) => {
559                    let stats: crate::services::rpc_types::InferenceComplete =
560                        serde_json::from_slice(&meta).unwrap_or_else(|e| {
561                            tracing::warn!("Failed to parse InferenceComplete: {e}");
562                            crate::services::rpc_types::InferenceComplete::empty()
563                        });
564                    return Ok(crate::config::GenerationResult {
565                        text,
566                        tokens_generated: stats.tokens_generated,
567                        finish_reason: crate::config::FinishReason::Stop,
568                        generation_time_ms: stats.generation_time_ms,
569                        tokens_per_second: stats.tokens_per_second,
570                        quality_metrics: None,
571                        prefill_tokens: stats.prefill_tokens,
572                        prefill_time_ms: stats.prefill_time_ms,
573                        prefill_tokens_per_sec: stats.prefill_tokens_per_sec,
574                        inference_tokens: stats.inference_tokens,
575                        inference_time_ms: stats.inference_time_ms,
576                        inference_tokens_per_sec: stats.inference_tokens_per_sec,
577                        ttt_metrics: stats.ttt_metrics,
578                    });
579                }
580                StreamPayload::Error(msg) => {
581                    bail!("Generation error: {msg}");
582                }
583                StreamPayload::Tagged { .. } => {
584                    // encrypted event payload, skip
585                    continue;
586                }
587            },
588            None => {
589                bail!("Stream ended without completion");
590            }
591        }
592    }
593}
594
595/// Handle `training batch` command
596///
597/// Batch training: starts InferenceService once and processes all files with TTT.
598/// Each file's content is sent as a generation request, triggering TTT adaptation.
599#[allow(clippy::too_many_arguments)]
600pub async fn handle_training_batch(
601    registry: &RegistryClient,
602    policy_transport: hyprstream_rpc::transport::TransportConfig,
603    model_ref_str: &str,
604    input_files: Vec<String>,
605    input_dir: Option<PathBuf>,
606    pattern: &str,
607    format: &str,
608    max_tokens: usize,
609    chunk_size: usize,
610    skip: usize,
611    limit: Option<usize>,
612    progress_interval: usize,
613    checkpoint_interval: usize,
614    test_set: Option<String>,
615    max_context: Option<usize>,
616    kv_quant: KVQuantArg,
617    signing_key: SigningKey,
618    verifying_key: VerifyingKey,
619) -> Result<()> {
620    use crate::runtime::RuntimeConfig;
621    use crate::storage::AdapterManager;
622    use glob::glob;
623
624    info!("Batch training: model={}, format={}", model_ref_str, format);
625
626    let model_ref = ModelRef::parse(model_ref_str)?;
627    let tracked = registry
628        .get_by_name(&model_ref.model)
629        .await
630        .map_err(|e| anyhow::anyhow!("Model '{}' not found: {}", model_ref.model, e))?;
631    let repo_client = registry.repo(&tracked.id);
632    let branch_name_resolved = match &model_ref.git_ref {
633        git2db::GitRef::Branch(name) => name.clone(),
634        _ => repo_client
635            .get_head()
636            .await
637            .unwrap_or_else(|_| "main".to_owned()),
638    };
639    let worktrees = repo_client.list_worktrees().await?;
640    if !worktrees
641        .iter()
642        .any(|wt| wt.branch_name == branch_name_resolved)
643    {
644        bail!(
645            "Worktree '{}' not found for model {}.\n\
646             Run: hyprstream training init {} --branch {}",
647            branch_name_resolved,
648            model_ref.model,
649            model_ref.model,
650            branch_name_resolved,
651        );
652    }
653
654    // Derive worktree path locally
655    let storage_paths = crate::storage::StoragePaths::new()?;
656    let model_path = storage_paths.worktree_path(&model_ref.model, &branch_name_resolved)?;
657
658    // Collect input files
659    let mut files: Vec<PathBuf> = Vec::new();
660
661    for file_pattern in &input_files {
662        for path in (glob(file_pattern)?).flatten() {
663            if path.is_file() {
664                files.push(path);
665            }
666        }
667    }
668
669    if let Some(dir) = input_dir {
670        if !dir.exists() {
671            bail!("Input directory does not exist: {}", dir.display());
672        }
673        let full_pattern = dir.join(pattern);
674        for path in (glob(&full_pattern.to_string_lossy())?).flatten() {
675            if path.is_file() {
676                files.push(path);
677            }
678        }
679    }
680
681    if files.is_empty() {
682        bail!("No files found matching input patterns");
683    }
684
685    files.sort();
686    let files: Vec<_> = files
687        .into_iter()
688        .skip(skip)
689        .take(limit.unwrap_or(usize::MAX))
690        .collect();
691
692    let total_files = files.len();
693    println!("Found {total_files} files to process");
694
695    // Ensure TTT is enabled in model config
696    ensure_ttt_enabled(&model_path)?;
697
698    // Load test set (for future validation)
699    let test_files: Vec<PathBuf> = if let Some(ref pattern) = test_set {
700        glob(pattern)?
701            .filter_map(std::result::Result::ok)
702            .filter(|p| p.is_file())
703            .collect()
704    } else {
705        vec![]
706    };
707
708    if !test_files.is_empty() {
709        println!("Loaded {} test files for validation", test_files.len());
710    }
711
712    // Start InferenceService with TTT enabled
713    println!("Starting InferenceService for {model_ref_str}...");
714    let runtime_config = RuntimeConfig {
715        max_context: max_context.map(|v| v as u32),
716        kv_quant_type: kv_quant.into(),
717        ..Default::default()
718    };
719
720    // Start InferenceService with TTT enabled
721    let transport = hyprstream_rpc::transport::TransportConfig::from_endpoint(INFERENCE_ENDPOINT);
722    let service_config = InferenceServiceConfig::new(
723        &model_path,
724        runtime_config,
725        verifying_key,
726        signing_key.clone(),
727        transport,
728        policy_transport,
729        None, // CLI: no FsOps
730    );
731    let spawner = ServiceSpawner::threaded();
732    let mut service_handle = spawner
733        .spawn(service_config)
734        .await
735        .map_err(|e| anyhow::anyhow!("Failed to spawn inference service: {}", e))?;
736
737    let client = InferenceClient::for_local_bootstrap(
738        signing_key.clone(),
739        signing_key.verifying_key(),
740        None,
741    )?;
742
743    // Get adapter info for checkpoint saves
744    let adapter_manager = AdapterManager::new(&model_path);
745    let adapters = adapter_manager.list_adapters().unwrap_or_default();
746
747    println!("\nStarting batch training...\n");
748
749    let mut processed = 0;
750    let mut total_input_tokens = 0usize;
751    let mut total_output_tokens = 0usize;
752    let start_time = std::time::Instant::now();
753
754    for file_path in &files {
755        // Read file content
756        let content = std::fs::read_to_string(file_path)?;
757        let chunk: String = content.chars().take(chunk_size).collect();
758
759        if chunk.is_empty() {
760            continue;
761        }
762
763        // Build generation request - TTT is applied by InferenceService before generation
764        let prompt = match format {
765            "jsonl" => {
766                // Parse JSONL format - expect {"prompt": "...", "completion": "..."} or similar
767                if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(&chunk) {
768                    parsed
769                        .get("prompt")
770                        .or_else(|| parsed.get("text"))
771                        .or_else(|| parsed.get("content"))
772                        .and_then(|v| v.as_str())
773                        .unwrap_or(&chunk)
774                        .to_owned()
775                } else {
776                    chunk.clone()
777                }
778            }
779            _ => chunk.clone(), // "text" format - use as-is
780        };
781
782        let request = hyprstream_rpc_std::inference_client::GenerationRequest {
783            prompt,
784            max_tokens: Some(max_tokens as u32),
785            temperature: Some(0.7),
786            ..Default::default()
787        };
788
789        // Send generation request via collect-stream — InferenceService applies TTT during this call
790        match collect_inference_stream(&client, &request).await {
791            Ok(response) => {
792                // Track tokens from response
793                total_input_tokens += response.prefill_tokens;
794                total_output_tokens += response.tokens_generated;
795            }
796            Err(e) => {
797                warn!("Generation failed for {}: {}", file_path.display(), e);
798            }
799        }
800
801        processed += 1;
802
803        // Progress update
804        if processed % progress_interval == 0 {
805            let elapsed = start_time.elapsed().as_secs_f32();
806            let rate = processed as f32 / elapsed;
807            let tokens_per_sec = (total_input_tokens + total_output_tokens) as f32 / elapsed;
808            println!(
809                "[{processed}/{total_files}] {rate:.1} files/sec, {total_input_tokens} input tokens, {total_output_tokens} output tokens ({tokens_per_sec:.0} tok/s)"
810            );
811        }
812
813        // Checkpoint - save adapter weights
814        if processed % checkpoint_interval == 0 {
815            println!("\n→ Checkpoint at {processed} files...");
816            // Request the service to save weights (if supported), or save manually
817            if let Some(adapter) = adapters.first() {
818                println!("  Adapter: {}", adapter.name);
819            }
820            println!("  ✓ Weights updated (will be saved on service stop)");
821        }
822    }
823
824    // Stop service - this saves final adapter weights
825    println!("\nStopping InferenceService and saving weights...");
826    let _ = service_handle.stop().await;
827
828    let elapsed = start_time.elapsed();
829    let total_tokens = total_input_tokens + total_output_tokens;
830    let tokens_per_sec = total_tokens as f32 / elapsed.as_secs_f32();
831    println!("\n✓ Batch training complete!");
832    println!("  Processed: {processed} files");
833    println!("  Input tokens: {total_input_tokens}");
834    println!("  Output tokens: {total_output_tokens}");
835    println!("  Total tokens: {total_tokens} ({tokens_per_sec:.0} tok/s)");
836    println!("  Time: {:.1}s", elapsed.as_secs_f32());
837    println!("\n→ Run 'hyprstream training checkpoint {model_ref}' to commit");
838
839    Ok(())
840}
841
842/// Handle `training checkpoint` command
843///
844/// Commits dirty adapter changes to git.
845pub async fn handle_training_checkpoint(
846    registry: &RegistryClient,
847    model_ref_str: &str,
848    message: Option<String>,
849    push: bool,
850    remote: &str,
851) -> Result<()> {
852    use crate::storage::AdapterManager;
853
854    let model_ref = ModelRef::parse(model_ref_str)?;
855    let tracked = registry
856        .get_by_name(&model_ref.model)
857        .await
858        .map_err(|e| anyhow::anyhow!("Model '{}' not found: {}", model_ref, e))?;
859    let repo_client = registry.repo(&tracked.id);
860
861    let branch_for_fs = match &model_ref.git_ref {
862        git2db::GitRef::Branch(name) => name.clone(),
863        _ => repo_client
864            .get_head()
865            .await
866            .unwrap_or_else(|_| "main".to_owned()),
867    };
868    let fs = repo_client.worktree(&branch_for_fs);
869
870    let adapter_manager = AdapterManager::with_fs(fs.clone());
871    let adapters = adapter_manager.list_adapters_async().await?;
872
873    if adapters.is_empty() {
874        bail!("No adapters found for model {}", model_ref);
875    }
876
877    let status = repo_client
878        .status()
879        .await
880        .map_err(|e| anyhow::anyhow!("Failed to get repository status: {}", e))?;
881
882    // Filter for dirty adapter files
883    let dirty_adapters: Vec<_> = status
884        .modified_files
885        .into_iter()
886        .filter(|p| p.contains("adapters/") && p.ends_with(".safetensors"))
887        .collect();
888
889    if dirty_adapters.is_empty() {
890        println!("No dirty adapter files to commit.");
891        return Ok(());
892    }
893
894    println!("Found {} modified adapter files:", dirty_adapters.len());
895    for path in &dirty_adapters {
896        println!("  {}", path);
897    }
898
899    // Stage adapter files using service layer
900    let mut files_to_stage: Vec<String> = Vec::new();
901    for adapter in &adapters {
902        // adapter.path is relative for FsOps-backed managers (e.g., "adapters/00_base.safetensors")
903        let rel_path = adapter.path.to_string_lossy().to_string();
904        files_to_stage.push(rel_path);
905
906        // Also stage config if exists (check via FsOps)
907        let base_name = adapter.filename.trim_end_matches(".safetensors");
908        let rel_config = format!("adapters/{base_name}.config.json");
909        if fs
910            .stat_path(&rel_config)
911            .await
912            .map(|s| s.exists)
913            .unwrap_or(false)
914        {
915            files_to_stage.push(rel_config);
916        }
917    }
918
919    // Use worktree-scoped client for staging and commit
920    let wt = repo_client.worktree(&branch_for_fs);
921    wt.stage_files(&StageFilesRequest {
922        files: files_to_stage.clone(),
923    })
924    .await
925    .map_err(|e| anyhow::anyhow!("Failed to stage files: {}", e))?;
926
927    // Create commit using service layer
928    let commit_message = message.unwrap_or_else(|| {
929        format!(
930            "Training checkpoint: {} adapters updated\n\n\
931             🤖 Generated with hyprstream training",
932            adapters.len()
933        )
934    });
935
936    let commit_id = wt
937        .commit(&CommitRequest {
938            message: commit_message,
939            author: String::new(),
940            email: String::new(),
941        })
942        .await
943        .map_err(|e| anyhow::anyhow!("Failed to create commit: {}", e))?;
944
945    println!(
946        "\n✓ Created commit: {}",
947        &commit_id[..8.min(commit_id.len())]
948    );
949
950    // Push if requested
951    if push {
952        println!("Pushing to {remote}...");
953
954        let refspec = format!(
955            "refs/heads/{}:refs/heads/{}",
956            model_ref.git_ref, model_ref.git_ref
957        );
958        repo_client
959            .push(&PushRequest {
960                remote: remote.to_owned(),
961                refspec,
962                force: false,
963            })
964            .await
965            .map_err(|e| anyhow::anyhow!("Failed to push: {}", e))?;
966        println!("✓ Pushed to {remote}");
967    }
968
969    Ok(())
970}