Skip to main content

hyprstream_core/cli/
service_handlers.rs

1//! Handlers for service management commands
2//!
3//! Provides lifecycle management for hyprstream services including
4//! installation, upgrade, start/stop, and status display.
5// CLI handlers intentionally print to stdout/stderr for user interaction
6#![allow(clippy::print_stdout, clippy::print_stderr)]
7
8use std::ffi::OsString;
9use std::io::Write;
10use std::path::{Path, PathBuf};
11
12use anyhow::{Context, Result};
13use tracing::info;
14
15/// Reconcile the published local service roster before runtime startup.
16pub fn handle_service_reconcile_enrollment(config: &crate::config::HyprConfig) -> Result<()> {
17    let secrets = crate::config::HyprConfig::resolve_secrets_dir_for(Some(config))?;
18    let count = crate::auth::service_enrollment::reconcile_existing_services(
19        &secrets,
20        &config.oauth.issuer_url(),
21        chrono::Utc::now().timestamp(),
22    )?;
23    info!(
24        services = count,
25        "Reconciled retained service enrollment and credentials"
26    );
27    Ok(())
28}
29
30/// Persist requested built-in templates into the same policy store loaded by
31/// PolicyService, without starting a resolver or loading service credentials.
32pub async fn handle_service_provision_policy_templates(
33    models_dir: &Path,
34    template_names: &[String],
35) -> Result<()> {
36    provision_policy_templates(
37        models_dir,
38        template_names,
39        |_| Ok(()),
40        |_| Ok(()),
41        |_| Ok(()),
42    )
43    .await
44}
45
46async fn provision_policy_templates(
47    models_dir: &Path,
48    template_names: &[String],
49    after_staged_save: impl FnOnce(&Path) -> Result<()>,
50    after_snapshot: impl FnOnce(&Path) -> Result<()>,
51    mut after_publication_write: impl FnMut(&Path) -> Result<()>,
52) -> Result<()> {
53    use crate::auth::{get_template, PolicyManager, PolicyTemplate};
54    use anyhow::{bail, ensure};
55    use std::collections::BTreeSet;
56
57    ensure!(
58        !template_names.is_empty(),
59        "at least one policy template is required"
60    );
61
62    // Resolve and validate the complete request before PolicyManager::new can
63    // create or migrate anything on disk.
64    let mut unique = BTreeSet::new();
65    let mut templates: Vec<&'static PolicyTemplate> = Vec::with_capacity(template_names.len());
66    for name in template_names {
67        ensure!(
68            unique.insert(name.as_str()),
69            "duplicate policy template: {name}"
70        );
71        let Some(template) = get_template(name) else {
72            bail!("unknown policy template: {name}");
73        };
74        templates.push(template);
75    }
76
77    let registry_dir = models_dir.join(".registry");
78    tokio::fs::create_dir_all(&registry_dir)
79        .await
80        .context("create registry directory for policy provisioning")?;
81    let staging = tempfile::Builder::new()
82        .prefix(".policy-provision-")
83        .tempdir_in(&registry_dir)
84        .context("create policy staging directory")?;
85    let policies_dir = registry_dir.join("policies");
86    let staged_policies_dir = staging.path().join("policies");
87    tokio::fs::create_dir(&staged_policies_dir)
88        .await
89        .context("create staged policy directory")?;
90    for name in ["model.conf", "policy.csv"] {
91        let source = policies_dir.join(name);
92        match tokio::fs::read(&source).await {
93            Ok(content) => {
94                crate::auth::write_policy_file(&staged_policies_dir.join(name), content)
95                    .await
96                    .with_context(|| format!("stage retained policy file {name}"))?;
97            }
98            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
99            Err(error) => {
100                return Err(error)
101                    .with_context(|| format!("read retained policy file {}", source.display()));
102            }
103        }
104    }
105
106    // PolicyManager's FileAdapter truncates files when it saves. Keep every
107    // constructor migration and template save in this disposable directory.
108    let manager = PolicyManager::new(&staged_policies_dir)
109        .await
110        .context("open staged policy store")?;
111    for template in &templates {
112        manager
113            .apply_template(template)
114            .await
115            .with_context(|| format!("apply policy template '{}'", template.name))?;
116    }
117
118    let requested: BTreeSet<&str> = templates.iter().map(|template| template.name).collect();
119    let public_staging: BTreeSet<&str> = ["public-inference", "public-read"].into_iter().collect();
120    verify_requested_templates(&manager, &templates).await?;
121    if requested == public_staging {
122        verify_public_staging_policy(&manager).await?;
123    }
124    let intended_state = complete_policy_state(&manager).await;
125    drop(manager);
126
127    let staged_policy_path = staged_policies_dir.join("policy.csv");
128    after_staged_save(&staged_policy_path)?;
129
130    // Capture once, validate these immutable bytes, and publish these same
131    // buffers. A late write to staging cannot change the selected replacement.
132    let staged_model = tokio::fs::read(staged_policies_dir.join("model.conf"))
133        .await
134        .context("capture staged policy model")?;
135    let staged_policy = tokio::fs::read(&staged_policy_path)
136        .await
137        .context("capture staged policy")?;
138    anyhow::ensure!(
139        parse_complete_policy_state(&staged_model, &staged_policy).await? == intended_state,
140        "captured staged policy does not match the complete intended policy state"
141    );
142    after_snapshot(&staged_policy_path)?;
143    tokio::fs::create_dir_all(&policies_dir)
144        .await
145        .context("create live policy directory")?;
146    #[cfg(unix)]
147    {
148        use std::os::unix::fs::PermissionsExt as _;
149        tokio::fs::set_permissions(&policies_dir, std::fs::Permissions::from_mode(0o750))
150            .await
151            .context("set live policy directory permissions")?;
152    }
153
154    // The model is compatible with both the retained and replacement policy.
155    // Publish policy.csv last: its rename is the authority commit point.
156    atomic_publish_policy_file(
157        &policies_dir.join("model.conf"),
158        &staged_model,
159        &mut after_publication_write,
160    )
161    .context("publish verified policy model")?;
162    atomic_publish_policy_file(
163        &policies_dir.join("policy.csv"),
164        &staged_policy,
165        &mut after_publication_write,
166    )
167    .context("publish verified policy")?;
168
169    let published = PolicyManager::new(&policies_dir)
170        .await
171        .context("reopen published policy store")?;
172    verify_requested_templates(&published, &templates).await?;
173    if requested == public_staging {
174        verify_public_staging_policy(&published).await?;
175    }
176
177    println!(
178        "verified {} policy template(s): {}",
179        templates.len(),
180        template_names.join(",")
181    );
182    Ok(())
183}
184
185#[derive(Debug, Eq, PartialEq)]
186struct CompletePolicyState {
187    policies: std::collections::BTreeSet<Vec<String>>,
188    groupings: std::collections::BTreeSet<Vec<String>>,
189    domain_groupings: std::collections::BTreeSet<Vec<String>>,
190}
191
192async fn complete_policy_state(manager: &crate::auth::PolicyManager) -> CompletePolicyState {
193    CompletePolicyState {
194        policies: manager.get_policy().await.into_iter().collect(),
195        groupings: manager.get_grouping_policy().await.into_iter().collect(),
196        domain_groupings: manager
197            .get_domain_grouping_policy()
198            .await
199            .into_iter()
200            .collect(),
201    }
202}
203
204async fn parse_complete_policy_state(model: &[u8], policy: &[u8]) -> Result<CompletePolicyState> {
205    use casbin::{CoreApi as _, DefaultModel, Enforcer, MgmtApi as _, StringAdapter};
206
207    let model = std::str::from_utf8(model).context("captured policy model is not UTF-8")?;
208    let policy = std::str::from_utf8(policy).context("captured policy is not UTF-8")?;
209    let model = DefaultModel::from_str(model)
210        .await
211        .context("parse captured policy model")?;
212    let enforcer = Enforcer::new(model, StringAdapter::new(policy))
213        .await
214        .context("parse captured policy")?;
215    Ok(CompletePolicyState {
216        policies: enforcer.get_policy().into_iter().collect(),
217        groupings: enforcer.get_grouping_policy().into_iter().collect(),
218        domain_groupings: enforcer
219            .get_named_grouping_policy("g2")
220            .into_iter()
221            .collect(),
222    })
223}
224
225fn atomic_publish_policy_file(
226    path: &Path,
227    content: &[u8],
228    after_write: &mut impl FnMut(&Path) -> Result<()>,
229) -> Result<()> {
230    let parent = path
231        .parent()
232        .context("policy publication path has no parent")?;
233    let mut staged = tempfile::NamedTempFile::new_in(parent)
234        .with_context(|| format!("create publication file for {}", path.display()))?;
235    #[cfg(unix)]
236    {
237        use std::os::unix::fs::PermissionsExt as _;
238        staged
239            .as_file()
240            .set_permissions(std::fs::Permissions::from_mode(0o640))
241            .with_context(|| format!("set publication permissions for {}", path.display()))?;
242    }
243    staged
244        .write_all(content)
245        .with_context(|| format!("write publication file for {}", path.display()))?;
246    after_write(path)?;
247    staged
248        .as_file_mut()
249        .flush()
250        .with_context(|| format!("flush publication file for {}", path.display()))?;
251    staged
252        .as_file()
253        .sync_all()
254        .with_context(|| format!("sync publication file for {}", path.display()))?;
255    staged
256        .persist(path)
257        .map_err(|error| error.error)
258        .with_context(|| format!("publish policy file {}", path.display()))?;
259    #[cfg(unix)]
260    std::fs::File::open(parent)
261        .and_then(|directory| directory.sync_all())
262        .with_context(|| format!("sync policy directory {}", parent.display()))?;
263    Ok(())
264}
265
266async fn verify_requested_templates(
267    manager: &crate::auth::PolicyManager,
268    templates: &[&crate::auth::PolicyTemplate],
269) -> Result<()> {
270    use anyhow::ensure;
271    use std::collections::BTreeSet;
272
273    let policies = manager.get_policy().await;
274    let groupings = manager.get_grouping_policy().await;
275    let domain_groupings = manager.get_domain_grouping_policy().await;
276    for template in templates {
277        let expanded = template.expanded_policies();
278        let tenant_domains: BTreeSet<&str> = expanded
279            .iter()
280            .map(|policy| policy.domain)
281            .filter(|domain| *domain != "*")
282            .collect();
283        for expected in expanded {
284            ensure!(
285                policies.contains(&expected.to_vec()),
286                "policy template '{}' did not persist its canonical rule",
287                template.name
288            );
289        }
290        if let Some(expected_groupings) = template.groupings {
291            for expected in expected_groupings {
292                if let Some(domain) = tenant_domains.first() {
293                    ensure!(
294                        domain_groupings.contains(&vec![
295                            expected.user.to_owned(),
296                            expected.role.to_owned(),
297                            (*domain).to_owned(),
298                        ]),
299                        "policy template '{}' did not persist its canonical domain grouping",
300                        template.name
301                    );
302                } else {
303                    ensure!(
304                        groupings.contains(&expected.to_vec()),
305                        "policy template '{}' did not persist its canonical grouping",
306                        template.name
307                    );
308                }
309            }
310        }
311    }
312    Ok(())
313}
314
315async fn verify_public_staging_policy(manager: &crate::auth::PolicyManager) -> Result<()> {
316    use anyhow::ensure;
317
318    let policies = manager.get_policy().await;
319    let mut expected = Vec::new();
320    for name in ["public-inference", "public-read"] {
321        let template = crate::auth::get_template(name)
322            .with_context(|| format!("compiled-in public staging template missing: {name}"))?;
323        expected.extend(
324            template
325                .expanded_policies()
326                .into_iter()
327                .map(|rule| rule.to_vec()),
328        );
329    }
330    ensure!(
331        expected.len() == 3,
332        "public staging templates must define exactly three rules"
333    );
334    for rule in &expected {
335        ensure!(
336            policies
337                .iter()
338                .filter(|candidate| candidate.as_slice() == rule.as_slice())
339                .count()
340                == 1,
341            "public staging policy must persist each of its three canonical rules exactly once"
342        );
343    }
344
345    let checks = [
346        ("model:policy-bootstrap-probe", "infer.generate"),
347        ("model:policy-bootstrap-probe", "query.status"),
348        ("registry:policy-bootstrap-probe", "query.status"),
349    ];
350    for (resource, action) in checks {
351        ensure!(
352            manager
353                .check_with_domain("anonymous", "*", resource, action)
354                .await,
355            "public staging policy is not effective for {action} on {resource}"
356        );
357    }
358    ensure!(
359        !manager
360            .check_with_domain(
361                "anonymous",
362                "*",
363                "model:policy-bootstrap-probe",
364                "ttt.writeback",
365            )
366            .await,
367        "public staging policy must not grant anonymous ttt.writeback"
368    );
369    Ok(())
370}
371
372/// Handle `service install` - Idempotent setup and optional restart
373///
374/// 1. Run repair checks (dirs, registry, policy, signing key, git identity)
375/// 2. Install command alias (~/.local/bin/hyprstream)
376/// 3. Install/update systemd units (if systemd available)
377/// 4. If `start`: stop → start all target services
378pub async fn handle_service_install(
379    models_dir: &Path,
380    config_services: &[String],
381    services_filter: Option<Vec<String>>,
382    start: bool,
383    enable: bool,
384    target: hyprstream_service::ServiceTarget,
385    verbose: bool,
386    explicit_config: Option<&Path>,
387    iroh_required: bool,
388) -> Result<()> {
389    let target_services = services_filter.unwrap_or_else(|| config_services.to_vec());
390
391    println!("Installing hyprstream...\n");
392
393    // 1. Run repair checks to bootstrap the environment
394    run_repair_checks(models_dir, verbose).await?;
395    println!();
396
397    // 2. Install command alias to user's executable directory
398    println!("  Installing command...");
399    match InstallPlan::prepare() {
400        Ok(plan) => {
401            println!(
402                "    Source: {} ({})",
403                plan.source.display(),
404                plan.type_label()
405            );
406            match plan.execute() {
407                Ok(result) => {
408                    println!(
409                        "    {} ({})",
410                        result.version_dir.display(),
411                        result.type_label()
412                    );
413                    println!(
414                        "    {} -> ...",
415                        result.bin_dir.join("hyprstream.appimage").display()
416                    );
417                    println!(
418                        "    {} -> hyprstream.appimage",
419                        result.bin_dir.join("hyprstream").display()
420                    );
421                    if !result.updated_profiles.is_empty() {
422                        println!("    PATH updated: {}", result.updated_profiles.join(" "));
423                    }
424                }
425                Err(e) => println!("    install failed ({})", e),
426            }
427        }
428        Err(e) => println!("    skipped ({})", e),
429    }
430    println!();
431
432    // 3. Install/update systemd units if available
433    if hyprstream_rpc::has_systemd() {
434        // Installed units run their own stored configuration (fixed ExecStart,
435        // no config provenance, raw dependency order). Installing, enabling,
436        // or starting such a unit under an explicit --config selector or the
437        // required-native profile would silently run a different identity
438        // than this process loaded. Refuse before any unit mutation; the
439        // direct launch path is the supported route.
440        if explicit_config.is_some() || iroh_required {
441            anyhow::bail!(
442                "installed hyprstream service units cannot receive an explicit --config \
443                 selector or the required-native profile, so this command cannot install \
444                 or start units for that configuration. Launch provisioned services \
445                 directly instead, e.g. `hyprstream{} service start <service> --daemon`. \
446                 (Plain `hyprstream service install` without the custom selector or \
447                 required profile still installs default-configuration units.)",
448                match explicit_config {
449                    Some(path) => format!(" --config {}", path.display()),
450                    None => String::new(),
451                }
452            );
453        }
454
455        let manager = hyprstream_service::detect_service_manager_with_mode(target).await?;
456
457        // Encrypt secrets into the systemd user credstore before generating units.
458        // This must happen before manager.install() so that install() can see the
459        // .cred files when deciding whether to emit ImportCredential= directives.
460        #[cfg(feature = "systemd")]
461        {
462            let secrets_dir = crate::config::HyprConfig::load()
463                .ok()
464                .and_then(|c| crate::config::HyprConfig::resolve_secrets_dir_for(Some(&c)).ok());
465            hyprstream_service::encrypt_credentials_if_available(secrets_dir.as_deref());
466        }
467
468        // If --start, stop all target services first so they pick up changes.
469        // Stop is legitimately idempotent here (already-stopped units are the
470        // common case on reinstall), so failures stay non-fatal.
471        if start {
472            println!("  Stopping services...");
473            for service in &target_services {
474                let _ = manager.stop(service).await;
475            }
476        }
477
478        println!("  Installing systemd units...");
479        for service in &target_services {
480            print!("    \u{25CB} {}... ", service);
481            manager
482                .install(service)
483                .await
484                .map_err(|e| anyhow::anyhow!("installing unit for {service}: {e}"))?;
485            println!("\u{2713}");
486        }
487
488        // If --enable, register units for autostart at boot
489        if enable {
490            println!("  Enabling services for autostart...");
491            for service in &target_services {
492                print!("    \u{25CB} {}... ", service);
493                manager
494                    .enable(service)
495                    .await
496                    .map_err(|e| anyhow::anyhow!("enabling unit for {service}: {e}"))?;
497                println!("\u{2713}");
498            }
499        }
500
501        // 4. If --start, start all target services. A queued start request is
502        // not success: require the unit to reach the active state (Type=notify)
503        // within the bounded startup budget (#1585).
504        if start {
505            println!("  Starting services...");
506            start_units_to_active(&*manager, &target_services, CHILD_READINESS_TIMEOUT).await?;
507        }
508    } else if start {
509        // No systemd: the same factored direct launch path as `service start`
510        // — profile-aware ordering, config forwarding, notification readiness.
511        println!("  Starting services (standalone)...\n");
512
513        let exe = hyprstream_rpc::paths::installed_executable_path()
514            .unwrap_or_else(|| hyprstream_rpc::paths::executable_path().unwrap_or_default());
515
516        let spawner = hyprstream_service::ProcessSpawner::standalone();
517
518        launch_direct_children(
519            &target_services,
520            iroh_required,
521            explicit_config,
522            &exe,
523            &spawner,
524        )
525        .await?;
526    }
527
528    println!("\n\u{2713} Install complete");
529    if !start {
530        println!();
531        println!("Next steps:");
532        println!("  1. Open a new shell to use the 'hyprstream' command");
533        println!("  2. Start services with: hyprstream service start");
534        println!("     Or reinstall with:   hyprstream service install --start");
535    }
536    Ok(())
537}
538
539/// Handle `service uninstall` - Stop and remove units
540pub async fn handle_service_uninstall(
541    config_services: &[String],
542    services_filter: Option<Vec<String>>,
543) -> Result<()> {
544    let target_services = services_filter.unwrap_or_else(|| config_services.to_vec());
545
546    if !hyprstream_rpc::has_systemd() {
547        println!("Systemd not available. No units to uninstall.");
548        return Ok(());
549    }
550
551    let manager = hyprstream_service::detect_service_manager().await?;
552
553    println!("Uninstalling hyprstream services...\n");
554
555    for service in &target_services {
556        print!("  \u{25CB} {}... ", service);
557
558        // Stop first, then uninstall
559        let _ = manager.stop(service).await;
560
561        match manager.uninstall(service).await {
562            Ok(_) => println!("\u{2713}"),
563            Err(e) => println!("\u{2717} {}", e),
564        }
565    }
566
567    // Reload daemon
568    manager.reload().await?;
569
570    println!("\n\u{2713} Uninstall complete");
571    Ok(())
572}
573
574/// Budget a spawned child gets to report genuine readiness (#1585).
575const CHILD_READINESS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
576
577/// Build one child invocation for the direct launch path (#1585).
578///
579/// Required-native children run `hyprstream [--config PATH] service start NAME
580/// --foreground` — no `--ipc`, no local endpoint fallback; their readiness is
581/// the child's own notification boundary, never a UDS socket. Compatibility
582/// children keep the historical `--ipc` shape. The explicit config path (when
583/// the operator supplied one) is forwarded so the child loads, validates, and
584/// pins the same configuration the launcher did; it is canonical absolute, so
585/// it stays meaningful regardless of the child's working directory, and it is
586/// one argv element, so spaces need no quoting. Config contents, keys, and
587/// JWTs never enter argv or the environment.
588pub fn direct_child_process_config(
589    service: &str,
590    iroh_required: bool,
591    explicit_config: Option<&Path>,
592    exe: &Path,
593) -> Result<hyprstream_service::ProcessConfig> {
594    let mut args: Vec<OsString> = Vec::new();
595    if let Some(config_path) = explicit_config {
596        args.push(OsString::from("--config"));
597        args.push(config_path.as_os_str().to_owned());
598    }
599    args.push(OsString::from("service"));
600    args.push(OsString::from("start"));
601    args.push(OsString::from(service));
602    args.push(OsString::from("--foreground"));
603    if !iroh_required {
604        args.push(OsString::from("--ipc"));
605    }
606
607    let mut config = hyprstream_service::ProcessConfig::new(service, exe);
608    config.args = args;
609    if iroh_required {
610        // The child's own authenticated startup boundary is the readiness
611        // signal (sd_notify READY after Iroh bind and first publication).
612        config.readiness = hyprstream_service::ProcessReadiness::Notify {
613            timeout: CHILD_READINESS_TIMEOUT,
614        };
615    }
616    Ok(config)
617}
618
619/// Launch children directly in profile-aware dependency order (#1585).
620///
621/// Shared by `service start --daemon`, no-systemd `service start`, and the
622/// no-systemd `service install --start` fallback, so every direct launch gets
623/// the same ordering, config forwarding, and readiness contract.
624///
625/// Required-native: every child must report genuine readiness before the next
626/// child (and stage) starts; any spawn, early-exit, or timeout failure stops
627/// already-started children in reverse start order and returns `Err` — the
628/// caller never prints success for a partial launch. Compatibility keeps the
629/// historical best-effort behavior (per-service error lines, UDS-presence
630/// stage barrier with warn-and-continue) so existing deployments are
631/// unchanged.
632pub async fn launch_direct_children(
633    targets: &[String],
634    iroh_required: bool,
635    explicit_config: Option<&Path>,
636    exe: &Path,
637    spawner: &hyprstream_service::ProcessSpawner,
638) -> Result<()> {
639    if iroh_required {
640        for service in targets {
641            anyhow::ensure!(
642                hyprstream_service::get_factory(service).is_some(),
643                "unknown native service: {service}"
644            );
645        }
646    }
647    let stages = hyprstream_service::startup_stages_for_profile(targets, iroh_required);
648    // Flatten the ordered stages into the serial launch plan (same order the
649    // loop below would spawn in), so the launch core is injectably testable
650    // without the real service binary.
651    let mut plans: Vec<Vec<(String, hyprstream_service::ProcessConfig)>> = Vec::new();
652    for stage in &stages {
653        let mut stage_plans = Vec::new();
654        for service in stage {
655            stage_plans.push((
656                service.clone(),
657                direct_child_process_config(service, iroh_required, explicit_config, exe)?,
658            ));
659        }
660        plans.push(stage_plans);
661    }
662    launch_planned_children(plans, iroh_required, spawner).await
663}
664
665/// Cancellation-safe ownership of the children of a Required launch
666/// transaction (#1585).
667///
668/// Every child that spawns successfully — including children that already
669/// reached READY and were adopted by the backend — is recorded here until the
670/// WHOLE launch commits. If the orchestration future is cancelled (dropped at
671/// any await: a later child's readiness, or a rollback stop), `Drop` runs the
672/// synchronous bounded tracked-child stop
673/// ([`hyprstream_service::ProcessSpawner::stop_tracked_child_sync`]) over the
674/// children it still owns, newest first: the backend stops them through the
675/// retained `Child` handles (never a blind by-PID signal) and removes their
676/// PID artifacts. That cleanup is plain synchronous code on the drop path —
677/// it does not depend on any async cleanup future surviving runtime shutdown.
678/// On whole-launch success [`Self::commit`] disarms the guard so the adopted
679/// daemons intentionally keep running, stoppable as before. Compatibility
680/// launches are never armed: their historical behavior, including a
681/// cancelled launch's historical leak, is unchanged.
682struct RequiredLaunchGuard {
683    started: Vec<(String, hyprstream_service::SpawnedProcess)>,
684    /// Children whose async rollback stop FAILED: still fully owned by the
685    /// guard (a local collection would recreate the ownership-transfer hole
686    /// across the remaining rollback awaits), retried by the synchronous
687    /// bounded pass.
688    failed: Vec<(String, hyprstream_service::SpawnedProcess)>,
689    /// Clone of the launching spawner: the tracked `Child` handles live in
690    /// its backend, so `Drop` cleanup must go through THIS backend to keep
691    /// the stronger tracked ownership.
692    spawner: hyprstream_service::ProcessSpawner,
693    armed: bool,
694    committed: bool,
695}
696
697impl RequiredLaunchGuard {
698    /// Arm only for Required launches; Compatibility ownership stays exactly
699    /// as it was before #1585's transaction guard.
700    fn arm(iroh_required: bool, spawner: &hyprstream_service::ProcessSpawner) -> Self {
701        Self {
702            started: Vec::new(),
703            failed: Vec::new(),
704            spawner: spawner.clone(),
705            armed: iroh_required,
706            committed: false,
707        }
708    }
709
710    /// Record an adopted child under transaction ownership.
711    fn adopt(&mut self, service: String, process: hyprstream_service::SpawnedProcess) {
712        if self.armed {
713            self.started.push((service, process));
714        }
715    }
716
717    /// The newest owned child, without surrendering ownership: the async stop
718    /// below works on cloned metadata, so a cancellation mid-await leaves the
719    /// original owned and the guard's `Drop` responsible for it.
720    fn newest(&self) -> Option<&(String, hyprstream_service::SpawnedProcess)> {
721        self.started.last()
722    }
723
724    /// Surrender ownership of the newest child — ONLY after a CONFIRMED
725    /// successful stop.
726    fn release_newest(&mut self) {
727        self.started.pop();
728    }
729
730    /// Demote the newest child after a FAILED async stop: it stays
731    /// guard-owned (moved into `failed`) so every later rollback await, and
732    /// cancellation during any of them, still leaves it under `Drop` cleanup.
733    fn demote_newest_to_failed(&mut self) {
734        if let Some(entry) = self.started.pop() {
735            self.failed.push(entry);
736        }
737    }
738
739    /// Reverse-order rollback with cancellation-safe ownership: a stop that
740    /// errors demotes the child to guard-owned `failed` state — never a
741    /// local unguarded collection — while confirmed successes are released.
742    /// The `stop` seam is the production async rollback stop (the tracked
743    /// path terminates via the retained handle with SIGKILL and reaps within
744    /// a bounded budget — it is not a graceful TERM shutdown); tests inject
745    /// a controllable boundary.
746    async fn rollback_owned_children<S, F>(&mut self, mut stop: S)
747    where
748        S: FnMut(hyprstream_service::SpawnedProcess) -> F,
749        F: std::future::Future<Output = anyhow::Result<()>>,
750    {
751        while let Some((service, process)) = self.newest().cloned() {
752            print!("  \u{25CB} stopping {} after failed launch... ", service);
753            match stop(process).await {
754                Ok(()) => {
755                    println!("\u{2713}");
756                    self.release_newest();
757                }
758                Err(e) => {
759                    println!("\u{2717} {}", e);
760                    self.demote_newest_to_failed();
761                }
762            }
763        }
764    }
765
766    /// Synchronously stop everything still owned — unreleased children and
767    /// failed-stop demotions, newest first — and disarm. Returns one
768    /// residual-report string per child whose bounded stop did not confirm
769    /// termination or whose PID-artifact cleanup failed (the artifact is then
770    /// deliberately retained — it may still name a live process).
771    fn stop_all_sync_and_disarm(&mut self) -> Vec<String> {
772        let mut residuals = Vec::new();
773        // Deterministic reverse-start-order contract across BOTH ownership
774        // collections: failed demotions were appended newest-first during the
775        // async rollback, so they are revisited in insertion order first,
776        // then the still-unattempted started entries in reverse order.
777        let owned = self
778            .failed
779            .iter()
780            .chain(self.started.iter().rev())
781            .collect::<Vec<_>>();
782        for (service, process) in owned {
783            print!("  \u{25CB} stopping {} (synchronous pass)... ", service);
784            match self.spawner.stop_tracked_child_sync(process) {
785                Ok(()) => println!("\u{2713}"),
786                Err(e) => {
787                    println!("\u{2717} {}", e);
788                    residuals.push(format!("{service}: {e}"));
789                    tracing::error!(
790                        service = %service,
791                        error = %e,
792                        "aborted required launch left residual child state; PID artifact retained"
793                    );
794                }
795            }
796        }
797        self.started.clear();
798        self.failed.clear();
799        self.committed = true;
800        residuals
801    }
802
803    /// Mark the whole launch committed: adopted daemons stay intentionally
804    /// alive and the `Drop` path becomes a no-op.
805    fn commit(&mut self) {
806        self.committed = true;
807    }
808}
809
810impl Drop for RequiredLaunchGuard {
811    fn drop(&mut self) {
812        if self.committed || !self.armed || (self.started.is_empty() && self.failed.is_empty()) {
813            return;
814        }
815        // Cancellation (task abort at any await — later-child readiness or a
816        // rollback stop) lands here: the synchronous bounded pass is the
817        // cleanup owner of last resort for everything not yet released,
818        // including children demoted to `failed` by earlier stop errors.
819        self.stop_all_sync_and_disarm();
820    }
821}
822
823/// Serially launch pre-built child plans (stage-ordered) with the
824/// required-native readiness contract and reverse-order rollback. Narrow seam
825/// (#1585): plan building is separate so causal tests can inject concrete
826/// children without the real service binary.
827///
828/// Required-native cancellation safety: started children stay owned by the
829/// [`RequiredLaunchGuard`] until the whole launch commits, so a cancelled
830/// future still cleans them through the backend's retained child handles; an
831/// explicit failure runs the async rollback stop first (tracked path
832/// terminates via SIGKILL through the retained handle — not a graceful TERM
833/// shutdown), with the synchronous tracked-child pass as the
834/// accurate-residual fallback.
835async fn launch_planned_children(
836    plans: Vec<Vec<(String, hyprstream_service::ProcessConfig)>>,
837    iroh_required: bool,
838    spawner: &hyprstream_service::ProcessSpawner,
839) -> Result<()> {
840    let mut guard = RequiredLaunchGuard::arm(iroh_required, spawner);
841    let mut launch_error: Option<anyhow::Error> = None;
842
843    'stages: for stage in &plans {
844        for (service, config) in stage {
845            print!("  \u{25CB} {}... ", service);
846
847            match spawner.spawn(config.clone()).await {
848                Ok(process) => {
849                    info!("Spawned {} service: {:?}", service, process.kind);
850                    println!("\u{2713} (pid {:?})", process.pid());
851                    guard.adopt(service.clone(), process);
852                }
853                Err(e) => {
854                    println!("\u{2717} {}", e);
855                    if iroh_required {
856                        // A required-native failure aborts the WHOLE launch:
857                        // later stages depend on earlier ones, so none of
858                        // them may spawn before the rollback runs.
859                        launch_error = Some(e.into());
860                        break 'stages;
861                    }
862                }
863            }
864        }
865
866        // Compatibility stage barrier: wait for IPC sockets before the next
867        // stage. Required-native readiness was already enforced per child
868        // above; it never depends on a service UDS socket.
869        if !iroh_required {
870            let runtime_dir = hyprstream_rpc::paths::runtime_dir();
871            for (service, _) in stage {
872                let sock = runtime_dir.join(format!("{service}.sock"));
873                let deadline = std::time::Instant::now() + CHILD_READINESS_TIMEOUT;
874                while !sock.exists() && std::time::Instant::now() < deadline {
875                    tokio::time::sleep(std::time::Duration::from_millis(100)).await;
876                }
877                if !sock.exists() {
878                    tracing::warn!("Timeout waiting for {service} socket; continuing");
879                }
880            }
881        }
882    }
883
884    if let Some(error) = launch_error {
885        if iroh_required {
886            // Deterministic rollback: stop what we started, newest first; each
887            // stop removes that child's PID artifact. Every child stays
888            // guard-owned until its stop CONFIRMS — a failed stop is demoted
889            // to guard-owned `failed` state, so a cancellation during ANY of
890            // these awaits (or the synchronous pass) still cleans everything
891            // unconfirmed through the guard's `Drop`.
892            guard
893                .rollback_owned_children(|process| async move {
894                    spawner.stop(&process).await.map_err(anyhow::Error::from)
895                })
896                .await;
897            // Synchronous bounded pass over guard-owned state: retries failed
898            // stops and sweeps unreleased children. Only children whose
899            // bounded stop did not confirm termination (or whose PID-artifact
900            // cleanup failed) are residual — reported honestly, never
901            // silently dropped.
902            let residuals = guard.stop_all_sync_and_disarm();
903            if !residuals.is_empty() {
904                return Err(error.context(format!(
905                    "launch rollback left residual state: {}",
906                    residuals.join("; ")
907                )));
908            }
909        }
910        return Err(error);
911    }
912    // A child may exit after its readiness notification while a later stage
913    // is still starting. Recheck every adopted child immediately before the
914    // transaction commits so we never report success for a deployment whose
915    // earlier service has already died.
916    if iroh_required {
917        let liveness_error = {
918            let mut failure = None;
919            for (service, process) in &guard.started {
920                match spawner.is_running(process).await {
921                    Ok(true) => {}
922                    Ok(false) => {
923                        failure = Some(anyhow::anyhow!(
924                            "required-native child '{service}' exited before launch commit"
925                        ));
926                        break;
927                    }
928                    Err(e) => {
929                        failure = Some(anyhow::anyhow!(
930                            "could not verify required-native child '{service}' before launch commit: {e}"
931                        ));
932                        break;
933                    }
934                }
935            }
936            failure
937        };
938        if let Some(error) = liveness_error {
939            guard
940                .rollback_owned_children(|process| async move {
941                    spawner.stop(&process).await.map_err(anyhow::Error::from)
942                })
943                .await;
944            let residuals = guard.stop_all_sync_and_disarm();
945            if !residuals.is_empty() {
946                return Err(error.context(format!(
947                    "launch rollback left residual state: {}",
948                    residuals.join("; ")
949                )));
950            }
951            return Err(error);
952        }
953    }
954    guard.commit();
955    Ok(())
956}
957
958/// Handle `service start` (non-foreground) - Start via systemd or spawn
959///
960/// `explicit_config` is the operator's canonical absolute `--config` selector
961/// (CLI value or the `HYPRSTREAM_CONFIG` env selector); `iroh_required` is the
962/// loaded config's native-network profile. Required-native or custom-config
963/// starts never select an installed systemd unit — those units encode their
964/// own stored configuration and cannot receive this process's configuration —
965/// and are routed to the direct launch path instead.
966pub async fn handle_service_start(
967    config_services: &[String],
968    name: Option<String>,
969    daemon: bool,
970    explicit_config: Option<&Path>,
971    iroh_required: bool,
972) -> Result<()> {
973    let target_services: Vec<String> = if let Some(name) = name {
974        vec![name]
975    } else {
976        config_services.to_vec()
977    };
978
979    // MAC genesis coverage gate: logged by the shared `ServiceAction::Start`
980    // dispatch in `bin/main.rs` (before the foreground/systemd branching), so
981    // every startup mode — including foreground boots that bypass this
982    // handler — emits the report exactly once per process.
983
984    // Use systemd if available and --daemon not specified
985    if hyprstream_rpc::has_systemd() && !daemon {
986        if explicit_config.is_some() || iroh_required {
987            anyhow::bail!(
988                "installed hyprstream service units run their own stored configuration; \
989                 an explicit --config selector or the required-native profile cannot be \
990                 applied through them. Launch provisioned services directly instead, \
991                 e.g. `hyprstream{} service start <service> --daemon`.",
992                match explicit_config {
993                    Some(path) => format!(" --config {}", path.display()),
994                    None => String::new(),
995                }
996            );
997        }
998
999        let manager = hyprstream_service::detect_service_manager().await?;
1000
1001        println!("Starting services (systemd)...\n");
1002
1003        start_units_to_active(&*manager, &target_services, CHILD_READINESS_TIMEOUT).await?;
1004    } else {
1005        // Direct launch: profile-aware ordering, config forwarding, and the
1006        // per-child notification readiness contract.
1007        println!("Starting services (standalone)...\n");
1008
1009        let spawner = hyprstream_service::ProcessSpawner::standalone();
1010        let exe = hyprstream_rpc::paths::executable_path()?;
1011        launch_direct_children(
1012            &target_services,
1013            iroh_required,
1014            explicit_config,
1015            &exe,
1016            &spawner,
1017        )
1018        .await?;
1019    }
1020
1021    println!("\n\u{2713} Start complete");
1022    Ok(())
1023}
1024
1025/// Start installed units and require each to reach the active state within
1026/// `active_timeout` before reporting success (#1585).
1027///
1028/// The permitted Compatibility-only existing-unit path: mutation failures
1029/// propagate (a pre-existing active unit must never mask a failed start), and
1030/// a queued start request is not completion — the unit is Type=notify.
1031/// Injectable over [`hyprstream_service::ServiceManager`] for causal tests.
1032async fn start_units_to_active(
1033    manager: &dyn hyprstream_service::ServiceManager,
1034    targets: &[String],
1035    active_timeout: std::time::Duration,
1036) -> Result<()> {
1037    // Submit every start request concurrently before waiting on any unit.
1038    // Compatibility services may probe Policy during their own startup;
1039    // waiting for an earlier Event unit to become active before submitting
1040    // Policy would deadlock when the units were stopped. Systemd's
1041    // StartUnit call waits for its job result, so concurrent requests are
1042    // required to let Policy and Event make progress together. Mutation
1043    // failures still propagate before any active-state success is reported.
1044    for service in targets {
1045        print!("  \u{25CB} {}... ", service);
1046    }
1047    futures::future::try_join_all(targets.iter().map(|service| async move {
1048        manager
1049            .start(service)
1050            .await
1051            .map_err(|e| anyhow::anyhow!("starting {service}: {e}"))
1052    }))
1053    .await?;
1054
1055    for service in targets {
1056        let deadline = std::time::Instant::now() + active_timeout;
1057        loop {
1058            match manager.is_active(service).await {
1059                Ok(true) => break,
1060                Ok(false) => {
1061                    if std::time::Instant::now() >= deadline {
1062                        anyhow::bail!(
1063                            "service {service} unit did not reach the active state within {}s",
1064                            active_timeout.as_secs()
1065                        );
1066                    }
1067                    tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1068                }
1069                Err(e) => return Err(e),
1070            }
1071        }
1072        println!("\u{2713}");
1073    }
1074    Ok(())
1075}
1076
1077/// Handle `service stop` - Stop services
1078///
1079/// Attempts to stop services via both systemd and PID files.
1080pub async fn handle_service_stop(config_services: &[String], name: Option<String>) -> Result<()> {
1081    let target_services: Vec<String> = if let Some(name) = name {
1082        vec![name]
1083    } else {
1084        config_services.to_vec()
1085    };
1086
1087    println!("Stopping services...\n");
1088
1089    // Try systemd if available
1090    if hyprstream_rpc::has_systemd() {
1091        let manager = hyprstream_service::detect_service_manager().await?;
1092
1093        for service in &target_services {
1094            print!("  \u{25CB} {} (systemd)... ", service);
1095            match manager.stop(service).await {
1096                Ok(_) => println!("\u{2713}"),
1097                Err(_) => println!("-"),
1098            }
1099        }
1100    }
1101
1102    // Stop daemon processes via PID files
1103    let runtime_dir = hyprstream_rpc::paths::runtime_dir();
1104    for service in &target_services {
1105        let pid_file = runtime_dir.join(format!("{}.pid", service));
1106
1107        if pid_file.exists() {
1108            print!("  \u{25CB} {} (daemon)... ", service);
1109            if let Ok(pid_str) = std::fs::read_to_string(&pid_file) {
1110                if let Ok(pid) = pid_str.trim().parse::<i32>() {
1111                    let nix_pid = nix::unistd::Pid::from_raw(pid);
1112                    match nix::sys::signal::kill(nix_pid, nix::sys::signal::Signal::SIGTERM) {
1113                        Ok(_) => {
1114                            let _ = std::fs::remove_file(&pid_file);
1115                            println!("\u{2713}");
1116                        }
1117                        Err(nix::errno::Errno::ESRCH) => {
1118                            let _ = std::fs::remove_file(&pid_file);
1119                            println!("- (stale)");
1120                        }
1121                        Err(e) => println!("\u{2717} {}", e),
1122                    }
1123                } else {
1124                    println!("\u{2717} invalid pid");
1125                }
1126            } else {
1127                println!("\u{2717} read error");
1128            }
1129        }
1130    }
1131
1132    println!("\n\u{2713} Stop complete");
1133    Ok(())
1134}
1135
1136/// Handle `service status` - Show current state
1137pub async fn handle_service_status(config_services: &[String], verbose: bool) -> Result<()> {
1138    println!("Hyprstream Service Status");
1139    println!("{}", "=".repeat(50));
1140
1141    // Execution context
1142    println!("\nExecution Context:");
1143    if let Ok(appimage) = std::env::var("APPIMAGE") {
1144        println!("  Executable:    {} (AppImage)", appimage);
1145    } else if let Ok(exe) = std::env::current_exe() {
1146        println!("  Executable:    {}", exe.display());
1147    }
1148
1149    // Service manager type
1150    println!("\nService Manager:");
1151    if hyprstream_rpc::has_systemd() {
1152        println!("  Type:          systemd (user session)");
1153
1154        if let Some(config_dir) = dirs::config_dir() {
1155            let units_dir = config_dir.join("systemd/user");
1156            println!("  Units dir:     {}", units_dir.display());
1157        }
1158    } else {
1159        println!("  Type:          standalone (process spawner)");
1160    }
1161
1162    // Service status - check both systemd and daemon PID files
1163    println!("\nService Status:");
1164    println!("  {:<15} {:<10} MODE", "SERVICE", "STATUS");
1165    println!("  {}", "-".repeat(45));
1166
1167    let runtime_dir = hyprstream_rpc::paths::runtime_dir();
1168    let manager = if hyprstream_rpc::has_systemd() {
1169        Some(hyprstream_service::detect_service_manager().await?)
1170    } else {
1171        None
1172    };
1173
1174    for service in config_services {
1175        // Check systemd status
1176        let systemd_running = if let Some(ref mgr) = manager {
1177            mgr.is_active(service).await.unwrap_or(false)
1178        } else {
1179            false
1180        };
1181
1182        // Check daemon PID file
1183        let daemon_running = {
1184            let pid_file = runtime_dir.join(format!("{}.pid", service));
1185            if pid_file.exists() {
1186                if let Ok(pid_str) = std::fs::read_to_string(&pid_file) {
1187                    if let Ok(pid) = pid_str.trim().parse::<i32>() {
1188                        let nix_pid = nix::unistd::Pid::from_raw(pid);
1189                        // Signal 0 checks if process exists
1190                        matches!(
1191                            nix::sys::signal::kill(nix_pid, None),
1192                            Ok(()) | Err(nix::errno::Errno::EPERM)
1193                        )
1194                    } else {
1195                        false
1196                    }
1197                } else {
1198                    false
1199                }
1200            } else {
1201                false
1202            }
1203        };
1204
1205        let (icon, status, mode) = match (systemd_running, daemon_running) {
1206            (true, true) => ("\u{2713}", "running", "systemd+daemon"),
1207            (true, false) => ("\u{2713}", "running", "systemd"),
1208            (false, true) => ("\u{2713}", "running", "daemon"),
1209            (false, false) => ("\u{25CB}", "stopped", ""),
1210        };
1211        println!("  {:<15} {} {:<10} {}", service, icon, status, mode);
1212    }
1213
1214    // Verbose: show unit file contents
1215    if verbose && hyprstream_rpc::has_systemd() {
1216        println!("\n{}", "=".repeat(50));
1217        println!("Unit File Contents:\n");
1218
1219        if let Some(config_dir) = dirs::config_dir() {
1220            let units_dir = config_dir.join("systemd/user");
1221            for service in config_services {
1222                let unit_path = units_dir.join(format!("hyprstream-{}.service", service));
1223                if unit_path.exists() {
1224                    println!("--- {} ---", unit_path.display());
1225                    if let Ok(content) = std::fs::read_to_string(&unit_path) {
1226                        println!("{}", content);
1227                    }
1228                }
1229            }
1230        }
1231    }
1232
1233    Ok(())
1234}
1235
1236/// Run repair checks: directories, registry, policy, signing key, git identity.
1237///
1238/// Extracted from the old `handle_service_repair` so it can be called as part
1239/// of `handle_service_install` without the surrounding summary chrome.
1240pub async fn run_repair_checks(models_dir: &Path, verbose: bool) -> Result<()> {
1241    use crate::auth::PolicyManager;
1242    use crate::cli::policy_handlers::load_or_generate_signing_key;
1243
1244    println!("  Repair checks\n");
1245
1246    let mut all_passed = true;
1247    let mut warnings = Vec::new();
1248
1249    // 1. Directories
1250    {
1251        let label = "Directories";
1252        let registry_path = models_dir.join(".registry");
1253        let policies_dir = registry_path.join("policies");
1254        let keys_dir = registry_path.join("keys");
1255
1256        let dirs_to_create = [
1257            models_dir,
1258            registry_path.as_path(),
1259            policies_dir.as_path(),
1260            keys_dir.as_path(),
1261        ];
1262
1263        let mut fixed = false;
1264        for dir in &dirs_to_create {
1265            if !dir.exists() {
1266                std::fs::create_dir_all(dir)
1267                    .with_context(|| format!("Failed to create directory: {}", dir.display()))?;
1268                fixed = true;
1269            }
1270        }
1271
1272        if fixed {
1273            print_check(
1274                label,
1275                CheckStatus::Fixed,
1276                &format!("created missing directories under {}", models_dir.display()),
1277            );
1278        } else {
1279            print_check(label, CheckStatus::Ok, &format!("{}", models_dir.display()));
1280        }
1281
1282        if verbose {
1283            for dir in &dirs_to_create {
1284                println!("      {}", dir.display());
1285            }
1286        }
1287    }
1288
1289    // 2. Registry initialized
1290    {
1291        let label = "Registry";
1292        let git_dir = models_dir.join(".registry").join(".git");
1293
1294        if git_dir.exists() {
1295            // Verify it's a valid git repo
1296            match git2::Repository::open(models_dir.join(".registry")) {
1297                Ok(repo) => {
1298                    let count = match repo.revwalk() {
1299                        Ok(mut walk) => {
1300                            let _ = walk.push_head();
1301                            walk.count()
1302                        }
1303                        Err(_) => 0,
1304                    };
1305                    print_check(
1306                        label,
1307                        CheckStatus::Ok,
1308                        &format!(".registry initialized ({count} commits)"),
1309                    );
1310                }
1311                Err(e) => {
1312                    print_check(
1313                        label,
1314                        CheckStatus::Fail,
1315                        &format!(".registry git repo corrupt: {e}"),
1316                    );
1317                    all_passed = false;
1318                }
1319            }
1320        } else {
1321            // Initialize via Git2DB
1322            match git2db::Git2DB::open(models_dir).await {
1323                Ok(_) => {
1324                    print_check(
1325                        label,
1326                        CheckStatus::Fixed,
1327                        ".registry initialized via Git2DB",
1328                    );
1329                }
1330                Err(e) => {
1331                    // Fall back to raw git init
1332                    match git2::Repository::init(models_dir.join(".registry")) {
1333                        Ok(_) => print_check(
1334                            label,
1335                            CheckStatus::Fixed,
1336                            ".registry initialized (git init)",
1337                        ),
1338                        Err(e2) => {
1339                            print_check(
1340                                label,
1341                                CheckStatus::Fail,
1342                                &format!("failed to init: Git2DB: {e}, git init: {e2}"),
1343                            );
1344                            all_passed = false;
1345                        }
1346                    }
1347                }
1348            }
1349        }
1350    }
1351
1352    // 3. Policy files
1353    {
1354        let label = "Policy files";
1355        let policies_dir = models_dir.join(".registry").join("policies");
1356
1357        let model_conf = policies_dir.join("model.conf");
1358        let policy_csv = policies_dir.join("policy.csv");
1359
1360        if model_conf.exists() && policy_csv.exists() {
1361            print_check(label, CheckStatus::Ok, "model.conf + policy.csv present");
1362        } else {
1363            // PolicyManager::new creates defaults
1364            match PolicyManager::new(&policies_dir).await {
1365                Ok(_) => print_check(label, CheckStatus::Fixed, "created default policy files"),
1366                Err(e) => {
1367                    print_check(label, CheckStatus::Fail, &format!("failed to create: {e}"));
1368                    all_passed = false;
1369                }
1370            }
1371        }
1372    }
1373
1374    // 4. Signing key
1375    {
1376        let label = "Signing key";
1377        let keys_dir = models_dir.join(".registry").join("keys");
1378        let key_path = keys_dir.join("signing.key");
1379
1380        if key_path.exists() {
1381            match tokio::fs::read(&key_path).await {
1382                Ok(bytes) if bytes.len() == 32 => {
1383                    print_check(label, CheckStatus::Ok, "Ed25519 key loaded (32 bytes)");
1384                }
1385                Ok(bytes) => {
1386                    print_check(
1387                        label,
1388                        CheckStatus::Fail,
1389                        &format!("invalid key: {} bytes (expected 32)", bytes.len()),
1390                    );
1391                    all_passed = false;
1392                }
1393                Err(e) => {
1394                    print_check(label, CheckStatus::Fail, &format!("read error: {e}"));
1395                    all_passed = false;
1396                }
1397            }
1398        } else {
1399            match load_or_generate_signing_key(&keys_dir).await {
1400                Ok(_) => print_check(label, CheckStatus::Fixed, "generated new Ed25519 key"),
1401                Err(e) => {
1402                    print_check(
1403                        label,
1404                        CheckStatus::Fail,
1405                        &format!("failed to generate: {e}"),
1406                    );
1407                    all_passed = false;
1408                }
1409            }
1410        }
1411    }
1412
1413    // 4b. TLS materials (HTTP + QUIC) — generate into secrets dir so they are
1414    //     available for systemd-creds encryption in phase 6.
1415    {
1416        match crate::config::HyprConfig::resolve_secrets_dir() {
1417            Ok(secrets_dir) => {
1418                // HTTP TLS (365-day self-signed)
1419                match crate::auth::identity_store::load_or_generate_tls_materials(
1420                    &secrets_dir,
1421                    "localhost",
1422                    365,
1423                ) {
1424                    Ok(_) => print_check("TLS key+cert", CheckStatus::Ok, "HTTP (365d)"),
1425                    Err(e) => {
1426                        print_check("TLS key+cert", CheckStatus::Fail, &format!("{e}"));
1427                        all_passed = false;
1428                    }
1429                }
1430                // QUIC TLS (14-day per WebTransport spec)
1431                match crate::auth::identity_store::load_or_generate_tls_materials_named(
1432                    &secrets_dir,
1433                    "localhost",
1434                    14,
1435                    "quic-key",
1436                    "quic-cert",
1437                ) {
1438                    Ok(_) => print_check("QUIC key+cert", CheckStatus::Ok, "WebTransport (14d)"),
1439                    Err(e) => {
1440                        print_check("QUIC key+cert", CheckStatus::Fail, &format!("{e}"));
1441                        all_passed = false;
1442                    }
1443                }
1444            }
1445            Err(e) => {
1446                print_check("Secrets directory", CheckStatus::Fail, &format!("{e}"));
1447                all_passed = false;
1448            }
1449        }
1450    }
1451
1452    // 4c. RSA key for RS256 JWT signing (OIDC interop)
1453    {
1454        match crate::config::HyprConfig::resolve_secrets_dir() {
1455            Ok(secrets_dir) => {
1456                match crate::auth::identity_store::load_or_generate_rsa_key(&secrets_dir) {
1457                    Ok(_) => print_check("RSA key", CheckStatus::Ok, "RS256 (2048-bit)"),
1458                    Err(e) => {
1459                        // Non-fatal: EdDSA still works, RS256 is for interop
1460                        print_check("RSA key", CheckStatus::Warn, &format!("{e}"));
1461                    }
1462                }
1463            }
1464            Err(e) => {
1465                print_check("Secrets directory", CheckStatus::Warn, &format!("{e}"));
1466            }
1467        }
1468    }
1469
1470    // 5. Git config (warning only, don't modify)
1471    {
1472        let label = "Git identity";
1473        match git2::Config::open_default() {
1474            Ok(config) => {
1475                let has_name = config.get_string("user.name").is_ok();
1476                let has_email = config.get_string("user.email").is_ok();
1477
1478                if has_name && has_email {
1479                    let name = config.get_string("user.name").unwrap_or_default();
1480                    print_check(label, CheckStatus::Ok, &name);
1481                } else {
1482                    let mut missing = Vec::new();
1483                    if !has_name {
1484                        missing.push("user.name");
1485                    }
1486                    if !has_email {
1487                        missing.push("user.email");
1488                    }
1489                    let msg = format!("{} not set", missing.join(", "));
1490                    print_check(label, CheckStatus::Warn, &msg);
1491                    warnings.push(
1492                        "Set git identity: git config --global user.name \"Your Name\" && git config --global user.email \"you@example.com\"".to_owned()
1493                    );
1494                }
1495            }
1496            Err(_) => {
1497                print_check(label, CheckStatus::Warn, "could not read git config");
1498                warnings.push("Set git identity for policy versioning".to_owned());
1499            }
1500        }
1501    }
1502
1503    // 6. Service JWT presence
1504    {
1505        let credentials_dir = crate::auth::identity_store::credentials_dir()?;
1506
1507        let mut missing_jwts = Vec::new();
1508        for factory in hyprstream_service::list_factories() {
1509            let svc = factory.name;
1510            if svc == "policy" {
1511                continue; // PolicyService uses the root CA key, not a per-service JWT
1512            }
1513            match crate::auth::identity_store::load_service_jwt(&credentials_dir, svc) {
1514                Ok(Some(_)) => {}
1515                _ => missing_jwts.push(svc),
1516            }
1517        }
1518
1519        if missing_jwts.is_empty() {
1520            print_check("Service JWTs", CheckStatus::Ok, "all present");
1521        } else {
1522            print_check(
1523                "Service JWTs",
1524                CheckStatus::Warn,
1525                &format!(
1526                    "missing for: {}. Run: hyprstream wizard",
1527                    missing_jwts.join(", ")
1528                ),
1529            );
1530            warnings.push("Run 'hyprstream wizard' to generate missing service JWTs".to_owned());
1531        }
1532    }
1533
1534    // 7. Policy active
1535    {
1536        let label = "Policy active";
1537        let policies_dir = models_dir.join(".registry").join("policies");
1538        let policy_csv = policies_dir.join("policy.csv");
1539
1540        if policy_csv.exists() {
1541            match tokio::fs::read_to_string(&policy_csv).await {
1542                Ok(content) => {
1543                    let rule_count = content
1544                        .lines()
1545                        .filter(|l| l.starts_with("p,") || l.starts_with("p "))
1546                        .count();
1547
1548                    if rule_count > 0 {
1549                        // Get first subject for display
1550                        let first_subject = content
1551                            .lines()
1552                            .find(|l| l.starts_with("p,") || l.starts_with("p "))
1553                            .and_then(|l| l.split(',').nth(1))
1554                            .map(|s| s.trim().to_owned())
1555                            .unwrap_or_default();
1556                        print_check(
1557                            label,
1558                            CheckStatus::Ok,
1559                            &format!("{rule_count} allow rule(s) ({first_subject})"),
1560                        );
1561                    } else {
1562                        print_check(label, CheckStatus::Warn, "no allow rules (deny-by-default)");
1563                        warnings.push(
1564                            "Apply a template: hyprstream quick policy list-templates".to_owned(),
1565                        );
1566                    }
1567                }
1568                Err(e) => {
1569                    print_check(label, CheckStatus::Fail, &format!("read error: {e}"));
1570                    all_passed = false;
1571                }
1572            }
1573        } else {
1574            print_check(label, CheckStatus::Warn, "policy.csv not found");
1575            warnings.push(
1576                "Run 'hyprstream service install' again after fixing policy files".to_owned(),
1577            );
1578        }
1579    }
1580
1581    // 8. Bootstrap-pubkeys hybrid posture
1582    //
1583    // A stale Ed25519-only file (from a pre-hybrid provisioning run) blocks
1584    // startup: service identities can never be anchored for post-quantum
1585    // verification. Detect it here and name the recovery, rather than letting
1586    // the operator discover it from a startup refusal.
1587    {
1588        let label = "Hybrid bootstrap";
1589        match crate::config::HyprConfig::resolve_secrets_dir() {
1590            Ok(secrets_dir) => {
1591                match crate::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir) {
1592                    Ok(entries) if entries.is_empty() => {
1593                        print_check(label, CheckStatus::Ok, "unprovisioned (clean)");
1594                    }
1595                    Ok(entries) => {
1596                        match crate::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)
1597                        {
1598                            Ok(()) => {
1599                                let n = entries.len();
1600                                let noun = if n == 1 { "entry" } else { "entries" };
1601                                print_check(
1602                                    label,
1603                                    CheckStatus::Ok,
1604                                    &format!("{n} hybrid {noun} bound"),
1605                                );
1606                            }
1607                            Err(e) => {
1608                                print_check(label, CheckStatus::Fail, &format!("{e}"));
1609                                all_passed = false;
1610                                warnings.push(
1611                                    "Re-provision with 'hyprstream wizard' to bind \
1612                                     ML-DSA-65 keys for every service"
1613                                        .to_owned(),
1614                                );
1615                            }
1616                        }
1617                    }
1618                    Err(e) => {
1619                        print_check(label, CheckStatus::Fail, &format!("parse error: {e}"));
1620                        all_passed = false;
1621                        warnings.push(
1622                            "Fix the file or re-provision with 'hyprstream wizard'".to_owned(),
1623                        );
1624                    }
1625                }
1626            }
1627            Err(e) => {
1628                print_check(label, CheckStatus::Warn, &format!("secrets dir: {e}"));
1629            }
1630        }
1631    }
1632
1633    // Summary
1634    println!();
1635    if !warnings.is_empty() {
1636        println!("  Suggestions:");
1637        for w in &warnings {
1638            println!("    {w}");
1639        }
1640        println!();
1641    }
1642
1643    if all_passed && warnings.is_empty() {
1644        println!("    All checks passed.");
1645    } else if all_passed {
1646        println!("    All checks passed (with warnings).");
1647    } else {
1648        println!("    Some checks failed. Review output above.");
1649    }
1650
1651    if all_passed {
1652        Ok(())
1653    } else {
1654        Err(anyhow::anyhow!(
1655            "one or more repair checks failed; review output above"
1656        ))
1657    }
1658}
1659
1660#[allow(dead_code)]
1661pub(crate) enum CheckStatus {
1662    Ok,
1663    Fixed,
1664    Warn,
1665    Fail,
1666    Info,
1667}
1668
1669pub(crate) fn print_check(label: &str, status: CheckStatus, detail: &str) {
1670    let (icon, color) = match status {
1671        CheckStatus::Ok => ("\u{2713}", "\x1b[32m"), // green checkmark
1672        CheckStatus::Fixed => ("\u{2713}", "\x1b[33m"), // yellow checkmark (fixed)
1673        CheckStatus::Warn => ("\u{26A0}", "\x1b[33m"), // yellow warning
1674        CheckStatus::Fail => ("\u{2717}", "\x1b[31m"), // red X
1675        CheckStatus::Info => ("\u{25CB}", "\x1b[36m"), // cyan circle (informational)
1676    };
1677    println!("  {color}{icon}\x1b[0m {:<20} {detail}", label);
1678}
1679
1680// =============================================================================
1681// Version helpers
1682// =============================================================================
1683
1684/// Get the full build version string
1685///
1686/// Format: `{cargo_version}+{branch}.g{sha7}[.dirty]`
1687/// Example: `0.1.0-alpha-7+main.gabc1234.dirty`
1688///
1689/// Uses BUILD_VERSION from build.rs, falls back to CARGO_PKG_VERSION.
1690pub(crate) fn build_version() -> &'static str {
1691    option_env!("BUILD_VERSION").unwrap_or(env!("CARGO_PKG_VERSION"))
1692}
1693
1694// =============================================================================
1695// Command alias installation helpers
1696// =============================================================================
1697
1698// =============================================================================
1699// InstallPlan: unified binary installation pipeline
1700// =============================================================================
1701
1702/// Plan for installing the hyprstream binary/AppImage.
1703///
1704/// Separates detection and validation (`prepare()`) from side effects (`execute()`).
1705/// Both `service install` and the wizard share this pipeline.
1706pub(crate) struct InstallPlan {
1707    pub(crate) source: PathBuf,
1708    pub(crate) is_appimage: bool,
1709    pub(crate) source_size: u64,
1710    pub(crate) version: &'static str,
1711    pub(crate) filename: &'static str,
1712    pub(crate) version_dir: PathBuf,
1713    pub(crate) bin_dir: PathBuf,
1714    pub(crate) available_space: u64,
1715}
1716
1717/// Result of a successful `InstallPlan::execute()`.
1718pub(crate) struct InstallResult {
1719    pub(crate) bin_dir: PathBuf,
1720    pub(crate) version_dir: PathBuf,
1721    pub(crate) is_appimage: bool,
1722    pub(crate) updated_profiles: Vec<String>,
1723}
1724
1725impl InstallPlan {
1726    /// Detect source, validate it, resolve paths, check disk space.
1727    /// No side effects — safe to call and discard.
1728    pub(crate) fn prepare() -> Result<Self> {
1729        let (source, is_appimage) = binary_copy_source()?;
1730        let source_size = validate_source(&source)?;
1731
1732        let version = build_version();
1733        let filename = if is_appimage {
1734            "hyprstream.appimage"
1735        } else {
1736            "hyprstream"
1737        };
1738
1739        let bin_dir = hyprstream_rpc::paths::bin_dir()
1740            .ok_or_else(|| anyhow::anyhow!("Cannot determine user executable directory"))?;
1741        let version_dir = hyprstream_rpc::paths::version_dir(version)
1742            .ok_or_else(|| anyhow::anyhow!("Cannot determine version directory"))?;
1743
1744        let available_space = available_space(&version_dir).unwrap_or(0);
1745
1746        Ok(Self {
1747            source,
1748            is_appimage,
1749            source_size,
1750            version,
1751            filename,
1752            version_dir,
1753            bin_dir,
1754            available_space,
1755        })
1756    }
1757
1758    /// Execute the install: copy, symlink, update shell profiles.
1759    /// Consumes the plan to prevent double-execution.
1760    pub(crate) fn execute(self) -> Result<InstallResult> {
1761        std::fs::create_dir_all(&self.bin_dir)
1762            .with_context(|| format!("Failed to create directory: {}", self.bin_dir.display()))?;
1763        std::fs::create_dir_all(&self.version_dir).with_context(|| {
1764            format!("Failed to create directory: {}", self.version_dir.display())
1765        })?;
1766
1767        // Copy binary to versioned directory (skip if already in place)
1768        let versioned_binary = self.version_dir.join(self.filename);
1769        let same_file = std::fs::canonicalize(&self.source).ok()
1770            == std::fs::canonicalize(&versioned_binary).ok();
1771        if !same_file {
1772            remove_if_exists(&versioned_binary)?;
1773            std::fs::copy(&self.source, &versioned_binary).with_context(|| {
1774                format!(
1775                    "Failed to copy {} -> {}",
1776                    self.source.display(),
1777                    versioned_binary.display()
1778                )
1779            })?;
1780        }
1781
1782        #[cfg(unix)]
1783        {
1784            use std::os::unix::fs::PermissionsExt;
1785            std::fs::set_permissions(&versioned_binary, std::fs::Permissions::from_mode(0o755))?;
1786        }
1787
1788        // Create symlinks in bin_dir
1789        let bin_appimage = self.bin_dir.join("hyprstream.appimage");
1790        let bin_hyprstream = self.bin_dir.join("hyprstream");
1791
1792        // Calculate relative path from bin to versioned binary
1793        let relative_path = Path::new("..")
1794            .join("share")
1795            .join("hyprstream")
1796            .join("versions")
1797            .join(self.version)
1798            .join(self.filename);
1799
1800        remove_if_exists(&bin_appimage)?;
1801        remove_if_exists(&bin_hyprstream)?;
1802
1803        #[cfg(unix)]
1804        {
1805            std::os::unix::fs::symlink(&relative_path, &bin_appimage)
1806                .with_context(|| format!("Failed to create symlink: {}", bin_appimage.display()))?;
1807            std::os::unix::fs::symlink(Path::new("hyprstream.appimage"), &bin_hyprstream)
1808                .with_context(|| {
1809                    format!("Failed to create symlink: {}", bin_hyprstream.display())
1810                })?;
1811        }
1812
1813        let updated_profiles = if let Some(home) = dirs::home_dir() {
1814            update_shell_profiles(&home, &self.bin_dir).unwrap_or_default()
1815        } else {
1816            Vec::new()
1817        };
1818
1819        Ok(InstallResult {
1820            bin_dir: self.bin_dir,
1821            version_dir: self.version_dir,
1822            is_appimage: self.is_appimage,
1823            updated_profiles,
1824        })
1825    }
1826
1827    pub(crate) fn has_sufficient_space(&self) -> bool {
1828        self.available_space == 0 || self.available_space >= self.source_size + 1024 * 1024
1829    }
1830
1831    pub(crate) fn type_label(&self) -> &'static str {
1832        if self.is_appimage {
1833            "AppImage"
1834        } else {
1835            "binary"
1836        }
1837    }
1838}
1839
1840impl InstallResult {
1841    pub(crate) fn type_label(&self) -> &'static str {
1842        if self.is_appimage {
1843            "AppImage"
1844        } else {
1845            "binary"
1846        }
1847    }
1848}
1849
1850// =============================================================================
1851// Source detection and validation helpers
1852// =============================================================================
1853
1854/// Get the copy source: `$APPIMAGE` if set, otherwise `argv[0]`.
1855///
1856/// `$APPIMAGE` is set by the AppImage runtime and points to the stable
1857/// AppImage file (not the temporary FUSE mount). `argv[0]` preserves
1858/// what the shell resolved.
1859fn binary_copy_source() -> Result<(PathBuf, bool)> {
1860    if let Ok(appimage) = std::env::var("APPIMAGE") {
1861        let path = PathBuf::from(&appimage);
1862        if path.exists() {
1863            return Ok((path, true));
1864        }
1865    }
1866
1867    // Fall back to argv[0]
1868    let argv0 = std::env::args_os().next().context("No argv[0] available")?;
1869    let path = PathBuf::from(&argv0);
1870
1871    // Resolve relative paths against CWD
1872    let path = if path.is_relative() {
1873        std::env::current_dir()?.join(&path)
1874    } else {
1875        path
1876    };
1877
1878    let is_appimage = is_appimage_file(&path).unwrap_or(false);
1879    Ok((path, is_appimage))
1880}
1881
1882/// Validate the source file before copying:
1883/// - Must be a regular file (not symlink, directory, device, etc.)
1884/// - Must not be empty
1885/// - Must be owned by current user or root
1886fn validate_source(path: &Path) -> Result<u64> {
1887    use std::os::unix::fs::MetadataExt;
1888
1889    let meta = std::fs::symlink_metadata(path)
1890        .with_context(|| format!("Cannot stat source: {}", path.display()))?;
1891
1892    if !meta.file_type().is_file() {
1893        anyhow::bail!(
1894            "Source is not a regular file: {} (type: {:?})",
1895            path.display(),
1896            meta.file_type()
1897        );
1898    }
1899
1900    let size = meta.len();
1901    if size == 0 {
1902        anyhow::bail!("Source file is empty: {}", path.display());
1903    }
1904
1905    // Ownership: must be current user or root
1906    let file_uid = meta.uid();
1907    let my_uid = nix::unistd::getuid().as_raw();
1908    if file_uid != my_uid && file_uid != 0 {
1909        anyhow::bail!(
1910            "Source file owned by uid {} (expected {} or root): {}",
1911            file_uid,
1912            my_uid,
1913            path.display()
1914        );
1915    }
1916
1917    Ok(size)
1918}
1919
1920/// Check available disk space at the given path using `statvfs`.
1921pub(crate) fn available_space(path: &Path) -> Result<u64> {
1922    let check_path = if path.exists() {
1923        path.to_path_buf()
1924    } else {
1925        path.ancestors()
1926            .find(|p| p.exists())
1927            .unwrap_or_else(|| Path::new("/"))
1928            .to_path_buf()
1929    };
1930    let stat = nix::sys::statvfs::statvfs(&check_path)
1931        .with_context(|| format!("statvfs failed on {}", check_path.display()))?;
1932    Ok(stat.blocks_available() as u64 * stat.fragment_size() as u64)
1933}
1934
1935/// Format a byte count as a human-readable string (powers of 1024, matching `df -h`).
1936pub(crate) fn format_size(bytes: u64) -> String {
1937    const KIB: u64 = 1024;
1938    const MIB: u64 = KIB * 1024;
1939    const GIB: u64 = MIB * 1024;
1940
1941    if bytes >= GIB {
1942        let val = bytes as f64 / GIB as f64;
1943        if val >= 100.0 {
1944            format!("{:.0} GB", val)
1945        } else {
1946            format!("{:.1} GB", val)
1947        }
1948    } else if bytes >= MIB {
1949        let val = bytes as f64 / MIB as f64;
1950        if val >= 100.0 {
1951            format!("{:.0} MB", val)
1952        } else {
1953            format!("{:.1} MB", val)
1954        }
1955    } else if bytes >= KIB {
1956        format!("{:.0} KB", bytes as f64 / KIB as f64)
1957    } else {
1958        format!("{} B", bytes)
1959    }
1960}
1961
1962/// Check if the running binary is already in a known installed location.
1963///
1964/// Uses `current_exe()` (reads `/proc/self/exe` on Linux — kernel-maintained,
1965/// cannot be spoofed) to determine the real binary path, then checks if it
1966/// lives under any of the standard install locations.
1967pub(crate) fn is_binary_installed() -> Option<PathBuf> {
1968    let exe = std::env::current_exe().ok()?;
1969    let canonical = exe.canonicalize().ok()?;
1970
1971    // Check 1: Under the XDG data dir version store
1972    if let Some(versions_dir) = hyprstream_rpc::paths::versions_dir() {
1973        if let Ok(versions_canonical) = versions_dir.canonicalize() {
1974            if canonical.starts_with(&versions_canonical) {
1975                return Some(canonical);
1976            }
1977        }
1978    }
1979
1980    // Check 2: Under the XDG executable dir
1981    if let Some(bin_dir) = dirs::executable_dir() {
1982        if let Ok(bin_canonical) = bin_dir.canonicalize() {
1983            if canonical.starts_with(&bin_canonical) {
1984                return Some(canonical);
1985            }
1986        }
1987    }
1988
1989    // Check 3: ~/bin (traditional Unix)
1990    if let Some(home) = dirs::home_dir() {
1991        let home_bin = home.join("bin");
1992        if let Ok(home_bin_canonical) = home_bin.canonicalize() {
1993            if canonical.starts_with(&home_bin_canonical) {
1994                return Some(canonical);
1995            }
1996        }
1997
1998        // Check 4: ~/Applications (AppImage community convention)
1999        let applications = home.join("Applications");
2000        if let Ok(app_canonical) = applications.canonicalize() {
2001            if canonical.starts_with(&app_canonical) {
2002                return Some(canonical);
2003            }
2004        }
2005    }
2006
2007    // Check 5: Inode match against any PATH entry
2008    if let Ok(exe_meta) = std::fs::metadata(&canonical) {
2009        use std::os::unix::fs::MetadataExt;
2010        let exe_dev = exe_meta.dev();
2011        let exe_ino = exe_meta.ino();
2012
2013        if let Ok(path_var) = std::env::var("PATH") {
2014            for dir in path_var.split(':') {
2015                if dir.is_empty() || !Path::new(dir).is_absolute() {
2016                    continue;
2017                }
2018                let candidate = Path::new(dir).join("hyprstream");
2019                if let Ok(meta) = std::fs::metadata(&candidate) {
2020                    if meta.dev() == exe_dev && meta.ino() == exe_ino {
2021                        return Some(canonical);
2022                    }
2023                }
2024            }
2025        }
2026    }
2027
2028    None
2029}
2030
2031/// Check if a file is an AppImage by reading its magic bytes
2032///
2033/// AppImage Type 2 has:
2034/// - ELF magic at offset 0: 0x7f 'E' 'L' 'F'
2035/// - AppImage magic at offset 8: 'A' 'I' 0x02
2036pub(crate) fn is_appimage_file(path: &Path) -> Result<bool> {
2037    use std::io::Read;
2038
2039    let mut file =
2040        std::fs::File::open(path).with_context(|| format!("Failed to open: {}", path.display()))?;
2041
2042    let mut header = [0u8; 11];
2043    if file.read_exact(&mut header).is_err() {
2044        return Ok(false);
2045    }
2046
2047    // Check ELF magic
2048    let is_elf = header[0..4] == [0x7f, b'E', b'L', b'F'];
2049
2050    // Check AppImage Type 2 magic at offset 8
2051    let is_appimage = header[8..11] == [b'A', b'I', 0x02];
2052
2053    Ok(is_elf && is_appimage)
2054}
2055
2056/// Remove a file if it exists (handles both regular files and symlinks)
2057fn remove_if_exists(path: &Path) -> Result<()> {
2058    if path.symlink_metadata().is_ok() {
2059        std::fs::remove_file(path)
2060            .with_context(|| format!("Failed to remove: {}", path.display()))?;
2061    }
2062    Ok(())
2063}
2064
2065/// Handle `--print-cert-hash` — output the SHA-256 hash of the QUIC certificate.
2066///
2067/// Loads or generates the TLS certificate from QuicConfig and prints
2068/// the base64-encoded SHA-256 hash, suitable for use in the browser's
2069/// `serverCertificateHashes` WebTransport option.
2070pub fn handle_print_cert_hash(quic_config: &crate::config::QuicConfig) -> Result<()> {
2071    let (cert_chain, _key_der) = quic_config
2072        .load_tls_materials()
2073        .context("Failed to load/generate QUIC TLS certificate")?;
2074
2075    let hash = hyprstream_rpc::transport::zmtp_quic::cert_hash(&cert_chain[0]);
2076    println!("{}", hash);
2077    Ok(())
2078}
2079
2080/// Handle `service ensure-key` — materialize a service's signing key and its
2081/// public sidecars without starting any services.
2082///
2083/// Runs the same loader the service itself uses (`resolve_service_signing_key`,
2084/// so `policy` resolves to the flat node/CA key), then guarantees the public
2085/// sidecars exist next to the seed:
2086///
2087/// - `signing-key.pub` — 32-byte Ed25519 verifying key (mode 0644)
2088/// - `service-pubkey.hybrid` — 1984-byte hybrid bootstrap entry (mode 0644)
2089///
2090/// Idempotent: an existing key is loaded, never rotated, and up-to-date
2091/// sidecars are left untouched. Prints the base64 public key and the sidecar
2092/// paths for provisioning units that wire them into a credential mint.
2093pub fn handle_service_ensure_key(
2094    config: Option<&crate::config::HyprConfig>,
2095    name: &str,
2096) -> Result<()> {
2097    use crate::auth::identity_store;
2098    use base64::Engine as _;
2099
2100    identity_store::validate_service_name(name)?;
2101    let secrets_dir = identity_store::credentials_dir_for_config(config)?;
2102    let profile = identity_store::SecretsProfile::from_env()?;
2103    let key = identity_store::resolve_service_signing_key(&secrets_dir, name, profile)?;
2104    let key_dir = identity_store::service_signing_key_dir(&secrets_dir, name, profile);
2105    identity_store::ensure_service_key_sidecars(&key_dir, &key)?;
2106
2107    let pubkey_b64 =
2108        base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(key.verifying_key().as_bytes());
2109    println!("service '{name}' signing key ensured");
2110    println!("  public key (base64): {pubkey_b64}");
2111    println!(
2112        "  {}",
2113        key_dir.join(identity_store::SIGNING_KEY_PUB_NAME).display()
2114    );
2115    println!(
2116        "  {}",
2117        key_dir
2118            .join(identity_store::SERVICE_PUBKEY_HYBRID_NAME)
2119            .display()
2120    );
2121    Ok(())
2122}
2123
2124/// Update shell profiles to include bin_dir in PATH
2125fn update_shell_profiles(home: &Path, bin_dir: &Path) -> Result<Vec<String>> {
2126    let path_line = format!(r#"export PATH="{}:$PATH""#, bin_dir.display());
2127    let bin_dir_str = bin_dir.to_string_lossy();
2128    let mut updated = Vec::new();
2129
2130    for profile in &[".bashrc", ".zshrc", ".profile"] {
2131        let profile_path = home.join(profile);
2132        if profile_path.exists() {
2133            let content = std::fs::read_to_string(&profile_path)
2134                .with_context(|| format!("Failed to read: {}", profile_path.display()))?;
2135            // Check if bin_dir already in PATH
2136            if !content.contains(bin_dir_str.as_ref()) {
2137                let mut file = std::fs::OpenOptions::new()
2138                    .append(true)
2139                    .open(&profile_path)
2140                    .with_context(|| format!("Failed to open: {}", profile_path.display()))?;
2141                writeln!(file, "\n# Added by hyprstream\n{}", path_line)?;
2142                updated.push((*profile).to_owned());
2143            }
2144        }
2145    }
2146
2147    Ok(updated)
2148}
2149
2150#[cfg(test)]
2151#[allow(clippy::expect_used)]
2152mod offline_policy_provision_tests {
2153    use super::*;
2154    use crate::auth::{get_template, PolicyManager};
2155
2156    fn public_templates() -> Vec<String> {
2157        vec!["public-inference".to_owned(), "public-read".to_owned()]
2158    }
2159
2160    #[tokio::test]
2161    async fn staging_e2e_template_grants_only_the_test_did_and_pinned_resources() {
2162        assert_staging_owner_template(
2163            "cyberdione-staging-e2e",
2164            "did:plc:3f7ldey5bivhfjapwqi2sljy",
2165            "did:plc:rjqczurevrpoz4mcnax2kgw4",
2166        )
2167        .await;
2168    }
2169
2170    #[tokio::test]
2171    async fn staging_operator_template_grants_only_the_operator_did_and_pinned_resources() {
2172        assert_staging_owner_template(
2173            "cyberdione-staging-operator",
2174            "did:plc:rjqczurevrpoz4mcnax2kgw4",
2175            "did:plc:3f7ldey5bivhfjapwqi2sljy",
2176        )
2177        .await;
2178    }
2179
2180    async fn assert_staging_owner_template(template: &str, did: &str, other_owner: &str) {
2181        let root = tempfile::tempdir().expect("temporary models root");
2182        let templates = vec![template.to_owned()];
2183        handle_service_provision_policy_templates(root.path(), &templates)
2184            .await
2185            .expect("provision exact staging grant");
2186        let manager = PolicyManager::new(root.path().join(".registry/policies"))
2187            .await
2188            .expect("reopen policy store");
2189        for (resource, action) in [
2190            ("registry:qwen2.5-0.5b-instruct", "query"),
2191            ("model:Load", "write"),
2192            ("model:Status", "query"),
2193            ("model:qwen2.5-0.5b-instruct:main", "infer"),
2194        ] {
2195            for denied in [other_owner, "anonymous", "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"] {
2196                assert!(
2197                    !manager
2198                        .check_with_domain(denied, "local", resource, action)
2199                        .await,
2200                    "owner template must not grant {denied} {resource} {action}",
2201                );
2202            }
2203            assert!(
2204                manager
2205                    .check_with_domain(did, "local", resource, action)
2206                    .await
2207            );
2208            assert!(
2209                !manager
2210                    .check_with_domain(did, "other", resource, action)
2211                    .await
2212            );
2213            assert!(
2214                !manager
2215                    .check_with_domain(
2216                        "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa",
2217                        "local",
2218                        resource,
2219                        action
2220                    )
2221                    .await
2222            );
2223        }
2224        assert!(
2225            !manager
2226                .check_with_domain(did, "local", "registry:other", "query")
2227                .await
2228        );
2229        assert!(
2230            !manager
2231                .check_with_domain(did, "local", "model:qwen2.5-0.5b-instruct:other", "infer")
2232                .await
2233        );
2234    }
2235
2236    #[tokio::test]
2237    async fn fresh_and_repeat_public_staging_policy_is_canonical() {
2238        let root = tempfile::tempdir().expect("temporary models root");
2239        handle_service_provision_policy_templates(root.path(), &public_templates())
2240            .await
2241            .expect("fresh provision");
2242        let policy_path = root.path().join(".registry/policies/policy.csv");
2243        let first = tokio::fs::read(&policy_path).await.expect("first policy");
2244
2245        handle_service_provision_policy_templates(root.path(), &public_templates())
2246            .await
2247            .expect("repeat provision");
2248        let second = tokio::fs::read(&policy_path)
2249            .await
2250            .expect("repeated policy");
2251        assert_eq!(first, second, "repeat must leave serialized policy stable");
2252
2253        let manager = PolicyManager::new(root.path().join(".registry/policies"))
2254            .await
2255            .expect("reopen policy store");
2256        let policies = manager.get_policy().await;
2257        let expected = public_templates()
2258            .iter()
2259            .flat_map(|name| get_template(name).expect("template").expanded_policies())
2260            .map(|rule| rule.to_vec())
2261            .collect::<Vec<_>>();
2262        assert_eq!(expected.len(), 3);
2263        for rule in expected {
2264            assert_eq!(
2265                policies
2266                    .iter()
2267                    .filter(|candidate| candidate.as_slice() == rule.as_slice())
2268                    .count(),
2269                1
2270            );
2271        }
2272        assert!(
2273            !manager
2274                .check_with_domain(
2275                    "anonymous",
2276                    "*",
2277                    "model:policy-bootstrap-probe",
2278                    "ttt.writeback",
2279                )
2280                .await
2281        );
2282    }
2283
2284    #[tokio::test]
2285    async fn retained_partial_template_converges() {
2286        let root = tempfile::tempdir().expect("temporary models root");
2287        let policies_dir = root.path().join(".registry/policies");
2288        let manager = PolicyManager::new(&policies_dir)
2289            .await
2290            .expect("policy manager");
2291        manager
2292            .add_policy_with_domain("anonymous", "*", "model:*", "infer.generate", "allow")
2293            .await
2294            .expect("partial rule");
2295        manager.save().await.expect("save partial state");
2296        drop(manager);
2297
2298        handle_service_provision_policy_templates(root.path(), &public_templates())
2299            .await
2300            .expect("partial state must converge");
2301        let verified = PolicyManager::new(&policies_dir)
2302            .await
2303            .expect("reopen converged policy");
2304        assert!(
2305            verified
2306                .check_with_domain(
2307                    "anonymous",
2308                    "*",
2309                    "model:policy-bootstrap-probe",
2310                    "query.status",
2311                )
2312                .await
2313        );
2314        assert!(
2315            verified
2316                .check_with_domain(
2317                    "anonymous",
2318                    "*",
2319                    "registry:policy-bootstrap-probe",
2320                    "query.status",
2321                )
2322                .await
2323        );
2324    }
2325
2326    #[tokio::test]
2327    async fn retained_partial_domain_grouping_converges() {
2328        let root = tempfile::tempdir().expect("temporary models root");
2329        let policies_dir = root.path().join(".registry/policies");
2330        let manager = PolicyManager::new(&policies_dir)
2331            .await
2332            .expect("policy manager");
2333        manager
2334            .add_role_for_user_in_domain("service:inference:host-1", "mesh-readers", "acme")
2335            .await
2336            .expect("partial domain grouping");
2337        manager.save().await.expect("save partial grouping");
2338        drop(manager);
2339
2340        handle_service_provision_policy_templates(root.path(), &["mesh-host-group".to_owned()])
2341            .await
2342            .expect("partial grouping must converge");
2343        let verified = PolicyManager::new(&policies_dir)
2344            .await
2345            .expect("reopen grouping policy");
2346        let groupings = verified.get_domain_grouping_policy().await;
2347        for host in ["service:inference:host-1", "service:inference:host-2"] {
2348            assert!(groupings.contains(&vec![
2349                host.to_owned(),
2350                "mesh-readers".to_owned(),
2351                "acme".to_owned(),
2352            ]));
2353        }
2354    }
2355
2356    #[tokio::test]
2357    async fn invalid_requests_fail_before_storage_mutation() {
2358        for templates in [
2359            vec!["not-a-template".to_owned()],
2360            vec!["public-read".to_owned(), "public-read".to_owned()],
2361        ] {
2362            let root = tempfile::tempdir().expect("temporary models root");
2363            assert!(
2364                handle_service_provision_policy_templates(root.path(), &templates)
2365                    .await
2366                    .is_err()
2367            );
2368            assert!(!root.path().join(".registry").exists());
2369        }
2370    }
2371
2372    #[tokio::test]
2373    async fn malformed_or_explicitly_denied_policy_fails_closed() {
2374        let malformed_root = tempfile::tempdir().expect("malformed models root");
2375        let malformed_dir = malformed_root.path().join(".registry/policies");
2376        PolicyManager::new(&malformed_dir)
2377            .await
2378            .expect("initialize policy");
2379        let malformed_path = malformed_dir.join("policy.csv");
2380        let malformed = b"p, malformed\n";
2381        tokio::fs::write(&malformed_path, malformed)
2382            .await
2383            .expect("write malformed policy");
2384        assert!(handle_service_provision_policy_templates(
2385            malformed_root.path(),
2386            &public_templates(),
2387        )
2388        .await
2389        .is_err());
2390        assert_eq!(
2391            tokio::fs::read(&malformed_path)
2392                .await
2393                .expect("read malformed"),
2394            malformed
2395        );
2396
2397        let denied_root = tempfile::tempdir().expect("denied models root");
2398        let denied_dir = denied_root.path().join(".registry/policies");
2399        let denied = PolicyManager::new(&denied_dir)
2400            .await
2401            .expect("initialize denied policy");
2402        denied
2403            .add_policy_with_domain("anonymous", "*", "model:*", "infer.generate", "deny")
2404            .await
2405            .expect("add explicit deny");
2406        denied.save().await.expect("persist explicit deny");
2407        drop(denied);
2408        let denied_path = denied_dir.join("policy.csv");
2409        let denied_original = tokio::fs::read(&denied_path)
2410            .await
2411            .expect("retained anonymous deny");
2412        assert!(
2413            handle_service_provision_policy_templates(denied_root.path(), &public_templates(),)
2414                .await
2415                .is_err(),
2416            "explicit deny must block verified staging readiness"
2417        );
2418        assert_eq!(
2419            tokio::fs::read(&denied_path)
2420                .await
2421                .expect("anonymous deny after refused retry"),
2422            denied_original
2423        );
2424        let reopened = PolicyManager::new(&denied_dir)
2425            .await
2426            .expect("reopen denied policy");
2427        assert!(
2428            !reopened
2429                .check_with_domain(
2430                    "anonymous",
2431                    "*",
2432                    "model:policy-bootstrap-probe",
2433                    "infer.generate",
2434                )
2435                .await
2436        );
2437    }
2438
2439    #[tokio::test]
2440    async fn prepublication_failure_and_retry_preserve_retained_deny() {
2441        let root = tempfile::tempdir().expect("retained-policy models root");
2442        let policies_dir = root.path().join(".registry/policies");
2443        let denied = PolicyManager::new(&policies_dir)
2444            .await
2445            .expect("initialize policy");
2446        let deny = vec![
2447            "service:retained".to_owned(),
2448            "*".to_owned(),
2449            "model:*".to_owned(),
2450            "ttt.writeback".to_owned(),
2451            "deny".to_owned(),
2452        ];
2453        denied
2454            .add_policy_with_domain("service:retained", "*", "model:*", "ttt.writeback", "deny")
2455            .await
2456            .expect("add retained deny");
2457        denied.save().await.expect("persist retained deny");
2458        drop(denied);
2459        let policy_path = policies_dir.join("policy.csv");
2460        let original = tokio::fs::read(&policy_path)
2461            .await
2462            .expect("retained policy");
2463        assert!(!original.is_empty());
2464
2465        let mut fault_reached = false;
2466        assert!(provision_policy_templates(
2467            root.path(),
2468            &public_templates(),
2469            |_| {
2470                fault_reached = true;
2471                anyhow::bail!("injected failure after staged write before publication")
2472            },
2473            |_| Ok(()),
2474            |_| Ok(()),
2475        )
2476        .await
2477        .is_err());
2478        assert!(fault_reached, "prepublication fault seam must be reached");
2479        assert_eq!(
2480            tokio::fs::read(&policy_path)
2481                .await
2482                .expect("policy after injected failure"),
2483            original
2484        );
2485
2486        let retained = PolicyManager::new(&policies_dir)
2487            .await
2488            .expect("reopen retained policy");
2489        assert!(retained.get_policy().await.contains(&deny));
2490        drop(retained);
2491
2492        handle_service_provision_policy_templates(root.path(), &public_templates())
2493            .await
2494            .expect("retry must converge");
2495        let after_retry = PolicyManager::new(&policies_dir)
2496            .await
2497            .expect("reopen retry policy");
2498        assert!(after_retry.get_policy().await.contains(&deny));
2499    }
2500
2501    #[tokio::test]
2502    async fn atomic_policy_publication_failure_preserves_retained_deny_and_retry() {
2503        let root = tempfile::tempdir().expect("retained-policy models root");
2504        let policies_dir = root.path().join(".registry/policies");
2505        let retained = PolicyManager::new(&policies_dir)
2506            .await
2507            .expect("initialize policy");
2508        let deny = vec![
2509            "service:retained".to_owned(),
2510            "*".to_owned(),
2511            "model:*".to_owned(),
2512            "ttt.writeback".to_owned(),
2513            "deny".to_owned(),
2514        ];
2515        retained
2516            .add_policy_with_domain("service:retained", "*", "model:*", "ttt.writeback", "deny")
2517            .await
2518            .expect("add retained deny");
2519        retained.save().await.expect("persist retained deny");
2520        drop(retained);
2521        let policy_path = policies_dir.join("policy.csv");
2522        let original = tokio::fs::read(&policy_path)
2523            .await
2524            .expect("retained policy bytes");
2525
2526        let mut wrote_model_temp = false;
2527        let mut failed_policy_write = false;
2528        let result = provision_policy_templates(
2529            root.path(),
2530            &public_templates(),
2531            |_| Ok(()),
2532            |_| Ok(()),
2533            |destination| {
2534                if destination
2535                    .file_name()
2536                    .is_some_and(|name| name == "model.conf")
2537                {
2538                    wrote_model_temp = true;
2539                } else if destination
2540                    .file_name()
2541                    .is_some_and(|name| name == "policy.csv")
2542                {
2543                    failed_policy_write = true;
2544                    anyhow::bail!("injected failure after policy temp-file write")
2545                }
2546                Ok(())
2547            },
2548        )
2549        .await;
2550        assert!(result.is_err());
2551        assert!(wrote_model_temp, "model publication must precede policy");
2552        assert!(failed_policy_write, "fault must reach policy publication");
2553        assert_eq!(
2554            tokio::fs::read(&policy_path)
2555                .await
2556                .expect("policy after publication failure"),
2557            original
2558        );
2559        let after_failure = PolicyManager::new(&policies_dir)
2560            .await
2561            .expect("reopen policy after failure");
2562        assert!(after_failure.get_policy().await.contains(&deny));
2563        drop(after_failure);
2564
2565        handle_service_provision_policy_templates(root.path(), &public_templates())
2566            .await
2567            .expect("retry converges without losing retained deny");
2568        let after_retry = PolicyManager::new(&policies_dir)
2569            .await
2570            .expect("reopen policy after retry");
2571        assert!(after_retry.get_policy().await.contains(&deny));
2572        assert!(
2573            !after_retry
2574                .check_with_domain(
2575                    "service:retained",
2576                    "*",
2577                    "model:policy-bootstrap-probe",
2578                    "ttt.writeback",
2579                )
2580                .await
2581        );
2582    }
2583
2584    #[tokio::test]
2585    async fn incomplete_staged_snapshot_cannot_drop_retained_deny() {
2586        let root = tempfile::tempdir().expect("incomplete-snapshot models root");
2587        let policies_dir = root.path().join(".registry/policies");
2588        let retained = PolicyManager::new(&policies_dir)
2589            .await
2590            .expect("initialize policy");
2591        retained
2592            .add_policy_with_domain("service:retained", "*", "model:*", "ttt.writeback", "deny")
2593            .await
2594            .expect("add retained deny");
2595        retained.save().await.expect("persist retained deny");
2596        drop(retained);
2597        let policy_path = policies_dir.join("policy.csv");
2598        let original = tokio::fs::read(&policy_path)
2599            .await
2600            .expect("retained policy bytes");
2601
2602        let result = provision_policy_templates(
2603            root.path(),
2604            &public_templates(),
2605            |staged_path| {
2606                let staged = std::fs::read_to_string(staged_path)?;
2607                for name in public_templates() {
2608                    let template = get_template(&name).expect("public template");
2609                    for rule in template.expanded_policies() {
2610                        assert!(staged.contains(&format!("p, {}", rule.to_vec().join(", "))));
2611                    }
2612                }
2613                let mut removed = 0;
2614                let incomplete = staged
2615                    .lines()
2616                    .filter(|line| {
2617                        let keep = !line.contains("service:retained")
2618                            || !line.contains("ttt.writeback")
2619                            || !line.ends_with("deny");
2620                        if !keep {
2621                            removed += 1;
2622                        }
2623                        keep
2624                    })
2625                    .collect::<Vec<_>>()
2626                    .join("\n")
2627                    + "\n";
2628                assert_eq!(removed, 1, "fault must omit exactly the retained DENY");
2629                std::fs::write(staged_path, incomplete)?;
2630                Ok(())
2631            },
2632            |_| Ok(()),
2633            |_| Ok(()),
2634        )
2635        .await;
2636        assert!(result
2637            .expect_err("incomplete snapshot must be rejected")
2638            .to_string()
2639            .contains("complete intended policy state"));
2640        assert_eq!(
2641            tokio::fs::read(&policy_path)
2642                .await
2643                .expect("live policy after rejected snapshot"),
2644            original
2645        );
2646    }
2647
2648    #[tokio::test]
2649    async fn mutation_after_snapshot_cannot_change_published_policy() {
2650        let root = tempfile::tempdir().expect("post-snapshot-mutation models root");
2651        let mut captured = None;
2652        provision_policy_templates(
2653            root.path(),
2654            &public_templates(),
2655            |_| Ok(()),
2656            |staged_path| {
2657                captured = Some(std::fs::read(staged_path)?);
2658                std::fs::write(staged_path, b"p, malformed\n")?;
2659                Ok(())
2660            },
2661            |_| Ok(()),
2662        )
2663        .await
2664        .expect("immutable captured policy must publish successfully");
2665        assert_eq!(
2666            tokio::fs::read(root.path().join(".registry/policies/policy.csv"))
2667                .await
2668                .expect("published policy"),
2669            captured.expect("snapshot callback captured bytes")
2670        );
2671    }
2672
2673    #[cfg(unix)]
2674    #[tokio::test]
2675    async fn unreadable_retained_policy_is_not_replaced_with_defaults() {
2676        use std::os::unix::fs::PermissionsExt as _;
2677
2678        let root = tempfile::tempdir().expect("unreadable-policy models root");
2679        let policies_dir = root.path().join(".registry/policies");
2680        PolicyManager::new(&policies_dir)
2681            .await
2682            .expect("initialize policy");
2683        let policy_path = policies_dir.join("policy.csv");
2684        let original = tokio::fs::read(&policy_path)
2685            .await
2686            .expect("retained policy bytes");
2687        tokio::fs::set_permissions(&policy_path, std::fs::Permissions::from_mode(0o000))
2688            .await
2689            .expect("make retained policy unreadable");
2690
2691        assert!(
2692            handle_service_provision_policy_templates(root.path(), &public_templates())
2693                .await
2694                .is_err()
2695        );
2696        tokio::fs::set_permissions(&policy_path, std::fs::Permissions::from_mode(0o640))
2697            .await
2698            .expect("restore retained policy permissions");
2699        assert_eq!(
2700            tokio::fs::read(&policy_path)
2701                .await
2702                .expect("policy after read failure"),
2703            original
2704        );
2705    }
2706}
2707
2708#[cfg(test)]
2709mod launcher_tests {
2710    #![allow(clippy::expect_used, clippy::unwrap_used)]
2711
2712    use super::*;
2713    use hyprstream_service::ProcessReadiness;
2714
2715    #[test]
2716    fn direct_child_process_config_builds_profile_specific_invocation() -> anyhow::Result<()> {
2717        let root = tempfile::tempdir()?;
2718        // A relative-looking config directory containing spaces: the launcher
2719        // hands the child one canonical absolute argv element that keeps the
2720        // spaces (no shell splitting, no quoting).
2721        let config_path = root.path().join("my configs/custom.toml");
2722        std::fs::create_dir_all(config_path.parent().expect("parent"))?;
2723        std::fs::write(&config_path, "[secrets]\n")?;
2724        let canonical = std::fs::canonicalize(&config_path)?;
2725
2726        // Required-native: no --ipc, config forwarded, notification readiness.
2727        let required = direct_child_process_config(
2728            "model",
2729            true,
2730            Some(&canonical),
2731            Path::new("/usr/local/bin/hyprstream"),
2732        )?;
2733        let expected_tail = [
2734            OsString::from("service"),
2735            OsString::from("start"),
2736            OsString::from("model"),
2737            OsString::from("--foreground"),
2738        ];
2739        assert_eq!(&required.args[required.args.len() - 4..], &expected_tail);
2740        assert!(
2741            !required.args.iter().any(|arg| arg == "--ipc"),
2742            "required-native child must not be forced onto the local IPC endpoint"
2743        );
2744        assert!(
2745            required
2746                .args
2747                .windows(2)
2748                .any(|pair| pair[0] == "--config" && pair[1] == canonical.to_str().expect("utf-8")),
2749            "canonical config path (spaces intact) must be forwarded as one argv element"
2750        );
2751        assert!(matches!(
2752            required.readiness,
2753            ProcessReadiness::Notify { .. }
2754        ));
2755
2756        // Compatibility keeps the historical shape and immediate reporting.
2757        let compat_with_config = direct_child_process_config(
2758            "registry",
2759            false,
2760            Some(&canonical),
2761            Path::new("/usr/local/bin/hyprstream"),
2762        )?;
2763        assert_eq!(
2764            &compat_with_config.args[compat_with_config.args.len() - 5..],
2765            &[
2766                OsString::from("service"),
2767                OsString::from("start"),
2768                OsString::from("registry"),
2769                OsString::from("--foreground"),
2770                OsString::from("--ipc"),
2771            ]
2772        );
2773        assert_eq!(compat_with_config.readiness, ProcessReadiness::Immediate);
2774
2775        // No explicit selector: argv identical to the legacy launcher.
2776        let compat_default =
2777            direct_child_process_config("policy", false, None, Path::new("/bin/hyprstream"))?;
2778        assert_eq!(
2779            compat_default.args,
2780            [
2781                OsString::from("service"),
2782                OsString::from("start"),
2783                OsString::from("policy"),
2784                OsString::from("--foreground"),
2785                OsString::from("--ipc"),
2786            ]
2787        );
2788        Ok(())
2789    }
2790
2791    #[cfg(unix)]
2792    #[test]
2793    fn direct_child_preserves_non_utf8_explicit_config_path() -> anyhow::Result<()> {
2794        use std::os::unix::ffi::{OsStrExt, OsStringExt};
2795
2796        let root = tempfile::tempdir()?;
2797        let config_path = root
2798            .path()
2799            .join(OsString::from_vec(b"custom-\xff.toml".to_vec()));
2800        std::fs::write(&config_path, "[secrets]\n")?;
2801        let canonical = std::fs::canonicalize(&config_path)?;
2802        assert!(
2803            canonical.to_str().is_none(),
2804            "fixture path must be non-UTF-8"
2805        );
2806
2807        let loaded = crate::config::HyprConfig::from_file(&canonical)?;
2808        loaded.validate()?;
2809
2810        for (iroh_required, expects_ipc) in [(true, false), (false, true)] {
2811            let plan = direct_child_process_config(
2812                "model",
2813                iroh_required,
2814                Some(&canonical),
2815                Path::new("/usr/local/bin/hyprstream"),
2816            )?;
2817            let selectors: Vec<_> = plan
2818                .args
2819                .windows(2)
2820                .filter(|pair| pair[0] == "--config")
2821                .collect();
2822            assert_eq!(
2823                selectors.len(),
2824                1,
2825                "one explicit selector must be forwarded"
2826            );
2827            assert_eq!(
2828                selectors[0][1].as_os_str().as_bytes(),
2829                canonical.as_os_str().as_bytes(),
2830                "the canonical selector must be preserved byte for byte"
2831            );
2832            assert_eq!(
2833                plan.args.iter().any(|arg| arg == "--ipc"),
2834                expects_ipc,
2835                "Required and Compatibility invocation shapes must remain distinct"
2836            );
2837            assert_eq!(
2838                matches!(plan.readiness, ProcessReadiness::Notify { .. }),
2839                iroh_required,
2840                "Required and Compatibility readiness policies must remain distinct"
2841            );
2842        }
2843        Ok(())
2844    }
2845
2846    #[test]
2847    fn known_required_roster_retains_dependency_order() {
2848        let targets = ["model", "registry", "policy", "discovery"];
2849        for service in targets {
2850            assert!(
2851                hyprstream_service::get_factory(service).is_some(),
2852                "test roster service {service} must be compiled"
2853            );
2854        }
2855        assert_eq!(
2856            hyprstream_service::startup_stages_for_profile(&targets, true),
2857            vec![
2858                vec!["discovery".to_owned()],
2859                vec!["policy".to_owned()],
2860                vec!["registry".to_owned()],
2861                vec!["model".to_owned()],
2862            ]
2863        );
2864    }
2865
2866    #[cfg(unix)]
2867    #[tokio::test]
2868    async fn unknown_required_roster_is_rejected_before_any_child_spawns() -> anyhow::Result<()> {
2869        use std::os::unix::fs::PermissionsExt;
2870
2871        let root = tempfile::tempdir()?;
2872        let executable = root.path().join("spawn-sentinel.sh");
2873        let marker = root.path().join("spawn-sentinel.sh.spawned");
2874        std::fs::write(&executable, b"#!/bin/sh\ntouch \"$0.spawned\"\n")?;
2875        let mut permissions = std::fs::metadata(&executable)?.permissions();
2876        permissions.set_mode(0o700);
2877        std::fs::set_permissions(&executable, permissions)?;
2878
2879        let unknown = "not-a-compiled-native-service";
2880        let targets = vec!["policy".to_owned(), unknown.to_owned()];
2881        let spawner = hyprstream_service::ProcessSpawner::standalone();
2882        let error = launch_direct_children(&targets, true, None, &executable, &spawner)
2883            .await
2884            .expect_err("an unknown Required target must reject the whole plan");
2885        assert_eq!(
2886            error.to_string(),
2887            format!("unknown native service: {unknown}")
2888        );
2889        assert!(
2890            !marker.exists(),
2891            "validation must finish before any configured executable runs"
2892        );
2893        Ok(())
2894    }
2895
2896    /// Mock manager: scripted start/is_active behavior for the permitted
2897    /// Compatibility unit-start lifecycle contract.
2898    struct MockManager {
2899        start_error: Option<&'static str>,
2900        active_after: std::sync::atomic::AtomicU32,
2901        calls: parking_lot::Mutex<Vec<String>>,
2902    }
2903
2904    impl MockManager {
2905        fn failing() -> Self {
2906            Self {
2907                start_error: Some("unit is masked"),
2908                active_after: std::sync::atomic::AtomicU32::new(0),
2909                calls: parking_lot::Mutex::new(Vec::new()),
2910            }
2911        }
2912        fn never_active() -> Self {
2913            Self {
2914                start_error: None,
2915                active_after: std::sync::atomic::AtomicU32::new(u32::MAX),
2916                calls: parking_lot::Mutex::new(Vec::new()),
2917            }
2918        }
2919        fn immediate() -> Self {
2920            Self {
2921                start_error: None,
2922                active_after: std::sync::atomic::AtomicU32::new(0),
2923                calls: parking_lot::Mutex::new(Vec::new()),
2924            }
2925        }
2926    }
2927
2928    #[async_trait::async_trait]
2929    impl hyprstream_service::ServiceManager for MockManager {
2930        async fn install(&self, _service: &str) -> anyhow::Result<()> {
2931            Ok(())
2932        }
2933        async fn uninstall(&self, _service: &str) -> anyhow::Result<()> {
2934            Ok(())
2935        }
2936        async fn start(&self, service: &str) -> anyhow::Result<()> {
2937            self.calls.lock().push(format!("start:{service}"));
2938            match self.start_error {
2939                Some(reason) => anyhow::bail!("{reason}"),
2940                None => Ok(()),
2941            }
2942        }
2943        async fn stop(&self, _service: &str) -> anyhow::Result<()> {
2944            Ok(())
2945        }
2946        async fn is_active(&self, service: &str) -> anyhow::Result<bool> {
2947            self.calls.lock().push(format!("active:{service}"));
2948            use std::sync::atomic::Ordering;
2949            let remaining = self.active_after.load(Ordering::SeqCst);
2950            if remaining > 0 && remaining != u32::MAX {
2951                self.active_after.store(remaining - 1, Ordering::SeqCst);
2952                return Ok(false);
2953            }
2954            Ok(remaining == 0)
2955        }
2956        async fn reload(&self) -> anyhow::Result<()> {
2957            Ok(())
2958        }
2959
2960        async fn spawn(
2961            &self,
2962            _spawnable: Box<dyn hyprstream_rpc::Spawnable>,
2963        ) -> anyhow::Result<hyprstream_service::SpawnedService> {
2964            anyhow::bail!("mock manager does not host services")
2965        }
2966    }
2967
2968    #[tokio::test]
2969    async fn unit_start_mutation_failure_propagates_without_active_poll() {
2970        let manager = MockManager::failing();
2971        let error = start_units_to_active(
2972            &manager,
2973            &["model".to_owned()],
2974            std::time::Duration::from_secs(1),
2975        )
2976        .await
2977        .expect_err("start mutation failure must propagate");
2978        assert!(
2979            error.to_string().contains("unit is masked"),
2980            "real failure expected, got: {error}"
2981        );
2982    }
2983
2984    #[tokio::test]
2985    async fn unit_start_requires_bounded_active_state() {
2986        let manager = MockManager::never_active();
2987        let error = start_units_to_active(
2988            &manager,
2989            &["model".to_owned()],
2990            std::time::Duration::from_millis(300),
2991        )
2992        .await
2993        .expect_err("a unit that never activates must fail the bounded wait");
2994        assert!(
2995            error.to_string().contains("did not reach the active state"),
2996            "active-state timeout expected, got: {error}"
2997        );
2998
2999        let manager = MockManager::immediate();
3000        start_units_to_active(
3001            &manager,
3002            &["model".to_owned()],
3003            std::time::Duration::from_secs(1),
3004        )
3005        .await
3006        .expect("immediately-active unit must complete");
3007    }
3008
3009    #[tokio::test]
3010    async fn unit_starts_are_queued_before_active_waits() {
3011        let manager = MockManager::immediate();
3012        start_units_to_active(
3013            &manager,
3014            &["event".to_owned(), "policy".to_owned()],
3015            std::time::Duration::from_secs(1),
3016        )
3017        .await
3018        .expect("queued unit starts must complete");
3019
3020        assert_eq!(
3021            *manager.calls.lock(),
3022            [
3023                "start:event".to_owned(),
3024                "start:policy".to_owned(),
3025                "active:event".to_owned(),
3026                "active:policy".to_owned(),
3027            ]
3028        );
3029    }
3030
3031    /// Required roster rollback: a later child failing must stop earlier
3032    /// children in reverse order — proven with real supervised processes via
3033    /// the injected-plan seam.
3034    #[cfg(any(target_os = "linux", target_os = "android"))]
3035    #[tokio::test]
3036    async fn required_rollback_stops_started_children_on_later_failure() -> anyhow::Result<()> {
3037        use hyprstream_service::{ProcessConfig, ProcessReadiness, ProcessSpawner};
3038        #[allow(unused_imports)]
3039        use ProcessReadiness as _ProcessReadinessMarker;
3040
3041        const ROLLBACK_HELPER: &str = "HYPRSTREAM_LAUNCHER_ROLLBACK_HELPER";
3042
3043        // Helper child used as the first plan: READY then stays alive.
3044        if std::env::var_os(ROLLBACK_HELPER).is_some() {
3045            hyprstream_rpc::notify::ready()?;
3046            std::thread::sleep(std::time::Duration::from_secs(30));
3047            return Ok(());
3048        }
3049
3050        let exe = std::env::current_exe()?;
3051        let spawner = ProcessSpawner::standalone();
3052        let supervisor_a = "rollback-alive";
3053        let supervisor_b = "rollback-fail";
3054
3055        let plans = vec![vec![
3056            (
3057                supervisor_a.to_owned(),
3058                ProcessConfig::new(supervisor_a, &exe)
3059                    .args([
3060                        "--exact",
3061                        "cli::service_handlers::launcher_tests::required_rollback_stops_started_children_on_later_failure",
3062                        "--nocapture",
3063                    ])
3064                    .env(ROLLBACK_HELPER, "1")
3065                    .with_notify_ready(std::time::Duration::from_secs(30)),
3066            ),
3067            (
3068                supervisor_b.to_owned(),
3069                ProcessConfig::new(supervisor_b, Path::new("/bin/sh"))
3070                    .args(["-c", "exit 7"])
3071                    .with_notify_ready(std::time::Duration::from_secs(10)),
3072            ),
3073        ]];
3074
3075        let error = launch_planned_children(plans, true, &spawner)
3076            .await
3077            .expect_err("later-child failure must fail the launch");
3078        assert!(
3079            error.to_string().contains("exited during startup"),
3080            "launch failure expected, got: {error}"
3081        );
3082
3083        // The alive first child was stopped by the rollback (reverse order):
3084        // its PID file was removed and the process is reaped. The failing
3085        // child never published anything.
3086        assert!(
3087            !hyprstream_rpc::paths::service_pid_file(supervisor_a).exists(),
3088            "rolled-back child must leave no PID file"
3089        );
3090        assert!(
3091            !hyprstream_rpc::paths::service_pid_file(supervisor_b).exists(),
3092            "failed child must leave no PID file"
3093        );
3094        Ok(())
3095    }
3096
3097    /// Multi-stage required launch: a failure in one stage must abort the
3098    /// WHOLE launch — later dependent stages must never spawn — and every
3099    /// already-started child must be genuinely reaped.
3100    ///
3101    /// Fixture observes, per root dispatch `pr1585-launch-stage-failure-root.md`:
3102    /// two live predecessors (each records its own PID to a marker file, so
3103    /// termination is confirmed by `kill(pid, 0)` → ESRCH, not by PID-file
3104    /// absence alone), the original launch error surfaced unchanged, and a
3105    /// sentinel in a LATER stage whose spawn marker must never appear (the
3106    /// single-stage test cannot distinguish a full abort from a per-stage
3107    /// break). Stop ORDER is not externally observable here: rollback kills
3108    /// are SIGKILL, so reverse ordering remains a construction guarantee of
3109    /// `started.iter().rev()`, not an observed event.
3110    #[cfg(any(target_os = "linux", target_os = "android"))]
3111    #[tokio::test]
3112    async fn required_stage_failure_aborts_launch_and_never_spawns_later_stages(
3113    ) -> anyhow::Result<()> {
3114        use hyprstream_service::{ProcessConfig, ProcessReadiness, ProcessSpawner};
3115        #[allow(unused_imports)]
3116        use ProcessReadiness as _ProcessReadinessMarker;
3117
3118        const MARKER_HELPER: &str = "HYPRSTREAM_LAUNCHER_MARKER_HELPER";
3119
3120        // Helper child used as a live predecessor: records its PID to the
3121        // marker BEFORE its READY send, so the parent's continuation past
3122        // this child deterministically proves the marker exists.
3123        if let Some(path) = std::env::var_os(MARKER_HELPER) {
3124            std::fs::write(&path, std::process::id().to_string())?;
3125            hyprstream_rpc::notify::ready()?;
3126            std::thread::sleep(std::time::Duration::from_secs(30));
3127            return Ok(());
3128        }
3129
3130        let exe = std::env::current_exe()?;
3131        let spawner = ProcessSpawner::standalone();
3132        let dir = tempfile::tempdir()?;
3133        // Unique supervisor names per test process: PID files land in the
3134        // shared runtime dir, and libtest runs suites in parallel.
3135        let unique = std::process::id();
3136        let stage0a = format!("launch-stage-a-{unique}");
3137        let stage0b = format!("launch-stage-b-{unique}");
3138        let stage1fail = format!("launch-stage-fail-{unique}");
3139        let stage2sentinel = format!("launch-stage-sentinel-{unique}");
3140
3141        let marker = |name: &str| dir.path().join(format!("{name}.marker"));
3142        let marker_pid = |path: &Path| -> anyhow::Result<u32> {
3143            std::fs::read_to_string(path)?
3144                .trim()
3145                .parse::<u32>()
3146                .context("marker must hold the child pid")
3147        };
3148        // Reap confirmation: ESRCH, not merely an absent PID file.
3149        let assert_reaped = |pid: u32| {
3150            let gone = nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), None)
3151                .is_err_and(|e| e == nix::errno::Errno::ESRCH);
3152            assert!(gone, "child pid {pid} must be reaped after rollback");
3153        };
3154
3155        let plans = vec![
3156            // Stage 0: two live predecessors (marker→READY→stay alive) —
3157            // both must be rolled back and genuinely reaped.
3158            vec![
3159                (
3160                    stage0a.clone(),
3161                    ProcessConfig::new(&stage0a, &exe)
3162                        .args([
3163                            "--exact",
3164                            "cli::service_handlers::launcher_tests::required_stage_failure_aborts_launch_and_never_spawns_later_stages",
3165                            "--nocapture",
3166                        ])
3167                        .env(MARKER_HELPER, marker(&stage0a).display().to_string())
3168                        .with_notify_ready(std::time::Duration::from_secs(30)),
3169                ),
3170                (
3171                    stage0b.clone(),
3172                    ProcessConfig::new(&stage0b, &exe)
3173                        .args([
3174                            "--exact",
3175                            "cli::service_handlers::launcher_tests::required_stage_failure_aborts_launch_and_never_spawns_later_stages",
3176                            "--nocapture",
3177                        ])
3178                        .env(MARKER_HELPER, marker(&stage0b).display().to_string())
3179                        .with_notify_ready(std::time::Duration::from_secs(30)),
3180                ),
3181            ],
3182            // Stage 1: required child fails during startup.
3183            vec![(
3184                stage1fail.clone(),
3185                ProcessConfig::new(&stage1fail, Path::new("/bin/sh"))
3186                    .args(["-c", "exit 7"])
3187                    .with_notify_ready(std::time::Duration::from_secs(10)),
3188            )],
3189            // Stage 2 sentinel: same marker-before-READY helper with Notify
3190            // readiness. If the abort ever leaked past the failing stage, the
3191            // buggy path would WAIT for this child's READY — which the helper
3192            // sends only after writing the marker — so an incorrect later-stage
3193            // spawn deterministically leaves a marker (an Immediate child could
3194            // be SIGKILLed by the buggy rollback before it ever wrote one,
3195            // giving a false pass).
3196            vec![(
3197                stage2sentinel.clone(),
3198                ProcessConfig::new(&stage2sentinel, &exe)
3199                    .args([
3200                        "--exact",
3201                        "cli::service_handlers::launcher_tests::required_stage_failure_aborts_launch_and_never_spawns_later_stages",
3202                        "--nocapture",
3203                    ])
3204                    .env(MARKER_HELPER, marker(&stage2sentinel).display().to_string())
3205                    .with_notify_ready(std::time::Duration::from_secs(30)),
3206            )],
3207        ];
3208
3209        let error = launch_planned_children(plans, true, &spawner)
3210            .await
3211            .expect_err("stage-1 failure must fail the whole launch");
3212        assert!(
3213            error.to_string().contains("exited during startup"),
3214            "original launch error expected, got: {error}"
3215        );
3216
3217        // Both started predecessors: actually reaped (ESRCH), PID files gone.
3218        for (name, path) in [(&stage0a, marker(&stage0a)), (&stage0b, marker(&stage0b))] {
3219            assert!(path.exists(), "predecessor {name} must have run");
3220            assert_reaped(marker_pid(&path)?);
3221            assert!(
3222                !hyprstream_rpc::paths::service_pid_file(name).exists(),
3223                "rolled-back {name} must leave no PID file"
3224            );
3225        }
3226        // The later-stage sentinel never spawned.
3227        assert!(
3228            !marker(&stage2sentinel).exists(),
3229            "sentinel stage must never spawn after a required stage failure"
3230        );
3231        assert!(
3232            !hyprstream_rpc::paths::service_pid_file(&stage2sentinel).exists(),
3233            "sentinel must publish no PID file"
3234        );
3235        Ok(())
3236    }
3237
3238    /// Cancellation while a later Required child is pending must retain
3239    /// cleanup ownership of the already-adopted READY children: aborting the
3240    /// orchestration future (the real cancellation boundary — `BootstrapManager::drop`
3241    /// aborts this exact task shape) reaps the adopted child through the
3242    /// backend's retained handle and removes its PID artifact. The later
3243    /// child is proven genuinely pending: its marker proves it spawned (so
3244    /// the first child was adopted), and it never sends READY.
3245    #[cfg(any(target_os = "linux", target_os = "android"))]
3246    #[tokio::test]
3247    async fn required_cancellation_reaps_adopted_children_and_artifacts() -> anyhow::Result<()> {
3248        use hyprstream_service::{ProcessConfig, ProcessReadiness, ProcessSpawner};
3249        #[allow(unused_imports)]
3250        use ProcessReadiness as _ProcessReadinessMarker;
3251
3252        const HELPER: &str = "HYPRSTREAM_LAUNCHER_CANCEL_HELPER";
3253        const PENDING: &str = "HYPRSTREAM_LAUNCHER_CANCEL_PENDING";
3254
3255        // Helper child: writes its PID marker first; the READY variant then
3256        // sends READY and stays alive, the pending variant never sends READY.
3257        if let Some(path) = std::env::var_os(HELPER) {
3258            std::fs::write(&path, std::process::id().to_string())?;
3259            if std::env::var_os(PENDING).is_none() {
3260                hyprstream_rpc::notify::ready()?;
3261            }
3262            std::thread::sleep(std::time::Duration::from_secs(30));
3263            return Ok(());
3264        }
3265
3266        let exe = std::env::current_exe()?;
3267        let dir = tempfile::tempdir()?;
3268        let unique = std::process::id();
3269        let ready_name = format!("cancel-ready-{unique}");
3270        let pending_name = format!("cancel-pending-{unique}");
3271        let marker = |name: &str| dir.path().join(format!("{name}.marker"));
3272        let marker_pid = |path: &Path| -> anyhow::Result<u32> {
3273            std::fs::read_to_string(path)?
3274                .trim()
3275                .parse::<u32>()
3276                .context("marker must hold the child pid")
3277        };
3278        let assert_reaped = |pid: u32| {
3279            let gone = nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), None)
3280                .is_err_and(|e| e == nix::errno::Errno::ESRCH);
3281            assert!(gone, "cancelled child pid {pid} must be genuinely reaped");
3282        };
3283        let helper_cfg = |name: &str, pending: bool| {
3284            let mut config = ProcessConfig::new(name, &exe)
3285                .args([
3286                    "--exact",
3287                    "cli::service_handlers::launcher_tests::required_cancellation_reaps_adopted_children_and_artifacts",
3288                    "--nocapture",
3289                ])
3290                .env(HELPER, marker(name).display().to_string())
3291                .with_notify_ready(std::time::Duration::from_secs(30));
3292            if pending {
3293                config = config.env(PENDING, "1");
3294            }
3295            config
3296        };
3297        let plans = vec![vec![
3298            (ready_name.clone(), helper_cfg(&ready_name, false)),
3299            (pending_name.clone(), helper_cfg(&pending_name, true)),
3300        ]];
3301
3302        // The spawner lives inside the task exactly as in production
3303        // (`handle_service_start` owns it), so cancellation also drops the
3304        // backend while the guard's synchronous cleanup runs.
3305        let task = tokio::spawn(async move {
3306            let spawner = ProcessSpawner::standalone();
3307            launch_planned_children(plans, true, &spawner).await
3308        });
3309
3310        // Bounded wait for the cancellation point: the pending child's
3311        // marker proves the launcher adopted the READY first child and is
3312        // now awaiting the second child's readiness.
3313        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);
3314        while !marker(&ready_name).exists() || !marker(&pending_name).exists() {
3315            anyhow::ensure!(
3316                std::time::Instant::now() < deadline,
3317                "launch never reached the pending second child"
3318            );
3319            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3320        }
3321        task.abort();
3322        let joined = task.await;
3323        assert!(
3324            matches!(&joined, Err(join) if join.is_cancelled()),
3325            "the orchestration future must have been cancelled, got: {joined:?}"
3326        );
3327
3328        // Both children reaped (ESRCH — observed termination, not an
3329        // artifact absence) and no PID artifacts remain: the adopted READY
3330        // child through the transaction guard's synchronous tracked-child
3331        // pass, the pending child through its own armed startup guard.
3332        assert_reaped(marker_pid(&marker(&ready_name))?);
3333        assert_reaped(marker_pid(&marker(&pending_name))?);
3334        assert!(
3335            !hyprstream_rpc::paths::service_pid_file(&ready_name).exists(),
3336            "cancelled READY child must leave no PID artifact"
3337        );
3338        assert!(
3339            !hyprstream_rpc::paths::service_pid_file(&pending_name).exists(),
3340            "cancelled pending child must leave no PID artifact"
3341        );
3342        Ok(())
3343    }
3344
3345    /// Whole-launch success must disarm the transaction guard: adopted
3346    /// daemons intentionally survive — both a normal launcher/spawner drop
3347    /// and later `stop` through the established contract. (Pair A proves
3348    /// stoppability through the launching backend's tracked stop; pair C is
3349    /// dropped with its spawner and must still be running afterwards.)
3350    #[cfg(any(target_os = "linux", target_os = "android"))]
3351    #[tokio::test]
3352    async fn required_success_commit_leaves_adopted_daemons_alive_and_stoppable(
3353    ) -> anyhow::Result<()> {
3354        use hyprstream_service::{
3355            ProcessConfig, ProcessKind, ProcessReadiness, ProcessSpawner, SpawnedProcess,
3356        };
3357        #[allow(unused_imports)]
3358        use ProcessReadiness as _ProcessReadinessMarker;
3359
3360        const HELPER: &str = "HYPRSTREAM_LAUNCHER_COMMIT_HELPER";
3361
3362        if let Some(path) = std::env::var_os(HELPER) {
3363            std::fs::write(&path, std::process::id().to_string())?;
3364            hyprstream_rpc::notify::ready()?;
3365            std::thread::sleep(std::time::Duration::from_secs(30));
3366            return Ok(());
3367        }
3368
3369        let exe = std::env::current_exe()?;
3370        let dir = tempfile::tempdir()?;
3371        let unique = std::process::id();
3372        let marker = |name: &str| dir.path().join(format!("{name}.marker"));
3373        let marker_pid = |path: &Path| -> anyhow::Result<u32> {
3374            std::fs::read_to_string(path)?
3375                .trim()
3376                .parse::<u32>()
3377                .context("marker must hold the child pid")
3378        };
3379        let alive =
3380            |pid: u32| nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), None).is_ok();
3381        let helper_cfg = |name: &str| {
3382            ProcessConfig::new(name, &exe)
3383                .args([
3384                    "--exact",
3385                    "cli::service_handlers::launcher_tests::required_success_commit_leaves_adopted_daemons_alive_and_stoppable",
3386                    "--nocapture",
3387                ])
3388                .env(HELPER, marker(name).display().to_string())
3389                .with_notify_ready(std::time::Duration::from_secs(30))
3390        };
3391
3392        // Pair A: committed, then stopped through the launching backend.
3393        let spawner = ProcessSpawner::standalone();
3394        let a = format!("commit-a-{unique}");
3395        let b = format!("commit-b-{unique}");
3396        launch_planned_children(
3397            vec![vec![
3398                (a.clone(), helper_cfg(&a)),
3399                (b.clone(), helper_cfg(&b)),
3400            ]],
3401            true,
3402            &spawner,
3403        )
3404        .await
3405        .expect("whole-launch success must commit");
3406        // Bounded wait until both helpers recorded their PIDs (the launch
3407        // only returns after both are READY, so the markers already exist).
3408        for name in [&a, &b] {
3409            let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
3410            while !marker(name).exists() {
3411                anyhow::ensure!(
3412                    std::time::Instant::now() < deadline,
3413                    "committed child {name} never recorded its PID"
3414                );
3415                tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3416            }
3417        }
3418        // Stop through the launching backend's established tracked contract.
3419        for (name, pid) in [
3420            (&a, marker_pid(&marker(&a))?),
3421            (&b, marker_pid(&marker(&b))?),
3422        ] {
3423            assert!(alive(pid), "committed daemon {name} must still be running");
3424            let meta = SpawnedProcess::new(format!("{name}-{pid}"), ProcessKind::Direct(pid))
3425                .with_pid_file(hyprstream_rpc::paths::service_pid_file(name));
3426            spawner
3427                .stop(&meta)
3428                .await
3429                .expect("committed daemon must remain stoppable");
3430            assert!(
3431                nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), None)
3432                    .is_err_and(|e| e == nix::errno::Errno::ESRCH),
3433                "stopped committed daemon {name} must be reaped"
3434            );
3435            assert!(
3436                !hyprstream_rpc::paths::service_pid_file(name).exists(),
3437                "stopped committed daemon {name} must leave no PID artifact"
3438            );
3439        }
3440
3441        // Pair C: committed, then the launching spawner is dropped normally —
3442        // the adopted daemons must survive it (kill_on_drop stays false).
3443        let c = format!("commit-c-{unique}");
3444        let d = format!("commit-d-{unique}");
3445        let drop_spawner = ProcessSpawner::standalone();
3446        launch_planned_children(
3447            vec![vec![
3448                (c.clone(), helper_cfg(&c)),
3449                (d.clone(), helper_cfg(&d)),
3450            ]],
3451            true,
3452            &drop_spawner,
3453        )
3454        .await
3455        .expect("whole-launch success must commit");
3456        let dropped_pids = [marker_pid(&marker(&c))?, marker_pid(&marker(&d))?];
3457        drop(drop_spawner);
3458        for (name, pid) in [(&c, dropped_pids[0]), (&d, dropped_pids[1])] {
3459            assert!(
3460                alive(pid),
3461                "committed daemon {name} must survive a normal launcher drop"
3462            );
3463        }
3464        // Fixture hygiene: reap the surviving pair directly (the test is the
3465        // parent); not a product-contract assertion. Afterwards remove only
3466        // the matching fixture-owned PID artifacts and establish their
3467        // absence, so a passing test leaves no stale files; any replacement
3468        // artifact would be preserved.
3469        for (name, pid) in [(&c, dropped_pids[0]), (&d, dropped_pids[1])] {
3470            let raw = nix::unistd::Pid::from_raw(pid as i32);
3471            let _ = nix::sys::signal::kill(raw, nix::sys::signal::Signal::SIGKILL);
3472            let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
3473            loop {
3474                match nix::sys::wait::waitpid(raw, Some(nix::sys::wait::WaitPidFlag::WNOHANG)) {
3475                    Ok(status) if status != nix::sys::wait::WaitStatus::StillAlive => break,
3476                    Err(_) => break,
3477                    Ok(_) => {}
3478                }
3479                anyhow::ensure!(
3480                    std::time::Instant::now() < deadline,
3481                    "fixture child {pid} could not be reaped"
3482                );
3483                std::thread::sleep(std::time::Duration::from_millis(20));
3484            }
3485            let pid_file = hyprstream_rpc::paths::service_pid_file(name);
3486            if let Ok(content) = std::fs::read_to_string(&pid_file) {
3487                if content.trim() == pid.to_string() {
3488                    let _ = std::fs::remove_file(&pid_file);
3489                }
3490            }
3491            assert!(
3492                !pid_file.exists(),
3493                "fixture hygiene: {name} must leave no stale PID artifact"
3494            );
3495        }
3496        Ok(())
3497    }
3498
3499    /// Cancellation around a required startup failure cleans the spawned
3500    /// predecessor at the point where abort lands. Its PID marker is written
3501    /// before READY, so marker presence proves the child ran and wrote its
3502    /// PID; it does not prove READY was sent, that the launcher adopted it,
3503    /// or that rollback was entered. The landing is unspecified. The
3504    /// deterministic rollback-boundary evidence is provided by
3505    /// `rollback_failed_and_pending_stops_stay_guard_owned_through_cancellation`.
3506    #[cfg(any(target_os = "linux", target_os = "android"))]
3507    #[tokio::test]
3508    async fn required_cancellation_around_startup_failure_cleans_predecessor_at_any_landing(
3509    ) -> anyhow::Result<()> {
3510        use hyprstream_service::{ProcessConfig, ProcessReadiness, ProcessSpawner};
3511        #[allow(unused_imports)]
3512        use ProcessReadiness as _ProcessReadinessMarker;
3513
3514        const HELPER: &str = "HYPRSTREAM_LAUNCHER_CANCEL_ROLLBACK_HELPER";
3515
3516        if let Some(path) = std::env::var_os(HELPER) {
3517            std::fs::write(&path, std::process::id().to_string())?;
3518            hyprstream_rpc::notify::ready()?;
3519            std::thread::sleep(std::time::Duration::from_secs(30));
3520            return Ok(());
3521        }
3522
3523        let exe = std::env::current_exe()?;
3524        let dir = tempfile::tempdir()?;
3525        let unique = std::process::id();
3526        let ready_name = format!("cancel-rollback-ready-{unique}");
3527        let fail_name = format!("cancel-rollback-fail-{unique}");
3528        let marker = dir.path().join(format!("{ready_name}.marker"));
3529        let marker_pid = |path: &Path| -> anyhow::Result<u32> {
3530            std::fs::read_to_string(path)?
3531                .trim()
3532                .parse::<u32>()
3533                .context("marker must hold the child pid")
3534        };
3535
3536        let plans = vec![vec![
3537            (
3538                ready_name.clone(),
3539                ProcessConfig::new(&ready_name, &exe)
3540                    .args([
3541                        "--exact",
3542                        "cli::service_handlers::launcher_tests::required_cancellation_around_startup_failure_cleans_predecessor_at_any_landing",
3543                        "--nocapture",
3544                    ])
3545                    .env(HELPER, marker.display().to_string())
3546                    .with_notify_ready(std::time::Duration::from_secs(30)),
3547            ),
3548            (
3549                fail_name.clone(),
3550                ProcessConfig::new(&fail_name, Path::new("/bin/sh"))
3551                    .args(["-c", "exit 7"])
3552                    .with_notify_ready(std::time::Duration::from_secs(10)),
3553            ),
3554        ]];
3555
3556        let task = tokio::spawn(async move {
3557            let spawner = ProcessSpawner::standalone();
3558            launch_planned_children(plans, true, &spawner).await
3559        });
3560
3561        // Abort after the predecessor's PID marker appears. The marker
3562        // precedes READY; it does not prove notification, adoption, or
3563        // rollback entry. This test checks cleanup at the observed,
3564        // unspecified cancellation point.
3565        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);
3566        while !marker.exists() {
3567            anyhow::ensure!(
3568                std::time::Instant::now() < deadline,
3569                "READY predecessor never recorded its PID"
3570            );
3571            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3572        }
3573        task.abort();
3574        let joined = task.await;
3575        // Either the future was cancelled (cancellation cleanup ran) or the
3576        // rollback completed before the abort landed (explicit-failure
3577        // rollback ran) — both paths must end with the predecessor cleaned.
3578        match &joined {
3579            Err(join) => assert!(join.is_cancelled(), "unexpected join error: {join:?}"),
3580            Ok(Err(error)) => assert!(
3581                error.to_string().contains("exited during startup"),
3582                "completed rollback must carry the launch error, got: {error}"
3583            ),
3584            Ok(Ok(())) => panic!("a required launch with a failing child must not succeed"),
3585        }
3586        let pid = marker_pid(&marker)?;
3587        let gone = nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), None)
3588            .is_err_and(|e| e == nix::errno::Errno::ESRCH);
3589        assert!(
3590            gone,
3591            "predecessor pid {pid} must be cleaned at every landing point"
3592        );
3593        assert!(
3594            !hyprstream_rpc::paths::service_pid_file(&ready_name).exists(),
3595            "predecessor must leave no PID artifact"
3596        );
3597        Ok(())
3598    }
3599
3600    /// Rollback ownership-boundary regression (root-requested): a stop that
3601    /// FAILS must leave its child guard-owned — never moved into a local
3602    /// unguarded collection — and a cancellation while a LATER stop is still
3603    /// PENDING must attempt synchronous cleanup of BOTH. The failure/pending
3604    /// boundary is injected at the narrow `rollback_owned_children` stop seam; the
3605    /// two children are REAL spawned processes, so ownership and cleanup are
3606    /// evidenced by observed reaps (ESRCH) and PID-artifact removal.
3607    #[cfg(any(target_os = "linux", target_os = "android"))]
3608    #[tokio::test]
3609    async fn rollback_failed_and_pending_stops_stay_guard_owned_through_cancellation(
3610    ) -> anyhow::Result<()> {
3611        use hyprstream_service::{ProcessConfig, ProcessSpawner};
3612        use std::sync::atomic::{AtomicUsize, Ordering};
3613
3614        const HELPER: &str = "HYPRSTREAM_LAUNCHER_RBOWN_HELPER";
3615
3616        if let Some(path) = std::env::var_os(HELPER) {
3617            std::fs::write(&path, std::process::id().to_string())?;
3618            hyprstream_rpc::notify::ready()?;
3619            std::thread::sleep(std::time::Duration::from_secs(30));
3620            return Ok(());
3621        }
3622
3623        let exe = std::env::current_exe()?;
3624        let dir = tempfile::tempdir()?;
3625        let unique = std::process::id();
3626        let older = format!("rbown-older-{unique}");
3627        let newer = format!("rbown-newer-{unique}");
3628        let marker = |name: &str| dir.path().join(format!("{name}.marker"));
3629        let marker_pid = |path: &Path| -> anyhow::Result<u32> {
3630            std::fs::read_to_string(path)?
3631                .trim()
3632                .parse::<u32>()
3633                .context("marker must hold the child pid")
3634        };
3635        let assert_reaped = |pid: u32| {
3636            let gone = nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), None)
3637                .is_err_and(|e| e == nix::errno::Errno::ESRCH);
3638            assert!(gone, "guard-owned child pid {pid} must be genuinely reaped");
3639        };
3640        let helper_cfg = |name: &str| {
3641            ProcessConfig::new(name, &exe)
3642                .args([
3643                    "--exact",
3644                    "cli::service_handlers::launcher_tests::rollback_failed_and_pending_stops_stay_guard_owned_through_cancellation",
3645                    "--nocapture",
3646                ])
3647                .env(HELPER, marker(name).display().to_string())
3648                .with_notify_ready(std::time::Duration::from_secs(30))
3649        };
3650
3651        let spawner = ProcessSpawner::standalone();
3652        let mut guard = RequiredLaunchGuard::arm(true, &spawner);
3653        // Real adoption, start order older → newer; the rollback walks
3654        // newest first, so the NEWER child hits the injected failed stop and
3655        // the OLDER child the injected pending stop.
3656        for name in [&older, &newer] {
3657            let process = spawner.spawn(helper_cfg(name)).await?;
3658            guard.adopt(name.to_owned(), process);
3659        }
3660
3661        let calls = std::sync::Arc::new(AtomicUsize::new(0));
3662        let boundary: std::sync::Arc<parking_lot::Mutex<Vec<&'static str>>> =
3663            std::sync::Arc::default();
3664        let calls_task = calls.clone();
3665        let boundary_task = boundary.clone();
3666        let task = tokio::spawn(async move {
3667            guard
3668                .rollback_owned_children(move |_process| {
3669                    let call = calls_task.fetch_add(1, Ordering::SeqCst);
3670                    let boundary = boundary_task.clone();
3671                    async move {
3672                        if call == 0 {
3673                            boundary.lock().push("first-stop-failed");
3674                            Err(anyhow::anyhow!("injected stop failure"))
3675                        } else {
3676                            boundary.lock().push("next-stop-pending-entered");
3677                            // Pending forever: only the task abort below
3678                            // ends this await.
3679                            std::future::pending::<()>().await;
3680                            Ok(())
3681                        }
3682                    }
3683                })
3684                .await;
3685            // Unreachable under the abort; the guard's Drop owns cleanup.
3686        });
3687
3688        // Deterministic boundary, no sleep-only timing assumption: abort
3689        // only after BOTH stop invocations happened — the first returned the
3690        // injected failure (its child demoted to guard-owned `failed`) and
3691        // the second is suspended inside the pending stop.
3692        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
3693        while calls.load(Ordering::SeqCst) < 2 {
3694            anyhow::ensure!(
3695                std::time::Instant::now() < deadline,
3696                "rollback never reached the pending second stop"
3697            );
3698            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3699        }
3700        task.abort();
3701        let joined = task.await;
3702        assert!(
3703            matches!(&joined, Err(join) if join.is_cancelled()),
3704            "the rollback future must have been cancelled, got: {joined:?}"
3705        );
3706        // The claimed causal boundary happened: the first stop FAILED and
3707        // the next stop was entered and PENDING at cancellation time.
3708        assert_eq!(
3709            *boundary.lock(),
3710            vec!["first-stop-failed", "next-stop-pending-entered"],
3711            "the first stop must have failed and the next pending stop must have been \
3712             entered before cancellation"
3713        );
3714
3715        // BOTH children — the failed-stop one and the pending-stop one — are
3716        // reaped with their PID artifacts removed by the guard's synchronous
3717        // pass through the backend's retained handles.
3718        assert_reaped(marker_pid(&marker(&newer))?);
3719        assert_reaped(marker_pid(&marker(&older))?);
3720        for name in [&older, &newer] {
3721            assert!(
3722                !hyprstream_rpc::paths::service_pid_file(name).exists(),
3723                "cancelled rollback child {name} must leave no PID artifact"
3724            );
3725        }
3726        Ok(())
3727    }
3728
3729    /// Installed units encode their own stored configuration; an explicit
3730    /// config selector or the required-native profile must be refused on the
3731    /// existing-unit path before any unit is started.
3732    #[tokio::test]
3733    async fn installed_unit_start_rejects_explicit_config_and_required_profile() {
3734        if !hyprstream_rpc::has_systemd() {
3735            return;
3736        }
3737        let root = tempfile::tempdir().unwrap();
3738        let config_path = root.path().join("custom.toml");
3739        std::fs::write(&config_path, "[secrets]\n").unwrap();
3740
3741        let explicit_error = handle_service_start(
3742            &["model".to_owned()],
3743            Some("model".to_owned()),
3744            false,
3745            Some(config_path.as_path()),
3746            false,
3747        )
3748        .await
3749        .expect_err("explicit config selector must not start an installed unit");
3750        assert!(
3751            explicit_error
3752                .to_string()
3753                .contains("installed hyprstream service units"),
3754            "actionable rejection expected, got: {explicit_error}"
3755        );
3756
3757        let required_error = handle_service_start(
3758            &["model".to_owned()],
3759            Some("model".to_owned()),
3760            false,
3761            None,
3762            true,
3763        )
3764        .await
3765        .expect_err("required-native profile must not start an installed unit");
3766        assert!(
3767            required_error.to_string().contains("--daemon"),
3768            "rejection must direct the operator to the direct launch path"
3769        );
3770    }
3771}