Skip to main content

hyprstream/
main.rs

1//! Hyprstream binary - ML inference server
2//!
3//! This binary provides the main entry point for the Hyprstream service.
4// CLI binary intentionally prints to stdout/stderr for user interaction
5#![allow(clippy::print_stdout, clippy::print_stderr)]
6
7// Standard library imports
8use anyhow::{Context as _, Result};
9use clap::{
10    Arg, Args as ClapArgs, Command as ClapCommand, FromArgMatches, Subcommand as ClapSubcommand,
11};
12use tracing::info;
13
14// Core application imports
15use hyprstream_core::cli::commands::{
16    ExecutionMode, FlightArgs, ImageCommand, ServiceAction, TrainingAction, TrustCommand,
17    TuiAction, WorkerAction,
18};
19use hyprstream_core::cli::commands::{
20    PolicyCommand, RoleCommand, TokenCommand, UserCommand, UserKeysCommand,
21};
22use hyprstream_core::cli::quick::{QuickCommand, RemoteQuickCommand, WorktreeQuickCommand};
23use hyprstream_core::cli::schema_cli;
24use hyprstream_core::cli::{
25    handle_branch,
26    handle_checkout,
27    handle_clone,
28    // Worker handlers
29    handle_images_df,
30    handle_images_list,
31    handle_images_pull,
32    handle_images_rm,
33    handle_infer,
34    handle_info,
35    handle_list,
36    handle_load,
37    handle_notify_command,
38    handle_policy_apply,
39    handle_policy_apply_template,
40    handle_policy_check,
41    handle_policy_diff,
42    handle_policy_edit,
43    handle_policy_history,
44    handle_policy_list_templates,
45    handle_policy_role_add,
46    handle_policy_role_list,
47    handle_policy_role_remove,
48    handle_policy_rollback,
49    handle_policy_show,
50    handle_pull,
51    handle_remote_add,
52    handle_remote_list,
53    handle_remote_remove,
54    handle_remote_rename,
55    handle_remote_set_url,
56    handle_remove,
57    // Service handlers
58    handle_service_install,
59    handle_service_provision_policy_templates,
60    handle_service_start,
61    handle_service_status,
62    handle_service_stop,
63    handle_service_uninstall,
64    handle_sign_challenge,
65    handle_status,
66    handle_token_create,
67    handle_training_batch,
68    handle_training_checkpoint,
69    handle_training_infer,
70    handle_training_init,
71    handle_unload,
72    handle_user_create,
73    handle_user_keys_import,
74    handle_user_keys_list,
75    handle_user_keys_remove,
76    handle_user_list,
77    handle_user_register,
78    handle_user_remove,
79    handle_worker_exec,
80    handle_worker_list,
81    handle_worker_restart,
82    handle_worker_rm,
83    handle_worker_run,
84    handle_worker_start,
85    handle_worker_stats,
86    handle_worker_status,
87    handle_worker_stop,
88    handle_worker_terminal,
89    handle_worktree_add,
90    handle_worktree_info,
91    handle_worktree_list,
92    handle_worktree_remove,
93    load_or_generate_signing_key,
94    parse_filters,
95    parse_status_filter,
96    AppContext,
97    DeviceConfig,
98    DevicePreference,
99    RuntimeConfig,
100};
101
102#[cfg(feature = "experimental")]
103use hyprstream_core::cli::{
104    handle_commit, handle_merge, handle_promote, handle_push, MergeOptions,
105};
106use hyprstream_core::config::HyprConfig;
107use hyprstream_core::storage::{GitRef, ModelRef};
108
109// Registry and policy services
110use hyprstream_rpc_std::registry_client::RegistryClient;
111// Worker service for Kata-based workload execution
112use hyprstream_workers::runtime::WorkerService;
113#[cfg(feature = "oci-image")]
114use hyprstream_workers::ImageConfig;
115use hyprstream_workers::PoolConfig;
116use std::sync::Arc;
117// Unified service manager API
118use hyprstream_rpc::events::EventPublisher;
119use hyprstream_rpc::registry::SocketKind;
120use hyprstream_rpc::transport::TransportConfig;
121use hyprstream_rpc::{SigningKey, VerifyingKey};
122use hyprstream_service::{get_factory, InprocManager, ServiceContext, ServiceManager};
123
124fn supports_tui() -> bool {
125    use std::io::IsTerminal;
126    if !std::io::stdout().is_terminal() || !std::io::stdin().is_terminal() {
127        return false;
128    }
129    match std::env::var("TERM").as_deref() {
130        Ok("dumb") | Ok("") | Err(_) => return false,
131        _ => {}
132    }
133    #[cfg(unix)]
134    {
135        let mut ws: libc::winsize = unsafe { std::mem::zeroed() };
136        let ok = unsafe { libc::ioctl(libc::STDOUT_FILENO, libc::TIOCGWINSZ, &mut ws) };
137        if ok == 0 {
138            return ws.ws_col >= 60 && ws.ws_row >= 20;
139        }
140    }
141    true
142}
143// Tracing imports (feature-gated)
144#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
145use opentelemetry::trace::TracerProvider;
146#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
147use opentelemetry::KeyValue;
148#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
149use opentelemetry_otlp::{SpanExporter, WithExportConfig};
150#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
151use opentelemetry_sdk::Resource;
152use tracing_appender::rolling::{RollingFileAppender, Rotation};
153#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
154use tracing_opentelemetry::OpenTelemetryLayer;
155#[cfg(any(not(feature = "otel"), target_arch = "wasm32"))]
156use tracing_subscriber::EnvFilter;
157#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
158use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
159
160// ─────────────────────────────────────────────────────────────────────────────
161// CLI builder
162// ─────────────────────────────────────────────────────────────────────────────
163
164/// Build the CLI using clap builder API with hybrid derive-based subcommands.
165fn build_cli() -> ClapCommand {
166    let mut app = ClapCommand::new("hyprstream")
167        .version(env!("BUILD_VERSION"))
168        .about("Real-time adaptive ML inference server with dynamic sparse weight adjustments")
169        .arg(
170            Arg::new("config")
171                .long("config")
172                .global(true)
173                .env("HYPRSTREAM_CONFIG")
174                .value_parser(clap::value_parser!(std::path::PathBuf))
175                .help("Path to configuration file (overrides default config locations)"),
176        )
177        .arg(
178            Arg::new("gpu_device")
179                .long("gpu-device")
180                .global(true)
181                .env("HYPRSTREAM_GPU_DEVICE")
182                .value_parser(clap::value_parser!(usize))
183                .help("GPU device ID to use (e.g., 0, 1). Defaults to auto-detect."),
184        );
185
186    // Schema-driven: hyprstream tool <service> <method>
187    app = app.subcommand(schema_cli::build_tool_command());
188
189    // Quick workflows (derive-based subcommands)
190    app = app.subcommand(<QuickCommand as ClapSubcommand>::augment_subcommands(
191        ClapCommand::new("quick").about("Quick workflows \u{2014} multi-step convenience commands"),
192    ));
193
194    // Service management (derive-based subcommands)
195    app = app.subcommand(<ServiceAction as ClapSubcommand>::augment_subcommands(
196        ClapCommand::new("service").about("Service management and lifecycle commands"),
197    ));
198
199    // Out-of-band deployment trust authority tooling.
200    app = app.subcommand(<TrustCommand as ClapSubcommand>::augment_subcommands(
201        ClapCommand::new("trust").about("Mint and verify deployment trust artifacts"),
202    ));
203
204    // TUI display server (derive-based subcommands)
205    app = app.subcommand(<TuiAction as ClapSubcommand>::augment_subcommands(
206        ClapCommand::new("tui")
207            .about("TUI display server \u{2014} terminal multiplexer with session persistence"),
208    ));
209
210    // Flight SQL client (derive-based args)
211    app = app.subcommand(<FlightArgs as ClapArgs>::augment_args(
212        ClapCommand::new("flight").about("Flight SQL client to query datasets"),
213    ));
214
215    // User management (derive-based subcommands)
216    app = app.subcommand(<UserCommand as ClapSubcommand>::augment_subcommands(
217        ClapCommand::new("user").about("Manage local user credentials"),
218    ));
219
220    // PDS home attachment (RFC 8628 device authorization)
221    app = app.subcommand(
222        ClapCommand::new("pds")
223            .about("Attach this host to its home personal data server")
224            .subcommand(
225                ClapCommand::new("init-deployment-store")
226                    .about("Initialize an explicitly provisioned checkpoint store")
227                    .arg(
228                        Arg::new("fresh-genesis")
229                            .long("fresh-genesis")
230                            .action(clap::ArgAction::SetTrue)
231                            .help(
232                                "Clear only the configured RDS records KV before first-boot initialization; staging deployment bootstrap only",
233                            ),
234                    ),
235            )
236            .subcommand(
237                ClapCommand::new("inspect-services")
238                    .about("Read existing checkpoint-verified service identities as public JSON without provisioning or renewal")
239                    .arg(Arg::new("service").long("service").required(true)
240                        .action(clap::ArgAction::Append).value_delimiter(',')),
241            )
242            .subcommand(
243                ClapCommand::new("provision-services")
244                    .about("Admit existing local service identities before starting the registry")
245                    .arg(Arg::new("service").long("service").required(true)
246                        .action(clap::ArgAction::Append).value_delimiter(','))
247                    .arg(Arg::new("valid-for-seconds").long("valid-for-seconds")
248                        .value_parser(clap::value_parser!(i64)).default_value("86400"))
249                    .arg(Arg::new("roster-export").long("roster-export").value_name("PATH")
250                        .value_parser(clap::value_parser!(std::path::PathBuf))
251                        .help("Opt-in: atomically write a JSON manifest of the verified accepted roster (public fields only; not a trust root) to PATH. Behavior is unchanged when absent.")),
252            )
253            .subcommand(
254                ClapCommand::new("join")
255                    .visible_alias("attach")
256                    .about("Authorize and attach this host to one home PDS")
257                    .arg(
258                        Arg::new("url")
259                            .required(true)
260                            .value_name("PDS_URL")
261                            .help("HTTPS origin of the home PDS"),
262                    )
263                    .arg(
264                        Arg::new("scope")
265                            .long("scope")
266                            .value_name("SCOPE")
267                            .help("Space-delimited OAuth scopes requested from the PDS"),
268                    ),
269            ),
270    );
271
272    // Interactive setup wizard
273    app = app.subcommand(
274        ClapCommand::new("wizard")
275            .about("Interactive setup wizard — configure policies, users, and API tokens")
276            .arg(
277                Arg::new("non_interactive")
278                    .long("non-interactive")
279                    .short('y')
280                    .action(clap::ArgAction::SetTrue)
281                    .help("Accept defaults without prompting"),
282            )
283            .arg(
284                Arg::new("start")
285                    .long("start")
286                    .action(clap::ArgAction::SetTrue)
287                    .help("Start services after setup"),
288            )
289            .arg(
290                Arg::new("tui")
291                    .long("tui")
292                    .action(clap::ArgAction::SetTrue)
293                    .help("Force TUI wizard (auto-detected when terminal supports it)"),
294            )
295            .arg(
296                Arg::new("bootstrap_only")
297                    .long("bootstrap-only")
298                    .action(clap::ArgAction::SetTrue)
299                    .help("Run only the trust-root bootstrap (phase 1); skip binary install, policy templates, user creation, token mint, and systemd"),
300            )
301            .arg(
302                Arg::new("enable_federation")
303                    .long("enable-federation")
304                    .action(clap::ArgAction::SetTrue)
305                    .help("Apply the federation-open policy template — accept third-party apps AND remote peer servers from any HTTPS origin (atproto-style federation). Default is disabled; under -y this flag is the only way to enable it."),
306            )
307            .arg(
308                Arg::new("pds_url")
309                    .long("pds-url")
310                    .value_name("PDS_URL")
311                    .help("Attach this host to a home PDS after wizard setup (same as `pds join`)")
312                    .conflicts_with("bootstrap_only")
313                    .conflicts_with("tui"),
314            )
315            .arg(
316                Arg::new("initial_user_role")
317                    .long("initial-user-role")
318                    .value_name("ROLE")
319                    .default_value("admin")
320                    .help("Role to assign the local user under --non-interactive (admin|operator|trainer|viewer). Default is admin; use operator/viewer in tests for least-privilege."),
321            )
322            .arg(
323                Arg::new("deployment_trust")
324                    .long("deployment-trust")
325                    .action(clap::ArgAction::SetTrue)
326                    .help("Run the deployment-trust ceremony after node bootstrap — detects an attached YubiKey and mints the deployment root, delegated signer, and registry credential. Opt-in: node bootstrap alone is unchanged without it."),
327            )
328            .arg(
329                Arg::new("deployment_trust_dir")
330                    .long("deployment-trust-dir")
331                    .value_name("DIR")
332                    .requires("deployment_trust")
333                    .help("Absolute ceremony working directory (default: <models-dir>/deployment-ceremony). Destroy it once the artifacts are distributed."),
334            )
335            .arg(
336                Arg::new("deployment_trust_software")
337                    .long("deployment-trust-software")
338                    .action(clap::ArgAction::SetTrue)
339                    .requires("deployment_trust")
340                    .help("Mint a DEV-GRADE software deployment root even if a hardware token is attached. Never selected on your behalf."),
341            )
342            .arg(
343                Arg::new("deployment_trust_serial")
344                    .long("deployment-trust-serial")
345                    .value_name("SERIAL")
346                    .requires("deployment_trust")
347                    .help("Bind the deployment root to the token with this serial when several are attached."),
348            )
349            .arg(
350                Arg::new("deployment_trust_piv_slot")
351                    .long("deployment-trust-piv-slot")
352                    .value_name("SLOT")
353                    .requires("deployment_trust")
354                    .help("PIV slot holding the Ed25519 leg on firmware 5.7.4+ (default: 9c)."),
355            )
356            .arg(
357                Arg::new("deployment_trust_allow_plaintext_break_glass")
358                    .long("deployment-trust-allow-plaintext-break-glass")
359                    .action(clap::ArgAction::SetTrue)
360                    .requires("deployment_trust")
361                    .help("Accept an unencrypted break-glass identity. Only for a throwaway root you destroy immediately afterwards."),
362            ),
363    );
364
365    // Sign challenge (OAuth device flow and auth code flow)
366    app = app.subcommand(
367        ClapCommand::new("sign-challenge")
368            .about("Sign an Ed25519 challenge for OAuth device or auth code flow")
369            .arg(
370                Arg::new("user_code")
371                    .index(1)
372                    .required(false)
373                    .help("User code from device flow (e.g., ABCD-EFGH)"),
374            )
375            .arg(
376                Arg::new("nonce")
377                    .long("nonce")
378                    .required(false)
379                    .allow_hyphen_values(true)
380                    .help("Nonce from the browser authorization challenge form"),
381            )
382            .arg(
383                Arg::new("code_challenge")
384                    .long("code-challenge")
385                    .required(false)
386                    .allow_hyphen_values(true)
387                    .help("PKCE code_challenge from the browser authorization URL"),
388            )
389            .arg(
390                Arg::new("server")
391                    .long("server")
392                    .required(false)
393                    .help("OAuth server URL (default: from config or http://localhost:6791)"),
394            )
395            .arg(
396                Arg::new("insecure")
397                    .long("insecure")
398                    .action(clap::ArgAction::SetTrue)
399                    .help("Disable TLS certificate verification (use only against a trusted local dev server). By default the local self-signed dev cert is trusted automatically."),
400            ),
401    );
402
403    // Self-update
404    app = app.subcommand(
405        ClapCommand::new("update")
406            .about("Check for and install updates")
407            .arg(
408                Arg::new("cleanup")
409                    .long("cleanup")
410                    .action(clap::ArgAction::SetTrue)
411                    .help("Remove old version worktrees"),
412            ),
413    );
414
415    // Native MAC status/inspection (epic #547).
416    app =
417        app.subcommand(
418            ClapCommand::new("mac")
419                .about("Native MAC (mandatory access control) status and inspection")
420                .subcommand(ClapCommand::new("genesis").about(
421                    "Show the MAC genesis coverage report (activation coverage-gate evidence)",
422                )),
423        );
424
425    app
426}
427
428// ─────────────────────────────────────────────────────────────────────────────
429// Config & runtime helpers
430// ─────────────────────────────────────────────────────────────────────────────
431
432/// Load HyprConfig from file path or default locations
433fn load_config(path: Option<&std::path::Path>) -> Result<HyprConfig> {
434    match path {
435        Some(p) => {
436            info!("Loading config from: {}", p.display());
437            HyprConfig::from_file(p)
438                .with_context(|| format!("Failed to load config from {}", p.display()))
439        }
440        None => {
441            info!("Loading config from default locations");
442            // Optional files and defaults are handled inside load(). A load
443            // error means invalid configuration, not an absent file; replacing
444            // it with defaults can silently change the selected account store.
445            HyprConfig::load().context("Failed to load default-location configuration")
446        }
447    }
448}
449
450/// Execute a command with the appropriate runtime configuration
451fn with_runtime<F, Fut>(config: RuntimeConfig, handler: F) -> Result<()>
452where
453    F: FnOnce() -> Fut,
454    Fut: std::future::Future<Output = Result<()>>,
455{
456    // Check GPU availability early if required
457    if matches!(config.device.preference, DevicePreference::RequireGPU)
458        && std::env::var("CUDA_VISIBLE_DEVICES").is_err()
459        && std::env::var("HIP_VISIBLE_DEVICES").is_err()
460        && !std::path::Path::new("/usr/local/cuda").exists()
461        && !std::path::Path::new("/opt/rocm").exists()
462    {
463        anyhow::bail!("GPU required but no CUDA or ROCm installation detected. Set CUDA_VISIBLE_DEVICES or HIP_VISIBLE_DEVICES, or install CUDA/ROCm.");
464    }
465
466    // Set environment hints for the runtime config
467    match config.device.preference {
468        DevicePreference::RequestCPU => {
469            std::env::set_var("HYPRSTREAM_FORCE_CPU", "1");
470        }
471        DevicePreference::RequestGPU | DevicePreference::RequireGPU => {
472            std::env::remove_var("HYPRSTREAM_FORCE_CPU");
473        }
474    }
475
476    // Create appropriate runtime
477    if config.multi_threaded {
478        let rt =
479            tokio::runtime::Runtime::new().context("Failed to create multi-threaded runtime")?;
480        rt.block_on(handler())
481    } else {
482        let rt = tokio::runtime::Builder::new_current_thread()
483            .enable_all()
484            .build()
485            .context("Failed to create single-threaded runtime")?;
486        rt.block_on(handler())
487    }
488}
489
490// ─────────────────────────────────────────────────────────────────────────────
491// Telemetry
492// ─────────────────────────────────────────────────────────────────────────────
493
494#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
495/// Telemetry provider type
496#[derive(Debug, Clone, Copy)]
497enum TelemetryProvider {
498    /// Use OTLP exporter for production telemetry
499    Otlp,
500    /// Use stdout exporter for local debugging
501    Stdout,
502}
503
504#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
505/// Initialize OpenTelemetry with the specified provider
506fn init_telemetry(provider: TelemetryProvider) -> Result<()> {
507    let service_name =
508        std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "hyprstream".to_owned());
509
510    let resource = Resource::builder()
511        .with_service_name(service_name.clone())
512        .with_attribute(KeyValue::new("service.version", env!("CARGO_PKG_VERSION")))
513        .build();
514
515    let endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")
516        .unwrap_or_else(|_| "http://localhost:4317".to_owned());
517
518    let tracer_provider = match provider {
519        TelemetryProvider::Otlp => {
520            info!("Using OTLP exporter for OpenTelemetry at {}", endpoint);
521
522            let exporter = SpanExporter::builder()
523                .with_tonic()
524                .with_endpoint(endpoint.clone())
525                .build()
526                .context("Failed to create OTLP exporter")?;
527
528            opentelemetry_sdk::trace::SdkTracerProvider::builder()
529                .with_resource(resource.clone())
530                .with_simple_exporter(exporter)
531                .build()
532        }
533        TelemetryProvider::Stdout => {
534            info!("Using stdout exporter for OpenTelemetry (debug)");
535
536            opentelemetry_sdk::trace::SdkTracerProvider::builder()
537                .with_resource(resource.clone())
538                .with_simple_exporter(opentelemetry_stdout::SpanExporter::default())
539                .build()
540        }
541    };
542
543    // Token-burn + other OTel metrics (#1261) only flow to a collector when a
544    // meter provider is registered; otherwise the global meter is a no-op. Build
545    // a periodic-exporting provider mirroring the tracer provider's transport.
546    let meter_provider = match provider {
547        TelemetryProvider::Otlp => {
548            let exporter = opentelemetry_otlp::MetricExporter::builder()
549                .with_tonic()
550                .with_endpoint(endpoint)
551                .build()
552                .context("Failed to create OTLP metric exporter")?;
553
554            opentelemetry_sdk::metrics::SdkMeterProvider::builder()
555                .with_resource(resource.clone())
556                .with_reader(opentelemetry_sdk::metrics::PeriodicReader::builder(exporter).build())
557                .build()
558        }
559        TelemetryProvider::Stdout => opentelemetry_sdk::metrics::SdkMeterProvider::builder()
560            .with_resource(resource)
561            .with_reader(
562                opentelemetry_sdk::metrics::PeriodicReader::builder(
563                    opentelemetry_stdout::MetricExporter::default(),
564                )
565                .build(),
566            )
567            .build(),
568    };
569    opentelemetry::global::set_meter_provider(meter_provider);
570
571    let tracer = tracer_provider.tracer("hyprstream");
572    let otel_layer = OpenTelemetryLayer::new(tracer);
573
574    let default_log_level = match provider {
575        TelemetryProvider::Otlp => "hyprstream=info,opentelemetry=info",
576        TelemetryProvider::Stdout => "hyprstream=warn",
577    };
578
579    let filter = EnvFilter::builder()
580        .parse_lossy(std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()));
581
582    let fmt_layer = tracing_subscriber::fmt::layer()
583        .with_target(true)
584        .with_file(true)
585        .with_line_number(true);
586
587    tracing_subscriber::registry()
588        .with(filter)
589        .with(fmt_layer)
590        .with(otel_layer)
591        .init();
592
593    info!("OpenTelemetry initialized with {:?} provider", provider);
594    Ok(())
595}
596
597// ─────────────────────────────────────────────────────────────────────────────
598// Quick command handler
599// ─────────────────────────────────────────────────────────────────────────────
600
601/// Handle all `quick` subcommands — orchestrated multi-step workflows.
602fn handle_quick_command(
603    cmd: QuickCommand,
604    ctx: AppContext,
605    signing_key: SigningKey,
606    verifying_key: VerifyingKey,
607) -> Result<()> {
608    match cmd {
609        QuickCommand::Infer {
610            model,
611            prompt,
612            image,
613            max_tokens,
614            temperature,
615            top_p,
616            top_k,
617            repeat_penalty,
618            seed,
619            sync,
620        } => {
621            // Read prompt from stdin if not provided via --prompt
622            let prompt = match prompt {
623                Some(p) => p,
624                None => {
625                    use std::io::Read;
626                    let mut buffer = String::new();
627                    std::io::stdin().read_to_string(&mut buffer)?;
628                    buffer.trim().to_owned()
629                }
630            };
631
632            with_runtime(
633                RuntimeConfig {
634                    device: DeviceConfig::request_gpu(),
635                    multi_threaded: true,
636                },
637                || async move {
638                    handle_infer(
639                        &model,
640                        &prompt,
641                        image,
642                        max_tokens,
643                        temperature,
644                        top_p,
645                        top_k,
646                        repeat_penalty,
647                        seed,
648                        sync,
649                        signing_key,
650                    )
651                    .await
652                },
653            )
654        }
655
656        QuickCommand::Clone {
657            repo_url,
658            name,
659            branch,
660            depth,
661            full,
662            quiet,
663            verbose,
664            policy,
665        } => with_runtime(
666            RuntimeConfig {
667                device: DeviceConfig::request_cpu(),
668                multi_threaded: true,
669            },
670            || async move {
671                handle_clone(
672                    ctx.registry(),
673                    &repo_url,
674                    name,
675                    branch,
676                    depth,
677                    full,
678                    quiet,
679                    verbose,
680                    policy,
681                )
682                .await
683            },
684        ),
685
686        QuickCommand::List { filter, status } => with_runtime(
687            RuntimeConfig {
688                device: DeviceConfig::request_cpu(),
689                multi_threaded: true,
690            },
691            || async move {
692                let keys_dir = ctx.models_dir().join(".registry").join("keys");
693                let signing_key = load_or_generate_signing_key(&keys_dir).await?;
694                let model_client = hyprstream_rpc_std::model_client::ModelClient::from_provider(
695                    &hyprstream_discovery::ProductionRpcClientProvider,
696                    signing_key,
697                    None,
698                )?;
699                let filters = parse_filters(&filter)?;
700                let status_filter = parse_status_filter(&status)?;
701                handle_list(ctx.registry(), model_client, &filters, &status_filter).await
702            },
703        ),
704
705        QuickCommand::Info {
706            model,
707            verbose,
708            adapters_only,
709        } => with_runtime(
710            RuntimeConfig {
711                device: DeviceConfig::request_cpu(),
712                multi_threaded: true,
713            },
714            || async move {
715                match handle_info(ctx.registry(), &model, verbose, adapters_only).await {
716                    Ok(()) => Ok(()),
717                    Err(e) => {
718                        eprintln!("Warning: Some operations failed: {}", e);
719                        Ok(())
720                    }
721                }
722            },
723        ),
724
725        QuickCommand::Status { model, verbose } => with_runtime(
726            RuntimeConfig {
727                device: DeviceConfig::request_cpu(),
728                multi_threaded: true,
729            },
730            || async move { handle_status(ctx.registry(), model, verbose).await },
731        ),
732
733        QuickCommand::Branch {
734            model,
735            name,
736            from,
737            policy,
738        } => with_runtime(
739            RuntimeConfig {
740                device: DeviceConfig::request_cpu(),
741                multi_threaded: true,
742            },
743            || async move { handle_branch(ctx.registry(), &model, &name, from, policy).await },
744        ),
745
746        QuickCommand::Checkout {
747            model,
748            git_ref,
749            create_branch,
750            force,
751        } => with_runtime(
752            RuntimeConfig {
753                device: DeviceConfig::request_cpu(),
754                multi_threaded: true,
755            },
756            || async move {
757                let git_ref_parsed = match git_ref {
758                    Some(ref r) => GitRef::parse(r)?,
759                    None => GitRef::DefaultBranch,
760                };
761                let model_ref = ModelRef::with_ref(model, git_ref_parsed);
762                handle_checkout(ctx.registry(), &model_ref.to_string(), create_branch, force).await
763            },
764        ),
765
766        QuickCommand::Pull {
767            model,
768            remote,
769            branch,
770            rebase,
771        } => with_runtime(
772            RuntimeConfig {
773                device: DeviceConfig::request_cpu(),
774                multi_threaded: true,
775            },
776            || async move { handle_pull(ctx.registry(), &model, remote, branch, rebase).await },
777        ),
778
779        QuickCommand::Remove {
780            model,
781            force,
782            registry_only,
783            files_only,
784        } => with_runtime(
785            RuntimeConfig {
786                device: DeviceConfig::request_cpu(),
787                multi_threaded: true,
788            },
789            || async move {
790                handle_remove(ctx.registry(), &model, force, registry_only, files_only).await
791            },
792        ),
793
794        QuickCommand::Load {
795            model,
796            max_context,
797            kv_quant,
798            wait,
799        } => with_runtime(
800            RuntimeConfig {
801                device: DeviceConfig::request_gpu(),
802                multi_threaded: true,
803            },
804            || async move { handle_load(&model, max_context, kv_quant.into(), wait, signing_key).await },
805        ),
806
807        QuickCommand::Unload { model } => with_runtime(
808            RuntimeConfig {
809                device: DeviceConfig::request_cpu(),
810                multi_threaded: true,
811            },
812            || async move { handle_unload(&model, signing_key).await },
813        ),
814
815        QuickCommand::Notify { command } => with_runtime(
816            RuntimeConfig {
817                device: DeviceConfig::request_cpu(),
818                multi_threaded: true,
819            },
820            || async move { handle_notify_command(command, signing_key).await },
821        ),
822
823        QuickCommand::Worktree { command } => with_runtime(
824            RuntimeConfig {
825                device: DeviceConfig::request_cpu(),
826                multi_threaded: true,
827            },
828            || async move {
829                match command {
830                    WorktreeQuickCommand::Add {
831                        model,
832                        branch,
833                        policy,
834                    } => handle_worktree_add(ctx.registry(), &model, &branch, policy).await,
835                    WorktreeQuickCommand::List { model, all } => {
836                        handle_worktree_list(ctx.registry(), &model, all).await
837                    }
838                    WorktreeQuickCommand::Info { model, branch } => {
839                        handle_worktree_info(ctx.registry(), &model, &branch).await
840                    }
841                    WorktreeQuickCommand::Remove {
842                        model,
843                        branch,
844                        force,
845                    } => handle_worktree_remove(ctx.registry(), &model, &branch, force).await,
846                }
847            },
848        ),
849
850        #[cfg(feature = "experimental")]
851        QuickCommand::Commit {
852            model,
853            message,
854            all,
855            all_untracked,
856            amend,
857            author,
858            author_name,
859            author_email,
860            allow_empty,
861            dry_run,
862            verbose,
863        } => with_runtime(
864            RuntimeConfig {
865                device: DeviceConfig::request_cpu(),
866                multi_threaded: true,
867            },
868            || async move {
869                handle_commit(
870                    ctx.registry(),
871                    &model,
872                    &message,
873                    all,
874                    all_untracked,
875                    amend,
876                    author,
877                    author_name,
878                    author_email,
879                    allow_empty,
880                    dry_run,
881                    verbose,
882                )
883                .await
884            },
885        ),
886
887        #[cfg(feature = "experimental")]
888        QuickCommand::Promote {
889            model,
890            branch,
891            author_name,
892            author_email,
893        } => with_runtime(
894            RuntimeConfig {
895                device: DeviceConfig::request_cpu(),
896                multi_threaded: true,
897            },
898            || async move {
899                handle_promote(ctx.registry(), &model, &branch, author_name, author_email).await
900            },
901        ),
902
903        #[cfg(feature = "experimental")]
904        QuickCommand::Push {
905            model,
906            remote,
907            branch,
908            set_upstream,
909            force,
910        } => with_runtime(
911            RuntimeConfig {
912                device: DeviceConfig::request_cpu(),
913                multi_threaded: true,
914            },
915            || async move {
916                handle_push(ctx.registry(), &model, remote, branch, set_upstream, force).await
917            },
918        ),
919
920        #[cfg(feature = "experimental")]
921        QuickCommand::Merge {
922            target,
923            source,
924            ff,
925            no_ff,
926            ff_only,
927            no_commit,
928            squash,
929            message,
930            abort,
931            continue_merge,
932            quit,
933            no_stat,
934            quiet,
935            verbose,
936            strategy,
937            strategy_option,
938            allow_unrelated_histories,
939            no_verify,
940        } => with_runtime(
941            RuntimeConfig {
942                device: DeviceConfig::request_cpu(),
943                multi_threaded: true,
944            },
945            || async move {
946                let options = MergeOptions {
947                    ff,
948                    no_ff,
949                    ff_only,
950                    no_commit,
951                    squash,
952                    message: message.clone(),
953                    abort,
954                    continue_merge,
955                    quit,
956                    no_stat,
957                    quiet,
958                    verbose,
959                    strategy: strategy.clone(),
960                    strategy_option: strategy_option.clone(),
961                    allow_unrelated_histories,
962                    no_verify,
963                };
964                handle_merge(ctx.registry(), &target, &source, options).await
965            },
966        ),
967
968        QuickCommand::Training(training_cmd) => with_runtime(
969            RuntimeConfig {
970                device: DeviceConfig::request_gpu(),
971                multi_threaded: true,
972            },
973            || async move {
974                let registry = ctx.registry();
975                match training_cmd.action {
976                    TrainingAction::Init {
977                        model,
978                        branch,
979                        adapter,
980                        index,
981                        rank,
982                        alpha,
983                        mode,
984                        learning_rate,
985                    } => {
986                        handle_training_init(
987                            registry,
988                            &model,
989                            branch,
990                            adapter,
991                            index,
992                            rank,
993                            alpha,
994                            &mode,
995                            learning_rate,
996                        )
997                        .await
998                    }
999                    TrainingAction::Infer {
1000                        model,
1001                        prompt,
1002                        image,
1003                        max_tokens,
1004                        temperature,
1005                        top_p,
1006                        top_k,
1007                        repeat_penalty,
1008                        sync,
1009                        max_context,
1010                        kv_quant,
1011                    } => {
1012                        let prompt_text = match prompt {
1013                            Some(p) => p,
1014                            None => {
1015                                use std::io::Read;
1016                                let mut buffer = String::new();
1017                                std::io::stdin().read_to_string(&mut buffer)?;
1018                                buffer.trim().to_owned()
1019                            }
1020                        };
1021                        handle_training_infer(
1022                            registry,
1023                            hyprstream_rpc::registry::global().endpoint("policy", SocketKind::Rep),
1024                            &model,
1025                            &prompt_text,
1026                            image,
1027                            max_tokens,
1028                            temperature,
1029                            top_p,
1030                            top_k,
1031                            repeat_penalty,
1032                            sync,
1033                            max_context,
1034                            kv_quant,
1035                            signing_key,
1036                            verifying_key,
1037                        )
1038                        .await
1039                    }
1040                    TrainingAction::Batch {
1041                        model,
1042                        input,
1043                        input_dir,
1044                        pattern,
1045                        format,
1046                        max_tokens,
1047                        chunk_size,
1048                        skip,
1049                        limit,
1050                        progress_interval,
1051                        checkpoint_interval,
1052                        test_set,
1053                        max_context,
1054                        kv_quant,
1055                    } => {
1056                        handle_training_batch(
1057                            registry,
1058                            hyprstream_rpc::registry::global().endpoint("policy", SocketKind::Rep),
1059                            &model,
1060                            input,
1061                            input_dir,
1062                            &pattern,
1063                            &format,
1064                            max_tokens,
1065                            chunk_size,
1066                            skip,
1067                            limit,
1068                            progress_interval,
1069                            checkpoint_interval,
1070                            test_set,
1071                            max_context,
1072                            kv_quant,
1073                            signing_key,
1074                            verifying_key,
1075                        )
1076                        .await
1077                    }
1078                    TrainingAction::Checkpoint {
1079                        model,
1080                        message,
1081                        push,
1082                        remote,
1083                    } => handle_training_checkpoint(registry, &model, message, push, &remote).await,
1084                }
1085            },
1086        ),
1087
1088        QuickCommand::Policy { command } => with_runtime(
1089            RuntimeConfig {
1090                device: DeviceConfig::request_cpu(),
1091                multi_threaded: true,
1092            },
1093            || async move {
1094                let registry_path = ctx.models_dir().join(".registry");
1095                let policies_dir = registry_path.join("policies");
1096                let keys_dir = registry_path.join("keys");
1097
1098                // Load signing key for RPC authentication
1099                let signing_key = load_or_generate_signing_key(&keys_dir).await?;
1100
1101                // Ensure policies directory exists for the editor path
1102                tokio::fs::create_dir_all(&policies_dir).await?;
1103                let policy_csv_path = policies_dir.join("policy.csv");
1104
1105                match command {
1106                    PolicyCommand::Show { raw } => handle_policy_show(&signing_key, raw).await,
1107                    PolicyCommand::History { count, oneline } => {
1108                        handle_policy_history(&signing_key, count, oneline).await
1109                    }
1110                    PolicyCommand::Edit => handle_policy_edit(&signing_key, &policy_csv_path).await,
1111                    PolicyCommand::Diff { against } => {
1112                        handle_policy_diff(&signing_key, against).await
1113                    }
1114                    PolicyCommand::Apply { dry_run, message } => {
1115                        handle_policy_apply(&signing_key, dry_run, message).await
1116                    }
1117                    PolicyCommand::Rollback { git_ref, dry_run } => {
1118                        handle_policy_rollback(&signing_key, &git_ref, dry_run).await
1119                    }
1120                    PolicyCommand::Check {
1121                        user,
1122                        resource,
1123                        action,
1124                    } => handle_policy_check(&signing_key, &user, &resource, &action).await,
1125                    PolicyCommand::Token { command: token_cmd } => match token_cmd {
1126                        TokenCommand::Create {
1127                            user,
1128                            name,
1129                            expires,
1130                            scope,
1131                        } => handle_token_create(&signing_key, &user, name, &expires, scope).await,
1132                        TokenCommand::List { user: _ } => {
1133                            println!("JWT tokens are stateless and cannot be listed.");
1134                            println!();
1135                            println!("Tokens are validated by signature and expiry time.");
1136                            println!("To see who has access, review the policy:");
1137                            println!("  hyprstream quick policy show");
1138                            Ok(())
1139                        }
1140                        TokenCommand::Revoke {
1141                            token: _,
1142                            name: _,
1143                            revoke_user: _,
1144                            force: _,
1145                        } => {
1146                            println!("JWT tokens cannot be revoked individually.");
1147                            println!();
1148                            println!("Options for token invalidation:");
1149                            println!("  1. Use short expiry times (e.g., --expires 1d)");
1150                            println!("  2. Regenerate the signing key to invalidate all tokens:");
1151                            println!(
1152                                "     rm ~/.local/share/hyprstream/.registry/keys/signing.key"
1153                            );
1154                            println!("  3. Remove user permissions via policy:");
1155                            println!("     hyprstream quick policy edit");
1156                            println!();
1157                            println!("A token blocklist feature may be added in the future.");
1158                            Ok(())
1159                        }
1160                    },
1161                    PolicyCommand::ApplyTemplate { template, dry_run } => {
1162                        handle_policy_apply_template(&signing_key, &template, dry_run).await
1163                    }
1164                    PolicyCommand::ListTemplates => handle_policy_list_templates().await,
1165                    PolicyCommand::Role { command } => match command {
1166                        RoleCommand::Add {
1167                            user,
1168                            role,
1169                            dry_run,
1170                        } => handle_policy_role_add(&signing_key, &user, &role, dry_run).await,
1171                        RoleCommand::Remove { user, role, force } => {
1172                            handle_policy_role_remove(&signing_key, &user, &role, force).await
1173                        }
1174                        RoleCommand::List { user, role } => {
1175                            handle_policy_role_list(&signing_key, user.as_deref(), role.as_deref())
1176                                .await
1177                        }
1178                    },
1179                }
1180            },
1181        ),
1182
1183        QuickCommand::Worker { command: action } => {
1184            let signing_key = signing_key.clone();
1185            with_runtime(
1186                RuntimeConfig {
1187                    device: DeviceConfig::request_cpu(),
1188                    multi_threaded: true,
1189                },
1190                || async move {
1191                    let worker_already_running =
1192                        ctx.config().services.startup.contains(&"worker".to_owned());
1193
1194                    if worker_already_running {
1195                        info!(
1196                            "WorkerService is managed by systemd, connecting to existing service"
1197                        );
1198                    } else {
1199                        info!("Starting WorkerService for CLI worker commands");
1200                    }
1201
1202                    let data_dir = dirs::data_local_dir()
1203                        .unwrap_or_else(|| std::path::PathBuf::from("."))
1204                        .join("hyprstream");
1205                    let runtime_dir = dirs::runtime_dir()
1206                        .unwrap_or_else(std::env::temp_dir)
1207                        .join("hyprstream");
1208
1209                    let kata_boot_path = std::env::var("KATA_BOOT_PATH")
1210                        .map(std::path::PathBuf::from)
1211                        .unwrap_or_else(|_| {
1212                            std::path::PathBuf::from("/opt/kata/share/kata-containers")
1213                        });
1214
1215                    let pool_config = PoolConfig {
1216                        warm_pool_size: 0,
1217                        runtime_dir: runtime_dir.join("sandboxes"),
1218                        kernel_path: kata_boot_path.join("vmlinux.container"),
1219                        vm_image: kata_boot_path.join("kata-containers.img"),
1220                        cloud_init_dir: data_dir.join("cloud-init"),
1221                        ..PoolConfig::default()
1222                    };
1223                    #[cfg(feature = "oci-image")]
1224                    let image_config = ImageConfig {
1225                        blobs_dir: data_dir.join("images/blobs"),
1226                        bootstrap_dir: data_dir.join("images/bootstrap"),
1227                        refs_dir: data_dir.join("images/refs"),
1228                        cache_dir: data_dir.join("images/cache"),
1229                        runtime_dir: runtime_dir.join("nydus"),
1230                        ..ImageConfig::default()
1231                    };
1232
1233                    let _worker_handle = if !worker_already_running {
1234                        use hyprstream_workers::runtime::{
1235                            resolve_backend, BackendCtx, SandboxBackend,
1236                        };
1237
1238                        // RAFS image store is built whenever the image filesystem
1239                        // service is compiled in (`oci-image`), so both kata
1240                        // (virtio-fs) and nspawn (FUSE tenant-VFS root, Model B
1241                        // #715) can compose a per-sandbox VFS from it.
1242                        #[cfg(feature = "oci-image")]
1243                        let rafs_store = {
1244                            use hyprstream_workers::image::RafsStore;
1245                            Arc::new(RafsStore::new(image_config.clone())?)
1246                        };
1247
1248                        // Resolve + construct the backend fail-closed against the
1249                        // inventory registry: "auto" (default) picks the strongest
1250                        // available backend; an explicit name must be registered
1251                        // and available, else error. No silent nspawn fallback.
1252                        let backend_name: String = ctx
1253                            .config()
1254                            .worker
1255                            .as_ref()
1256                            .map(|w| w.backend.clone())
1257                            .unwrap_or_else(|| "auto".to_owned());
1258                        let ninep_decider = hyprstream_core::mac::production_ninep_decider(
1259                            signing_key.clone(),
1260                            &ctx.config().oauth,
1261                            "ninep-worker",
1262                        )
1263                        .await
1264                        .context("construct worker 9P MAC PEP")?;
1265                        // #1269: install the full ReferenceMonitor at every
1266                        // production 9P constructor. Its anonymous attach
1267                        // authenticator denies the uncredentialed UDS/vsock
1268                        // boundary locally; do not narrow process-global RPC
1269                        // activation and deny authenticated Worker methods.
1270                        let ninep_monitor =
1271                            Some(hyprstream_core::mac::enrollment_ninep_reference_monitor(
1272                                Arc::clone(&ninep_decider),
1273                            ));
1274                        let backend_ctx = BackendCtx {
1275                            pool_config: pool_config.clone(),
1276                            ninep_decider,
1277                            ninep_monitor,
1278                            #[cfg(feature = "oci-image")]
1279                            image_config,
1280                            #[cfg(feature = "oci-image")]
1281                            rafs_store: Arc::clone(&rafs_store),
1282                        };
1283                        let backend: Arc<dyn SandboxBackend> =
1284                            resolve_backend(&backend_name, &backend_ctx)?;
1285
1286                        let worker_transport = TransportConfig::inproc("hyprstream/workers");
1287                        let mut worker_service = WorkerService::new(
1288                            pool_config,
1289                            backend,
1290                            // Wire rafs_store for the canonical `kata` feature as
1291                            // well as its `kata-vm` compat alias — `kata-vm = ["kata"]`
1292                            // is one-way, so a `--features kata` build must not fall
1293                            // through to the `None` arm (#518).
1294                            #[cfg(any(feature = "kata", feature = "kata-vm"))]
1295                            Some(rafs_store),
1296                            #[cfg(not(any(feature = "kata", feature = "kata-vm")))]
1297                            None,
1298                            worker_transport,
1299                            signing_key.clone(),
1300                        )?;
1301
1302                        // Wire up policy-backed authorization
1303                        let worker_policy_client =
1304                            hyprstream_core::services::policy_client_for_process(
1305                                signing_key.clone(),
1306                                resolve_service_vk("policy", Some(ctx.config())).ok_or_else(
1307                                    || anyhow::anyhow!("Cannot resolve policy pubkey. Run wizard."),
1308                                )?,
1309                                None,
1310                            )?;
1311                        worker_service.set_authorize_fn(
1312                            hyprstream_core::services::build_authorize_fn(worker_policy_client),
1313                        );
1314
1315                        // Standalone worker entrypoint serves RPC, so it must
1316                        // install the verify config too (#160) — otherwise the
1317                        // fail-closed default rejects all mesh traffic. No config
1318                        // is in scope here, so the mesh PQ store is empty (#157):
1319                        // identical to prior behavior (Hybrid fails closed for
1320                        // unknown peers).
1321                        install_envelope_verify_config(None, None);
1322
1323                        let manager = InprocManager::new();
1324                        Some(
1325                            manager
1326                                .spawn(Box::new(worker_service))
1327                                .await
1328                                .context("Failed to start worker service")?,
1329                        )
1330                    } else {
1331                        None
1332                    };
1333
1334                    use hyprstream_rpc_std::worker_client::WorkerClient;
1335                    let worker_client = if worker_already_running {
1336                        WorkerClient::from_provider(
1337                            &hyprstream_discovery::ProductionRpcClientProvider,
1338                            signing_key,
1339                            None,
1340                        )?
1341                    } else {
1342                        let destination = signing_key.verifying_key();
1343                        WorkerClient::for_local_bootstrap(signing_key, destination, None)?
1344                    };
1345
1346                    match action {
1347                        WorkerAction::List {
1348                            sandbox,
1349                            container,
1350                            containers,
1351                            sandboxes,
1352                            state,
1353                            verbose,
1354                        } => {
1355                            handle_worker_list(
1356                                &worker_client,
1357                                sandbox,
1358                                container,
1359                                containers,
1360                                sandboxes,
1361                                state,
1362                                verbose,
1363                            )
1364                            .await
1365                        }
1366                        WorkerAction::Run {
1367                            image,
1368                            command,
1369                            sandbox,
1370                            name,
1371                            env,
1372                            workdir,
1373                            detach,
1374                            rm,
1375                        } => {
1376                            handle_worker_run(
1377                                &worker_client,
1378                                &image,
1379                                command,
1380                                sandbox,
1381                                name,
1382                                env,
1383                                workdir,
1384                                detach,
1385                                rm,
1386                            )
1387                            .await
1388                        }
1389                        WorkerAction::Stop { id, timeout, force } => {
1390                            handle_worker_stop(&worker_client, &id, timeout, force).await
1391                        }
1392                        WorkerAction::Start { container_id } => {
1393                            handle_worker_start(&worker_client, &container_id).await
1394                        }
1395                        WorkerAction::Restart { id, timeout } => {
1396                            handle_worker_restart(&worker_client, &id, timeout).await
1397                        }
1398                        WorkerAction::Rm {
1399                            ids,
1400                            force,
1401                            volumes: _,
1402                        } => handle_worker_rm(&worker_client, ids, force).await,
1403                        WorkerAction::Status { id, verbose } => {
1404                            handle_worker_status(&worker_client, &id, verbose).await
1405                        }
1406                        WorkerAction::Stats {
1407                            ids,
1408                            stream: _,
1409                            no_header,
1410                        } => handle_worker_stats(&worker_client, ids, no_header).await,
1411                        WorkerAction::Exec {
1412                            container_id,
1413                            command,
1414                            timeout,
1415                        } => {
1416                            handle_worker_exec(&worker_client, &container_id, command, timeout)
1417                                .await
1418                        }
1419                        WorkerAction::Terminal {
1420                            container_id,
1421                            detach_keys,
1422                        } => {
1423                            handle_worker_terminal(&worker_client, &container_id, &detach_keys)
1424                                .await
1425                        }
1426                        WorkerAction::Images(img_cmd) => match img_cmd {
1427                            ImageCommand::List { verbose, filter: _ } => {
1428                                handle_images_list(&worker_client, verbose).await
1429                            }
1430                            ImageCommand::Pull {
1431                                image,
1432                                username,
1433                                password,
1434                            } => {
1435                                handle_images_pull(&worker_client, &image, username, password).await
1436                            }
1437                            ImageCommand::Rm { images, force } => {
1438                                handle_images_rm(&worker_client, images, force).await
1439                            }
1440                            ImageCommand::Df => handle_images_df(&worker_client).await,
1441                        },
1442                    }
1443                },
1444            )
1445        }
1446
1447        QuickCommand::Remote { command } => with_runtime(
1448            RuntimeConfig {
1449                device: DeviceConfig::request_cpu(),
1450                multi_threaded: true,
1451            },
1452            || async move {
1453                match command {
1454                    RemoteQuickCommand::Add { model, name, url } => {
1455                        handle_remote_add(ctx.registry(), &model, &name, &url).await
1456                    }
1457                    RemoteQuickCommand::List { model, verbose } => {
1458                        handle_remote_list(ctx.registry(), &model, verbose).await
1459                    }
1460                    RemoteQuickCommand::Remove { model, name } => {
1461                        handle_remote_remove(ctx.registry(), &model, &name).await
1462                    }
1463                    RemoteQuickCommand::SetUrl { model, name, url } => {
1464                        handle_remote_set_url(ctx.registry(), &model, &name, &url).await
1465                    }
1466                    RemoteQuickCommand::Rename {
1467                        model,
1468                        old_name,
1469                        new_name,
1470                    } => handle_remote_rename(ctx.registry(), &model, &old_name, &new_name).await,
1471                }
1472            },
1473        ),
1474    }
1475}
1476
1477// ─────────────────────────────────────────────────────────────────────────────
1478// CLI pubkey resolution helper
1479// ─────────────────────────────────────────────────────────────────────────────
1480
1481/// Resolve a service's verifying key via the global trust store.
1482///
1483/// On first call in CLI mode, seeds the trust store from bootstrap-pubkeys.
1484/// Used by CLI command handlers that don't have a ServiceContext but need
1485/// to verify responses from a target service.
1486///
1487/// Returns `None` if bootstrap-pubkeys is not available (wizard not run).
1488/// Resolve the configured `[quic] relay` URI into the wire-reach
1489/// [`TransportConfig`] a producer advertises as its `Role::Relay` reach (#358).
1490///
1491/// Reuses the SAME [`hyprstream_rpc::service_entry`] codec the DID document uses
1492/// (by constructing the equivalent `QuicTransport` service entry and decoding it),
1493/// then projecting to wire-reach form via
1494/// [`hyprstream_rpc::moq_stream::relay_reach_from_decoded`] — so the advertised
1495/// stream relay and the DID transport address are produced by one codec and never
1496/// drift. WebPKI (public PDS / federation anchor) is the default; a self-signed
1497/// mesh relay carries its leaf-cert SHA-256 pins after `#` in the URI fragment.
1498fn resolve_moq_relay_reach(
1499    relay_uri: &str,
1500) -> Result<hyprstream_rpc::stream_info::TransportConfig> {
1501    use serde_json::json;
1502    // Optional `#<multibase certHash>[,<multibase certHash>...]` fragment pins a
1503    // self-signed mesh relay; absent ⇒ WebPKI (public CA-fronted relay).
1504    let (uri, cert_hashes): (&str, Vec<String>) = match relay_uri.split_once('#') {
1505        Some((u, frag)) => (u, frag.split(',').map(str::to_owned).collect()),
1506        None => (relay_uri, Vec::new()),
1507    };
1508    let webpki = cert_hashes.is_empty();
1509    let entry = json!({
1510        "type": "QuicTransport",
1511        "serviceEndpoint": {
1512            "uri": uri,
1513            "webpki": webpki,
1514            "certHashes": cert_hashes,
1515            "accept": ["moql"],
1516        },
1517    });
1518    let decoded = hyprstream_rpc::service_entry::decode_service_entry(&entry)
1519        .map_err(|e| anyhow::anyhow!("relay URI decode: {e}"))?;
1520    hyprstream_rpc::moq_stream::relay_reach_from_decoded(&decoded.config)
1521        .ok_or_else(|| anyhow::anyhow!("relay URI is not a network-routable transport"))
1522}
1523
1524/// Resolve the services hosted by this foreground process independently of
1525/// transport profile. Background start commands do not host their children.
1526fn foreground_service_process_names(
1527    matches: &clap::ArgMatches,
1528    config: &HyprConfig,
1529) -> Result<Option<Vec<String>>> {
1530    let Some(("service", service_matches)) = matches.subcommand() else {
1531        return Ok(None);
1532    };
1533    let action = ServiceAction::from_arg_matches(service_matches)
1534        .map_err(|error| anyhow::anyhow!("{error}"))?;
1535    let ServiceAction::Start {
1536        name,
1537        foreground,
1538        standalone,
1539        services,
1540        ..
1541    } = action
1542    else {
1543        return Ok(None);
1544    };
1545    if !foreground && !standalone {
1546        return Ok(None);
1547    }
1548    let names = if standalone {
1549        config.services.startup.clone()
1550    } else {
1551        services.unwrap_or_else(|| name.into_iter().collect())
1552    };
1553    Ok(Some(names))
1554}
1555
1556/// Process termination is safe only when this process hosts one known service.
1557fn dedicated_service_process_name(
1558    matches: &clap::ArgMatches,
1559    config: &HyprConfig,
1560) -> Result<Option<String>> {
1561    let Some(names) = foreground_service_process_names(matches, config)? else {
1562        return Ok(None);
1563    };
1564    match names.as_slice() {
1565        [service] if get_factory(service).is_some() => Ok(Some(service.clone())),
1566        _ => Ok(None),
1567    }
1568}
1569
1570/// Select the process identity before constructing any resolver/client. Transport
1571/// flags do not decide whose identity a required-native split service uses.
1572fn native_service_process_name(
1573    matches: &clap::ArgMatches,
1574    config: &HyprConfig,
1575) -> Result<Option<String>> {
1576    if !config.quic.iroh_required() {
1577        return Ok(None);
1578    }
1579    let Some(names) = foreground_service_process_names(matches, config)? else {
1580        return Ok(None);
1581    };
1582    anyhow::ensure!(names.len() == 1,
1583        "network-iroh-required requires exactly one service per foreground process; launch each provisioned service separately");
1584    let service = names
1585        .into_iter()
1586        .next()
1587        .context("native service name missing")?;
1588    anyhow::ensure!(
1589        hyprstream_service::get_factory(&service).is_some(),
1590        "unknown native service: {service}"
1591    );
1592    Ok(Some(service))
1593}
1594
1595/// The identity a foreground process installs into the process-global
1596/// Discovery resolver and later uses for service startup. A native
1597/// (iroh-required) selection takes precedence; otherwise a dedicated
1598/// single-service compatibility process uses its provisioned service identity
1599/// so the resolver client and the service share one key and enrolled
1600/// credential. Plain CLI commands get neither.
1601fn process_identity_name<'a>(
1602    native_service_name: Option<&'a str>,
1603    dedicated_service_name: Option<&'a str>,
1604) -> Option<&'a str> {
1605    native_service_name.or(dedicated_service_name)
1606}
1607
1608/// The enrolled credential a process-global resolver presents when the process
1609/// has no selected service identity (a multi-service foreground graph or a
1610/// plain CLI command). On a flat-key deployment `policy` resolves to the node
1611/// root/CA key (#759), so that key IS the policy identity: the resolver may
1612/// present policy's enrolled JWT. Without it, cross-process Discovery
1613/// resolution from such a process goes out anonymous and dies at the MAC
1614/// dispatch PEP (NoClearance) on labeled nodes. Split-key deployments are
1615/// unchanged: key mismatch or a missing credential yields None.
1616fn flat_key_policy_resolver_credential(
1617    secrets: &std::path::Path,
1618    profile: hyprstream_core::auth::identity_store::SecretsProfile,
1619    process_key: &SigningKey,
1620) -> Option<String> {
1621    let policy_key = hyprstream_core::auth::identity_store::resolve_service_signing_key(
1622        secrets, "policy", profile,
1623    )
1624    .ok()?;
1625    if policy_key.verifying_key() != process_key.verifying_key() {
1626        return None;
1627    }
1628    hyprstream_core::auth::identity_store::load_service_jwt_for_profile(secrets, "policy", profile)
1629        .ok()
1630        .flatten()
1631}
1632
1633/// Required service startup consumes the provisioner's retained key. Missing or
1634/// malformed material is never a reason to generate a new identity or use the
1635/// CLI/root key. Compatibility commands retain their existing key behavior.
1636async fn load_process_signing_key(
1637    config: &HyprConfig,
1638    native_service: Option<&str>,
1639) -> Result<SigningKey> {
1640    if let Some(service) = native_service {
1641        let secrets = HyprConfig::resolve_secrets_dir_for(Some(config))?;
1642        return hyprstream_core::auth::identity_store::load_existing_service_signing_key(
1643            &secrets,
1644            service,
1645            hyprstream_core::auth::identity_store::SecretsProfile::from_env()?,
1646        )
1647        .with_context(|| format!("load provisioned native identity for {service}"));
1648    }
1649    let keys_dir = config.models_dir().join(".registry").join("keys");
1650    load_or_generate_signing_key(&keys_dir).await
1651}
1652
1653/// A QUIC process currently owns one native MoQ dialer and its admission-proof
1654/// slot. Sharing that slot across separately checkpointed services would make a
1655/// later service dial as the first service's DID. Refuse that topology until the
1656/// dialer is service-scoped; checkpointed services must run separately.
1657fn select_single_process_moql_admission_proof<T>(
1658    proofs: impl IntoIterator<Item = (String, Option<T>)>,
1659) -> Result<Option<T>> {
1660    let proofs: Vec<_> = proofs.into_iter().collect();
1661    let authenticated: Vec<_> = proofs
1662        .iter()
1663        .filter_map(|(service, proof)| proof.as_ref().map(|_| service.as_str()))
1664        .collect();
1665
1666    if authenticated.is_empty() {
1667        return Ok(None);
1668    }
1669    if proofs.len() != 1 {
1670        anyhow::bail!(
1671            "native MoQ admission proofs are service-scoped, but this QUIC process contains \
1672             checkpointed service(s) [{}] with {} total services; run each checkpointed service \
1673             in a separate process",
1674            authenticated.join(", "),
1675            proofs.len(),
1676        );
1677    }
1678
1679    Ok(proofs.into_iter().next().and_then(|(_, proof)| proof))
1680}
1681
1682/// Quinn's CONNECT path has no process-global MoQL proof slot. Defer proof
1683/// collection itself to an enabled Iroh profile so Quinn-only multi-service
1684/// startup does not inherit Iroh's one-service dialer restriction.
1685fn select_iroh_moql_admission_proof<T>(
1686    iroh_enabled: bool,
1687    proofs: impl FnOnce() -> Result<Vec<(String, Option<T>)>>,
1688) -> Result<Option<T>> {
1689    if iroh_enabled {
1690        select_single_process_moql_admission_proof(proofs()?)
1691    } else {
1692        Ok(None)
1693    }
1694}
1695
1696fn resolve_service_vk(service_name: &str, config: Option<&HyprConfig>) -> Option<VerifyingKey> {
1697    let trust = hyprstream_service::global_trust_store();
1698    // Fast path: already populated (service startup seeded it)
1699    if let Some(vk) = trust.resolve_one(service_name) {
1700        return Some(vk);
1701    }
1702    // CLI mode: seed from bootstrap-pubkeys on first use. When startup already
1703    // loaded a config (including --config), keep that path authoritative.
1704    let Ok(secrets_dir) = HyprConfig::resolve_secrets_dir_for(config) else {
1705        return None;
1706    };
1707    // Load the full entries so the hybrid requirement can be enforced before
1708    // anything is trusted. Only the Ed25519 half is inserted here: the
1709    // ML-DSA-65 half of these same entries is anchored into the process-global
1710    // PQ trust store when the envelope verify config is installed (that store
1711    // is immutable afterwards, so it is seeded once at construction rather
1712    // than lazily here).
1713    let entries =
1714        match hyprstream_core::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir) {
1715            Ok(entries) => entries,
1716            Err(e) => {
1717                tracing::error!("bootstrap-pubkeys could not be read: {e:#}");
1718                return None;
1719            }
1720        };
1721    // A classical-only service entry is refused rather than seeded. Seeding it
1722    // would not fail safe by half-measures: the service would be trusted
1723    // classically here and then denied at envelope verification for having no
1724    // anchored post-quantum key, with an error that names neither the file nor
1725    // the fix. Refusing here reports the actual provisioning fault.
1726    if let Err(e) = hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)
1727    {
1728        tracing::error!("{e:#}");
1729        return None;
1730    }
1731    for (name, entry) in &entries {
1732        trust.insert(
1733            entry.ed25519,
1734            hyprstream_service::Attestation {
1735                scopes: std::iter::once(name.clone()).collect(),
1736                subject: None,
1737                jwt: None,
1738                expires_at: 0,
1739                attested_by: None,
1740            },
1741        );
1742    }
1743    trust.resolve_one(service_name)
1744}
1745
1746/// Install the process production resolver and return whether the
1747/// authenticated trust source was the OS-owned bootstrap (`true`) or a
1748/// DID-anchored source (`false`). The caller retains this selection so the
1749/// QUIC startup gate can scope first-boot deferral to the OS-owned path only.
1750async fn install_process_production_resolver(
1751    signing_key: &SigningKey,
1752    config: &HyprConfig,
1753    service_token: Option<String>,
1754    ephemeral_outbound_transport: bool,
1755) -> Result<bool> {
1756    let trust_source = hyprstream_discovery::DeploymentTrustSource::from_anchors(
1757        config.cluster_at9p_did.as_deref(),
1758        config.cluster_did_web.as_deref(),
1759    )?;
1760    // Every service identity this node provisions is hybrid, so a classical
1761    // service entry means the node was provisioned by a pre-hybrid wizard and
1762    // its services can never be anchored for post-quantum verification. Fail
1763    // startup here, where the fix can be named, instead of letting each RPC
1764    // fail later with an opaque "no anchored ML-DSA-65 signer key". An
1765    // unprovisioned node has no entries and is unaffected.
1766    //
1767    // Scoped to this node's OWN service identities. External classical clients
1768    // and federated peers do not appear in this file and are not affected.
1769    if let Ok(secrets_dir) = HyprConfig::resolve_secrets_dir_for(Some(config)) {
1770        let entries =
1771            hyprstream_core::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir)?;
1772        hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)?;
1773        // hyprstream#1562 H3: an OS-owned deployment must enroll its discovery/
1774        // policy service keys into the ceremony signature chain — unsigned-TOFU
1775        // bootstrap-pubkeys are refused. DID-anchored and wizard/dev
1776        // deployments are unchanged.
1777        if matches!(
1778            trust_source,
1779            hyprstream_discovery::DeploymentTrustSource::OsOwnedFiles
1780        ) {
1781            hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_enrolled(
1782                &secrets_dir,
1783                &entries,
1784            )?;
1785        }
1786    }
1787
1788    // The bootstrap pins the discovery service key from the process trust
1789    // store; in CLI/service-start mode nothing has seeded it yet. Seed from
1790    // the node's own bootstrap-pubkeys (the same source resolve_service_vk
1791    // uses on first use) — a no-op when already populated or unprovisioned.
1792    let _ = resolve_service_vk("discovery", Some(config));
1793    // Private-PKI deployments may terminate the did:web host with an internal
1794    // CA; the extra root is additive (never disables verification), and an
1795    // unreadable file is a hard configuration error, not a silent skip.
1796    let trust_source = match (&trust_source, &config.cluster_anchor_root_cert) {
1797        (hyprstream_discovery::DeploymentTrustSource::DidAnchored(anchors), Some(path)) => {
1798            let pem = std::fs::read(path).with_context(|| {
1799                format!(
1800                    "cluster_anchor_root_cert is not readable: {}",
1801                    path.display()
1802                )
1803            })?;
1804            hyprstream_discovery::DeploymentTrustSource::DidAnchored(
1805                anchors.clone().with_root_cert_pem(pem),
1806            )
1807        }
1808        (hyprstream_discovery::DeploymentTrustSource::OsOwnedFiles, Some(path)) => {
1809            anyhow::bail!(
1810                "cluster_anchor_root_cert ({}) is set but no DID anchors are configured",
1811                path.display()
1812            )
1813        }
1814        _ => trust_source,
1815    };
1816    // Retain the authenticated selection before it is consumed by the
1817    // bootstrap. The QUIC startup gate uses this to scope first-boot
1818    // deferral to the OS-owned path only.
1819    let is_os_owned_bootstrap = matches!(
1820        trust_source,
1821        hyprstream_discovery::DeploymentTrustSource::OsOwnedFiles
1822    );
1823    // `remote_node` only has meaning for a DID-anchored bootstrap (it selects
1824    // between the lazy same-node fabric and a required, liveness-checked
1825    // dial of the DID-advertised transport). The OS-owned bootstrap always
1826    // installs the lazy local client regardless of this flag — silently, so
1827    // an operator who set `cluster_remote_node = true` expecting a remote
1828    // Discovery reach would otherwise get no error and no such reach. Fail
1829    // closed instead, mirroring the `cluster_anchor_root_cert` check above.
1830    if is_os_owned_bootstrap && config.cluster_remote_node {
1831        anyhow::bail!(
1832            "cluster_remote_node is set but no DID anchors are configured; \
1833             remote-node Discovery reach requires cluster_at9p_did and \
1834             cluster_did_web (DidAnchored mode) — the OS-owned trust source \
1835             has no remote-Discovery story and would silently ignore this flag"
1836        );
1837    }
1838    if config.quic.iroh_required() {
1839        hyprstream_rpc::moq_stream::require_native_iroh();
1840    }
1841    // The records-role binding selects the accepted-state authority backend
1842    // inside the discovery crate: a configured RDS Postgres store must serve
1843    // the production resolver, roster, and bootstrap endpoints (not only the
1844    // app PdsRecordStore) — with no local RocksDB fallback on error.
1845    let records = config
1846        .rds
1847        .resolved_from_env()
1848        .context("failed to resolve the records-role binding")?;
1849    if ephemeral_outbound_transport {
1850        hyprstream_discovery::bootstrap_deployment_process_with_ephemeral_client_transport(
1851            signing_key.clone(),
1852            service_token,
1853            trust_source,
1854            config.cluster_remote_node,
1855            config.quic.iroh_required(),
1856            &records,
1857        )
1858        .await?;
1859    } else {
1860        hyprstream_discovery::bootstrap_deployment_process(
1861            signing_key.clone(),
1862            service_token,
1863            trust_source,
1864            config.cluster_remote_node,
1865            config.quic.iroh_required(),
1866            &records,
1867        )
1868        .await?;
1869    }
1870    hyprstream_rpc::envelope::install_browser_currentness_verifier(
1871        hyprstream_discovery::production_browser_currentness_verifier()?,
1872    )?;
1873    Ok(is_os_owned_bootstrap)
1874}
1875
1876/// The only startup policy that consumes PDS lifecycle classification.
1877///
1878/// `is_os_owned_bootstrap` is derived from the authenticated deployment trust
1879/// source before bootstrap consumes it. Keeping the classifier inside this
1880/// gate is intentional: DID-anchored deployments must take the ordinary
1881/// checkpointed path without reading lifecycle state or mutating QUIC.
1882#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1883enum QuicCheckpointPolicy {
1884    CheckpointedAnnouncements,
1885    DeferForFirstBoot,
1886}
1887
1888fn quic_checkpoint_policy(
1889    is_os_owned_bootstrap: bool,
1890    classify: impl FnOnce() -> Result<hyprstream_core::services::factories::PdsBootState>,
1891) -> Result<QuicCheckpointPolicy> {
1892    if !is_os_owned_bootstrap {
1893        return Ok(QuicCheckpointPolicy::CheckpointedAnnouncements);
1894    }
1895
1896    use hyprstream_core::services::factories::PdsBootState;
1897    match classify()? {
1898        PdsBootState::Populated => Ok(QuicCheckpointPolicy::CheckpointedAnnouncements),
1899        PdsBootState::FirstBoot => Ok(QuicCheckpointPolicy::DeferForFirstBoot),
1900    }
1901}
1902
1903/// Install the process-global envelope verify configuration (#152, #160).
1904///
1905/// MUST be called once at startup on **every** process entrypoint that serves
1906/// RPC — the inproc daemon AND standalone service entrypoints (e.g. a lone
1907/// worker). When uninstalled, `global_verify_policy()` fail-closes to Hybrid
1908/// (#160), so a missing call here breaks mesh verification loudly rather than
1909/// silently downgrading to EdDSA-only. `install_verify_config` is first-write-
1910/// wins, so calling it from multiple stages within one process is harmless.
1911///
1912/// Policy is pinned to Hybrid (SNS nested COSE). With no anchored peer key the
1913/// verifier fails closed (correct, by design).
1914/// Install the process-global envelope verify configuration.
1915///
1916/// When `oauth` is `Some`, the kid-anchored PQ trust store is populated eagerly
1917/// from `mesh_peers` (#157). When `None` (e.g. the standalone worker entrypoint,
1918/// which has no config in scope), the store is empty — identical to the prior
1919/// behavior. `config` carries the already-loaded CLI config so its explicit
1920/// secrets path remains authoritative. Either way the store is immutable after
1921/// install.
1922fn install_envelope_verify_config(
1923    oauth: Option<&hyprstream_core::config::OAuthConfig>,
1924    config: Option<&HyprConfig>,
1925) {
1926    use hyprstream_rpc::envelope::{
1927        install_response_verify_config, install_verify_config, mandatory_envelope_policy,
1928        EnvelopeVerifyConfig, KeyedPqTrustStore, PqTrustStore, ResponseVerifyConfig,
1929    };
1930
1931    // The application suite is pinned: requests and responses both require
1932    // Ed25519 + ML-DSA-65, with no runtime downgrade selector.
1933    let policy = mandatory_envelope_policy();
1934
1935    // Mesh kid-anchored PQ trust store (#157, Option A): populated eagerly from
1936    // admin-configured `mesh_peers`, immutable after install. An empty store
1937    // under Hybrid fails closed for unknown peers (correct, by design).
1938    let mut keyed_store = match oauth {
1939        Some(oauth) => hyprstream_core::auth::mesh_trust::build_mesh_pq_trust_store(oauth),
1940        None => KeyedPqTrustStore::new(),
1941    };
1942
1943    // The node's own OS-owned bootstrap-pubkeys anchor the local services'
1944    // Ed25519 identities; hybrid entries additionally carry the ML-DSA-65 key
1945    // bound to that identity. Anchor those here, into the SAME store the
1946    // envelope verify path consults, so supplying hybrid material actually
1947    // turns PQ enforcement on for those services. A classical-only file
1948    // anchors nothing and changes nothing.
1949    if let Ok(secrets_dir) = HyprConfig::resolve_secrets_dir_for(config) {
1950        let anchored = hyprstream_core::auth::mesh_trust::seed_bootstrap_pq_bindings(
1951            &mut keyed_store,
1952            &secrets_dir,
1953        );
1954        tracing::info!("bootstrap PQ trust store seeded with {anchored} service binding(s)");
1955    } else {
1956        tracing::warn!(
1957            "secrets directory unresolvable; bootstrap PQ bindings not seeded. \
1958             Under Hybrid envelope policy local services may be unverifiable."
1959        );
1960    }
1961
1962    tracing::info!(
1963        "mesh PQ trust store installed with {} peer binding(s)",
1964        keyed_store.len()
1965    );
1966
1967    // Per-host mesh identity roster (#328): bind each enrolled peer's Ed25519
1968    // signer key to its OWN per-host subject (`service:inference:host-<label>`)
1969    // in the global trust store, so a verified mesh peer resolves to its
1970    // granular principal — never the `"system"` god principal. Reuses the SAME
1971    // `mesh_peers` enrollment record as the PQ store above (no new roster type).
1972    // A networked peer whose key is NOT enrolled resolves to anonymous
1973    // (deny-by-default, fail-closed).
1974    if let Some(oauth) = oauth {
1975        let roster = hyprstream_core::auth::mesh_trust::build_mesh_identity_roster(oauth);
1976        let trust = hyprstream_service::global_trust_store();
1977        for (ed_pubkey, subject) in &roster {
1978            if let Ok(vk) = VerifyingKey::from_bytes(ed_pubkey) {
1979                trust.insert(
1980                    vk,
1981                    hyprstream_service::Attestation {
1982                        scopes: std::iter::once("inference".to_owned()).collect(),
1983                        subject: subject.name().map(ToOwned::to_owned),
1984                        jwt: None,
1985                        // Admin-anchored, out-of-band enrollment: no expiry (0).
1986                        expires_at: 0,
1987                        attested_by: None,
1988                    },
1989                );
1990            }
1991        }
1992        tracing::info!(
1993            "mesh identity roster installed with {} per-host binding(s)",
1994            roster.len()
1995        );
1996    }
1997    // Shared Arc: the SAME admin-anchored store backs both the request-side and
1998    // response-side verify configs — built once, anchored once (#277 reuse of
1999    // #157). The server's `#mesh-pq` ML-DSA-65 key (keyed by its Ed25519 mesh
2000    // signer identity) anchors response verification just as it anchors request
2001    // verification.
2002    let pq_store: std::sync::Arc<dyn PqTrustStore> = std::sync::Arc::new(keyed_store);
2003
2004    // Install the RESPONSE-side process-global default (#277), mirroring the
2005    // request-side install below: fail-closed Hybrid by default with the same
2006    // anchored store, so native RPC clients consult it when no per-client store
2007    // was set.
2008    let _ = install_response_verify_config(ResponseVerifyConfig {
2009        policy,
2010        pq_store: Some(pq_store.clone()),
2011    });
2012
2013    if install_verify_config(EnvelopeVerifyConfig {
2014        policy,
2015        pq_store: Some(pq_store),
2016    })
2017    .is_ok()
2018    {
2019        tracing::info!(
2020            "envelope verify policy: HYBRID enforced (SNS nested COSE); \
2021             peer ML-DSA bindings required for cross-node traffic"
2022        );
2023    }
2024
2025    install_session_pq_overlay();
2026    install_proof_admission(oauth, config);
2027}
2028
2029/// PostgreSQL-backed replay admission for the configured shared RDS domain.
2030///
2031/// The database sees only a SHA-256 digest of a domain-separated replay tuple
2032/// and its signed expiry. It never receives a bearer, proof bytes, or request
2033/// body. `PgKv::replay_admit` executes the check-and-insert in one transaction.
2034#[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
2035struct PostgresProofReplayStore {
2036    kv: hyprstream_pds::pgsql_kv::PgKv,
2037}
2038
2039#[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
2040impl PostgresProofReplayStore {
2041    fn digest(parts: &[&[u8]]) -> [u8; 32] {
2042        use sha2::{Digest as _, Sha256};
2043        let mut digest = Sha256::new();
2044        digest.update(b"hyprstream/proof-replay-admission/v1\0");
2045        for part in parts {
2046            digest.update((part.len() as u64).to_be_bytes());
2047            digest.update(part);
2048        }
2049        digest.finalize().into()
2050    }
2051
2052    fn record(
2053        &self,
2054        parts: &[&[u8]],
2055        expires_at: u64,
2056    ) -> hyprstream_rpc::proof::admission::ProofAdmissionResult {
2057        match self.kv.replay_admit(Self::digest(parts), expires_at) {
2058            Ok(true) => hyprstream_rpc::proof::admission::ProofAdmissionResult::Admitted,
2059            Ok(false) => hyprstream_rpc::proof::admission::ProofAdmissionResult::Replayed,
2060            // Do not include a database error: a driver may retain connection
2061            // context. Dispatch receives a uniform failure/denial instead.
2062            Err(_) => hyprstream_rpc::proof::admission::ProofAdmissionResult::Failed,
2063        }
2064    }
2065}
2066
2067#[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
2068impl hyprstream_rpc::proof::admission::ProofReplayStore for PostgresProofReplayStore {
2069    fn domain_guarantee(&self) -> hyprstream_rpc::proof::admission::ReplayDomainGuarantee {
2070        hyprstream_rpc::proof::admission::ReplayDomainGuarantee::LinearizableSharedStore
2071    }
2072
2073    fn check_and_insert(
2074        &self,
2075        partition: hyprstream_rpc::proof::ProofDisposition,
2076        key: &hyprstream_rpc::proof::admission::ProofReplayKey,
2077        expires_at: u64,
2078    ) -> hyprstream_rpc::proof::admission::ProofAdmissionResult {
2079        let partition = match partition {
2080            hyprstream_rpc::proof::ProofDisposition::Authenticated => &b"authenticated"[..],
2081            hyprstream_rpc::proof::ProofDisposition::Unattributed => &b"unattributed"[..],
2082        };
2083        self.record(
2084            &[
2085                b"request-proof",
2086                partition,
2087                &key.signer_thumbprint,
2088                &key.request_id,
2089            ],
2090            expires_at,
2091        )
2092    }
2093
2094    fn check_and_insert_mediated(
2095        &self,
2096        key: &hyprstream_rpc::proof::admission::MediatedQueryReplayKey,
2097        expires_at: u64,
2098    ) -> hyprstream_rpc::proof::admission::ProofAdmissionResult {
2099        self.record(
2100            &[
2101                b"mediated-query",
2102                &key.signer_thumbprint,
2103                &key.request_id.to_be_bytes(),
2104                &key.request_nonce,
2105                key.mediator.as_bytes(),
2106                key.resource.as_bytes(),
2107                key.operation.as_bytes(),
2108            ],
2109            expires_at,
2110        )
2111    }
2112
2113    fn check_and_insert_credential(
2114        &self,
2115        key: &hyprstream_rpc::proof::admission::CredentialReplayKey,
2116        expires_at: u64,
2117    ) -> hyprstream_rpc::proof::admission::ProofAdmissionResult {
2118        self.record(
2119            &[
2120                b"source-credential",
2121                &key.issuer_client_namespace,
2122                &key.credential_identity,
2123            ],
2124            expires_at,
2125        )
2126    }
2127}
2128
2129/// Install the v16 proof admission substrate: the rotating server challenge
2130/// and the replay admission store (§4.6, §4.7).
2131///
2132/// Called from the same startup path as the envelope verify config, so every
2133/// entrypoint that serves RPC installs both. Both registrations are
2134/// first-write-wins and neither is auto-installed by dispatch: an absent
2135/// challenge manager or replay store denies at admission rather than admitting
2136/// under a guarantee nobody made.
2137///
2138/// **Deployment note.** The in-memory store declares
2139/// `SingleVerifierInstance`, which is the honest guarantee for a node that
2140/// admits requests for its own service domain by itself. A domain served by
2141/// several verifier instances needs a shared linearizable backend installed
2142/// here instead — the same trait, a different substrate. Installing this one
2143/// across several instances would silently weaken "admitted once per domain"
2144/// to "once per node", so the log line below states the guarantee in force.
2145/// The optional shared-postgres implementation uses a durable admission table
2146/// and role-scoped, CA-pinned PostgreSQL connection.
2147#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2148enum ProofReplayStoreSelection {
2149    #[cfg(feature = "postgres-replay")]
2150    DedicatedPostgresV1,
2151    #[cfg(not(feature = "postgres-replay"))]
2152    LegacyPdsPostgres,
2153    SingleVerifierMemory,
2154    Unavailable,
2155}
2156
2157fn select_proof_replay_store(
2158    guarantee: hyprstream_rpc::proof::admission::ReplayDomainGuarantee,
2159    legacy_pds_store_available: bool,
2160) -> ProofReplayStoreSelection {
2161    use hyprstream_rpc::proof::admission::ReplayDomainGuarantee;
2162
2163    match guarantee {
2164        #[cfg(feature = "postgres-replay")]
2165        ReplayDomainGuarantee::LinearizableSharedStore => {
2166            // The purpose-built v1 store must win even in a binary that also
2167            // has the PDS records-role feature compiled in.
2168            ProofReplayStoreSelection::DedicatedPostgresV1
2169        }
2170        #[cfg(not(feature = "postgres-replay"))]
2171        ReplayDomainGuarantee::LinearizableSharedStore => {
2172            if legacy_pds_store_available {
2173                ProofReplayStoreSelection::LegacyPdsPostgres
2174            } else {
2175                ProofReplayStoreSelection::Unavailable
2176            }
2177        }
2178        ReplayDomainGuarantee::SingleVerifierInstance if !legacy_pds_store_available => {
2179            ProofReplayStoreSelection::SingleVerifierMemory
2180        }
2181        _ => ProofReplayStoreSelection::Unavailable,
2182    }
2183}
2184
2185fn install_proof_admission(
2186    oauth: Option<&hyprstream_core::config::OAuthConfig>,
2187    config: Option<&HyprConfig>,
2188) {
2189    use hyprstream_rpc::proof::admission::{
2190        set_global_challenge_manager, set_global_proof_replay_store, InMemoryProofReplayStore,
2191        ProofReplayStore, ReplayDomainGuarantee,
2192    };
2193    use hyprstream_rpc::proof::challenge::{
2194        ChallengeManager, DEFAULT_CHALLENGE_OVERLAP_SECS, DEFAULT_CHALLENGE_WINDOW_SECS,
2195    };
2196
2197    #[cfg(feature = "postgres-replay")]
2198    let _ = config;
2199
2200    let now = std::time::SystemTime::now()
2201        .duration_since(std::time::UNIX_EPOCH)
2202        .map(|d| d.as_secs())
2203        .unwrap_or(0);
2204
2205    // The generated method-policy inventory (v16 §6.1): every linked
2206    // generated service module contributed one row per method leaf via
2207    // `inventory`; install the validated, deterministically sorted table as
2208    // the process policy. A failure installs nothing — proof-bearing dispatch
2209    // then denies at policy resolution, never serves a partial table.
2210    match hyprstream_rpc::proof::policy::install_generated_method_policy() {
2211        Ok(rows) => {
2212            tracing::info!("generated dispatch method policy installed ({rows} leaf row(s))");
2213        }
2214        Err(e) => tracing::error!(
2215            "generated dispatch method policy failed validation/install; \
2216             proof-bearing dispatch denies: {e:#}"
2217        ),
2218    }
2219
2220    // The dedicated v1 replay backend has its own schema and login role. When
2221    // it is compiled, do not resolve the records-role RDS binding here: the
2222    // later shared-postgres branch must select PostgresProofReplayStore from
2223    // its dedicated URL/CA files. Retain the historical PDS-backed adapter
2224    // only for pds-postgres builds that do not include postgres-replay.
2225    #[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
2226    let shared_store: Option<Box<dyn ProofReplayStore>> = match config {
2227        Some(config) => match config.rds.resolved_from_env() {
2228            Ok(rds) if rds.is_configured() => match rds.connect_replay_admission() {
2229                Ok(kv) => Some(Box::new(PostgresProofReplayStore { kv })),
2230                Err(_) => {
2231                    tracing::error!("configured shared Postgres replay store could not open; proof admission denies");
2232                    return;
2233                }
2234            },
2235            Ok(_) => None,
2236            Err(_) => {
2237                tracing::error!(
2238                    "records-role RDS binding could not be resolved; proof admission denies"
2239                );
2240                return;
2241            }
2242        },
2243        None => None,
2244    };
2245    #[cfg(feature = "postgres-replay")]
2246    let shared_store: Option<Box<dyn ProofReplayStore>> = None;
2247    #[cfg(all(not(feature = "pds-postgres"), not(feature = "postgres-replay")))]
2248    let shared_store: Option<Box<dyn ProofReplayStore>> = {
2249        if let Some(config) = config {
2250            match config.rds.resolved_from_env() {
2251                Ok(rds) if rds.is_configured() => {
2252                    tracing::error!("configured shared Postgres replay store requires the pds-postgres feature; proof admission denies");
2253                    return;
2254                }
2255                Ok(_) => {}
2256                Err(_) => {
2257                    tracing::error!(
2258                        "records-role RDS binding could not be resolved; proof admission denies"
2259                    );
2260                    return;
2261                }
2262            }
2263        }
2264        None
2265    };
2266
2267    // The replay admission domain is an operator statement about deployment
2268    // topology, not something startup may assume. "Admitted once" means once
2269    // per service domain, so a node that shares its service domain with other
2270    // verifiers cannot satisfy it with a process-local map — and startup
2271    // cannot tell the difference by inspection.
2272    //
2273    // The operator therefore declares it. Absent declaration, nothing is
2274    // installed and every proof-bearing request denies at admission, which is
2275    // the correct posture for an undeclared topology: it is visible and safe,
2276    // rather than a silent per-node guarantee that reads as domain-wide.
2277    let declared_domain = std::env::var("HYPRSTREAM_REPLAY_ADMISSION_DOMAIN").ok();
2278    let guarantee = if shared_store.is_some() {
2279        Some(ReplayDomainGuarantee::LinearizableSharedStore)
2280    } else {
2281        match declared_domain.as_deref() {
2282            Some("single-verifier-instance") => {
2283                // The single-verifier shape is only sound if this process is
2284                // genuinely the sole verifier for the domain. A bare env string
2285                // cannot establish that — two replicas can each set it. Require an
2286                // OS-enforced exclusive lease: acquire it here and hold it for the
2287                // process lifetime, so a second replica setting the same string
2288                // fails to acquire and its admission stays closed (§4.6).
2289                let lease_path = std::env::var("HYPRSTREAM_REPLAY_SINGLE_VERIFIER_LEASE").ok();
2290                let Some(lease_path) = lease_path else {
2291                    tracing::error!(
2292                        "HYPRSTREAM_REPLAY_ADMISSION_DOMAIN=single-verifier-instance requires \
2293                     HYPRSTREAM_REPLAY_SINGLE_VERIFIER_LEASE=<path> to an exclusive lease \
2294                     file: the env string alone cannot prove sole verification. Proof \
2295                     admission denies until the lease is configured."
2296                    );
2297                    return;
2298                };
2299                match hyprstream_rpc::proof::admission::SingleVerifierLease::acquire(
2300                    std::path::Path::new(&lease_path),
2301                ) {
2302                    Ok(lease) => {
2303                        tracing::info!(
2304                            "acquired exclusive single-verifier lease at {} \
2305                         (this process is the sole verifier for its domain)",
2306                            lease.path().display()
2307                        );
2308                        // Park the lease so its exclusive lock lives for the whole
2309                        // process, not just this function.
2310                        if hyprstream_rpc::proof::admission::hold_single_verifier_lease(lease)
2311                            .is_err()
2312                        {
2313                            tracing::error!(
2314                                "a single-verifier lease is already held; proof admission denies"
2315                            );
2316                            return;
2317                        }
2318                        Some(ReplayDomainGuarantee::SingleVerifierInstance)
2319                    }
2320                    Err(e) => {
2321                        tracing::error!(
2322                        "cannot claim sole-verifier lease; another verifier already holds it or \
2323                         the path is unusable. Proof admission denies: {e:#}"
2324                    );
2325                        return;
2326                    }
2327                }
2328            }
2329            Some("shared-postgres") => Some(ReplayDomainGuarantee::LinearizableSharedStore),
2330            Some(other) => {
2331                tracing::error!(
2332                    "HYPRSTREAM_REPLAY_ADMISSION_DOMAIN='{other}' names a topology this build \
2333                 cannot provide in-process: a shared linearizable backend or \
2334                 namespace-affine routing must be installed explicitly. Proof \
2335                 admission denies."
2336                );
2337                None
2338            }
2339            None => {
2340                tracing::warn!(
2341                    "no HYPRSTREAM_REPLAY_ADMISSION_DOMAIN declared; proof replay admission \
2342                 is not installed and proof-bearing requests will deny. Declare \
2343                 'shared-postgres' for a configured shared replay database, or \
2344                 'single-verifier-instance' only when this node is the sole verifier \
2345                 for its service domain."
2346                );
2347                None
2348            }
2349        }
2350    };
2351
2352    let Some(guarantee) = guarantee else {
2353        return;
2354    };
2355
2356    // The challenge is scoped to the SAME replay admission domain: one exact
2357    // deadline shared by verification and replay collection. A per-process
2358    // random challenge can only provide that for a single verifier, so any
2359    // other topology gets no challenge manager and unattributed proofs deny.
2360    match ChallengeManager::rotating_for_domain(
2361        guarantee,
2362        DEFAULT_CHALLENGE_WINDOW_SECS,
2363        DEFAULT_CHALLENGE_OVERLAP_SECS,
2364        now,
2365    ) {
2366        Some(manager) => {
2367            if set_global_challenge_manager(manager).is_ok() {
2368                tracing::info!(
2369                    "proof challenge manager installed for {guarantee:?}: \
2370                     {DEFAULT_CHALLENGE_WINDOW_SECS}s window, \
2371                     {DEFAULT_CHALLENGE_OVERLAP_SECS}s acceptance overlap"
2372                );
2373            }
2374        }
2375        None => tracing::error!(
2376            "replay domain {guarantee:?} needs a domain-wide challenge source; \
2377             unattributed proofs deny"
2378        ),
2379    }
2380
2381    // Request-proof signers are their own enrollment with their own
2382    // lifecycle, loaded from an operator-authored manifest. They are NOT
2383    // derived from the mesh/envelope identity: component-key separation is
2384    // normative, and a transport key carries no enrollment epoch, validity,
2385    // revocation state, approver role, or enrollment-policy identifier.
2386    //
2387    // The mesh roster is consulted only to learn which keys already belong to
2388    // another protocol, so a manifest cannot silently reuse one. Absent
2389    // manifest, nothing is enrolled and authenticated proofs deny.
2390    {
2391        let secrets_dir = HyprConfig::resolve_secrets_dir().ok();
2392        let manifest_path = secrets_dir
2393            .as_ref()
2394            .map(|dir| dir.join("proof-enrollment.toml"));
2395        match manifest_path {
2396            Some(path) if path.exists() => {
2397                match hyprstream_core::auth::proof_enrollment::ProofEnrollmentManifest::load(&path)
2398                {
2399                    Ok(manifest) => {
2400                        // Foreign keys come from every protocol source this
2401                        // node holds — the remote mesh roster and its own
2402                        // local bootstrap identities alike.
2403                        let foreign =
2404                            hyprstream_core::auth::proof_enrollment::foreign_protocol_keys(
2405                                oauth,
2406                                secrets_dir.as_deref(),
2407                            );
2408                        let entries = manifest.entries.len();
2409                        let resolver = hyprstream_core::auth::proof_enrollment::build_resolver(
2410                            &manifest,
2411                            &foreign,
2412                            &Default::default(),
2413                            now,
2414                        );
2415                        if hyprstream_rpc::proof::enrollment::set_global_enrollment_resolver(
2416                            Box::new(resolver),
2417                        )
2418                        .is_ok()
2419                        {
2420                            tracing::info!(
2421                                "proof enrollment manifest loaded from {} ({entries} entr(ies))",
2422                                path.display()
2423                            );
2424                        }
2425                    }
2426                    Err(e) => tracing::error!(
2427                        "proof enrollment manifest at {} is unusable; \
2428                         authenticated proofs will deny: {e:#}",
2429                        path.display()
2430                    ),
2431                }
2432            }
2433            _ => tracing::info!(
2434                "no proof enrollment manifest present; authenticated proofs deny \
2435                 until one is provisioned"
2436            ),
2437        }
2438    }
2439
2440    // Partitioned by disposition, fail-closed on capacity: an unexpired
2441    // accepted record is never evicted to make room.
2442    const REPLAY_CAPACITY_PER_PARTITION: usize = 100_000;
2443    let selection = select_proof_replay_store(guarantee, shared_store.is_some());
2444    let store: Box<dyn ProofReplayStore> = match selection {
2445        #[cfg(not(feature = "postgres-replay"))]
2446        ProofReplayStoreSelection::LegacyPdsPostgres => match shared_store {
2447            Some(store) => store,
2448            None => {
2449                tracing::error!("configured PDS replay store disappeared; admission denies");
2450                return;
2451            }
2452        },
2453        ProofReplayStoreSelection::SingleVerifierMemory => Box::new(
2454            InMemoryProofReplayStore::single_verifier_instance(REPLAY_CAPACITY_PER_PARTITION),
2455        ),
2456        #[cfg(feature = "postgres-replay")]
2457        ProofReplayStoreSelection::DedicatedPostgresV1 => {
2458            match hyprstream_core::auth::postgres_replay::PostgresProofReplayStore::from_env() {
2459                Ok(store) => Box::new(store),
2460                Err(error) => {
2461                    tracing::error!(
2462                        "shared Postgres replay admission unavailable; admission denies: {error:#}"
2463                    );
2464                    return;
2465                }
2466            }
2467        }
2468        ProofReplayStoreSelection::Unavailable => {
2469            tracing::error!("no replay store implementation for {guarantee:?}; admission denies");
2470            return;
2471        }
2472    };
2473    if set_global_proof_replay_store(store).is_ok() {
2474        tracing::info!("proof replay store installed: operator-declared {guarantee:?}");
2475    }
2476}
2477
2478/// Install the session PQ binding overlay, the runtime anchoring path for
2479/// clients no operator can pre-enroll (browsers and other device-generated
2480/// identities).
2481///
2482/// It is consulted only after the admin-anchored store above misses, so it
2483/// cannot displace an operator's enrollment, and a binding it establishes at
2484/// first contact is capped at `Classical` assurance — it makes a request
2485/// verifiable, it does not make the requester more trusted. Promotion to
2486/// `PqHybrid` requires an out-of-band fingerprint comparison.
2487///
2488/// Set `HYPRSTREAM_SESSION_PQ_OVERLAY=0` for a mesh-only deployment that wants
2489/// every unenrolled identity rejected at the envelope layer as before.
2490///
2491/// # Operator limitations at this revision — read before enabling
2492///
2493/// The overlay's approve / promote / revoke operations exist as library calls
2494/// and have **no operator surface yet**: nothing here, no RPC method, no CLI.
2495/// Two consequences an operator must know about up front:
2496///
2497/// - Every binding stays at first-contact provenance for its lifetime.
2498///   Out-of-band promotion is unreachable, so the overlay cannot raise MAC
2499///   assurance for anyone — a browser client is `Classical` and stays there.
2500/// - A refused rebinding is loud but **unanswerable**. A client that rotates
2501///   its ML-DSA-65 key while keeping its Ed25519 identity is refused here,
2502///   cannot be approved (no surface), and cannot be routed around through the
2503///   admin-anchored store (immutable after install). Restarting the daemon
2504///   clears the overlay and is currently the only way through. Until the
2505///   operator surface lands, plan rotations around a restart.
2506fn install_session_pq_overlay() {
2507    let disabled = std::env::var("HYPRSTREAM_SESSION_PQ_OVERLAY")
2508        .map(|v| matches!(v.trim(), "0" | "false" | "off" | "no"))
2509        .unwrap_or(false);
2510    if disabled {
2511        tracing::info!(
2512            "session PQ overlay disabled by configuration; identities without an \
2513             admin-anchored ML-DSA-65 key are rejected at the envelope layer"
2514        );
2515        return;
2516    }
2517    let overlay = std::sync::Arc::new(hyprstream_rpc::session_pq_overlay::SessionPqOverlay::new(
2518        std::sync::Arc::new(hyprstream_rpc::session_pq_overlay::TracingPqBindingEventSink),
2519    ));
2520    if hyprstream_rpc::session_pq_overlay::install_session_pq_overlay(overlay).is_ok() {
2521        tracing::info!(
2522            "session PQ overlay installed: unenrolled identities bind their ML-DSA-65 key \
2523             at first contact (verifiable, assurance capped at Classical); rebinding an \
2524             established identity is refused and surfaced, never silently applied. \
2525             No approve/promote/revoke surface exists yet: a refused rebinding requires \
2526             a daemon restart to clear."
2527        );
2528    }
2529}
2530
2531/// The only native-announcement refresh path used by the production publisher.
2532///
2533/// The operation is injected so the production cadence and terminal-expiry
2534/// behavior can be exercised without a live Discovery service. The production
2535/// caller still supplies [`hyprstream_rpc_std::discovery_client::DiscoveryClient::announce`].
2536#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2537enum NativeAnnouncementRefreshCompletion {
2538    Expired,
2539    Cancelled,
2540}
2541
2542async fn refresh_native_announcement<F, Fut, Error>(
2543    service_name: &str,
2544    socket_kind: &str,
2545    endpoint: &str,
2546    refresh_expires_at_unix_ms: i64,
2547    cancellation: tokio_util::sync::CancellationToken,
2548    mut announce: F,
2549) -> NativeAnnouncementRefreshCompletion
2550where
2551    F: FnMut() -> Fut,
2552    Fut: std::future::Future<Output = std::result::Result<(), Error>>,
2553    Error: std::fmt::Display,
2554{
2555    const ANNOUNCEMENT_REFRESH: std::time::Duration = std::time::Duration::from_secs(25);
2556    const RETRY_INITIAL: std::time::Duration = std::time::Duration::from_secs(5);
2557
2558    let mut delay = RETRY_INITIAL;
2559    loop {
2560        if chrono::Utc::now().timestamp_millis() >= refresh_expires_at_unix_ms {
2561            tracing::warn!(
2562                service = service_name,
2563                socket_kind,
2564                "Stopping native announcement refresh: accepted-state/JWT material expired"
2565            );
2566            return NativeAnnouncementRefreshCompletion::Expired;
2567        }
2568        let result = tokio::select! {
2569            biased;
2570            _ = cancellation.cancelled() => return NativeAnnouncementRefreshCompletion::Cancelled,
2571            result = async { announce().await } => result,
2572        };
2573        let sleep_delay = match result {
2574            Ok(()) => {
2575                tracing::info!(
2576                    service = service_name,
2577                    socket_kind,
2578                    endpoint,
2579                    "Announced native endpoint to DiscoveryService"
2580                );
2581                delay = RETRY_INITIAL;
2582                ANNOUNCEMENT_REFRESH
2583            }
2584            Err(error) => {
2585                tracing::warn!(
2586                    service = service_name,
2587                    socket_kind,
2588                    "Failed to announce native endpoint: {error}"
2589                );
2590                let sleep_delay = delay;
2591                delay = delay
2592                    .checked_mul(2)
2593                    .unwrap_or(ANNOUNCEMENT_REFRESH)
2594                    .min(ANNOUNCEMENT_REFRESH)
2595                    .max(RETRY_INITIAL);
2596                sleep_delay
2597            }
2598        };
2599        tokio::select! {
2600            biased;
2601            _ = cancellation.cancelled() => return NativeAnnouncementRefreshCompletion::Cancelled,
2602            _ = tokio::time::sleep(sleep_delay) => {}
2603        }
2604    }
2605}
2606
2607/// One-shot sender for the first native-announcement result.
2608type NativeAnnouncementFirstResult = std::result::Result<(), String>;
2609
2610/// Optional one-shot channel passed to the native-announcement loop body so it
2611/// can report the result of the first announcement.
2612type NativeAnnouncementFirstTx = Option<
2613    std::sync::Arc<
2614        parking_lot::Mutex<Option<tokio::sync::oneshot::Sender<NativeAnnouncementFirstResult>>>,
2615    >,
2616>;
2617
2618/// Use the current authority projection's service JWT for envelope identity.
2619fn native_announcement_bearer(
2620    announcement: &hyprstream_rpc_std::discovery_client::ServiceAnnouncement,
2621) -> anyhow::Result<String> {
2622    announcement
2623        .service_jwt
2624        .as_ref()
2625        .filter(|token| !token.trim().is_empty())
2626        .cloned()
2627        .context("Native announcement requires a service JWT as its RPC bearer")
2628}
2629
2630/// The required first result covers both fresh-authority projection and the
2631/// Discovery publication. An authority error must not bypass the handshake.
2632async fn publish_native_announcement_attempt<F, Fut>(
2633    announcement: anyhow::Result<hyprstream_rpc_std::discovery_client::ServiceAnnouncement>,
2634    announce_tx: NativeAnnouncementFirstTx,
2635    publish: F,
2636) -> anyhow::Result<()>
2637where
2638    F: FnOnce(hyprstream_rpc_std::discovery_client::ServiceAnnouncement) -> Fut,
2639    Fut: std::future::Future<Output = anyhow::Result<()>>,
2640{
2641    let result = match announcement {
2642        Ok(announcement) => publish(announcement).await,
2643        Err(error) => Err(error),
2644    };
2645    if let Some(tx) = announce_tx {
2646        if let Some(tx) = tx.lock().take() {
2647            let report = result
2648                .as_ref()
2649                .map(|_| ())
2650                .map_err(std::string::ToString::to_string);
2651            let _ = tx.send(report);
2652        }
2653    }
2654    result
2655}
2656
2657/// Policy must serve authority probes while Discovery starts, but is not READY
2658/// until Discovery accepts its announcement. Only typed pre-dispatch Iroh
2659/// unavailability can keep that initial publication pending.
2660const POLICY_INITIAL_PUBLICATION_BUDGET: std::time::Duration = std::time::Duration::from_secs(120);
2661const POLICY_INITIAL_PUBLICATION_RETRY: std::time::Duration = std::time::Duration::from_millis(500);
2662
2663async fn wait_for_policy_initial_publication<F, Fut>(
2664    cancellation: &tokio_util::sync::CancellationToken,
2665    budget: std::time::Duration,
2666    mut attempt: F,
2667) -> anyhow::Result<()>
2668where
2669    F: FnMut() -> Fut,
2670    Fut: std::future::Future<Output = anyhow::Result<()>>,
2671{
2672    let deadline = tokio::time::Instant::now() + budget;
2673    loop {
2674        let result = tokio::select! {
2675            biased;
2676            _ = cancellation.cancelled() => anyhow::bail!("Policy initial publication cancelled"),
2677            _ = tokio::time::sleep_until(deadline) => anyhow::bail!("Policy initial publication deadline exceeded"),
2678            result = async { attempt().await } => result,
2679        };
2680        match result {
2681            Ok(()) => return Ok(()),
2682            Err(error) if hyprstream_rpc::transport_traits::is_pre_dispatch_transport_error(&error)
2683                && error.chain().any(|cause| cause.downcast_ref::<hyprstream_rpc::transport::lazy_iroh::IrohPeerUnavailable>().is_some()) => {
2684                tracing::info!(%error, "Policy awaiting Discovery initial publication");
2685            }
2686            Err(error) => return Err(error),
2687        }
2688        tokio::select! {
2689            biased;
2690            _ = cancellation.cancelled() => anyhow::bail!("Policy initial publication cancelled"),
2691            _ = tokio::time::sleep_until(deadline) => anyhow::bail!("Policy initial publication deadline exceeded"),
2692            _ = tokio::time::sleep(POLICY_INITIAL_PUBLICATION_RETRY) => {}
2693        }
2694    }
2695}
2696
2697/// Spawn the native-announcement refresh loop on its own current-thread Tokio
2698/// runtime and return a channel for the first announcement result.
2699///
2700/// `loop_body` receives an optional one-shot sender for the first result; it
2701/// should send at most once. The runtime is kept alive until `loop_body`
2702/// completes or the bound service cancels it. Cancellation interrupts both the
2703/// first-result wait and subsequent refresh; it never satisfies readiness.
2704fn spawn_native_announcement_loop<F, Fut>(
2705    require_initial: bool,
2706    cancellation: tokio_util::sync::CancellationToken,
2707    loop_body: F,
2708) -> Option<std::sync::mpsc::Receiver<NativeAnnouncementFirstResult>>
2709where
2710    F: FnOnce(NativeAnnouncementFirstTx) -> Fut + Send + 'static,
2711    Fut: std::future::Future<Output = ()> + Send + 'static,
2712{
2713    let (initial_tx, initial_rx) = if require_initial {
2714        let (tx, rx) = std::sync::mpsc::sync_channel(1);
2715        (Some(tx), Some(rx))
2716    } else {
2717        (None, None)
2718    };
2719
2720    std::thread::spawn(move || {
2721        let mut initial_tx = initial_tx;
2722        let runtime = match tokio::runtime::Builder::new_current_thread()
2723            .enable_all()
2724            .build()
2725        {
2726            Ok(runtime) => runtime,
2727            Err(error) => {
2728                tracing::warn!("Failed to create announcement runtime: {error}");
2729                if let Some(tx) = initial_tx.take() {
2730                    let _ = tx.send(Err(error.to_string()));
2731                }
2732                return;
2733            }
2734        };
2735
2736        runtime.block_on(async move {
2737            let (announce_tx, announce_rx) = if require_initial {
2738                let (tx, rx) = tokio::sync::oneshot::channel();
2739                (
2740                    Some(std::sync::Arc::new(parking_lot::Mutex::new(Some(tx)))),
2741                    Some(rx),
2742                )
2743            } else {
2744                (None, None)
2745            };
2746
2747            let mut task = tokio::spawn(loop_body(announce_tx));
2748            if let Some(rx) = announce_rx {
2749                let first = tokio::select! {
2750                    biased;
2751                    _ = cancellation.cancelled() => Ok(Err(
2752                        "native announcement cancelled before first publication".to_owned(),
2753                    )),
2754                    result = rx => result,
2755                };
2756                match first {
2757                    Ok(Ok(())) => {
2758                        if let Some(tx) = initial_tx.take() {
2759                            let result = if cancellation.is_cancelled() {
2760                                Err("native announcement cancelled before readiness".to_owned())
2761                            } else {
2762                                Ok(())
2763                            };
2764                            let _ = tx.send(result);
2765                        }
2766                    }
2767                    Ok(Err(error)) => {
2768                        if let Some(tx) = initial_tx.take() {
2769                            let _ = tx.send(Err(error.clone()));
2770                        }
2771                        task.abort();
2772                    }
2773                    Err(_) => {
2774                        if let Some(tx) = initial_tx.take() {
2775                            let _ = tx
2776                                .send(Err("native announcement task exited before first result"
2777                                    .to_owned()));
2778                        }
2779                    }
2780                }
2781            }
2782            tokio::select! {
2783                biased;
2784                _ = cancellation.cancelled() => {
2785                    task.abort();
2786                    let _ = task.await;
2787                }
2788                _ = &mut task => {}
2789            }
2790        });
2791    });
2792
2793    initial_rx
2794}
2795
2796/// Production publisher startup boundary: only required Iroh waits for a
2797/// successful first publication. Compatibility keeps its retry task alive.
2798fn start_native_announcement_publisher<F, Fut>(
2799    reach: hyprstream_service::NativeAnnouncementReach,
2800    network_required: bool,
2801    cancellation: tokio_util::sync::CancellationToken,
2802    loop_body: F,
2803) -> anyhow::Result<()>
2804where
2805    F: FnOnce(NativeAnnouncementFirstTx) -> Fut + Send + 'static,
2806    Fut: std::future::Future<Output = ()> + Send + 'static,
2807{
2808    let require_initial = network_required
2809        && matches!(
2810            reach,
2811            hyprstream_service::NativeAnnouncementReach::Iroh { .. }
2812        );
2813    if let Some(rx) = spawn_native_announcement_loop(require_initial, cancellation, loop_body) {
2814        match rx.recv() {
2815            Ok(Ok(())) => {}
2816            Ok(Err(error)) => anyhow::bail!("initial Iroh announcement failed: {error}"),
2817            Err(_) => anyhow::bail!("initial Iroh announcement thread exited"),
2818        }
2819    }
2820    Ok(())
2821}
2822
2823fn main() -> Result<()> {
2824    // ROCm allocator and BLAS optimizations — must be set before any tch/libtorch init.
2825    // Expandable segments eliminates ~1,900 hipMalloc/hipFree calls per decode step.
2826    // hipBLASLt split-K improves M=1 GEMM (single-token decode) throughput.
2827    if std::env::var("HIP_VISIBLE_DEVICES").is_ok() {
2828        if std::env::var("PYTORCH_HIP_ALLOC_CONF").is_err() {
2829            // SAFETY: single-threaded at this point (before any thread spawning)
2830            unsafe {
2831                std::env::set_var(
2832                    "PYTORCH_HIP_ALLOC_CONF",
2833                    "expandable_segments:True,garbage_collection_threshold:0.8",
2834                );
2835            }
2836        }
2837        if std::env::var("TORCH_BLAS_PREFER_HIPBLASLT").is_err() {
2838            unsafe {
2839                std::env::set_var("TORCH_BLAS_PREFER_HIPBLASLT", "1");
2840            }
2841        }
2842        if std::env::var("HIPBLASLT_FORCE_SPLIT_K").is_err() {
2843            unsafe {
2844                std::env::set_var("HIPBLASLT_FORCE_SPLIT_K", "1");
2845            }
2846        }
2847    }
2848
2849    // CUDA allocator optimizations — must be set before any tch/libtorch init.
2850    // Expandable segments reduces fragmentation from growing KV cache tensors.
2851    // Garbage collection threshold triggers proactive reclamation at 80% usage.
2852    if std::env::var("HIP_VISIBLE_DEVICES").is_err()
2853        && std::env::var("PYTORCH_CUDA_ALLOC_CONF").is_err()
2854    {
2855        // SAFETY: single-threaded at this point (before any thread spawning)
2856        unsafe {
2857            std::env::set_var(
2858                "PYTORCH_CUDA_ALLOC_CONF",
2859                "expandable_segments:True,garbage_collection_threshold:0.8",
2860            );
2861        }
2862    }
2863
2864    // Set up panic handler to get better debugging information
2865    std::panic::set_hook(Box::new(|info| {
2866        eprintln!("\u{1f6a8} PANIC occurred:");
2867        if let Some(loc) = info.location() {
2868            eprintln!("   Location: {}:{}", loc.file(), loc.line());
2869        }
2870        if let Some(msg) = info.payload().downcast_ref::<&str>() {
2871            eprintln!("   Message: {}", msg);
2872        } else if let Some(msg) = info.payload().downcast_ref::<String>() {
2873            eprintln!("   Message: {}", msg);
2874        }
2875        eprintln!("\nThis was likely caused by a threading issue or memory corruption.");
2876        eprintln!("Please check for unsafe RefCell usage or race conditions.");
2877    }));
2878
2879    // Check if we should re-exec into an installed GPU variant.
2880    // Uses Unix execve() to replace this process with the GPU-optimized binary.
2881    // The binary path comes from our own data directory (not user input).
2882    {
2883        let data_dir = dirs::data_local_dir()
2884            .unwrap_or_else(|| std::path::PathBuf::from("."))
2885            .join("hyprstream");
2886        if let Some((variant_id, version)) =
2887            hyprstream_core::cli::update_handlers::check_should_reexec(&data_dir)
2888        {
2889            hyprstream_core::cli::update_handlers::re_exec_variant(
2890                &data_dir,
2891                &variant_id,
2892                &version,
2893            );
2894            // re_exec_variant never returns
2895        }
2896    }
2897
2898    // Parse CLI arguments using builder API
2899    let matches = build_cli().get_matches();
2900
2901    // Trust minting is intentionally independent of local service/config
2902    // bootstrap. It must work on an offline operator machine and must never
2903    // start a registry client or load instance credentials.
2904    if let Some(("trust", sub_m)) = matches.subcommand() {
2905        let command =
2906            TrustCommand::from_arg_matches(sub_m).map_err(|error| anyhow::anyhow!("{error}"))?;
2907        hyprstream_core::cli::trust::handle_trust_command(command)?;
2908        return Ok(());
2909    }
2910
2911    // Extract global args
2912    let config_path = matches
2913        .get_one::<std::path::PathBuf>("config")
2914        .map(std::path::PathBuf::as_path);
2915
2916    // Canonicalize the explicit selector (CLI `--config` or the
2917    // `HYPRSTREAM_CONFIG` env) BEFORE the first load, so the file that is
2918    // loaded, the snapshot provenance, and the path forwarded to spawned
2919    // children are one and the same value — and a relative selector stays
2920    // meaningful regardless of a child's working directory. `None` means
2921    // defaults-resolution, which children reproduce on their own.
2922    let explicit_config_path: Option<std::path::PathBuf> =
2923        match config_path {
2924            Some(path) => Some(std::fs::canonicalize(path).with_context(|| {
2925                format!("canonicalizing explicit config path {}", path.display())
2926            })?),
2927            None => None,
2928        };
2929    // Library-side launchers (bootstrap manager, wizard) forward the same
2930    // provenance their process pinned, so every launch path composes.
2931    if let Some(canonical) = &explicit_config_path {
2932        let _ = hyprstream_core::config::install_explicit_config_path(canonical.clone());
2933    }
2934
2935    // Load configuration early
2936    let config = load_config(explicit_config_path.as_deref())?;
2937
2938    // Validate configuration
2939    config
2940        .validate()
2941        .context("Configuration validation failed")?;
2942
2943    // Pin the validated configuration for the whole process (#1585): every
2944    // later `HyprConfig::load()` — service-factory reloads included — observes
2945    // this snapshot instead of re-deriving XDG defaults that would silently
2946    // drop an explicit `--config` deployment's settings. Write-once, immutable
2947    // afterwards; installed before any resolver, factory, or runtime thread.
2948    let _ = hyprstream_core::config::install_pinned_config(config.clone());
2949
2950    // `Option<&Path>` is Copy, so both service arms below can borrow it.
2951    let explicit_config: Option<&std::path::Path> = explicit_config_path.as_deref();
2952    let iroh_required = config.quic.iroh_required();
2953
2954    // Read-only accepted-state inspection must precede tracing (which may
2955    // create log files), endpoint/runtime initialization and all bootstrap/key
2956    // generation paths. Buffer the complete verified roster before stdout.
2957    if let Some(("pds", pds)) = matches.subcommand() {
2958        #[cfg(feature = "rocksdb")]
2959        if let Some(("inspect-services", inspect)) = pds.subcommand() {
2960            let services = inspect
2961                .get_many::<String>("service")
2962                .context("service roster is required")?
2963                .cloned()
2964                .collect::<Vec<_>>();
2965            // Inspection errors may include trusted-artifact or configured
2966            // secrets paths. Keep those details out of the process error sink
2967            // and log collectors while preserving the public JSON success
2968            // document unchanged.
2969            let bytes =
2970                hyprstream_core::cli::deployment_bootstrap::inspect_services(&config, &services)
2971                    .map_err(|_| anyhow::anyhow!("read-only service roster inspection failed"))?;
2972            std::io::Write::write_all(&mut std::io::stdout().lock(), &bytes)?;
2973            return Ok(());
2974        }
2975    }
2976
2977    // RPC clients are used by ordinary CLI commands (`quick`, `tui`, etc.),
2978    // not only by service entrypoints. Install both request- and response-side
2979    // verification defaults before dispatch so every command uses the
2980    // operator-configured mesh trust store (#1018). The installer is
2981    // first-write-wins, so the service-specific calls below remain harmless.
2982    install_envelope_verify_config(Some(&config.oauth), Some(&config));
2983
2984    // Install the per-service streaming-response concurrency cap from config
2985    // (#186) before any RPC service starts. First-write-wins; ignore if already
2986    // installed on this process.
2987    let _ = hyprstream_rpc::streaming::install_max_concurrent_streams_per_service(
2988        config.server.max_concurrent_streams_per_service,
2989    );
2990
2991    // ========== TRACING INITIALIZATION (before service startup) ==========
2992    let is_service_command = matches.subcommand_name() == Some("service");
2993    let _log_guard: Option<tracing_appender::non_blocking::WorkerGuard>;
2994
2995    #[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
2996    {
2997        let telemetry_provider = if is_service_command {
2998            TelemetryProvider::Otlp
2999        } else {
3000            TelemetryProvider::Stdout
3001        };
3002
3003        let otel_enabled = std::env::var("HYPRSTREAM_OTEL_ENABLE")
3004            .unwrap_or_else(|_| "false".to_owned())
3005            .parse::<bool>()
3006            .unwrap_or(false);
3007
3008        if otel_enabled {
3009            init_telemetry(telemetry_provider)?;
3010        } else {
3011            let default_log_level = match telemetry_provider {
3012                TelemetryProvider::Otlp => "hyprstream=info,hyprstream_core=info",
3013                TelemetryProvider::Stdout => "hyprstream=warn,hyprstream_core=warn",
3014            };
3015
3016            if let Ok(log_dir) = std::env::var("HYPRSTREAM_LOG_DIR") {
3017                let file_appender =
3018                    RollingFileAppender::new(Rotation::DAILY, &log_dir, "hyprstream.log");
3019                let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
3020                _log_guard = Some(guard);
3021
3022                tracing_subscriber::fmt()
3023                    .with_env_filter(EnvFilter::builder().parse_lossy(
3024                        std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()),
3025                    ))
3026                    .with_target(true)
3027                    .with_file(true)
3028                    .with_line_number(true)
3029                    .with_writer(non_blocking)
3030                    .init();
3031
3032                info!("File logging enabled to {}/hyprstream.log", log_dir);
3033            } else {
3034                _log_guard = None;
3035                tracing_subscriber::fmt()
3036                    .with_env_filter(EnvFilter::builder().parse_lossy(
3037                        std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()),
3038                    ))
3039                    .with_target(true)
3040                    .with_file(true)
3041                    .with_line_number(true)
3042                    .init();
3043            }
3044        }
3045    }
3046
3047    #[cfg(any(not(feature = "otel"), target_arch = "wasm32"))]
3048    {
3049        let default_log_level = if is_service_command {
3050            "hyprstream=info,hyprstream_core=info"
3051        } else {
3052            "hyprstream=warn,hyprstream_core=warn"
3053        };
3054
3055        if let Ok(log_dir) = std::env::var("HYPRSTREAM_LOG_DIR") {
3056            let file_appender =
3057                RollingFileAppender::new(Rotation::DAILY, &log_dir, "hyprstream.log");
3058            let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
3059            _log_guard = Some(guard);
3060
3061            tracing_subscriber::fmt()
3062                .with_env_filter(EnvFilter::builder().parse_lossy(
3063                    std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()),
3064                ))
3065                .with_target(true)
3066                .with_file(true)
3067                .with_line_number(true)
3068                .with_writer(non_blocking)
3069                .init();
3070
3071            info!("File logging enabled to {}/hyprstream.log", log_dir);
3072        } else {
3073            _log_guard = None;
3074            tracing_subscriber::fmt()
3075                .with_env_filter(EnvFilter::builder().parse_lossy(
3076                    std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()),
3077                ))
3078                .with_target(true)
3079                .with_file(true)
3080                .with_line_number(true)
3081                .init();
3082        }
3083    }
3084
3085    info!("Hyprstream v{} starting up", env!("CARGO_PKG_VERSION"));
3086    // ========== END TRACING INITIALIZATION ==========
3087
3088    // ========== EXECUTION MODE DETECTION ==========
3089    let execution_mode = ExecutionMode::detect();
3090    info!("Execution mode: {}", execution_mode);
3091
3092    // ========== ENDPOINT REGISTRY INITIALIZATION ==========
3093    {
3094        use hyprstream_rpc::registry::init as init_registry;
3095
3096        let mode = execution_mode.endpoint_mode();
3097        let runtime_dir = if execution_mode.uses_ipc() {
3098            Some(hyprstream_rpc::paths::runtime_dir())
3099        } else {
3100            None
3101        };
3102
3103        init_registry(mode, runtime_dir);
3104    }
3105    // ========== END ENDPOINT REGISTRY INITIALIZATION ==========
3106    // ── Wizard / first-run early dispatch ───────────────────────────────────
3107    // The wizard is a bootstrap command: it creates credentials that the
3108    // registry client init (below) depends on. Handle both `wizard` and the
3109    // no-subcommand first-run path here so they work on a completely fresh
3110    // machine with no credentials yet. Neither path needs registry_client/ctx.
3111    {
3112        let models_dir = config.models_dir().clone();
3113        let is_wizard = matches!(matches.subcommand_name(), Some("wizard"));
3114        let is_first_run = is_wizard
3115            || (matches.subcommand_name().is_none()
3116                && hyprstream_core::cli::bootstrap_manager::is_first_run(&models_dir));
3117
3118        if is_first_run {
3119            let services = config.services.startup.clone();
3120            if let Some(("wizard", sub_m)) = matches.subcommand() {
3121                let tui_mode = sub_m.get_flag("tui");
3122                let non_interactive = sub_m.get_flag("non_interactive");
3123                let start_services = sub_m.get_flag("start");
3124                let bootstrap_only = sub_m.get_flag("bootstrap_only");
3125                let enable_federation = sub_m.get_flag("enable_federation");
3126                let pds_url = sub_m.get_one::<String>("pds_url").cloned();
3127                let initial_user_role = sub_m
3128                    .get_one::<String>("initial_user_role")
3129                    .cloned()
3130                    .unwrap_or_else(|| "admin".to_owned());
3131                let deployment_trust =
3132                    hyprstream_core::cli::wizard_handlers::DeploymentTrustOptions {
3133                        enabled: sub_m.get_flag("deployment_trust"),
3134                        dir: sub_m
3135                            .get_one::<String>("deployment_trust_dir")
3136                            .map(std::path::PathBuf::from),
3137                        force_software: sub_m.get_flag("deployment_trust_software"),
3138                        serial: sub_m.get_one::<String>("deployment_trust_serial").cloned(),
3139                        piv_slot: sub_m
3140                            .get_one::<String>("deployment_trust_piv_slot")
3141                            .cloned(),
3142                        allow_plaintext_break_glass: sub_m
3143                            .get_flag("deployment_trust_allow_plaintext_break_glass"),
3144                    };
3145                let wizard_options = hyprstream_core::cli::WizardOptions {
3146                    non_interactive,
3147                    start_services,
3148                    bootstrap_only,
3149                    enable_federation,
3150                    initial_user_role,
3151                    deployment_trust,
3152                };
3153                // The TUI wizard has no ceremony flow yet, so an explicit
3154                // --deployment-trust keeps the text wizard.
3155                let use_tui = tui_mode
3156                    || (pds_url.is_none()
3157                        && !non_interactive
3158                        && !bootstrap_only
3159                        && !wizard_options.deployment_trust.enabled
3160                        && supports_tui());
3161                let config = config.clone();
3162                return with_runtime(
3163                    RuntimeConfig {
3164                        device: DeviceConfig::request_cpu(),
3165                        multi_threaded: true,
3166                    },
3167                    || async move {
3168                        if use_tui {
3169                            hyprstream_core::cli::handle_wizard_tui(&models_dir, &services).await
3170                        } else {
3171                            hyprstream_core::cli::handle_wizard(
3172                                &models_dir,
3173                                &services,
3174                                wizard_options,
3175                            )
3176                            .await?;
3177                            if let Some(pds_url) = pds_url {
3178                                hyprstream_core::cli::pds_handlers::handle_pds_join(
3179                                    &config, &pds_url, None,
3180                                )
3181                                .await?;
3182                            }
3183                            Ok(())
3184                        }
3185                    },
3186                );
3187            }
3188            // No subcommand + first run → TUI wizard (with text fallback)
3189            return with_runtime(
3190                RuntimeConfig {
3191                    device: DeviceConfig::request_cpu(),
3192                    multi_threaded: true,
3193                },
3194                || async move {
3195                    if supports_tui() {
3196                        hyprstream_core::cli::handle_wizard_tui(&models_dir, &services).await
3197                    } else {
3198                        // First-run fallback (no `wizard` subcommand) defaults
3199                        // federation to off — explicit opt-in only.
3200                        hyprstream_core::cli::handle_wizard(
3201                            &models_dir,
3202                            &services,
3203                            hyprstream_core::cli::WizardOptions::first_run(),
3204                        )
3205                        .await
3206                    }
3207                },
3208            );
3209        }
3210    }
3211
3212    // ── `mac` early dispatch ─────────────────────────────────────────────────
3213    // `mac genesis` is a read-only, in-memory diagnostic (compile-time service
3214    // inventory + site policy — no registry client, no credentials). Dispatch it
3215    // before credential loading and registry-key resolution so it works on a
3216    // fresh or broken installation, where its coverage evidence is most needed.
3217    if let Some(("mac", sub_m)) = matches.subcommand() {
3218        match sub_m.subcommand() {
3219            Some(("genesis", _)) => {
3220                // Build the boot-time gate and print the coverage report. This is
3221                // read-only inspection — it changes no enforcement state.
3222                let gate = hyprstream_core::mac::GenesisGate::production();
3223                print!("{}", gate.render_report());
3224            }
3225            _ => {
3226                eprintln!("usage: hyprstream mac genesis");
3227            }
3228        }
3229        return Ok(());
3230    }
3231
3232    // ── `pds` early dispatch ─────────────────────────────────────────────────
3233    // PDS attachment is independent of the local registry: it must work on a
3234    // newly provisioned host before local services are running.
3235    if let Some(("pds", sub_m)) = matches.subcommand() {
3236        match sub_m.subcommand() {
3237            #[cfg(feature = "rocksdb")]
3238            Some(("init-deployment-store", init_m)) => {
3239                if init_m.get_flag("fresh-genesis") {
3240                    hyprstream_core::cli::deployment_bootstrap::init_checkpoint_store_for_fresh_genesis(
3241                        &config,
3242                    )?;
3243                    println!("fresh-genesis deployment checkpoint store initialized");
3244                } else {
3245                    match hyprstream_core::cli::deployment_bootstrap::init_checkpoint_store(&config)? {
3246                        hyprstream_core::cli::deployment_bootstrap::DeploymentStoreInit::Created => {
3247                            println!("initialized empty deployment checkpoint store");
3248                        }
3249                        hyprstream_core::cli::deployment_bootstrap::DeploymentStoreInit::Reused => {
3250                            println!("reused healthy retained deployment checkpoint store");
3251                        }
3252                    }
3253                }
3254                return Ok(());
3255            }
3256            #[cfg(feature = "rocksdb")]
3257            Some(("provision-services", provision_m)) => {
3258                let services = provision_m.get_many::<String>("service")
3259                    .context("service roster is required")?.cloned().collect::<Vec<_>>();
3260                let lifetime = *provision_m.get_one::<i64>("valid-for-seconds")
3261                    .context("service identity lifetime is required")?;
3262                let roster_export = provision_m.get_one::<std::path::PathBuf>("roster-export");
3263                hyprstream_core::cli::deployment_bootstrap::provision_services(
3264                    &config,
3265                    &services,
3266                    lifetime,
3267                    roster_export.map(std::path::PathBuf::as_path),
3268                )?;
3269                if let Some(path) = roster_export {
3270                    println!("verified service roster manifest written to {}", path.display());
3271                }
3272                println!("checkpoint-accepted service roster ready ({} services)", services.len());
3273                return Ok(());
3274            }
3275            Some(("join", join_m)) => {
3276                let pds_url = join_m
3277                    .get_one::<String>("url")
3278                    .ok_or_else(|| anyhow::anyhow!("PDS URL is required"))?;
3279                let scope = join_m.get_one::<String>("scope").map(String::as_str);
3280                return with_runtime(
3281                    RuntimeConfig { device: DeviceConfig::request_cpu(), multi_threaded: true },
3282                    || hyprstream_core::cli::pds_handlers::handle_pds_join(&config, pds_url, scope),
3283                );
3284            }
3285            _ => anyhow::bail!("usage: hyprstream pds init-deployment-store | pds provision-services --service <NAMES> | pds join <PDS_URL> [--scope <SCOPE>]"),
3286        }
3287    }
3288
3289    // ── `service repair` early dispatch ──────────────────────────────────────
3290    // `service repair` must run on a partially-broken install — including one
3291    // whose bootstrap-pubkeys is stale (classical-only) and trips the hybrid
3292    // gate below. Dispatch it before the gate so the operator always has an
3293    // in-product diagnostic, and let run_repair_checks flag the stale file.
3294    if let Some(("service", sub_m)) = matches.subcommand() {
3295        if let Some(("repair", _)) = sub_m.subcommand() {
3296            let verbose = sub_m
3297                .subcommand_matches("repair")
3298                .map(|m| m.get_flag("verbose"))
3299                .unwrap_or(false);
3300            let models_dir = config.models_dir().clone();
3301            return with_runtime(
3302                RuntimeConfig {
3303                    device: DeviceConfig::request_cpu(),
3304                    multi_threaded: true,
3305                },
3306                || async move {
3307                    hyprstream_core::cli::service_handlers::run_repair_checks(&models_dir, verbose)
3308                        .await
3309                },
3310            );
3311        }
3312    }
3313
3314    // ── `service provision-policy-templates` early dispatch ─────────────────
3315    // The OS-owned bootstrap writes the configured policy store before either
3316    // Policy or Registry starts. It must never construct a resolver or borrow
3317    // a runtime service credential.
3318    if let Some(("service", sub_m)) = matches.subcommand() {
3319        if let Some(("provision-policy-templates", provision_m)) = sub_m.subcommand() {
3320            let templates = provision_m
3321                .get_many::<String>("template")
3322                .context("at least one policy template is required")?
3323                .cloned()
3324                .collect::<Vec<_>>();
3325            let models_dir = config.models_dir().clone();
3326            return with_runtime(
3327                RuntimeConfig {
3328                    device: DeviceConfig::request_cpu(),
3329                    multi_threaded: true,
3330                },
3331                || async move {
3332                    handle_service_provision_policy_templates(&models_dir, &templates).await
3333                },
3334            );
3335        }
3336    }
3337
3338    // ── Offline service-authority provisioning ─────────────────────────────
3339    if let Some(("service", sub_m)) = matches.subcommand() {
3340        if sub_m.subcommand_name() == Some("reconcile-enrollment") {
3341            return hyprstream_core::cli::service_handlers::handle_service_reconcile_enrollment(
3342                &config,
3343            );
3344        }
3345        if sub_m.subcommand_name() == Some("ensure-node-authority") {
3346            let secrets =
3347                hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3348            return hyprstream_core::auth::service_enrollment::ensure_node_authority(&secrets);
3349        }
3350    }
3351
3352    // Key materialization for provisioning/keygen units: it must work on a
3353    // fresh install (before any bootstrap-pubkeys exist) and must not start
3354    // any services, so dispatch before the registry bootstrap below — same
3355    // rationale as `service repair`.
3356    if let Some(("service", sub_m)) = matches.subcommand() {
3357        if let Some(("ensure-key", ek_m)) = sub_m.subcommand() {
3358            // `name` is a required positional; a missing value is a clap bug,
3359            // not operator error, so fail loudly rather than guess.
3360            let name = ek_m
3361                .get_one::<String>("name")
3362                .cloned()
3363                .ok_or_else(|| anyhow::anyhow!("service ensure-key: missing required <name>"))?;
3364            return hyprstream_core::cli::service_handlers::handle_service_ensure_key(
3365                Some(&config),
3366                &name,
3367            );
3368        }
3369    }
3370
3371    // The readback performs a same-identity Policy preflight before the one
3372    // Registry request. This command is intentionally dispatched before the
3373    // generic service-start path and never reaches Register.
3374    if let Some(("service", sub_m)) = matches.subcommand() {
3375        if let Some(("readback-local-canary", readback_m)) = sub_m.subcommand() {
3376            let path = readback_m
3377                .get_one::<std::path::PathBuf>("path")
3378                .cloned()
3379                .context("service readback-local-canary requires --path")?;
3380            let expected_head = readback_m
3381                .get_one::<String>("expected_head")
3382                .cloned()
3383                .context("service readback-local-canary requires --expected-head")?;
3384            return with_runtime(
3385                RuntimeConfig {
3386                    device: DeviceConfig::request_cpu(),
3387                    multi_threaded: true,
3388                },
3389                || async move {
3390                    let secrets = HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3391                    let profile =
3392                        hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3393                    let key =
3394                        hyprstream_core::auth::identity_store::load_existing_service_signing_key(
3395                            &secrets, "registry", profile,
3396                        )?;
3397                    let jwt = hyprstream_core::auth::identity_store::load_service_jwt_for_profile(
3398                        &secrets, "registry", profile,
3399                    )?
3400                    .context("enrolled Registry service credential is missing")?;
3401                    hyprstream_core::cli::registry_readback::require_registry_service_subject(&jwt)
3402                        .context("Registry service credential preflight failed")?;
3403                    install_process_production_resolver(&key, &config, Some(jwt.clone()), true)
3404                        .await?;
3405                    let provider = &hyprstream_discovery::ProductionRpcClientProvider;
3406                    let policy = hyprstream_rpc_std::policy_client::PolicyClient::from_provider(
3407                        provider,
3408                        key.clone(),
3409                        None,
3410                    )?;
3411                    let registry = RegistryClient::from_provider(provider, key, Some(jwt.clone()))?;
3412                    hyprstream_core::cli::registry_readback::readback_local_canary(
3413                        &policy,
3414                        &registry,
3415                        &jwt,
3416                        config.models_dir(),
3417                        &path,
3418                        &expected_head,
3419                    )
3420                    .await
3421                },
3422            )
3423            .map_err(|error| {
3424                anyhow::anyhow!(
3425                    "Registry canary readback inconclusive: {}",
3426                    hyprstream_core::cli::registry_readback::error_category(&error)
3427                )
3428            });
3429        }
3430    }
3431
3432    // Keep the enrolled Registry RPC authority while giving this one-shot
3433    // caller its own outbound carrier. Reusing the live Registry's N0 NodeId
3434    // could overwrite its published reach before Register is dispatched.
3435    if let Some(("service", sub_m)) = matches.subcommand() {
3436        if let Some(("register-local-canary", register_m)) = sub_m.subcommand() {
3437            let path = register_m
3438                .get_one::<std::path::PathBuf>("path")
3439                .cloned()
3440                .context("service register-local-canary requires --path")?;
3441            let expected_head = register_m
3442                .get_one::<String>("expected_head")
3443                .cloned()
3444                .context("service register-local-canary requires --expected-head")?;
3445            return with_runtime(
3446                RuntimeConfig {
3447                    device: DeviceConfig::request_cpu(),
3448                    multi_threaded: true,
3449                },
3450                || async move {
3451                    let secrets = HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3452                    let profile =
3453                        hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3454                    let key =
3455                        hyprstream_core::auth::identity_store::load_existing_service_signing_key(
3456                            &secrets, "registry", profile,
3457                        )?;
3458                    let jwt = hyprstream_core::auth::identity_store::load_service_jwt_for_profile(
3459                        &secrets, "registry", profile,
3460                    )?
3461                    .context("enrolled Registry service credential is missing")?;
3462                    hyprstream_core::cli::registry_readback::require_registry_service_subject(&jwt)
3463                        .context("Registry service credential preflight failed")?;
3464                    install_process_production_resolver(&key, &config, Some(jwt.clone()), true)
3465                        .await?;
3466                    let provider = &hyprstream_discovery::ProductionRpcClientProvider;
3467                    let policy = hyprstream_rpc_std::policy_client::PolicyClient::from_provider(
3468                        provider,
3469                        key.clone(),
3470                        None,
3471                    )?;
3472                    let registry = RegistryClient::from_provider(provider, key, Some(jwt.clone()))?;
3473                    hyprstream_core::cli::registry_readback::register_local_canary(
3474                        &policy,
3475                        &registry,
3476                        &jwt,
3477                        config.models_dir(),
3478                        &path,
3479                        &expected_head,
3480                    )
3481                    .await
3482                },
3483            )
3484            .map_err(|error| {
3485                anyhow::anyhow!(
3486                    "Registry canary registration inconclusive: {}",
3487                    hyprstream_core::cli::registry_readback::registration_error_category(&error)
3488                )
3489            });
3490        }
3491    }
3492
3493    let native_service_name = native_service_process_name(&matches, &config)?;
3494    let dedicated_service_name = dedicated_service_process_name(&matches, &config)?;
3495
3496    // Start registry service ONCE at CLI level
3497    let _registry_runtime = tokio::runtime::Builder::new_multi_thread()
3498        .enable_all()
3499        .build()
3500        .context("Failed to create registry runtime")?;
3501
3502    // Start services and create keypair
3503    let (registry_client, signing_key, verifying_key, is_os_owned_bootstrap): (
3504        RegistryClient,
3505        SigningKey,
3506        VerifyingKey,
3507        bool,
3508    ) = _registry_runtime
3509        .block_on(async {
3510            // A dedicated single-service foreground process owns that
3511            // service's identity regardless of transport profile: the
3512            // service-start path already swaps to the provisioned service key
3513            // for `--ipc`, and the revocation/session authority probe this
3514            // process must pass rides the process-owned Discovery client
3515            // installed below. Without this, a dedicated compatibility-mode
3516            // process (e.g. `service start oauth --foreground`, required for
3517            // hosted account workers) installs its resolver with the node
3518            // root key and no credential, so every cross-process dispatch —
3519            // starting with the mandatory authority probe — is denied
3520            // NoClearance.
3521            let process_identity = process_identity_name(
3522                native_service_name.as_deref(),
3523                dedicated_service_name.as_deref(),
3524            );
3525            let signing_key = load_process_signing_key(&config, process_identity).await?;
3526            let verifying_key = signing_key.verifying_key();
3527
3528            // The process-owned Discovery client is the first native RPC
3529            // client created. It must present the same enrolled, holder-bound
3530            // service JWT as the later factory clients: a signature alone
3531            // identifies the service but carries no MAC clearance.
3532            let service_token = match process_identity {
3533                Some(name) => {
3534                    let secrets = HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3535                    let profile =
3536                        hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3537                    let token =
3538                        hyprstream_core::auth::identity_store::load_service_jwt_for_profile(
3539                            &secrets, name, profile,
3540                        )?
3541                        .with_context(|| {
3542                            format!("missing enrolled service credential for {name}")
3543                        })?;
3544                    Some(token)
3545                }
3546                None => {
3547                    let secrets = HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3548                    let profile =
3549                        hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3550                    flat_key_policy_resolver_credential(&secrets, profile, &signing_key)
3551                }
3552            };
3553
3554            let is_os_owned_bootstrap = install_process_production_resolver(
3555                &signing_key,
3556                &config,
3557                service_token.clone(),
3558                false,
3559            )
3560            .await
3561            .context("Failed to install checkpoint-backed production resolver")?;
3562            let client = hyprstream_rpc_std::registry_client::RegistryClient::from_provider(
3563                &hyprstream_discovery::ProductionRpcClientProvider,
3564                signing_key.clone(),
3565                service_token,
3566            )?;
3567
3568            Ok::<_, anyhow::Error>((client, signing_key, verifying_key, is_os_owned_bootstrap))
3569        })
3570        .context("Failed to connect to services")?;
3571
3572    // Create application context with shared registry client
3573    let config_for_service = config.clone();
3574    let ctx = AppContext::with_client(config, Clone::clone(&registry_client));
3575
3576    // ========== TOP-LEVEL COMMAND DISPATCH ==========
3577    match matches.subcommand() {
3578        // ── Schema-driven tool commands ──────────────────────────────────
3579        Some(("tool", sub_m)) => {
3580            with_runtime(
3581                RuntimeConfig {
3582                    device: DeviceConfig::request_cpu(),
3583                    multi_threaded: true,
3584                },
3585                || async move { schema_cli::handle_tool_command(sub_m, signing_key).await },
3586            )?;
3587        }
3588
3589        // ── Quick workflows ─────────────────────────────────────────────
3590        Some(("quick", sub_m)) => {
3591            let cmd =
3592                QuickCommand::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
3593            handle_quick_command(cmd, ctx, signing_key, verifying_key)?;
3594        }
3595
3596        // ── Service management ──────────────────────────────────────────
3597        Some(("service", sub_m)) => {
3598            let action =
3599                ServiceAction::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
3600
3601            let services = config_for_service.services.startup.clone();
3602
3603            match action {
3604                ServiceAction::Install {
3605                    services: filter,
3606                    start,
3607                    enable,
3608                    system,
3609                    systemd_session,
3610                    verbose,
3611                } => {
3612                    let models_dir = config_for_service.models_dir().clone();
3613                    let target = if system {
3614                        hyprstream_service::ServiceTarget::System
3615                    } else if systemd_session {
3616                        hyprstream_service::ServiceTarget::UserSession
3617                    } else {
3618                        hyprstream_service::ServiceTarget::User
3619                    };
3620                    with_runtime(
3621                        RuntimeConfig {
3622                            device: DeviceConfig::request_cpu(),
3623                            multi_threaded: true,
3624                        },
3625                        || async move {
3626                            handle_service_install(
3627                                &models_dir,
3628                                &services,
3629                                filter,
3630                                start,
3631                                enable,
3632                                target,
3633                                verbose,
3634                                explicit_config,
3635                                iroh_required,
3636                            )
3637                            .await
3638                        },
3639                    )?;
3640                }
3641
3642                ServiceAction::Uninstall { services: filter } => {
3643                    with_runtime(
3644                        RuntimeConfig {
3645                            device: DeviceConfig::request_cpu(),
3646                            multi_threaded: false,
3647                        },
3648                        || async move { handle_service_uninstall(&services, filter).await },
3649                    )?;
3650                }
3651
3652                ServiceAction::Start {
3653                    name,
3654                    foreground,
3655                    daemon,
3656                    ipc,
3657                    services: multi_services,
3658                    quic_bind,
3659                    print_cert_hash,
3660                    standalone,
3661                } => {
3662                    // Validate configured artifacts on EVERY startup path, before
3663                    // any service spawn or baseline; no JWT key-store condition
3664                    // may silently skip the configured policy.
3665                    if let Some(artifacts) = &config_for_service.policy.boot_policy {
3666                        let secrets_dir =
3667                            HyprConfig::resolve_secrets_dir_for(Some(&config_for_service))?;
3668                        let policy =
3669                            hyprstream_core::mac::bootload::install_configured_boot_policy(
3670                                artifacts,
3671                                &secrets_dir,
3672                                is_os_owned_bootstrap,
3673                            )
3674                            .context("configured MAC boot policy rejected")?;
3675                        tracing::info!(
3676                            generation = policy.generation,
3677                            "Verified configured MAC boot policy installed"
3678                        );
3679                    }
3680                    // MAC genesis coverage gate (S1 activation, #567): log the
3681                    // boot-time coverage report on EVERY `service start` path —
3682                    // foreground (the mode that actually hosts services, incl.
3683                    // systemd ExecStart and standalone-spawned children),
3684                    // standalone, and the systemd/spawner dispatch below.
3685                    // Emit the coverage-gate evidence consumed by the active
3686                    // 9P translator PEP (see `mac::genesis`).
3687                    let mac_gate = hyprstream_core::mac::GenesisGate::production();
3688                    mac_gate.log_report();
3689                    // IdentityAware is the default. Refuse startup if the
3690                    // generated dispatch inventory is incomplete or invalid.
3691                    hyprstream_core::mac::install_production_rpc_dispatch_pep()?;
3692
3693                    if foreground || standalone {
3694                        // --foreground requires a service name or --services list;
3695                        // --standalone uses all configured services.
3696                        let (name, multi_services) = if standalone {
3697                            (String::from("standalone"), Some(services.clone()))
3698                        } else {
3699                            match (name, &multi_services) {
3700                                (Some(n), _) => (n, multi_services),
3701                                (None, Some(_)) => (String::from("multi"), multi_services),
3702                                (None, None) => {
3703                                    return Err(anyhow::anyhow!(
3704                                        "--foreground requires a service name or --services list"
3705                                    ))
3706                                }
3707                            }
3708                        };
3709
3710                        // --services with one member selects that service's identity,
3711                        // never the synthetic "multi"/"standalone" command label.
3712                        let name = native_service_name.clone().unwrap_or(name);
3713
3714                        // Standard foreground service startup
3715                        let rpc_mode = if ipc {
3716                            hyprstream_rpc::registry::EndpointMode::Ipc
3717                        } else {
3718                            hyprstream_rpc::registry::EndpointMode::Inproc
3719                        };
3720
3721                        let config = config_for_service;
3722
3723                        with_runtime(
3724                            RuntimeConfig {
3725                                device: DeviceConfig::request_cpu(),
3726                                multi_threaded: true,
3727                            },
3728                            || async move {
3729                                let (shutdown_tx, shutdown_rx) =
3730                                    tokio::sync::oneshot::channel::<()>();
3731
3732                                tokio::spawn(async move {
3733                                    let _ = tokio::signal::ctrl_c().await;
3734                                    info!("Received shutdown signal");
3735                                    let _ = shutdown_tx.send(());
3736                                });
3737
3738                                {
3739                                    use hyprstream_rpc::registry::init as init_registry;
3740                                    let runtime_dir = if ipc {
3741                                        Some(hyprstream_rpc::paths::runtime_dir())
3742                                    } else {
3743                                        None
3744                                    };
3745                                    init_registry(rpc_mode, runtime_dir);
3746                                }
3747
3748                                let models_dir = config.models_dir();
3749                                let keys_dir = models_dir.join(".registry").join("keys");
3750                                let signing_key = if process_identity_name(
3751                                    native_service_name.as_deref(),
3752                                    dedicated_service_name.as_deref(),
3753                                )
3754                                .is_some()
3755                                {
3756                                    // Same retained key already installed in the process resolver.
3757                                    signing_key.clone()
3758                                } else {
3759                                    load_or_generate_signing_key(&keys_dir).await?
3760                                };
3761                                let verifying_key = signing_key.verifying_key();
3762
3763                                let fed_src: Arc<dyn hyprstream_rpc::auth::FederationKeySource> =
3764                                    Arc::new(hyprstream_core::auth::FederationKeyResolver::new(
3765                                        &config.oauth.trusted_issuers,
3766                                    ));
3767                                let mut ctx = ServiceContext::new(
3768                                    signing_key.clone(),
3769                                    verifying_key,
3770                                    ipc,
3771                                    models_dir.clone(),
3772                                )
3773                                .with_oauth_issuer(config.oauth.issuer_url())
3774                                .with_federation_key_source(fed_src)
3775                                // Same authoritative resolver the startup key/JWT
3776                                // seeding above used (#759): without it, factory-time
3777                                // key registration and the hourly JWT renewal task
3778                                // would resolve a config-free default directory and
3779                                // silently skip custom `--config [secrets].path`
3780                                // deployments.
3781                                .with_secrets_dir(
3782                                    hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(
3783                                        Some(&config),
3784                                    )?,
3785                                );
3786
3787                                // Containment follows the actual single-service launch,
3788                                // independently of native/compatibility transport selection.
3789                                if let Some(service) = &dedicated_service_name {
3790                                    ctx = ctx.with_dedicated_process_service(service.clone());
3791                                }
3792
3793                                // Wire QUIC shared config from --quic-bind or [quic] config
3794                                let mut quic_cfg = if let Some(ref bind_addr) = quic_bind {
3795                                    let mut qc = hyprstream_core::config::QuicConfig::default();
3796                                    qc.enabled = true;
3797                                    qc.bind_addr = bind_addr.clone();
3798                                    qc.iroh = config.quic.iroh;
3799                                    qc.native_network_profile = config.quic.native_network_profile;
3800                                    qc
3801                                } else {
3802                                    config.quic.clone()
3803                                };
3804                                quic_cfg
3805                                    .validate_native_network_profile()
3806                                    .context("invalid native network profile")?;
3807
3808                                // Determine which services to start
3809                                let service_names: Vec<String> =
3810                                    if let Some(ref svc_list) = multi_services {
3811                                        svc_list.clone()
3812                                    } else {
3813                                        vec![name.clone()]
3814                                    };
3815
3816                                if ipc || native_service_name.is_some() {
3817                                    // Split-service identity is independent of local IPC transport.
3818                                    //
3819                                    // #759: resolve via the SAME authoritative path
3820                                    // `HyprConfig::resolve_secrets_dir()` uses elsewhere in this
3821                                    // function (e.g. the ML-DSA store below) and that
3822                                    // `bootstrap_manager::do_bootstrap` uses to write credentials —
3823                                    // NOT a hand-rolled `dirs::config_dir()` recomputation. The two
3824                                    // used to diverge under `HYPRSTREAM_INSTANCE` or a configured
3825                                    // `[secrets] path` (the manual version ignored both), so a
3826                                    // bootstrapped credential could land in a different directory
3827                                    // than this process reads from — the same "consumer silently
3828                                    // re-derives instead of reading the authoritative record"
3829                                    // disease #441 targets, just one directory earlier.
3830                                    let secrets_dir = hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3831                                    ctx = ctx.with_ca_verifying_key(
3832                                        hyprstream_core::auth::identity_store::load_ca_verifying_key(
3833                                            &secrets_dir,
3834                                        )?,
3835                                    );
3836                                    // The CA's derived ML-DSA-65 verifying key resolves the
3837                                    // composite kid of hybrid service JWTs. Absent only on
3838                                    // pre-hybrid credentials; re-running the wizard writes it.
3839                                    match hyprstream_core::auth::identity_store::load_ca_ml_dsa_verifying_key(&secrets_dir) {
3840                                        Ok(pq_vk) => ctx = ctx.with_ca_ml_dsa_verifying_key(pq_vk),
3841                                        Err(e) => tracing::warn!(
3842                                            "ca-mldsa-pubkey unavailable — hybrid service JWTs will not verify \
3843                                             until the wizard re-provisions credentials: {e}"
3844                                        ),
3845                                    }
3846
3847                                    // Load bootstrap pubkeys (all service pubkeys) into trust store.
3848                                    // Service identities are hybrid by construction; a classical
3849                                    // entry is stale provisioning and is refused, not trusted.
3850                                    if let Ok(entries) = hyprstream_core::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir) {
3851                                        hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)?;
3852                                        let pubkeys: std::collections::HashMap<String, VerifyingKey> =
3853                                            entries.into_iter().map(|(n, e)| (n, e.ed25519)).collect();
3854                                        for (svc_name, vk) in &pubkeys {
3855                                            hyprstream_service::global_trust_store().insert(
3856                                                *vk,
3857                                                hyprstream_service::Attestation {
3858                                                    scopes: std::iter::once(svc_name.clone()).collect(),
3859                                                    subject: None,
3860                                                    jwt: None,
3861                                                    expires_at: 0,
3862                                                    attested_by: None,
3863                                                },
3864                                            );
3865                                        }
3866                                    }
3867
3868                                    // C3 fix: scoped credential providers use flat
3869                                    // %d/signing-key, while standalone uses a subdirectory.
3870                                    // #759: "policy" always resolves to the flat root/CA key regardless of
3871                                    // secrets profile — see `resolve_service_signing_key` for why.
3872                                    let secrets_profile =
3873                                        hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3874                                    let own_key = if native_service_name.is_some() {
3875                                        signing_key.clone()
3876                                    } else {
3877                                        hyprstream_core::auth::identity_store::resolve_service_signing_key(
3878                                            &secrets_dir, &name, secrets_profile,
3879                                        )?
3880                                    };
3881
3882                                    if name == "policy" {
3883                                        // PolicyService: signing_key IS the CA key (already loaded).
3884                                        ctx = ctx.with_service_key(&name, own_key.clone());
3885                                    } else {
3886                                        // Non-policy: swap signing_key to service's own independent key.
3887                                        // CA key is no longer accessible via ctx.signing_key().
3888                                        ctx = ctx.swap_signing_key(own_key.clone());
3889                                        ctx = ctx.with_service_key(&name, own_key.clone());
3890                                    }
3891                                    // Both Policy and non-Policy factories need the
3892                                    // provisioned JWT. Policy's required-native factory
3893                                    // registers its own key too, so seed after the shared
3894                                    // key selection rather than only in the non-Policy arm.
3895                                    hyprstream_core::auth::identity_store::seed_service_jwt_into_trust_store(
3896                                        &name,
3897                                        &own_key,
3898                                        &secrets_dir,
3899                                        secrets_profile,
3900                                    );
3901                                } else {
3902                                    // Single-process mode: load keys from disk (same as IPC).
3903                                    // Wizard must have run to create credentials.
3904                                    //
3905                                    // #759: same authoritative resolver as the `--ipc` branch above —
3906                                    // see the comment there.
3907                                    let secrets_dir = hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3908                                    ctx = ctx.with_ca_verifying_key(
3909                                        hyprstream_core::auth::identity_store::load_ca_verifying_key(
3910                                            &secrets_dir,
3911                                        )?,
3912                                    );
3913                                    // The CA's derived ML-DSA-65 verifying key resolves the
3914                                    // composite kid of hybrid service JWTs. Absent only on
3915                                    // pre-hybrid credentials; re-running the wizard writes it.
3916                                    match hyprstream_core::auth::identity_store::load_ca_ml_dsa_verifying_key(&secrets_dir) {
3917                                        Ok(pq_vk) => ctx = ctx.with_ca_ml_dsa_verifying_key(pq_vk),
3918                                        Err(e) => tracing::warn!(
3919                                            "ca-mldsa-pubkey unavailable — hybrid service JWTs will not verify \
3920                                             until the wizard re-provisions credentials: {e}"
3921                                        ),
3922                                    }
3923
3924                                    // Load bootstrap pubkeys (all service pubkeys) into trust store.
3925                                    // Service identities are hybrid by construction; a classical
3926                                    // entry is stale provisioning and is refused, not trusted.
3927                                    let entries = hyprstream_core::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir)
3928                                        .context("Bootstrap pubkeys not found — run 'hyprstream wizard' first")?;
3929                                    if entries.is_empty() {
3930                                        anyhow::bail!("Bootstrap pubkeys file is empty — run 'hyprstream wizard' first");
3931                                    }
3932                                    hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)?;
3933                                    let pubkeys: std::collections::HashMap<String, VerifyingKey> =
3934                                        entries.into_iter().map(|(n, e)| (n, e.ed25519)).collect();
3935                                    for (svc_name, vk) in &pubkeys {
3936                                        hyprstream_service::global_trust_store().insert(
3937                                            *vk,
3938                                            hyprstream_service::Attestation {
3939                                                scopes: std::iter::once(svc_name.clone()).collect(),
3940                                                subject: None,
3941                                                jwt: None,
3942                                                expires_at: 0,
3943                                                attested_by: None,
3944                                            },
3945                                        );
3946                                    }
3947                                    tracing::debug!(
3948                                        "Loaded {} bootstrap pubkeys from {}",
3949                                        pubkeys.len(),
3950                                        secrets_dir.display()
3951                                    );
3952
3953                                    // Load signing keys for each service being started, and
3954                                    // seed own service-jwt into trust store so register_service_key
3955                                    // can find it on first call (trust store starts with jwt: None).
3956                                    //
3957                                    // #759: "policy" must resolve to the flat root/CA key (matching
3958                                    // `bootstrap_pubkeys["policy"]`), not an independent per-service
3959                                    // key — see `resolve_service_signing_key` for why. This mirrors the
3960                                    // same fix applied to the `--ipc` branch above.
3961                                    for svc_name in &service_names {
3962                                        let svc_key = hyprstream_core::auth::identity_store::resolve_service_signing_key(
3963                                            &secrets_dir,
3964                                            svc_name,
3965                                            hyprstream_core::auth::identity_store::SecretsProfile::SharedDirectory,
3966                                        )
3967                                            .with_context(|| format!("Failed to load signing key for service '{}'", svc_name))?;
3968                                        ctx = ctx.with_service_key(svc_name, svc_key.clone());
3969
3970                                        if svc_name != "policy" {
3971                                            if let Ok(Some(jwt_str)) = hyprstream_core::auth::identity_store::load_service_jwt(&secrets_dir, svc_name) {
3972                                                let exp = hyprstream_core::auth::identity_store::decode_jwt_exp_raw(&jwt_str).unwrap_or(0);
3973                                                hyprstream_service::global_trust_store().insert(
3974                                                    svc_key.verifying_key(),
3975                                                    hyprstream_service::Attestation {
3976                                                        scopes: std::iter::once(svc_name.clone()).collect(),
3977                                                        subject: None,
3978                                                        jwt: Some(jwt_str),
3979                                                        expires_at: exp,
3980                                                        attested_by: None,
3981                                                    },
3982                                                );
3983                                                tracing::info!(service = %svc_name, "Seeded trust store with own service-jwt from credential dir");
3984                                            }
3985                                        }
3986                                    }
3987                                }
3988
3989                                // ServiceEnrollmentManifest (v16 §11): load the
3990                                // authority manifest and bind it against the
3991                                // bootstrap trust anchors (both key halves).
3992                                // Present-but-invalid or key disagreement aborts
3993                                // startup (fail-closed); absence is
3994                                // legacy-tolerated for this staging wave with a
3995                                // loud warning — service credentials then carry
3996                                // no manifest-backed clearance.
3997                                {
3998                                    let secrets_dir =
3999                                        hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
4000                                    match hyprstream_core::auth::service_enrollment::ServiceEnrollmentManifest::load_and_validate(&secrets_dir)
4001                                        .context("service enrollment manifest validation failed")?
4002                                    {
4003                                        Some(manifest) => {
4004                                            let count = manifest.services.len();
4005                                            hyprstream_core::auth::service_enrollment::set_global_service_enrollment(std::sync::Arc::new(manifest))
4006                                                .map_err(|e| anyhow::anyhow!("{e}"))?;
4007                                            tracing::info!(services = count, "Service enrollment manifest installed");
4008                                        }
4009                                        None => {
4010                                            tracing::warn!(
4011                                                "No service-enrollment manifest — legacy mode: service credentials carry no manifest-backed clearance"
4012                                            );
4013                                        }
4014                                    }
4015                                }
4016
4017                                if quic_cfg.enabled {
4018                                    // The checkpoint gate: QUIC announcements require
4019                                    // checkpoint-verified accepted states.
4020                                    //
4021                                    // First-boot deferral is scoped to the OS-owned
4022                                    // bootstrap source ONLY. An OS-owned deployment
4023                                    // remains that source on restart, so it classifies
4024                                    // then takes the ordinary checkpointed path when
4025                                    // populated. DID-anchored deployments never read
4026                                    // lifecycle state or mutate QUIC here.
4027                                    match quic_checkpoint_policy(is_os_owned_bootstrap, || {
4028                                        hyprstream_core::services::factories::classify_pds_store_for_quic(&ctx)
4029                                    })
4030                                    .context(
4031                                        "PDS accepted-state store read failed; \
4032                                         refusing to start QUIC (fail-closed)",
4033                                    )? {
4034                                        QuicCheckpointPolicy::CheckpointedAnnouncements => {
4035                                                ctx = hyprstream_core::services::factories::with_checkpointed_native_announcements(
4036                                                    ctx,
4037                                                    &service_names,
4038                                                )?;
4039                                        }
4040                                        QuicCheckpointPolicy::DeferForFirstBoot => {
4041                                                if quic_cfg.iroh_required() {
4042                                                    anyhow::bail!(
4043                                                        "network-iroh-required requires a checkpoint-verified accepted state and initial Iroh announcement; first-boot local/QUIC deferral is forbidden"
4044                                                    );
4045                                                }
4046                                                // An explicit --quic-bind that cannot be
4047                                                // honored (no accepted states exist yet) is
4048                                                // an error, never a silent disable.
4049                                                if quic_bind.is_some() {
4050                                                    anyhow::bail!(
4051                                                        "QUIC requested via --quic-bind but the PDS \
4052                                                         accepted-state store is at first boot (no \
4053                                                         accepted states written yet). Complete \
4054                                                         discovery/registry provisioning, then restart \
4055                                                         to activate QUIC."
4056                                                    );
4057                                                }
4058                                                // Genuine first boot, no explicit network
4059                                                // request: defer QUIC for this run. Services
4060                                                // bind via inproc/IPC and serve HTTP. QUIC is
4061                                                // NOT re-enabled at runtime — a restart after
4062                                                // provisioning is required.
4063                                                tracing::warn!(
4064                                                    "QUIC disabled for this run: the PDS \
4065                                                     accepted-state store is at first boot (no \
4066                                                     accepted states written yet). Services will \
4067                                                     bind via inproc/IPC and serve HTTP. Restart \
4068                                                     after the discovery/registry pipeline writes \
4069                                                     accepted states to activate QUIC. No runtime \
4070                                                     auto-activation is implemented."
4071                                                );
4072                                                quic_cfg.enabled = false;
4073                                        }
4074                                    }
4075                                }
4076
4077                                // Wire QUIC shared config (must be after key generation so jwt_verifying_key is set)
4078                                if quic_cfg.enabled {
4079                                    let qc = quic_cfg;
4080                                    let (cert_chain, key_der) = qc
4081                                        .load_tls_materials()
4082                                        .context("Failed to load QUIC TLS materials")?;
4083                                    // Browser WebTransport authenticates short-lived self-signed
4084                                    // leaves by their DER SHA-256 hash. WebPKI chains instead use
4085                                    // ordinary CA validation and advertise no pin.
4086                                    let certificate_hashes = if qc.use_self_signed() {
4087                                        use sha2::Digest as _;
4088                                        vec![sha2::Sha256::digest(
4089                                            cert_chain
4090                                                .first()
4091                                                .context("QUIC TLS chain is empty")?,
4092                                        )
4093                                        .into()]
4094                                    } else {
4095                                        Vec::new()
4096                                    };
4097
4098                                    // Print cert hash if requested (hash of leaf cert)
4099                                    if print_cert_hash {
4100                                        let hash = hyprstream_rpc::transport::zmtp_quic::cert_hash(
4101                                            &cert_chain[0],
4102                                        );
4103                                        println!("QUIC/WebTransport cert hash: {}", hash);
4104                                    }
4105
4106                                    // #358: resolve the producer-chosen moq relay
4107                                    // (if configured) into wire-reach form via the
4108                                    // SAME service_entry codec the DID doc uses, so
4109                                    // the stream relay address and DID address can't
4110                                    // drift. Empty = direct-only.
4111                                    let moq_relay = if qc.relay.trim().is_empty() {
4112                                        None
4113                                    } else {
4114                                        match resolve_moq_relay_reach(&qc.relay) {
4115                                            Ok(reach) => Some(reach),
4116                                            Err(e) => {
4117                                                tracing::warn!(
4118                                                    relay = %qc.relay,
4119                                                    "ignoring invalid [quic] relay; continuing direct-only: {e}"
4120                                                );
4121                                                None
4122                                            }
4123                                        }
4124                                    };
4125                                    // #1542: a checkpointed native service has both an
4126                                    // accepted did:at9p identity and its accepted current
4127                                    // response signer. Use that existing state to prove Iroh
4128                                    // `moql` admission; do not fabricate credentials or fall
4129                                    // back to an anonymous Iroh handshake.
4130                                    let moq_admission_proof =
4131                                        select_iroh_moql_admission_proof(qc.iroh, || {
4132                                            service_names
4133                                                .iter()
4134                                                .map(|service_name| {
4135                                                    ctx.moql_admission_proof(service_name)
4136                                                        .map(|proof| (service_name.clone(), proof))
4137                                                })
4138                                                .collect::<Result<Vec<_>>>()
4139                                        })?;
4140                                    // Event clients start before factories, so the process
4141                                    // proof must be installed before any pre-factory dial.
4142                                    if let Some(proof) = moq_admission_proof.clone() {
4143                                        let _ = hyprstream_rpc::moq_stream::init_global_moq_admission_proof(proof);
4144                                    }
4145                                    let discovery_transport =
4146                                        ctx.transport("discovery", SocketKind::Rep);
4147                                    let shared = hyprstream_service::QuicSharedConfig {
4148                                        cert_chain,
4149                                        certificate_hashes,
4150                                        key_der,
4151                                        base_ip: qc.socket_addr()?.ip(),
4152                                        server_name: qc.server_name.clone(),
4153                                        oauth_issuer_url: Some(config.oauth.issuer_url()),
4154                                        jwt_verifying_key: Some(ctx.jwt_verifying_key()),
4155                                        // #282: bind iroh in parallel to quinn when opted in.
4156                                        iroh_enabled: qc.iroh,
4157                                        iroh_required: qc.iroh_required(),
4158                                        // #358: producer-chosen relay rendezvous (None = direct-only).
4159                                        moq_relay,
4160                                        // A relay's accepted-state witness must be supplied by a
4161                                        // verified resolver result; the URI alone is reachability,
4162                                        // never an application identity.
4163                                        moq_relay_server_identity: None,
4164                                        moq_admission: if moq_admission_proof.is_some() {
4165                                            Some(hyprstream_core::services::stream_network::production_stream_admission()?)
4166                                        } else { None },
4167                                        // The ingress authorizer only runs alongside
4168                                        // admission (it decides on already-admitted peers),
4169                                        // so it is constructed under the same guard and
4170                                        // derives tenants from the same live roster
4171                                        // source (#1652).
4172                                        moq_ingress_authorizer: if moq_admission_proof.is_some() {
4173                                            // Prime this roster instance synchronously at
4174                                            // construction too (review round 7): it is a
4175                                            // separate cache from the admission's, and the
4176                                            // first publisher ingress would otherwise pay
4177                                            // the synchronous first load on the serving
4178                                            // runtime.
4179                                            let roster = hyprstream_discovery::production_deployment_roster()?;
4180                                            let _ = roster.roster();
4181                                            Some(hyprstream_core::services::stream_network::stream_ingress_authorizer(
4182                                                &qc,
4183                                                roster,
4184                                            ))
4185                                        } else { None },
4186                                        moq_admission_proof,
4187                                        native_announcement_publisher: Some(std::sync::Arc::new(
4188                                            move |request: hyprstream_service::NativeAnnouncementRequest| {
4189                                                let discovery_transport = discovery_transport.clone();
4190                                                start_native_announcement_publisher(
4191                                                    request.reach.clone(),
4192                                                    hyprstream_discovery::native_network_required(),
4193                                                    request.cancellation.clone(),
4194                                                    move |announce_tx| async move {
4195                                                        let mut request = request;
4196                                                        let socket_kind = request.reach.socket_kind().to_owned();
4197                                                        let endpoint = request.reach.endpoint();
4198                                                        let service_name = request.service_name.clone();
4199                                                        let signing_key = request.signing_key.clone();
4200                                                        let discovery_verifying_key = request.discovery_verifying_key;
4201                                                        // Resolve a fresh client for EACH authority projection.
4202                                                        // The body JWT alone does not authenticate the envelope,
4203                                                        // and retaining a client would retain an expired bearer
4204                                                        // across credential renewal.
4205                                                        let make_client = |announcement: &hyprstream_rpc_std::discovery_client::ServiceAnnouncement| {
4206                                                            let token = native_announcement_bearer(announcement)?;
4207                                                            if hyprstream_discovery::native_network_required() {
4208                                                                hyprstream_rpc_std::discovery_client::DiscoveryClient::from_provider(
4209                                                                    &hyprstream_discovery::ProductionRpcClientProvider,
4210                                                                    signing_key.clone(), Some(token),
4211                                                                )
4212                                                            } else {
4213                                                                hyprstream_rpc_std::discovery_client::DiscoveryClient::for_local_transport_bootstrap(
4214                                                                    &discovery_transport, signing_key.clone(),
4215                                                                    discovery_verifying_key, Some(token),
4216                                                                )
4217                                                            }
4218                                                        };
4219                                                        if service_name == "policy"
4220                                                            && hyprstream_discovery::native_network_required()
4221                                                            && matches!(request.reach, hyprstream_service::NativeAnnouncementReach::Iroh { .. })
4222                                                        {
4223                                                            let cancellation = request.cancellation.clone();
4224                                                            let initial = wait_for_policy_initial_publication(
4225                                                                &cancellation,
4226                                                                POLICY_INITIAL_PUBLICATION_BUDGET,
4227                                                                || {
4228                                                                    // Re-read signed current state and the exact service
4229                                                                    // key's credential on every attempt, before any dial.
4230                                                                    let announcement = hyprstream_core::services::factories::current_native_announcement(&mut request);
4231                                                                    let client = announcement.as_ref().map_err(|e| anyhow::anyhow!("{e}")).and_then(&make_client);
4232                                                                    async move {
4233                                                                        let announcement = announcement?;
4234                                                                        let client = client?;
4235                                                                        let jwt_expiry = announcement.service_jwt.as_deref()
4236                                                                            .and_then(hyprstream_core::auth::identity_store::decode_jwt_exp_raw)
4237                                                                            .and_then(|seconds| seconds.checked_mul(1_000))
4238                                                                            .context("Policy initial announcement requires a bounded service JWT")?;
4239                                                                        let expiry = announcement.expires_at_unix_ms.min(jwt_expiry);
4240                                                                        let remaining = expiry.saturating_sub(chrono::Utc::now().timestamp_millis());
4241                                                                        anyhow::ensure!(remaining > 0, "Policy initial announcement authority expired");
4242                                                                        let duration = std::time::Duration::from_millis(u64::try_from(remaining)?);
4243                                                                        tokio::time::timeout(duration, client.announce(&announcement)).await
4244                                                                            .context("Policy initial announcement authority expired during publication")??;
4245                                                                        anyhow::ensure!(chrono::Utc::now().timestamp_millis() < expiry,
4246                                                                            "Policy initial announcement authority expired before readiness");
4247                                                                        Ok(())
4248                                                                    }
4249                                                                },
4250                                                            ).await;
4251                                                            // Report only success or terminal failure, never a
4252                                                            // transient availability error. The spawner's existing
4253                                                            // bind/publication READY and cancellation gates remain.
4254                                                            if let Some(tx) = announce_tx.as_ref() {
4255                                                                if let Some(tx) = tx.lock().take() {
4256                                                                    let _ = tx.send(initial.as_ref().map(|_| ()).map_err(ToString::to_string));
4257                                                                }
4258                                                            }
4259                                                            if initial.is_err() { return; }
4260                                                            tokio::select! {
4261                                                                biased;
4262                                                                _ = cancellation.cancelled() => return,
4263                                                                _ = tokio::time::sleep(std::time::Duration::from_secs(25)) => {}
4264                                                            }
4265                                                        }
4266                                                        let _completion = refresh_native_announcement(
4267                                                            &service_name,
4268                                                            &socket_kind,
4269                                                            &endpoint,
4270                                                            // Authority/JWT expiry is rechecked for each attempt.
4271                                                            // Never retain the original deadline after renewal.
4272                                                            i64::MAX,
4273                                                            request.cancellation.clone(),
4274                                                            || {
4275                                                                let announcement = hyprstream_core::services::factories::current_native_announcement(&mut request);
4276                                                                let announce_tx = announce_tx.as_ref().map(std::sync::Arc::clone);
4277                                                                let client = announcement.as_ref().map_err(|e| anyhow::anyhow!("{e}")).and_then(&make_client);
4278                                                                publish_native_announcement_attempt(announcement, announce_tx, move |announcement| async move {
4279                                                                    client?.announce(&announcement).await.map(|_| ())
4280                                                                })
4281                                                            },
4282                                                        )
4283                                                        .await;
4284                                                    },
4285                                                )
4286                                            },
4287                                        )),
4288                                    };
4289                                    ctx = ctx.with_quic(shared);
4290                                }
4291
4292                                // Wire JWKS-backed key resolution (kid-based, on-miss-refetch).
4293                                {
4294                                    let issuer_url = config.oauth.issuer_url();
4295                                    let jwks_url =
4296                                        format!("{}/oauth/jwks", issuer_url.trim_end_matches('/'));
4297                                    let fetcher =
4298                                        hyprstream_core::auth::jwks_fetcher::default_jwks_fetcher();
4299                                    ctx.set_jwks_fetcher(fetcher);
4300                                    tracing::debug!(
4301                                        "JWKS-backed key source configured: {}",
4302                                        jwks_url
4303                                    );
4304                                }
4305
4306                                // Populate ML-DSA-65 verifying keys for PQ-hybrid JWT verification.
4307                                {
4308                                    let secrets_dir = hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
4309                                    let ml_dsa_store = hyprstream_core::auth::key_rotation::global_ml_dsa_key_store(
4310                                        &secrets_dir,
4311                                        &config.oauth,
4312                                    );
4313                                    let vks: Vec<_> = tokio::task::block_in_place(|| {
4314                                        let rt = tokio::runtime::Handle::current();
4315                                        rt.block_on(ml_dsa_store.all_slots_snapshot())
4316                                    })
4317                                    .iter()
4318                                    .map(hyprstream_core::auth::MlDsaKeySlot::verifying_key)
4319                                    .collect();
4320                                    let shared_vks = hyprstream_core::auth::key_rotation::global_ml_dsa_verifying_keys();
4321                                    let _ = shared_vks.write().map(|mut guard| *guard = vks);
4322                                    ctx.set_ml_dsa_verifying_keys(shared_vks);
4323                                    let ed_store = hyprstream_core::auth::key_rotation::global_ed25519_key_store(
4324                                        &secrets_dir,
4325                                        &config.oauth,
4326                                    );
4327                                    let restore_result = tokio::task::block_in_place(|| {
4328                                        let rt = tokio::runtime::Handle::current();
4329                                        rt.block_on(hyprstream_core::auth::key_rotation::restore_composite_verifying_key_set(
4330                                            &secrets_dir,
4331                                            &ed_store,
4332                                            &ml_dsa_store,
4333                                            ctx.jwt_verifying_key(),
4334                                        ))
4335                                    });
4336                                    if let Err(error) = restore_result {
4337                                        tracing::warn!("exact composite pair ledger unavailable in verifier process: {error}");
4338                                    }
4339                                    tracing::info!("PQ-hybrid: ML-DSA-65 verifying keys loaded for JWT verification");
4340
4341                                    // Configured artifacts were already installed
4342                                    // at the startup gate; baseline is absent-config only.
4343                                    if config.policy.boot_policy.is_none() {
4344                                        // MAC S4 policy bootload (#570): compile → sign
4345                                        // → verify-once-at-load → install the node's
4346                                        // baseline CompiledPolicy so the MAC PDP inputs
4347                                        // are real. This is the one missing boot step
4348                                        // that populates `mac::COMPILED_POLICY`, flipping
4349                                        // `exchange_enrollment_resolver` from the deny-all
4350                                        // `DenyUnlabeledResolver` to the real (#698)
4351                                        // `EnrollmentSubjectContextResolver`.
4352                                        //
4353                                        // The 9P PEP is active and records the
4354                                        // bootloaded policy as audit provenance.
4355                                        //
4356                                        // Fail-closed: the baseline is hybrid-signed
4357                                        // (EdDSA + ML-DSA-65) and verified with
4358                                        // require_pq=true. With no active ML-DSA signing
4359                                        // key (Classical mode) nothing is installed —
4360                                        // never an Ed25519-only baseline (design §14) —
4361                                        // and the resolver keeps denying.
4362                                        let active_pq = tokio::task::block_in_place(|| {
4363                                            let rt = tokio::runtime::Handle::current();
4364                                            rt.block_on(ml_dsa_store.active_key())
4365                                        });
4366                                        match active_pq {
4367                                        Some(pq_sk) => match hyprstream_core::mac::install_baseline_boot_policy(
4368                                            &signing_key,
4369                                            &pq_sk,
4370                                        ) {
4371                                            Ok((policy, true)) => tracing::info!(
4372                                                "MAC S4: baseline compiled policy installed \
4373                                                 (generation {}, 9P enforcement active)",
4374                                                policy.generation
4375                                            ),
4376                                            // The seam is write-once: a policy was
4377                                            // already installed this process, so the
4378                                            // baseline did NOT replace it. Report it
4379                                            // honestly rather than claiming an install.
4380                                            Ok((_policy, false)) => tracing::warn!(
4381                                                "MAC S4: a compiled policy was already installed; \
4382                                                 baseline NOT installed (write-once seam unchanged)"
4383                                            ),
4384                                            Err(e) => tracing::error!(
4385                                                "MAC S4: baseline policy bootload failed \
4386                                                 (grant-path resolver stays fail-closed): {e}"
4387                                            ),
4388                                        },
4389                                        None => tracing::warn!(
4390                                            "MAC S4: no active ML-DSA signing key; baseline policy \
4391                                             NOT installed (fail-closed — grant-path resolver denies)"
4392                                        ),
4393                                    }
4394                                    }
4395                                }
4396
4397                                // M3 (#152): install the process-global envelope
4398                                // verify configuration that closes the fail-open
4399                                // at the ZMQ RequestLoop + StreamService verify
4400                                // sites.
4401                                //
4402                                // KEY SEPARATION (PQUIP key-reuse restriction):
4403                                // the mesh hybrid identity's ML-DSA key MUST be
4404                                // distinct from the JWT-signing ML-DSA keyset
4405                                // (`global_ml_dsa_verifying_keys`, loaded above).
4406                                // We therefore DO NOT seed the mesh PqTrustStore
4407                                // from the JWT keyset. The mesh store is keyed by
4408                                // Ed25519 signer identity.
4409                                //
4410                                // #157 (Option A — eager, admin-anchored): the
4411                                // store is populated EAGERLY here, before install,
4412                                // from the operator-configured `mesh_peers`. It is
4413                                // immutable after install (no lazy-at-verify
4414                                // resolution). Empty `mesh_peers` => empty store =>
4415                                // unchanged behavior.
4416                                //
4417                                // Policy: Hybrid is mandatory. With no anchored
4418                                // peer key the verifier FAILS CLOSED.
4419                                install_envelope_verify_config(Some(&config.oauth), Some(&config));
4420
4421                                // Install MoQ/event authorization in every
4422                                // production service process before any model,
4423                                // worker, workflow, or system publisher can be
4424                                // constructed. In multi-process mode only the
4425                                // event origin lives in the `event` process;
4426                                // authorization still belongs at each local
4427                                // publisher/subscriber choke point.
4428                                if !hyprstream_rpc::events::event_authz_installed() {
4429                                    let audit_stream =
4430                                        format!("moq-event-{}", service_names.join("-"));
4431                                    let authz = hyprstream_core::services::native_event_policy::production_event_authz(
4432                                        signing_key.clone(),
4433                                        &config,
4434                                        &audit_stream,
4435                                    )
4436                                    .await
4437                                    .context("construct process-wide MoQ/event MAC PEP")?;
4438                                    hyprstream_rpc::events::install_event_authz(authz)
4439                                        .map_err(anyhow::Error::msg)?;
4440                                }
4441
4442                                let manager = InprocManager::new();
4443                                let mut handles = Vec::new();
4444
4445                                // Compute dependency-aware startup stages.
4446                                let stages = hyprstream_service::service::ordering::startup_stages_for_profile(
4447                                    &service_names, ctx.iroh_required(),
4448                                );
4449
4450                                // In a split required-native launch each child
4451                                // sees only its own `service_names` entry, while
4452                                // the parent stages the configured roster. If
4453                                // this child is one of the services scheduled
4454                                // before Policy (currently event, discovery,
4455                                // or streams), defer its startup probe so the
4456                                // parent can reach the Policy stage. A
4457                                // standalone policy-dependent child retains
4458                                // the eager probe and cannot report ready with
4459                                // an unavailable authority.
4460                                let defer_policy_probe = if ctx.iroh_required()
4461                                    && service_names.len() == 1
4462                                {
4463                                    let configured_stages =
4464                                        hyprstream_service::service::ordering::startup_stages_for_profile(
4465                                            &services,
4466                                            true,
4467                                        );
4468                                    let policy_position =
4469                                        configured_stages.iter().enumerate().find_map(
4470                                            |(stage_index, stage)| {
4471                                                stage.iter().position(|name| name == "policy").map(
4472                                                    |service_index| (stage_index, service_index),
4473                                                )
4474                                            },
4475                                        );
4476                                    let service_position = configured_stages
4477                                        .iter()
4478                                        .enumerate()
4479                                        .find_map(|(stage_index, stage)| {
4480                                            stage
4481                                                .iter()
4482                                                .position(|name| name == &service_names[0])
4483                                                .map(|service_index| (stage_index, service_index))
4484                                        });
4485                                    match (service_position, policy_position) {
4486                                        (Some(service), Some(policy)) => service < policy,
4487                                        _ => false,
4488                                    }
4489                                } else {
4490                                    false
4491                                };
4492
4493                                // Publish the process-global authority stores
4494                                // (credential revocation + session registry)
4495                                // BEFORE any factory runs. The policy process
4496                                // owns the canonical stores (durable local
4497                                // files); every other process publishes
4498                                // policy-authority RPC client stores after
4499                                // probing the authority. Fail-closed: an
4500                                // unreachable authority or an unreadable
4501                                // durable file aborts startup.
4502                                hyprstream_core::services::revocation::init_process_authority_stores(
4503                                    &ctx,
4504                                    ctx.signing_key(),
4505                                    service_names.iter().any(|n| n == "policy"),
4506                                    defer_policy_probe,
4507                                )
4508                                .await
4509                                .context("revocation/session authority initialization failed")?;
4510
4511                                // #275: in the systemd / --ipc deployment each service
4512                                // runs in its OWN process. Only the `event` service's
4513                                // process initializes the process-global moq event bus
4514                                // origin. If THIS process does not host the `event`
4515                                // service, wire a CLIENT-mode event origin connected to
4516                                // the event service's UDS plane BEFORE any factory runs
4517                                // (so `EventPublisher::new` in e.g. the worker factory
4518                                // resolves the global). Idempotent + a no-op when `event`
4519                                // is co-located.
4520                                let hosts_event_service = stages
4521                                    .iter()
4522                                    .any(|stage| stage.iter().any(|s| s == "event"));
4523                                if !hosts_event_service {
4524                                    hyprstream_core::services::event_network::ensure_event_origin_for_profile()?;
4525                                }
4526
4527                                for stage in &stages {
4528                                    for svc_name in stage {
4529                                        let factory = get_factory(svc_name).ok_or_else(|| {
4530                                            let available: Vec<_> =
4531                                                hyprstream_service::list_factories()
4532                                                    .map(|f| f.name)
4533                                                    .collect();
4534                                            anyhow::anyhow!(
4535                                                "Unknown service: '{}'. Available services: {}",
4536                                                svc_name,
4537                                                available.join(", ")
4538                                            )
4539                                        })?;
4540
4541                                        info!(
4542                                            "Starting {} service in standalone mode (IPC: {})",
4543                                            svc_name, ipc
4544                                        );
4545
4546                                        let spawnable = (factory.factory)(&ctx).context(
4547                                            format!("Failed to create {} service", svc_name),
4548                                        )?;
4549                                        let handle = manager.spawn(spawnable).await.context(
4550                                            format!("Failed to start {} service", svc_name),
4551                                        )?;
4552                                        handles.push((svc_name.clone(), handle));
4553                                    }
4554
4555                                    // Publish ready events for this stage before starting the next.
4556                                    for svc_name in stage {
4557                                        if let Ok(publisher) = EventPublisher::new("system") {
4558                                            let _ = publisher
4559                                                .publish_raw(
4560                                                    &format!("system.{}.ready", svc_name),
4561                                                    b"",
4562                                                )
4563                                                .await;
4564                                        }
4565                                    }
4566                                }
4567
4568                                if handles.is_empty() {
4569                                    return Err(anyhow::anyhow!("No services to start"));
4570                                }
4571
4572                                let svc_names: Vec<_> =
4573                                    handles.iter().map(|(n, _)| n.as_str()).collect();
4574                                info!(
4575                                    "Services ready: [{}], waiting for shutdown signal",
4576                                    svc_names.join(", ")
4577                                );
4578
4579                                let _ = shutdown_rx.await;
4580
4581                                // Stop all services
4582                                for (svc_name, mut handle) in handles {
4583                                    if let Ok(publisher) = EventPublisher::new("system") {
4584                                        let _ = publisher
4585                                            .publish_raw(
4586                                                &format!("system.{}.stopping", svc_name),
4587                                                b"",
4588                                            )
4589                                            .await;
4590                                    }
4591                                    let _ = handle.stop().await;
4592                                    info!("{} service stopped", svc_name);
4593                                }
4594
4595                                Ok(())
4596                            },
4597                        )?;
4598                    } else {
4599                        with_runtime(
4600                            RuntimeConfig {
4601                                device: DeviceConfig::request_cpu(),
4602                                multi_threaded: false,
4603                            },
4604                            || async move {
4605                                handle_service_start(
4606                                    &services,
4607                                    name,
4608                                    daemon,
4609                                    explicit_config,
4610                                    iroh_required,
4611                                )
4612                                .await
4613                            },
4614                        )?;
4615                    }
4616                }
4617
4618                ServiceAction::Stop { name } => {
4619                    with_runtime(
4620                        RuntimeConfig {
4621                            device: DeviceConfig::request_cpu(),
4622                            multi_threaded: false,
4623                        },
4624                        || async move { handle_service_stop(&services, name).await },
4625                    )?;
4626                }
4627
4628                ServiceAction::Status { verbose } => {
4629                    with_runtime(
4630                        RuntimeConfig {
4631                            device: DeviceConfig::request_cpu(),
4632                            multi_threaded: false,
4633                        },
4634                        || async move { handle_service_status(&services, verbose).await },
4635                    )?;
4636                }
4637
4638                ServiceAction::Repair { verbose } => {
4639                    // Normally handled by the early dispatch above (before
4640                    // the hybrid gate). This arm is a defense-in-depth fallback
4641                    // that runs the same diagnostic checks if the early
4642                    // dispatch is ever bypassed.
4643                    let models_dir = config_for_service.models_dir().clone();
4644                    with_runtime(
4645                        RuntimeConfig {
4646                            device: DeviceConfig::request_cpu(),
4647                            multi_threaded: true,
4648                        },
4649                        || async move {
4650                            hyprstream_core::cli::service_handlers::run_repair_checks(
4651                                &models_dir,
4652                                verbose,
4653                            )
4654                            .await
4655                        },
4656                    )?;
4657                }
4658
4659                ServiceAction::EnsureKey { name } => {
4660                    // Normally handled by the early dispatch above (before any
4661                    // services start). Defense-in-depth fallback if that
4662                    // dispatch is ever bypassed.
4663                    hyprstream_core::cli::service_handlers::handle_service_ensure_key(
4664                        Some(ctx.config()),
4665                        &name,
4666                    )?;
4667                }
4668                ServiceAction::ReconcileEnrollment => {
4669                    hyprstream_core::cli::service_handlers::handle_service_reconcile_enrollment(
4670                        ctx.config(),
4671                    )?;
4672                }
4673                ServiceAction::EnsureNodeAuthority => {
4674                    let secrets = hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(
4675                        Some(ctx.config()),
4676                    )?;
4677                    hyprstream_core::auth::service_enrollment::ensure_node_authority(&secrets)?;
4678                }
4679                ServiceAction::ProvisionPolicyTemplates { template } => {
4680                    let models_dir = config_for_service.models_dir().clone();
4681                    with_runtime(
4682                        RuntimeConfig {
4683                            device: DeviceConfig::request_cpu(),
4684                            multi_threaded: true,
4685                        },
4686                        || async move {
4687                            handle_service_provision_policy_templates(&models_dir, &template).await
4688                        },
4689                    )?;
4690                }
4691                ServiceAction::ReadbackLocalCanary { .. } => {
4692                    anyhow::bail!("readback-local-canary must use its enrolled early dispatch")
4693                }
4694                ServiceAction::RegisterLocalCanary { .. } => {
4695                    anyhow::bail!("register-local-canary must use its enrolled early dispatch")
4696                }
4697            }
4698        }
4699
4700        // ── User management ─────────────────────────────────────────────
4701        Some(("user", sub_m)) => {
4702            let cmd = UserCommand::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
4703            // Deliberately resolve from `ctx`: it honors the `--config <path>`
4704            // CLI override. The operator's config must win here, or `--config`
4705            // invocations would resolve the user store from XDG defaults and
4706            // split it.
4707            let credentials_dir =
4708                hyprstream_core::auth::identity_store::credentials_dir_for_config(Some(
4709                    ctx.config(),
4710                ))?;
4711            // User handlers are async, run them in a minimal runtime
4712            let rt = tokio::runtime::Builder::new_current_thread()
4713                .enable_all()
4714                .build()
4715                .context("Failed to create runtime for user command")?;
4716            rt.block_on(async {
4717                match cmd {
4718                    UserCommand::Create {
4719                        username,
4720                        role,
4721                        generate,
4722                        key,
4723                        ssh,
4724                        no_role,
4725                    } => {
4726                        handle_user_create(
4727                            &credentials_dir,
4728                            &username,
4729                            &role,
4730                            no_role,
4731                            generate,
4732                            key.as_deref(),
4733                            ssh.as_deref(),
4734                        )
4735                        .await?;
4736                    }
4737                    UserCommand::Register { username } => {
4738                        handle_user_register(&credentials_dir, &username).await?;
4739                    }
4740                    UserCommand::List => {
4741                        handle_user_list(&credentials_dir).await?;
4742                    }
4743                    UserCommand::Remove { username, force } => {
4744                        handle_user_remove(&credentials_dir, &username, force).await?;
4745                    }
4746                    UserCommand::Keys { command } => match command {
4747                        UserKeysCommand::List { username } => {
4748                            handle_user_keys_list(&credentials_dir, &username).await?;
4749                        }
4750                        UserKeysCommand::Import { username, format } => {
4751                            handle_user_keys_import(&credentials_dir, &username, &format).await?;
4752                        }
4753                        UserKeysCommand::Remove {
4754                            username,
4755                            fingerprint,
4756                        } => {
4757                            handle_user_keys_remove(&credentials_dir, &username, &fingerprint)
4758                                .await?;
4759                        }
4760                    },
4761                }
4762                Ok::<_, anyhow::Error>(())
4763            })?;
4764        }
4765
4766        // ── TUI display server ──────────────────────────────────────────
4767        Some(("tui", sub_m)) => {
4768            let action =
4769                TuiAction::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
4770
4771            with_runtime(
4772                RuntimeConfig {
4773                    device: DeviceConfig::request_cpu(),
4774                    multi_threaded: true,
4775                },
4776                || async move {
4777                    use hyprstream_core::cli::tui_handlers;
4778
4779                    // Load signing key for RPC authentication
4780                    let data_dir = dirs::data_local_dir()
4781                        .unwrap_or_else(|| std::path::PathBuf::from("."))
4782                        .join("hyprstream");
4783                    let keys_dir = data_dir.join("keys");
4784                    let signing_key = load_or_generate_signing_key(&keys_dir).await?;
4785
4786                    match action {
4787                        TuiAction::Attach { session } => {
4788                            let sid = if session == 0 { None } else { Some(session) };
4789                            tui_handlers::handle_tui_attach(&signing_key, sid).await
4790                        }
4791                        TuiAction::New => tui_handlers::handle_tui_new(&signing_key).await,
4792                        TuiAction::List => tui_handlers::handle_tui_list(&signing_key).await,
4793                        TuiAction::Detach => tui_handlers::handle_tui_detach().await,
4794                        TuiAction::Play {
4795                            cast_file,
4796                            session,
4797                            loop_playback,
4798                        } => {
4799                            tui_handlers::handle_tui_play(
4800                                &signing_key,
4801                                &cast_file,
4802                                session,
4803                                loop_playback,
4804                            )
4805                            .await
4806                        }
4807                        TuiAction::Shell { session } => {
4808                            let models_dir = config_for_service.models_dir().clone();
4809                            hyprstream_core::cli::shell_handlers::handle_tui_shell(
4810                                &signing_key,
4811                                &models_dir,
4812                                session,
4813                            )
4814                            .await
4815                        }
4816                    }
4817                },
4818            )?;
4819        }
4820
4821        // ── Flight SQL client ───────────────────────────────────────────
4822        Some(("flight", sub_m)) => {
4823            #[cfg(not(feature = "metrics"))]
4824            {
4825                let _ = sub_m;
4826                return Err(anyhow::anyhow!(
4827                    "Flight SQL client requires the 'metrics' feature"
4828                ));
4829            }
4830            #[cfg(feature = "metrics")]
4831            {
4832                let args =
4833                    FlightArgs::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
4834
4835                with_runtime(
4836                    RuntimeConfig {
4837                        device: DeviceConfig::request_cpu(),
4838                        multi_threaded: true,
4839                    },
4840                    || async move {
4841                        use hyprstream_flight::client::{
4842                            format_batches_as_csv, format_batches_as_json, format_batches_as_table,
4843                            FlightClient,
4844                        };
4845
4846                        let mut client = FlightClient::connect(&args.addr)
4847                            .await
4848                            .map_err(|e| anyhow::anyhow!("Failed to connect: {}", e))?;
4849
4850                        if let Some(sql) = args.query {
4851                            let batches = client
4852                                .query(&sql)
4853                                .await
4854                                .map_err(|e| anyhow::anyhow!("Query failed: {}", e))?;
4855
4856                            let output = match args.format.as_str() {
4857                                "csv" => format_batches_as_csv(&batches),
4858                                "json" => format_batches_as_json(&batches)
4859                                    .map_err(|e| anyhow::anyhow!("JSON format error: {}", e))?,
4860                                _ => format_batches_as_table(&batches),
4861                            };
4862                            print!("{}", output);
4863                        } else {
4864                            println!("Connected to Flight SQL server at {}", args.addr);
4865                            println!(
4866                                "Interactive mode not yet implemented. Use --query to execute SQL."
4867                            );
4868                        }
4869                        Ok(())
4870                    },
4871                )?;
4872            }
4873        }
4874
4875        // ── Sign OAuth challenge ─────────────────────────────────────────
4876        Some(("sign-challenge", sub_m)) => {
4877            let user_code = sub_m.get_one::<String>("user_code").cloned();
4878            let nonce = sub_m.get_one::<String>("nonce").cloned();
4879            let code_challenge = sub_m.get_one::<String>("code_challenge").cloned();
4880            let server = sub_m.get_one::<String>("server").cloned();
4881            let insecure = sub_m.get_flag("insecure");
4882            // Pass the already-loaded config (honors `--config`) so the OAuth
4883            // issuer URL and the local self-signed cert's secrets dir are
4884            // resolved from the user's selected configuration, not re-derived
4885            // from defaults (#450). `config_for_service` is the surviving
4886            // clone (`config` itself is moved into AppContext above).
4887            let sign_cfg = config_for_service.clone();
4888            with_runtime(
4889                RuntimeConfig {
4890                    device: DeviceConfig::request_cpu(),
4891                    multi_threaded: false,
4892                },
4893                || async move {
4894                    handle_sign_challenge(
4895                        user_code,
4896                        nonce,
4897                        code_challenge,
4898                        server,
4899                        insecure,
4900                        Some(&sign_cfg),
4901                    )
4902                    .await
4903                },
4904            )?;
4905        }
4906
4907        // ── Wizard ── handled early (before registry init) above ────────────
4908        Some(("wizard", _)) => {
4909            unreachable!("wizard command is dispatched before registry init")
4910        }
4911
4912        // ── Self-update ──────────────────────────────────────────────────
4913        Some(("update", sub_m)) => {
4914            let cleanup = sub_m.get_flag("cleanup");
4915            let models_dir = config_for_service.models_dir().clone();
4916            with_runtime(
4917                RuntimeConfig {
4918                    device: DeviceConfig::request_cpu(),
4919                    multi_threaded: true,
4920                },
4921                || async move {
4922                    hyprstream_core::cli::update_handlers::handle_update(&models_dir, cleanup).await
4923                },
4924            )?;
4925        }
4926
4927        // ── Native MAC status/inspection ── handled early (before registry
4928        // init) above: it is a read-only diagnostic that must work on fresh
4929        // installations with no credentials.
4930        Some(("mac", _)) => {
4931            unreachable!("mac command is dispatched before registry init")
4932        }
4933
4934        // ── No subcommand → wizard (first run) or ShellClient ──────────
4935        _ => {
4936            let models_dir = config_for_service.models_dir().clone();
4937            if hyprstream_core::cli::bootstrap_manager::is_first_run(&models_dir) {
4938                let services = config_for_service.services.startup.clone();
4939                with_runtime(
4940                    RuntimeConfig {
4941                        device: DeviceConfig::request_cpu(),
4942                        multi_threaded: true,
4943                    },
4944                    || async move {
4945                        if supports_tui() {
4946                            hyprstream_core::cli::handle_wizard_tui(&models_dir, &services).await
4947                        } else {
4948                            // First-run auto-wizard defaults federation to off.
4949                            hyprstream_core::cli::handle_wizard(
4950                                &models_dir,
4951                                &services,
4952                                hyprstream_core::cli::WizardOptions::first_run(),
4953                            )
4954                            .await
4955                        }
4956                    },
4957                )?;
4958            } else {
4959                with_runtime(
4960                    RuntimeConfig {
4961                        device: DeviceConfig::request_cpu(),
4962                        multi_threaded: true,
4963                    },
4964                    || {
4965                        let md = models_dir.clone();
4966                        async move {
4967                            let keys_dir = md.join(".registry").join("keys");
4968                            let sk = load_or_generate_signing_key(&keys_dir).await?;
4969                            hyprstream_core::cli::shell_handlers::handle_shell_tui(&sk, &md).await
4970                        }
4971                    },
4972                )?;
4973            }
4974        }
4975    };
4976
4977    Ok(())
4978}
4979
4980#[cfg(test)]
4981mod process_identity_tests {
4982    use hyprstream_rpc::crypto::SigningKey;
4983
4984    #[test]
4985    fn dedicated_service_process_uses_service_identity_without_native_profile() {
4986        assert_eq!(
4987            super::process_identity_name(None, Some("oauth")),
4988            Some("oauth"),
4989            "a dedicated compatibility process must own its service identity"
4990        );
4991    }
4992
4993    #[test]
4994    fn native_selection_takes_precedence_over_dedicated_name() {
4995        assert_eq!(
4996            super::process_identity_name(Some("oauth"), Some("oauth")),
4997            Some("oauth")
4998        );
4999        assert_eq!(
5000            super::process_identity_name(Some("oauth"), None),
5001            Some("oauth")
5002        );
5003    }
5004
5005    #[test]
5006    fn plain_cli_process_has_no_service_identity() {
5007        assert_eq!(super::process_identity_name(None, None), None);
5008    }
5009
5010    #[test]
5011    fn flat_key_policy_credential_only_for_the_flat_key_with_a_jwt() -> anyhow::Result<()> {
5012        let dir = tempfile::tempdir()?;
5013        let profile = hyprstream_core::auth::identity_store::SecretsProfile::SharedDirectory;
5014        // Provision the flat node key (policy resolves to it, #759).
5015        let policy_key = hyprstream_core::auth::identity_store::resolve_service_signing_key(
5016            dir.path(),
5017            "policy",
5018            profile,
5019        )?;
5020        // No credential on disk yet: nothing to present.
5021        assert!(
5022            super::flat_key_policy_resolver_credential(dir.path(), profile, &policy_key).is_none()
5023        );
5024        // A non-policy process key never gets policy's credential.
5025        let other = SigningKey::from_bytes(&[0x41; 32]);
5026        let now = chrono::Utc::now().timestamp();
5027        let claims = hyprstream_rpc::auth::Claims::new("service:policy".to_owned(), now, now + 600)
5028            .with_cnf_jwk(policy_key.verifying_key().as_bytes());
5029        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &policy_key);
5030        hyprstream_core::auth::identity_store::write_service_jwt(dir.path(), "policy", &jwt)?;
5031        assert!(
5032            super::flat_key_policy_resolver_credential(dir.path(), profile, &other).is_none(),
5033            "split-key deployments never present policy's credential"
5034        );
5035        assert_eq!(
5036            super::flat_key_policy_resolver_credential(dir.path(), profile, &policy_key),
5037            Some(jwt),
5038            "the flat-key process presents policy's enrolled credential"
5039        );
5040        Ok(())
5041    }
5042}
5043
5044#[cfg(test)]
5045#[allow(clippy::expect_used)]
5046mod resolver_startup_controls {
5047    use clap::FromArgMatches as _;
5048
5049    #[test]
5050    fn canary_registration_cli_requires_exact_path_and_head_arguments() {
5051        let matches = super::build_cli()
5052            .try_get_matches_from([
5053                "hyprstream",
5054                "service",
5055                "register-local-canary",
5056                "--path",
5057                "/var/lib/hyprstream/models/canary",
5058                "--expected-head",
5059                "3f1d3a63258e132cd06314b736445a8bab13b9bd",
5060            ])
5061            .expect("canary registration CLI");
5062        let service = matches.subcommand_matches("service").expect("service");
5063        let action = hyprstream_core::cli::commands::ServiceAction::from_arg_matches(service)
5064            .expect("service action");
5065        assert!(matches!(
5066            action,
5067            hyprstream_core::cli::commands::ServiceAction::RegisterLocalCanary { .. }
5068        ));
5069        assert!(super::build_cli()
5070            .try_get_matches_from(["hyprstream", "service", "register-local-canary"])
5071            .is_err());
5072    }
5073
5074    #[test]
5075    fn offline_policy_provision_cli_requires_and_preserves_templates() {
5076        let matches = super::build_cli()
5077            .try_get_matches_from([
5078                "hyprstream",
5079                "service",
5080                "provision-policy-templates",
5081                "--template",
5082                "public-inference",
5083                "--template",
5084                "public-read",
5085            ])
5086            .expect("offline policy CLI");
5087        let service = matches.subcommand_matches("service").expect("service");
5088        let action = hyprstream_core::cli::commands::ServiceAction::from_arg_matches(service)
5089            .expect("service action");
5090        let hyprstream_core::cli::commands::ServiceAction::ProvisionPolicyTemplates { template } =
5091            action
5092        else {
5093            panic!("wrong service action");
5094        };
5095        assert_eq!(template, ["public-inference", "public-read"]);
5096        assert!(super::build_cli()
5097            .try_get_matches_from(["hyprstream", "service", "provision-policy-templates",])
5098            .is_err());
5099    }
5100
5101    #[test]
5102    fn native_service_identity_loads_existing_key_without_ipc() -> anyhow::Result<()> {
5103        const CHILD: &str = "HYPRSTREAM_NATIVE_SERVICE_IDENTITY_TEST";
5104        let Ok(profile) = std::env::var(CHILD) else {
5105            for profile in ["shared-directory", "per-service-scoped"] {
5106                let status = std::process::Command::new(std::env::current_exe()?)
5107                    .args(["--exact", "resolver_startup_controls::native_service_identity_loads_existing_key_without_ipc", "--nocapture"])
5108                    .env(CHILD, profile)
5109                    .env("HYPRSTREAM_SECRETS_PROFILE", profile)
5110                    .env_remove("HYPRSTREAM__SECRETS__PATH")
5111                    .env_remove("HYPRSTREAM__SIGNING_KEY")
5112                    .status()?;
5113                anyhow::ensure!(
5114                    status.success(),
5115                    "isolated {profile} identity regression failed"
5116                );
5117            }
5118            return Ok(());
5119        };
5120        let root = tempfile::tempdir()?;
5121        let runtime = tokio::runtime::Builder::new_current_thread()
5122            .enable_all()
5123            .build()?;
5124        for (service, seed) in [("policy", 31u8), ("model", 32u8), ("registry", 33u8)] {
5125            let secrets = root.path().join(format!("{service}-credentials"));
5126            std::fs::create_dir_all(secrets.join(service))?;
5127            let key_path = if service == "policy" || profile == "per-service-scoped" {
5128                // A misleading service subdirectory must not replace a scoped
5129                // identity, especially Policy's canonical flat key.
5130                std::fs::write(secrets.join(service).join("signing-key"), [99u8; 32])?;
5131                secrets.join("signing-key")
5132            } else {
5133                // A retained root/Policy key must not become a native Model or
5134                // Registry process identity merely because --ipc is absent.
5135                std::fs::write(secrets.join("signing-key"), [31u8; 32])?;
5136                secrets.join(service).join("signing-key")
5137            };
5138            std::fs::write(&key_path, [seed; 32])?;
5139            let mut configured = super::HyprConfig::default();
5140            configured.quic.native_network_profile =
5141                hyprstream_core::config::NativeNetworkProfile::NetworkIrohRequired;
5142            configured.secrets.path = Some(secrets.clone());
5143            configured.storage.models_dir = root.path().join(format!("{service}-models"));
5144            let config_path = root.path().join(format!("{service}.toml"));
5145            configured.to_file(&config_path)?;
5146            let matches = super::build_cli().try_get_matches_from([
5147                "hyprstream",
5148                "--config",
5149                config_path.to_str().expect("temporary UTF-8 path"),
5150                "service",
5151                "start",
5152                service,
5153                "--foreground",
5154            ])?;
5155            let config = super::load_config(
5156                matches
5157                    .get_one::<std::path::PathBuf>("config")
5158                    .map(std::path::PathBuf::as_path),
5159            )?;
5160            let selected = super::native_service_process_name(&matches, &config)?;
5161            assert_eq!(selected.as_deref(), Some(service));
5162            let key = runtime.block_on(super::load_process_signing_key(
5163                &config,
5164                selected.as_deref(),
5165            ))?;
5166            assert_eq!(key.to_bytes(), [seed; 32]);
5167            let ctx = super::ServiceContext::new(
5168                key.clone(),
5169                key.verifying_key(),
5170                false,
5171                config.models_dir().clone(),
5172            )
5173            .with_service_key(service, key.clone());
5174            assert_eq!(ctx.signing_key().verifying_key(), key.verifying_key());
5175            assert_eq!(
5176                ctx.service_signing_key(service).verifying_key(),
5177                key.verifying_key()
5178            );
5179            assert!(
5180                !config.models_dir().join(".registry/keys").exists(),
5181                "native startup must not materialize a CLI/root key"
5182            );
5183            std::fs::remove_file(&key_path)?;
5184            assert!(runtime
5185                .block_on(super::load_process_signing_key(
5186                    &config,
5187                    selected.as_deref()
5188                ))
5189                .is_err());
5190            assert!(
5191                !key_path.exists(),
5192                "missing provisioned key must not be regenerated"
5193            );
5194            std::fs::write(&key_path, [seed; 31])?;
5195            assert!(runtime
5196                .block_on(super::load_process_signing_key(
5197                    &config,
5198                    selected.as_deref()
5199                ))
5200                .is_err());
5201            assert_eq!(
5202                std::fs::read(&key_path)?,
5203                vec![seed; 31],
5204                "malformed key must not be repaired during startup"
5205            );
5206        }
5207        Ok(())
5208    }
5209
5210    #[test]
5211    fn dedicated_service_containment_is_independent_of_network_profile() -> anyhow::Result<()> {
5212        use hyprstream_core::config::NativeNetworkProfile;
5213        let mut config = super::HyprConfig::default();
5214        config.account.http = Some(hyprstream_core::account::AccountHttpConfig {
5215            host: "::1".to_owned(),
5216            port: 8443,
5217            tls_cert: "unused.pem".into(),
5218            tls_key: "unused.key".into(),
5219        });
5220        for profile in [
5221            NativeNetworkProfile::Compatibility,
5222            NativeNetworkProfile::NetworkIrohRequired,
5223        ] {
5224            config.quic.native_network_profile = profile;
5225            for argv in [
5226                vec!["hyprstream", "service", "start", "oauth", "--foreground"],
5227                vec![
5228                    "hyprstream",
5229                    "service",
5230                    "start",
5231                    "--foreground",
5232                    "--services",
5233                    "oauth",
5234                ],
5235                vec!["hyprstream", "service", "start", "--standalone"],
5236            ] {
5237                config.services.startup = vec!["oauth".into()];
5238                let matches = super::build_cli().try_get_matches_from(argv)?;
5239                let service = super::dedicated_service_process_name(&matches, &config)?
5240                    .expect("a single OAuth foreground service has process containment");
5241                assert_eq!(service, "oauth");
5242                let key = super::SigningKey::from_bytes(&[9; 32]);
5243                let context = super::ServiceContext::new(
5244                    key.clone(),
5245                    key.verifying_key(),
5246                    false,
5247                    "/unused".into(),
5248                )
5249                .with_dedicated_process_service(service);
5250                assert!(context.is_dedicated_process_for("oauth"));
5251                assert!(!context.is_dedicated_process_for("registry"));
5252                assert_eq!(
5253                    super::native_service_process_name(&matches, &config)?.is_some(),
5254                    config.quic.iroh_required(),
5255                    "containment must not change compatibility signing-key selection",
5256                );
5257            }
5258            config.services.startup = vec!["oauth".into(), "registry".into()];
5259            for argv in [
5260                vec!["hyprstream", "service", "start", "oauth"],
5261                vec!["hyprstream", "service", "start", "--standalone"],
5262                vec![
5263                    "hyprstream",
5264                    "service",
5265                    "start",
5266                    "--foreground",
5267                    "--services",
5268                    "oauth,registry",
5269                ],
5270                vec!["hyprstream", "service", "status"],
5271            ] {
5272                let matches = super::build_cli().try_get_matches_from(argv)?;
5273                assert_eq!(
5274                    super::dedicated_service_process_name(&matches, &config)?,
5275                    None
5276                );
5277            }
5278        }
5279        Ok(())
5280    }
5281
5282    #[test]
5283    fn native_service_identity_rejects_multi_and_selects_single_service_list() -> anyhow::Result<()>
5284    {
5285        let mut config = super::HyprConfig::default();
5286        config.quic.native_network_profile =
5287            hyprstream_core::config::NativeNetworkProfile::NetworkIrohRequired;
5288        let single = super::build_cli().try_get_matches_from([
5289            "hyprstream",
5290            "service",
5291            "start",
5292            "--foreground",
5293            "--services",
5294            "model",
5295        ])?;
5296        assert_eq!(
5297            super::native_service_process_name(&single, &config)?.as_deref(),
5298            Some("model")
5299        );
5300        let multi = super::build_cli().try_get_matches_from([
5301            "hyprstream",
5302            "service",
5303            "start",
5304            "--foreground",
5305            "--services",
5306            "model,registry",
5307        ])?;
5308        assert!(super::native_service_process_name(&multi, &config).is_err());
5309        let standalone = super::build_cli().try_get_matches_from([
5310            "hyprstream",
5311            "service",
5312            "start",
5313            "--standalone",
5314        ])?;
5315        config.services.startup = vec!["model".into(), "registry".into()];
5316        assert!(super::native_service_process_name(&standalone, &config).is_err());
5317        config.quic.native_network_profile =
5318            hyprstream_core::config::NativeNetworkProfile::Compatibility;
5319        assert_eq!(super::native_service_process_name(&multi, &config)?, None);
5320        assert_eq!(super::native_service_process_name(&single, &config)?, None);
5321        Ok(())
5322    }
5323
5324    #[test]
5325    fn native_service_identity_seeds_discovery_from_loaded_config() -> anyhow::Result<()> {
5326        const CHILD: &str = "HYPRSTREAM_NATIVE_DISCOVERY_SEED_TEST";
5327        if std::env::var_os(CHILD).is_none() {
5328            let status = std::process::Command::new(std::env::current_exe()?)
5329                .args([
5330                    "--exact",
5331                    "resolver_startup_controls::native_service_identity_seeds_discovery_from_loaded_config",
5332                    "--nocapture",
5333                ])
5334                .env(CHILD, "1")
5335                .env_remove("HYPRSTREAM__SECRETS__PATH")
5336                .env_remove("HYPRSTREAM_SECRETS_PROFILE")
5337                .status()?;
5338            anyhow::ensure!(
5339                status.success(),
5340                "isolated discovery seeding regression failed"
5341            );
5342            return Ok(());
5343        }
5344
5345        let root = tempfile::tempdir()?;
5346        let xdg_config_home = root.path().join("xdg-config");
5347        std::env::set_var("XDG_CONFIG_HOME", &xdg_config_home);
5348
5349        // Keep a different, valid bootstrap file at the default location so
5350        // this catches accidental re-resolution through HyprConfig::load().
5351        let default_secrets = xdg_config_home.join("hyprstream/credentials");
5352        let default_key = ed25519_dalek::SigningKey::from_bytes(&[91u8; 32]);
5353        let mut default_entries = std::collections::HashMap::new();
5354        default_entries.insert(
5355            "discovery".to_owned(),
5356            hyprstream_core::auth::identity_store::BootstrapPubkey::for_service_key(&default_key)?,
5357        );
5358        hyprstream_core::auth::identity_store::write_bootstrap_pubkeys_hybrid(
5359            &default_secrets,
5360            &default_entries,
5361        )?;
5362
5363        let custom_secrets = root.path().join("custom-credentials");
5364        let custom_key = ed25519_dalek::SigningKey::from_bytes(&[92u8; 32]);
5365        let mut custom_entries = std::collections::HashMap::new();
5366        custom_entries.insert(
5367            "discovery".to_owned(),
5368            hyprstream_core::auth::identity_store::BootstrapPubkey::for_service_key(&custom_key)?,
5369        );
5370        hyprstream_core::auth::identity_store::write_bootstrap_pubkeys_hybrid(
5371            &custom_secrets,
5372            &custom_entries,
5373        )?;
5374
5375        let mut config = super::HyprConfig::default();
5376        config.secrets.path = Some(custom_secrets);
5377        let selected = super::resolve_service_vk("discovery", Some(&config))
5378            .expect("configured bootstrap-pubkeys must seed discovery");
5379        assert_eq!(selected, custom_key.verifying_key());
5380        assert_ne!(selected, default_key.verifying_key());
5381        assert_eq!(
5382            hyprstream_service::global_trust_store().resolve_one("discovery"),
5383            Some(custom_key.verifying_key()),
5384        );
5385        Ok(())
5386    }
5387
5388    #[test]
5389    fn readonly_roster_cli_requires_services_and_has_no_mutation_options() {
5390        let matches = super::build_cli()
5391            .try_get_matches_from([
5392                "hyprstream",
5393                "pds",
5394                "inspect-services",
5395                "--service",
5396                "model,event",
5397            ])
5398            .expect("read-only roster CLI");
5399        let inspect = matches
5400            .subcommand_matches("pds")
5401            .expect("pds")
5402            .subcommand_matches("inspect-services")
5403            .expect("inspect");
5404        assert_eq!(
5405            inspect
5406                .get_many::<String>("service")
5407                .expect("services")
5408                .map(String::as_str)
5409                .collect::<Vec<_>>(),
5410            vec!["model", "event"]
5411        );
5412        for args in [
5413            vec!["hyprstream", "pds", "inspect-services"],
5414            vec![
5415                "hyprstream",
5416                "pds",
5417                "inspect-services",
5418                "--service",
5419                "model",
5420                "--valid-for-seconds",
5421                "86400",
5422            ],
5423            vec![
5424                "hyprstream",
5425                "pds",
5426                "inspect-services",
5427                "--service",
5428                "model",
5429                "--roster-export",
5430                "out.json",
5431            ],
5432        ] {
5433            assert!(super::build_cli().try_get_matches_from(args).is_err());
5434        }
5435    }
5436
5437    fn unavailable_policy_publication() -> anyhow::Error {
5438        hyprstream_rpc::transport_traits::PreDispatchTransportError::new(
5439            hyprstream_rpc::transport::lazy_iroh::IrohPeerUnavailable::ConnectTimeout.into(),
5440        )
5441        .into()
5442    }
5443
5444    #[tokio::test(start_paused = true)]
5445    async fn policy_initial_publication_retries_only_typed_availability_and_reprojects() {
5446        let cancellation = tokio_util::sync::CancellationToken::new();
5447        let mut projections = 0;
5448        let start = tokio::time::Instant::now();
5449        super::wait_for_policy_initial_publication(
5450            &cancellation,
5451            std::time::Duration::from_secs(120),
5452            || {
5453                projections += 1;
5454                let generation = projections;
5455                async move {
5456                    if generation < 3 {
5457                        Err(unavailable_policy_publication())
5458                    } else {
5459                        Ok(())
5460                    }
5461                }
5462            },
5463        )
5464        .await
5465        .expect("late Discovery should allow Policy to publish");
5466        assert_eq!(
5467            projections, 3,
5468            "every retry must request fresh authority material"
5469        );
5470        assert_eq!(start.elapsed(), std::time::Duration::from_secs(1));
5471    }
5472
5473    #[tokio::test(start_paused = true)]
5474    async fn policy_initial_publication_permanent_error_after_availability_is_terminal() {
5475        let cancellation = tokio_util::sync::CancellationToken::new();
5476        // These are intentionally untyped security/projection errors. Even
5477        // availability-looking text must never select the retry path.
5478        for terminal in [
5479            "expired current authority",
5480            "revoked service JWT",
5481            "iroh connect timed out",
5482        ] {
5483            let mut attempts = 0;
5484            let error = super::wait_for_policy_initial_publication(
5485                &cancellation,
5486                std::time::Duration::from_secs(120),
5487                || {
5488                    attempts += 1;
5489                    let first = attempts == 1;
5490                    async move {
5491                        if first {
5492                            Err(unavailable_policy_publication())
5493                        } else {
5494                            anyhow::bail!(terminal)
5495                        }
5496                    }
5497                },
5498            )
5499            .await
5500            .expect_err("security failures must be terminal");
5501            assert_eq!(error.to_string(), terminal);
5502            assert_eq!(attempts, 2);
5503        }
5504        let mut attempts = 0;
5505        let error = super::wait_for_policy_initial_publication(
5506            &cancellation,
5507            std::time::Duration::from_secs(120),
5508            || {
5509                attempts += 1;
5510                async {
5511                    Err(
5512                        hyprstream_rpc::transport_traits::PreDispatchTransportError::new(
5513                            anyhow::anyhow!("missing Iroh client endpoint"),
5514                        )
5515                        .into(),
5516                    )
5517                }
5518            },
5519        )
5520        .await
5521        .expect_err("pre-dispatch alone is not an availability proof");
5522        assert_eq!(attempts, 1);
5523        assert!(error.to_string().contains("missing Iroh client endpoint"));
5524    }
5525
5526    #[tokio::test(start_paused = true)]
5527    async fn policy_initial_publication_deadline_bounds_retries_and_inflight_call() {
5528        let cancellation = tokio_util::sync::CancellationToken::new();
5529        let start = tokio::time::Instant::now();
5530        let error = super::wait_for_policy_initial_publication(
5531            &cancellation,
5532            std::time::Duration::from_secs(2),
5533            || async { Err(unavailable_policy_publication()) },
5534        )
5535        .await
5536        .expect_err("absent Discovery cannot keep startup pending forever");
5537        assert!(error.to_string().contains("deadline exceeded"));
5538        assert_eq!(start.elapsed(), std::time::Duration::from_secs(2));
5539        let start = tokio::time::Instant::now();
5540        let error = super::wait_for_policy_initial_publication(
5541            &cancellation,
5542            std::time::Duration::from_secs(2),
5543            std::future::pending,
5544        )
5545        .await
5546        .expect_err("deadline also interrupts a stalled RPC");
5547        assert!(error.to_string().contains("deadline exceeded"));
5548        assert_eq!(start.elapsed(), std::time::Duration::from_secs(2));
5549    }
5550
5551    #[tokio::test(start_paused = true)]
5552    async fn policy_initial_publication_cancellation_precedes_success() {
5553        let cancellation = tokio_util::sync::CancellationToken::new();
5554        cancellation.cancel();
5555        let mut called = false;
5556        super::wait_for_policy_initial_publication(
5557            &cancellation,
5558            std::time::Duration::from_secs(120),
5559            || {
5560                called = true;
5561                async { Ok(()) }
5562            },
5563        )
5564        .await
5565        .expect_err("cancelled startup cannot publish or become ready");
5566        assert!(!called);
5567        let cancellation = tokio_util::sync::CancellationToken::new();
5568        let cancel = cancellation.clone();
5569        let cancel_task = tokio::spawn(async move {
5570            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
5571            cancel.cancel();
5572        });
5573        super::wait_for_policy_initial_publication(
5574            &cancellation,
5575            std::time::Duration::from_secs(120),
5576            std::future::pending,
5577        )
5578        .await
5579        .expect_err("shutdown interrupts an in-flight publication");
5580        cancel_task.await.expect("canceller");
5581    }
5582
5583    #[test]
5584    fn deployment_bootstrap_cli_requires_roster_and_parses_lifetime() {
5585        let matches = super::build_cli()
5586            .try_get_matches_from([
5587                "hyprstream",
5588                "pds",
5589                "provision-services",
5590                "--service",
5591                "model,event",
5592                "--valid-for-seconds",
5593                "3600",
5594            ])
5595            .expect("bootstrap CLI");
5596        let pds = matches.subcommand_matches("pds").expect("pds");
5597        let provision = pds
5598            .subcommand_matches("provision-services")
5599            .expect("provision");
5600        assert_eq!(
5601            provision
5602                .get_many::<String>("service")
5603                .expect("roster")
5604                .map(String::as_str)
5605                .collect::<Vec<_>>(),
5606            vec!["model", "event"]
5607        );
5608        assert_eq!(provision.get_one::<i64>("valid-for-seconds"), Some(&3600));
5609        assert!(provision
5610            .get_one::<std::path::PathBuf>("roster-export")
5611            .is_none());
5612        assert!(super::build_cli()
5613            .try_get_matches_from(["hyprstream", "pds", "provision-services",])
5614            .is_err());
5615        let matches = super::build_cli()
5616            .try_get_matches_from([
5617                "hyprstream",
5618                "pds",
5619                "provision-services",
5620                "--service",
5621                "model",
5622                "--roster-export",
5623                "/tmp/roster.json",
5624            ])
5625            .expect("bootstrap CLI with roster export");
5626        let provision = matches
5627            .subcommand_matches("pds")
5628            .expect("pds")
5629            .subcommand_matches("provision-services")
5630            .expect("provision");
5631        assert_eq!(
5632            provision
5633                .get_one::<std::path::PathBuf>("roster-export")
5634                .map(std::path::PathBuf::as_path),
5635            Some(std::path::Path::new("/tmp/roster.json"))
5636        );
5637    }
5638    const REFRESH_SCHEDULER_TURNS: usize = 32;
5639
5640    async fn assert_publication_ready(
5641        published_rx: &mut tokio::sync::mpsc::UnboundedReceiver<usize>,
5642        expected: usize,
5643        label: &str,
5644    ) {
5645        for _ in 0..REFRESH_SCHEDULER_TURNS {
5646            match published_rx.try_recv() {
5647                Ok(publication) => {
5648                    assert_eq!(publication, expected, "{label}");
5649                    return;
5650                }
5651                Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
5652                    tokio::task::yield_now().await;
5653                }
5654                Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
5655                    panic!("{label}: production refresh task exited before publishing");
5656                }
5657            }
5658        }
5659        panic!("{label}: publication was not ready after bounded scheduler turns");
5660    }
5661
5662    async fn publication_ready(
5663        published_rx: &mut tokio::sync::mpsc::UnboundedReceiver<usize>,
5664        label: &str,
5665    ) -> Option<usize> {
5666        for _ in 0..REFRESH_SCHEDULER_TURNS {
5667            match published_rx.try_recv() {
5668                Ok(publication) => return Some(publication),
5669                Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
5670                    tokio::task::yield_now().await;
5671                }
5672                Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
5673                    panic!("{label}: production refresh task exited before publishing");
5674                }
5675            }
5676        }
5677        None
5678    }
5679
5680    async fn assert_no_publication_ready(
5681        published_rx: &mut tokio::sync::mpsc::UnboundedReceiver<usize>,
5682        label: &str,
5683    ) {
5684        for _ in 0..REFRESH_SCHEDULER_TURNS {
5685            match published_rx.try_recv() {
5686                Ok(publication) => panic!("{label}: unexpected publication {publication}"),
5687                Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
5688                    tokio::task::yield_now().await;
5689                }
5690                Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
5691                    panic!("{label}: production refresh task exited unexpectedly");
5692                }
5693            }
5694        }
5695    }
5696
5697    #[test]
5698    fn checkpointed_moq_services_cannot_share_a_process_dial_proof() {
5699        let err = super::select_single_process_moql_admission_proof(vec![
5700            ("inference".to_owned(), Some(1_u8)),
5701            ("registry".to_owned(), Some(2_u8)),
5702        ])
5703        .expect_err("different checkpointed services need separate dialers");
5704        assert!(err.to_string().contains("separate process"));
5705
5706        let err = super::select_single_process_moql_admission_proof(vec![
5707            ("inference".to_owned(), Some(1_u8)),
5708            ("metrics".to_owned(), None),
5709        ])
5710        .expect_err("a proof cannot be inherited by an unauthenticated service");
5711        assert!(err.to_string().contains("service-scoped"));
5712
5713        assert_eq!(
5714            super::select_single_process_moql_admission_proof(vec![(
5715                "inference".to_owned(),
5716                Some(7_u8)
5717            ),])
5718            .expect("one service has one scoped proof"),
5719            Some(7),
5720        );
5721    }
5722
5723    #[test]
5724    fn quinn_only_multi_service_startup_never_selects_an_iroh_proof() {
5725        let selected = super::select_iroh_moql_admission_proof::<u8>(false, || {
5726            panic!("Quinn-only startup must not inspect Iroh admission proofs")
5727        })
5728        .expect("Quinn-only profile does not need an Iroh proof");
5729        assert_eq!(selected, None);
5730    }
5731
5732    #[test]
5733    fn command_and_service_processes_install_before_consumers() {
5734        let source = include_str!("main.rs");
5735        let production = source
5736            .split("mod resolver_startup_controls {")
5737            .next()
5738            .expect("production binary source")
5739            .split_whitespace()
5740            .collect::<String>();
5741        let startup = production
5742            .find("install_process_production_resolver(&signing_key,&config,service_token.clone(),false,).await")
5743            .expect("trusted startup boundary");
5744        let install = production[startup..]
5745            .find("install_process_production_resolver(&signing_key,&config,service_token.clone(),false,).await")
5746            .expect("process resolver install");
5747        let first_generated = production[startup..]
5748            .find(
5749                "RegistryClient::from_provider(&hyprstream_discovery::ProductionRpcClientProvider,",
5750            )
5751            .expect("first generated client");
5752        let command_dispatch = production[startup..]
5753            .find("matchmatches.subcommand()")
5754            .expect("top-level command dispatch");
5755        assert!(install < first_generated);
5756        assert!(first_generated < command_dispatch);
5757        assert!(production.contains("load_service_jwt_for_profile(&secrets,name,profile,"));
5758        assert!(production.contains("signing_key.clone(),service_token,"));
5759        assert_eq!(
5760            production
5761                .matches("install_process_production_resolver(&signing_key,&config,service_token.clone(),false,).await")
5762                .count(),
5763            1
5764        );
5765        assert!(!production.contains("install_process_production_resolver(&mut ctx)"));
5766        assert!(production.contains(
5767            "install_process_production_resolver(&key,&config,Some(jwt.clone()),true).await"
5768        ));
5769        assert!(
5770            production.contains("RegistryClient::from_provider(provider,key,Some(jwt.clone()))")
5771        );
5772
5773        let announcements = production
5774            .find("with_checkpointed_native_announcements(")
5775            .expect("authenticated announcements");
5776        let first_factory = production[announcements..]
5777            .find("get_factory(")
5778            .expect("first inventory factory invocation");
5779        assert!(first_factory > 0);
5780
5781        // These entry points all dispatch below the single command bootstrap:
5782        // quick, TUI, schema/tool, MCP/OpenAI services, shell, training, git,
5783        // and isolated `service start` IPC consumers.
5784        for entry in [
5785            "Some((\"tool\"",
5786            "Some((\"quick\"",
5787            "Some((\"tui\"",
5788            "handle_shell_tui",
5789            "handle_training_batch",
5790        ] {
5791            assert!(production.contains(entry), "missing {entry}");
5792        }
5793    }
5794
5795    #[test]
5796    fn configured_mac_boot_precedes_all_start_paths_and_never_falls_back() {
5797        let source = include_str!("main.rs");
5798        let production = source
5799            .split("mod resolver_startup_controls {")
5800            .next()
5801            .expect("production source");
5802        let configured = production
5803            .find("if let Some(artifacts) = &config_for_service.policy.boot_policy")
5804            .expect("configured boot branch");
5805        let start = &production[configured..];
5806        let install = start
5807            .find("install_configured_boot_policy(")
5808            .expect("configured install");
5809        let fatal = start
5810            .find(".context(\"configured MAC boot policy rejected\")?")
5811            .expect("fatal configured failure");
5812        let branch = start
5813            .find("if foreground || standalone {")
5814            .expect("startup branch");
5815        assert!(install < fatal && fatal < branch);
5816        let fallback = start
5817            .find("if config.policy.boot_policy.is_none() {")
5818            .expect("absent-only fallback");
5819        let baseline = start
5820            .find("install_baseline_boot_policy(")
5821            .expect("baseline install");
5822        assert!(branch < fallback && fallback < baseline);
5823        assert_eq!(
5824            production
5825                .matches("install_configured_boot_policy(")
5826                .count(),
5827            1
5828        );
5829    }
5830
5831    /// Lifecycle classification is consumed only for the authenticated
5832    /// OS-owned source. This is behavioral rather than positional: moving the
5833    /// classifier outside `quic_checkpoint_policy`'s source gate makes the
5834    /// DID-anchored case below invoke its deliberately failing classifier.
5835    #[test]
5836    fn first_boot_quic_deferral_is_scoped_to_os_owned_bootstrap() {
5837        use std::cell::Cell;
5838
5839        use hyprstream_core::services::factories::PdsBootState;
5840
5841        let os_owned_calls = Cell::new(0);
5842        assert_eq!(
5843            super::quic_checkpoint_policy(true, || {
5844                os_owned_calls.set(os_owned_calls.get() + 1);
5845                Ok(PdsBootState::FirstBoot)
5846            })
5847            .expect("OS-owned classification succeeds"),
5848            super::QuicCheckpointPolicy::DeferForFirstBoot,
5849        );
5850        assert_eq!(os_owned_calls.get(), 1, "OS-owned input classifies once");
5851
5852        let did_anchored_calls = Cell::new(0);
5853        assert_eq!(
5854            super::quic_checkpoint_policy(false, || {
5855                did_anchored_calls.set(did_anchored_calls.get() + 1);
5856                anyhow::bail!("DID-anchored startup must never classify lifecycle state")
5857            })
5858            .expect("DID-anchored startup bypasses the classifier"),
5859            super::QuicCheckpointPolicy::CheckpointedAnnouncements,
5860        );
5861        assert_eq!(
5862            did_anchored_calls.get(),
5863            0,
5864            "DID-anchored input must not invoke the classifier"
5865        );
5866    }
5867
5868    #[test]
5869    fn command_local_spawns_never_use_the_production_resolver() {
5870        let source = include_str!("main.rs");
5871        let worker = source
5872            .find("let worker_client = if worker_already_running")
5873            .expect("worker client selection");
5874        let worker_selection = source[worker..worker + 500]
5875            .split_whitespace()
5876            .collect::<String>();
5877        assert!(worker_selection.contains(
5878            "WorkerClient::from_provider(&hyprstream_discovery::ProductionRpcClientProvider"
5879        ));
5880        assert!(worker_selection.contains("WorkerClient::for_local_bootstrap"));
5881
5882        let training = include_str!("../cli/training_handlers.rs");
5883        assert_eq!(
5884            training.matches("InferenceClient::from_provider").count(),
5885            0
5886        );
5887        assert_eq!(
5888            training
5889                .matches("InferenceClient::for_local_bootstrap")
5890                .count(),
5891            2
5892        );
5893    }
5894
5895    #[tokio::test]
5896    async fn native_announcement_refresh_loop_publishes_immediately_and_after_two_intervals() {
5897        use std::sync::{
5898            atomic::{AtomicUsize, Ordering},
5899            Arc,
5900        };
5901        use std::time::Duration;
5902
5903        tokio::time::pause();
5904        // Tokio's timer wheel rounds an unaligned sleep deadline up to its next
5905        // millisecond tick. Align before establishing the cadence baseline;
5906        // each refresh then has an explicit one-tick observation window.
5907        tokio::time::sleep(Duration::ZERO).await;
5908        let initial_time = tokio::time::Instant::now();
5909        let just_before_refresh = Duration::from_secs(24) + Duration::from_millis(999);
5910        let timer_tick = Duration::from_millis(1);
5911        let publications = Arc::new(AtomicUsize::new(0));
5912        let (published_tx, mut published_rx) = tokio::sync::mpsc::unbounded_channel();
5913        let observed_publications = Arc::clone(&publications);
5914        let task = tokio::spawn(async move {
5915            super::refresh_native_announcement(
5916                "model",
5917                "iroh",
5918                "iroh://test-node",
5919                chrono::Utc::now().timestamp_millis() + 60_000,
5920                tokio_util::sync::CancellationToken::new(),
5921                move || {
5922                    let publication = observed_publications.fetch_add(1, Ordering::SeqCst) + 1;
5923                    published_tx
5924                        .send(publication)
5925                        .expect("test receiver remains live");
5926                    async { Ok::<(), std::convert::Infallible>(()) }
5927                },
5928            )
5929            .await
5930        });
5931
5932        assert_publication_ready(&mut published_rx, 1, "initial production publish").await;
5933        assert_no_publication_ready(&mut published_rx, "duplicate immediate publish").await;
5934        assert_eq!(tokio::time::Instant::now(), initial_time);
5935
5936        tokio::time::advance(just_before_refresh).await;
5937        assert_eq!(
5938            tokio::time::Instant::now(),
5939            initial_time + just_before_refresh
5940        );
5941        assert_no_publication_ready(&mut published_rx, "refresh before 25 virtual seconds").await;
5942        assert_eq!(
5943            tokio::time::Instant::now(),
5944            initial_time + just_before_refresh
5945        );
5946        tokio::time::advance(timer_tick).await;
5947        let first_nominal_wake = initial_time + Duration::from_secs(25);
5948        let second_wake =
5949            match publication_ready(&mut published_rx, "25-second production refresh").await {
5950                Some(publication) => {
5951                    assert_eq!(publication, 2, "25-second production refresh");
5952                    tokio::time::Instant::now()
5953                }
5954                None => {
5955                    tokio::time::advance(timer_tick).await;
5956                    assert_publication_ready(&mut published_rx, 2, "25-second production refresh")
5957                        .await;
5958                    tokio::time::Instant::now()
5959                }
5960            };
5961        assert!(
5962            second_wake >= first_nominal_wake && second_wake <= first_nominal_wake + timer_tick,
5963            "production refresh must occur at its 25-second cadence or within one Tokio timer tick"
5964        );
5965        assert_no_publication_ready(&mut published_rx, "duplicate 25-second refresh").await;
5966        assert_eq!(tokio::time::Instant::now(), second_wake);
5967
5968        tokio::time::advance(just_before_refresh).await;
5969        assert_eq!(
5970            tokio::time::Instant::now(),
5971            second_wake + just_before_refresh
5972        );
5973        assert_no_publication_ready(&mut published_rx, "refresh before 50 virtual seconds").await;
5974        assert_eq!(
5975            tokio::time::Instant::now(),
5976            second_wake + just_before_refresh
5977        );
5978        tokio::time::advance(timer_tick).await;
5979        let second_nominal_wake = second_wake + Duration::from_secs(25);
5980        let third_wake =
5981            match publication_ready(&mut published_rx, "50-second production refresh").await {
5982                Some(publication) => {
5983                    assert_eq!(publication, 3, "50-second production refresh");
5984                    tokio::time::Instant::now()
5985                }
5986                None => {
5987                    tokio::time::advance(timer_tick).await;
5988                    assert_publication_ready(&mut published_rx, 3, "50-second production refresh")
5989                        .await;
5990                    tokio::time::Instant::now()
5991                }
5992            };
5993        assert!(
5994            third_wake >= second_nominal_wake && third_wake <= second_nominal_wake + timer_tick,
5995            "next production refresh must use the prior actual wake and stay within one Tokio timer tick"
5996        );
5997        assert_eq!(tokio::time::Instant::now(), third_wake);
5998        assert_eq!(publications.load(Ordering::SeqCst), 3);
5999
6000        task.abort();
6001        assert!(task
6002            .await
6003            .expect_err("refresh task is cancelled")
6004            .is_cancelled());
6005    }
6006
6007    async fn expired_announcement_must_not_run() -> Result<(), std::convert::Infallible> {
6008        panic!("expired production refresh loop attempted an announcement");
6009    }
6010
6011    #[tokio::test(start_paused = true)]
6012    async fn native_announcement_refresh_loop_shutdown_cannot_overwrite_restart() {
6013        use parking_lot::Mutex;
6014        use std::sync::Arc;
6015        let publications = Arc::new(Mutex::new(Vec::new()));
6016        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
6017        let old_cancel = tokio_util::sync::CancellationToken::new();
6018        let new_cancel = tokio_util::sync::CancellationToken::new();
6019        let spawn = |endpoint: &'static str, cancellation| {
6020            let publications = Arc::clone(&publications);
6021            let tx = tx.clone();
6022            tokio::spawn(async move {
6023                super::refresh_native_announcement(
6024                    "model",
6025                    "quic",
6026                    endpoint,
6027                    i64::MAX,
6028                    cancellation,
6029                    move || {
6030                        publications.lock().push(endpoint);
6031                        tx.send(endpoint).expect("publication receiver");
6032                        async { Ok::<(), std::convert::Infallible>(()) }
6033                    },
6034                )
6035                .await
6036            })
6037        };
6038        let old = spawn("quic://old:10001", old_cancel.clone());
6039        assert_eq!(rx.recv().await, Some("quic://old:10001"));
6040        old_cancel.cancel();
6041        assert_eq!(
6042            old.await.expect("old task"),
6043            super::NativeAnnouncementRefreshCompletion::Cancelled
6044        );
6045        let new = spawn("quic://new:10002", new_cancel.clone());
6046        assert_eq!(rx.recv().await, Some("quic://new:10002"));
6047        tokio::time::advance(std::time::Duration::from_secs(26)).await;
6048        assert_eq!(rx.recv().await, Some("quic://new:10002"));
6049        new_cancel.cancel();
6050        assert_eq!(
6051            new.await.expect("new task"),
6052            super::NativeAnnouncementRefreshCompletion::Cancelled
6053        );
6054        assert_eq!(
6055            *publications.lock(),
6056            vec!["quic://old:10001", "quic://new:10002", "quic://new:10002"],
6057        );
6058    }
6059
6060    #[tokio::test]
6061    async fn native_announcement_refresh_loop_cancels_in_flight_publish() {
6062        use std::sync::{
6063            atomic::{AtomicBool, Ordering},
6064            Arc,
6065        };
6066        let cancellation = tokio_util::sync::CancellationToken::new();
6067        let completed = Arc::new(AtomicBool::new(false));
6068        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
6069        let task = tokio::spawn({
6070            let cancellation = cancellation.clone();
6071            let completed = Arc::clone(&completed);
6072            let mut started_tx = Some(started_tx);
6073            async move {
6074                super::refresh_native_announcement(
6075                    "model",
6076                    "iroh",
6077                    "iroh://test",
6078                    i64::MAX,
6079                    cancellation,
6080                    move || {
6081                        let started = started_tx.take();
6082                        let completed = Arc::clone(&completed);
6083                        async move {
6084                            if let Some(started) = started {
6085                                let _ = started.send(());
6086                            }
6087                            std::future::pending::<()>().await;
6088                            completed.store(true, Ordering::SeqCst);
6089                            Ok::<(), std::convert::Infallible>(())
6090                        }
6091                    },
6092                )
6093                .await
6094            }
6095        });
6096        started_rx.await.expect("publish started");
6097        cancellation.cancel();
6098        let completion = tokio::time::timeout(std::time::Duration::from_secs(1), task)
6099            .await
6100            .expect("cancellation must not wait for RPC completion")
6101            .expect("task");
6102        assert_eq!(
6103            completion,
6104            super::NativeAnnouncementRefreshCompletion::Cancelled
6105        );
6106        assert!(!completed.load(Ordering::SeqCst));
6107    }
6108
6109    #[tokio::test(start_paused = true)]
6110    async fn native_announcement_refresh_loop_backoff_starts_at_five_and_resets_after_success() {
6111        let cancellation = tokio_util::sync::CancellationToken::new();
6112        let (sent, mut received) = tokio::sync::mpsc::unbounded_channel();
6113        let task = tokio::spawn({
6114            let cancellation = cancellation.clone();
6115            let mut attempt = 0;
6116            async move {
6117                super::refresh_native_announcement(
6118                    "model",
6119                    "iroh",
6120                    "iroh://test",
6121                    i64::MAX,
6122                    cancellation,
6123                    move || {
6124                        attempt += 1;
6125                        sent.send(tokio::time::Instant::now())
6126                            .expect("receiver live");
6127                        let success = attempt == 5;
6128                        async move {
6129                            if success {
6130                                Ok(())
6131                            } else {
6132                                Err("retry fixture")
6133                            }
6134                        }
6135                    },
6136                )
6137                .await
6138            }
6139        });
6140        let mut previous = received.recv().await.expect("initial attempt");
6141        for seconds in [5, 10, 20, 25, 25, 5, 10] {
6142            let current = received.recv().await.expect("next attempt");
6143            let elapsed = current.duration_since(previous);
6144            let expected = std::time::Duration::from_secs(seconds);
6145            assert!(
6146                elapsed >= expected && elapsed <= expected + std::time::Duration::from_millis(1),
6147                "expected {expected:?} retry interval, got {elapsed:?}"
6148            );
6149            previous = current;
6150        }
6151        cancellation.cancel();
6152        assert_eq!(
6153            task.await.expect("refresh task"),
6154            super::NativeAnnouncementRefreshCompletion::Cancelled
6155        );
6156    }
6157
6158    #[tokio::test]
6159    async fn native_announcement_refresh_loop_stops_before_announce_after_absolute_expiry() {
6160        let completion = super::refresh_native_announcement(
6161            "model",
6162            "iroh",
6163            "iroh://test-node",
6164            chrono::Utc::now().timestamp_millis() - 1,
6165            tokio_util::sync::CancellationToken::new(),
6166            expired_announcement_must_not_run,
6167        )
6168        .await;
6169
6170        assert_eq!(
6171            completion,
6172            super::NativeAnnouncementRefreshCompletion::Expired
6173        );
6174    }
6175}
6176
6177#[cfg(test)]
6178#[allow(clippy::unwrap_used, clippy::expect_used)]
6179mod native_announcement_wiring {
6180    use std::time::Duration;
6181
6182    #[test]
6183    fn production_publisher_profile_controls_first_failure_and_retry() {
6184        use hyprstream_service::NativeAnnouncementReach;
6185        // Guard the real QuicSharedConfig publisher's inputs in addition to
6186        // exercising its extracted startup boundary below. A correct loop
6187        // test alone did not catch the old reach-only call-site decision.
6188        let source = include_str!("main.rs");
6189        let production = source
6190            .split("mod native_announcement_wiring")
6191            .next()
6192            .unwrap();
6193        let compact: String = production.split_whitespace().collect();
6194        assert!(compact.contains("start_native_announcement_publisher(request.reach.clone(),hyprstream_discovery::native_network_required(),request.cancellation.clone(),"));
6195
6196        for network_required in [false, true] {
6197            for reach in [
6198                NativeAnnouncementReach::Iroh { node_id: [7; 32] },
6199                NativeAnnouncementReach::Quic {
6200                    address: "127.0.0.1:12345".parse().unwrap(),
6201                    server_name: "fixture".to_owned(),
6202                    certificate_hashes: Vec::new(),
6203                },
6204            ] {
6205                let fatal =
6206                    network_required && matches!(reach, NativeAnnouncementReach::Iroh { .. });
6207                let cancellation = tokio_util::sync::CancellationToken::new();
6208                let loop_cancellation = cancellation.clone();
6209                let (retry_tx, retry_rx) = std::sync::mpsc::sync_channel(2);
6210                let (dropped_tx, dropped_rx) = std::sync::mpsc::sync_channel(1);
6211                struct Dropped(std::sync::mpsc::SyncSender<()>);
6212                impl Drop for Dropped {
6213                    fn drop(&mut self) {
6214                        let _ = self.0.send(());
6215                    }
6216                }
6217                let result = super::start_native_announcement_publisher(
6218                    reach,
6219                    network_required,
6220                    cancellation.clone(),
6221                    move |announce_tx| async move {
6222                        let _dropped = Dropped(dropped_tx);
6223                        let mut attempts = 0;
6224                        super::refresh_native_announcement(
6225                            "policy",
6226                            "iroh",
6227                            "iroh://fixture",
6228                            i64::MAX,
6229                            loop_cancellation,
6230                            || {
6231                                attempts += 1;
6232                                let attempt = attempts;
6233                                let tx = announce_tx.clone();
6234                                let retry_tx = retry_tx.clone();
6235                                async move {
6236                                    let outcome = if attempt == 1 {
6237                                        Err(anyhow::anyhow!("local Discovery is not bound yet"))
6238                                    } else {
6239                                        retry_tx.send(attempt).unwrap();
6240                                        Ok(())
6241                                    };
6242                                    send_first_result(
6243                                        &tx,
6244                                        outcome.as_ref().map(|_| ()).map_err(ToString::to_string),
6245                                    );
6246                                    outcome
6247                                }
6248                            },
6249                        )
6250                        .await;
6251                    },
6252                );
6253                if fatal {
6254                    assert!(result
6255                        .unwrap_err()
6256                        .to_string()
6257                        .contains("local Discovery is not bound yet"));
6258                    dropped_rx
6259                        .recv_timeout(Duration::from_secs(2))
6260                        .expect("failed required startup aborts the publisher");
6261                    assert!(
6262                        retry_rx.try_recv().is_err(),
6263                        "required startup cannot silently retry its first failure"
6264                    );
6265                } else {
6266                    result.expect("Compatibility/non-Iroh startup is nonfatal");
6267                    assert_eq!(retry_rx.recv_timeout(Duration::from_secs(8)).expect("publication retries at the production five-second cadence when Discovery becomes ready"), 2);
6268                    cancellation.cancel();
6269                    dropped_rx
6270                        .recv_timeout(Duration::from_secs(2))
6271                        .expect("service cancellation still owns the retry task");
6272                }
6273            }
6274        }
6275    }
6276
6277    fn send_first_result(
6278        announce_tx: &super::NativeAnnouncementFirstTx,
6279        result: super::NativeAnnouncementFirstResult,
6280    ) {
6281        if let Some(tx) = announce_tx {
6282            if let Some(tx) = tx.lock().take() {
6283                let _ = tx.send(result);
6284            }
6285        }
6286    }
6287
6288    #[test]
6289    fn required_path_handshakes_and_runs_multiple_cycles() {
6290        let (cycle_tx, cycle_rx) = std::sync::mpsc::sync_channel(10);
6291        let initial_rx = super::spawn_native_announcement_loop(
6292            true,
6293            tokio_util::sync::CancellationToken::new(),
6294            move |announce_tx| {
6295                let cycle_tx = cycle_tx.clone();
6296                async move {
6297                    for i in 0..3 {
6298                        let announcement =
6299                            hyprstream_rpc_std::discovery_client::ServiceAnnouncement {
6300                                service_name: "model".to_owned(),
6301                                socket_kind: "iroh".to_owned(),
6302                                endpoint: "iroh://bound-node".to_owned(),
6303                                service_jwt: Some(format!("jwt-{i}")),
6304                                service_did: "did:at9p:fixture".into(),
6305                                capabilities: vec!["model".to_owned()],
6306                                accepted_state_digest: vec![i as u8; 64],
6307                                accepted_state_epoch: i,
6308                                response_key_id: "#response".to_owned(),
6309                                request_kem_key_id: "#kem".to_owned(),
6310                                request_kem_recipient: vec![i as u8],
6311                                expires_at_unix_ms: 1000 + i as i64,
6312                            };
6313                        let cycle_tx = cycle_tx.clone();
6314                        super::publish_native_announcement_attempt(
6315                            Ok(announcement),
6316                            announce_tx.as_ref().map(std::sync::Arc::clone),
6317                            move |announcement| async move {
6318                                assert_eq!(announcement.service_jwt, Some(format!("jwt-{i}")));
6319                                assert_eq!(
6320                                    super::native_announcement_bearer(&announcement).unwrap(),
6321                                    format!("jwt-{i}")
6322                                );
6323                                let mut missing = announcement.clone();
6324                                missing.service_jwt = None;
6325                                assert!(super::native_announcement_bearer(&missing).is_err());
6326                                missing.service_jwt = Some(" ".into());
6327                                assert!(super::native_announcement_bearer(&missing).is_err());
6328                                assert_eq!(announcement.expires_at_unix_ms, 1000 + i as i64);
6329                                cycle_tx
6330                                    .send(announcement.accepted_state_epoch)
6331                                    .expect("test receiver is live");
6332                                Ok(())
6333                            },
6334                        )
6335                        .await
6336                        .expect("publication succeeds");
6337                        tokio::task::yield_now().await;
6338                    }
6339                }
6340            },
6341        );
6342
6343        let result = initial_rx
6344            .expect("required path has a handshake receiver")
6345            .recv_timeout(Duration::from_secs(2))
6346            .expect("handshake completes");
6347        assert!(result.is_ok(), "first announcement should succeed");
6348
6349        let mut seen = Vec::with_capacity(3);
6350        for _ in 0..3 {
6351            seen.push(
6352                cycle_rx
6353                    .recv_timeout(Duration::from_secs(2))
6354                    .expect("loop continues producing cycles"),
6355            );
6356        }
6357        assert_eq!(
6358            seen,
6359            vec![0, 1, 2],
6360            "observed three full publication cycles"
6361        );
6362    }
6363
6364    #[test]
6365    fn compat_path_publishes_without_handshake() {
6366        let (cycle_tx, cycle_rx) = std::sync::mpsc::sync_channel(10);
6367        let initial_rx = super::spawn_native_announcement_loop(
6368            false,
6369            tokio_util::sync::CancellationToken::new(),
6370            move |_announce_tx| {
6371                let cycle_tx = cycle_tx.clone();
6372                async move {
6373                    cycle_tx.send(0).expect("test receiver is live");
6374                }
6375            },
6376        );
6377
6378        assert!(
6379            initial_rx.is_none(),
6380            "compat path has no handshake receiver"
6381        );
6382        let got = cycle_rx
6383            .recv_timeout(Duration::from_secs(2))
6384            .expect("compat path publishes");
6385        assert_eq!(got, 0);
6386    }
6387
6388    #[test]
6389    fn required_path_cancels_pending_first_publication() {
6390        let cancellation = tokio_util::sync::CancellationToken::new();
6391        let loop_cancellation = cancellation.clone();
6392        let (started_tx, started_rx) = std::sync::mpsc::sync_channel(1);
6393        let (dropped_tx, dropped_rx) = std::sync::mpsc::sync_channel(1);
6394        struct Dropped(std::sync::mpsc::SyncSender<()>);
6395        impl Drop for Dropped {
6396            fn drop(&mut self) {
6397                let _ = self.0.send(());
6398            }
6399        }
6400        let initial = super::spawn_native_announcement_loop(
6401            true,
6402            cancellation.clone(),
6403            move |announce_tx| async move {
6404                let _dropped = Dropped(dropped_tx);
6405                super::refresh_native_announcement(
6406                    "model",
6407                    "iroh",
6408                    "iroh://test",
6409                    i64::MAX,
6410                    loop_cancellation,
6411                    || {
6412                        started_tx.send(()).expect("publication started");
6413                        std::future::pending::<Result<(), std::io::Error>>()
6414                    },
6415                )
6416                .await;
6417                // Keep the first-result sender alive during the pending RPC.
6418                drop(announce_tx);
6419            },
6420        )
6421        .expect("required handshake");
6422        started_rx
6423            .recv_timeout(Duration::from_secs(2))
6424            .expect("first publication entered");
6425        cancellation.cancel();
6426        assert!(initial
6427            .recv_timeout(Duration::from_secs(2))
6428            .expect("cancelled handshake must finish")
6429            .is_err());
6430        dropped_rx
6431            .recv_timeout(Duration::from_secs(2))
6432            .expect("publication task must be dropped");
6433    }
6434
6435    #[test]
6436    fn required_path_rejects_success_when_already_cancelled() {
6437        let cancellation = tokio_util::sync::CancellationToken::new();
6438        cancellation.cancel();
6439        let initial =
6440            super::spawn_native_announcement_loop(true, cancellation, |announce_tx| async move {
6441                send_first_result(&announce_tx, Ok(()));
6442            })
6443            .expect("required handshake");
6444        assert!(initial
6445            .recv_timeout(Duration::from_secs(2))
6446            .expect("cancelled handshake must finish")
6447            .is_err());
6448    }
6449
6450    #[test]
6451    fn required_path_aborts_on_first_failure() {
6452        let (cycle_tx, cycle_rx) = std::sync::mpsc::sync_channel(10);
6453        let initial_rx = super::spawn_native_announcement_loop(
6454            true,
6455            tokio_util::sync::CancellationToken::new(),
6456            move |announce_tx| {
6457                let cycle_tx = cycle_tx.clone();
6458                async move {
6459                    send_first_result(&announce_tx, Err("injected".to_owned()));
6460                    loop {
6461                        tokio::task::yield_now().await;
6462                        cycle_tx.send(1).expect("test receiver is live");
6463                    }
6464                }
6465            },
6466        );
6467
6468        let result = initial_rx
6469            .expect("required path has a handshake receiver")
6470            .recv_timeout(Duration::from_secs(2))
6471            .expect("handshake completes");
6472        assert!(result.is_err(), "first failure must be reported");
6473
6474        assert!(
6475            cycle_rx.recv_timeout(Duration::from_millis(200)).is_err(),
6476            "loop must abort after first failure; extra cycle observed"
6477        );
6478    }
6479
6480    #[test]
6481    fn required_path_reports_first_authority_failure_without_publication() {
6482        let published = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
6483        let observed = published.clone();
6484        let cancellation = tokio_util::sync::CancellationToken::new();
6485        let initial = super::spawn_native_announcement_loop(
6486            true,
6487            cancellation.clone(),
6488            move |announce_tx| async move {
6489                super::refresh_native_announcement(
6490                    "model",
6491                    "iroh",
6492                    "iroh://test",
6493                    i64::MAX,
6494                    cancellation,
6495                    || {
6496                        let tx = announce_tx.as_ref().map(std::sync::Arc::clone);
6497                        let published = published.clone();
6498                        super::publish_native_announcement_attempt(
6499                            Err(anyhow::anyhow!("fresh checkpoint unavailable")),
6500                            tx,
6501                            move |_| async move {
6502                                published.store(true, std::sync::atomic::Ordering::SeqCst);
6503                                Ok(())
6504                            },
6505                        )
6506                    },
6507                )
6508                .await;
6509            },
6510        )
6511        .expect("required handshake");
6512        let error = initial
6513            .recv_timeout(Duration::from_secs(2))
6514            .expect("authority failure must complete first handshake")
6515            .expect_err("startup must fail");
6516        assert!(error.contains("fresh checkpoint unavailable"));
6517        assert!(
6518            !observed.load(std::sync::atomic::Ordering::SeqCst),
6519            "failed authority projection must never publish stale reach"
6520        );
6521    }
6522}
6523
6524#[cfg(test)]
6525#[allow(clippy::expect_used)]
6526mod native_launcher {
6527    //! Causal coverage for the direct child launch path (#1585): the produced
6528    //! child invocation must load the operator's explicit custom config
6529    //! (relative path with spaces included), select the provisioned native
6530    //! identity — not defaults — through the real parser/identity path, and
6531    //! report readiness only through the authenticated notification boundary.
6532
6533    use super::*;
6534    use hyprstream_core::cli::service_handlers::direct_child_process_config;
6535    use std::path::Path;
6536
6537    const CAUSAL_CHILD: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_CHILD";
6538    const CAUSAL_MODE: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_MODE";
6539    const CAUSAL_ARGV: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_ARGV";
6540    const CAUSAL_PROOF: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_PROOF";
6541    const CAUSAL_STOP: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_STOP";
6542    const SERVICE: &str = "model";
6543    const KEY_SEED: u8 = 0x5A;
6544
6545    #[cfg(unix)]
6546    #[test]
6547    fn cli_parser_loads_non_utf8_config_path_without_changing_bytes() -> anyhow::Result<()> {
6548        use std::os::unix::ffi::{OsStrExt, OsStringExt};
6549
6550        let root = tempfile::tempdir()?;
6551        let config_path = root
6552            .path()
6553            .join(std::ffi::OsString::from_vec(b"parent-\xff.toml".to_vec()));
6554        HyprConfig::default().to_file(&config_path)?;
6555        let canonical = std::fs::canonicalize(&config_path)?;
6556        assert!(
6557            canonical.to_str().is_none(),
6558            "fixture path must be non-UTF-8"
6559        );
6560
6561        let matches = build_cli().try_get_matches_from([
6562            std::ffi::OsString::from("hyprstream"),
6563            std::ffi::OsString::from("--config"),
6564            canonical.as_os_str().to_owned(),
6565        ])?;
6566        let parsed = matches
6567            .get_one::<std::path::PathBuf>("config")
6568            .context("config PathBuf parsed by clap")?;
6569        assert_eq!(
6570            parsed.as_os_str().as_bytes(),
6571            canonical.as_os_str().as_bytes(),
6572            "clap must preserve the explicit selector byte for byte"
6573        );
6574        let config = load_config(Some(parsed))?;
6575        config.validate()?;
6576        Ok(())
6577    }
6578
6579    fn provision_custom_config(
6580        root: &tempfile::TempDir,
6581        seed: u8,
6582        with_key: bool,
6583    ) -> anyhow::Result<std::path::PathBuf> {
6584        let secrets = root.path().join("custom credentials");
6585        if with_key {
6586            std::fs::create_dir_all(secrets.join(SERVICE))?;
6587            std::fs::write(secrets.join(SERVICE).join("signing-key"), [seed; 32])?;
6588        }
6589        let mut configured = HyprConfig::default();
6590        configured.quic.native_network_profile =
6591            hyprstream_core::config::NativeNetworkProfile::NetworkIrohRequired;
6592        configured.secrets.path = Some(secrets);
6593        configured.storage.models_dir = root.path().join("custom-models");
6594        // A relative path containing spaces, relative to the launch directory.
6595        let launch_dir = root.path().join("launch dir");
6596        let config_dir = launch_dir.join("my configs");
6597        std::fs::create_dir_all(&config_dir)?;
6598        let config_path = config_dir.join("custom.toml");
6599        configured.to_file(&config_path)?;
6600        Ok(config_path)
6601    }
6602
6603    async fn launcher_child() -> anyhow::Result<()> {
6604        let mode = std::env::var(CAUSAL_MODE).unwrap_or_default();
6605        // The exact argv the launcher helper produced for the real binary.
6606        let args: Vec<String> = serde_json::from_str(&std::env::var(CAUSAL_ARGV)?)?;
6607        let matches = build_cli()
6608            .try_get_matches_from(std::iter::once("hyprstream".to_owned()).chain(args))?;
6609        let config = load_config(
6610            matches
6611                .get_one::<std::path::PathBuf>("config")
6612                .map(std::path::PathBuf::as_path),
6613        )?;
6614        config
6615            .validate()
6616            .context("child configuration validation")?;
6617        let _ = hyprstream_core::config::install_pinned_config(config.clone());
6618
6619        let selected = native_service_process_name(&matches, &config)?;
6620        anyhow::ensure!(
6621            selected.as_deref() == Some(SERVICE),
6622            "child selected {selected:?} instead of the configured native service"
6623        );
6624        if mode == "missing-key" {
6625            // Negative control: the provisioned key is absent, so the real
6626            // required-native identity path must refuse — never generate.
6627            let outcome = load_process_signing_key(&config, selected.as_deref()).await;
6628            anyhow::ensure!(
6629                outcome.is_err(),
6630                "missing provisioned key must fail required-native startup"
6631            );
6632            anyhow::bail!("missing-key control refused startup as required");
6633        }
6634
6635        let key = load_process_signing_key(&config, selected.as_deref()).await?;
6636        // Proof carries the PUBLIC verifying key only — never private material.
6637        let proof = std::env::var(CAUSAL_PROOF)?;
6638        std::fs::write(&proof, key.verifying_key().to_bytes())?;
6639        // Authentic readiness: only after the config/identity assertions.
6640        hyprstream_rpc::notify::ready()?;
6641        // Stay alive so the supervisor observes a genuinely RUNNING ready
6642        // child (its post-READY liveness recheck would otherwise race our
6643        // exit), and exit only on the parent's controlled stop signal.
6644        let stop_file = std::path::PathBuf::from(std::env::var(CAUSAL_STOP)?);
6645        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
6646        while !stop_file.exists() {
6647            if std::time::Instant::now() >= deadline {
6648                anyhow::bail!("supervisor stop signal never arrived");
6649            }
6650            std::thread::sleep(std::time::Duration::from_millis(50));
6651        }
6652        Ok(())
6653    }
6654
6655    async fn spawn_causal_child(
6656        supervisor: &hyprstream_service::ProcessSpawner,
6657        launch_args: &[std::ffi::OsString],
6658        mode: &str,
6659        proof: &Path,
6660        stop_file: &Path,
6661        working_dir: &Path,
6662        child_instance: &str,
6663    ) -> anyhow::Result<Result<hyprstream_service::SpawnedProcess, hyprstream_rpc::error::RpcError>>
6664    {
6665        let exe = std::env::current_exe()?;
6666        // Unique supervisor name: PID artifacts land under a test-owned
6667        // namespace, never the shared `model.pid` of a real deployment.
6668        let serialized_args: Vec<&str> = launch_args
6669            .iter()
6670            .map(|arg| {
6671                arg.to_str()
6672                    .context("UTF-8 fixture argument for launcher causal-test envelope")
6673            })
6674            .collect::<anyhow::Result<_>>()?;
6675        let mut child = hyprstream_service::ProcessConfig::new(child_instance, &exe)
6676            .args([
6677                "--exact",
6678                "native_launcher::launcher_child_loads_explicit_config_and_provisioned_identity",
6679                "--nocapture",
6680            ])
6681            .env(CAUSAL_CHILD, "1")
6682            .env(CAUSAL_MODE, mode)
6683            .env(CAUSAL_ARGV, serde_json::to_string(&serialized_args)?)
6684            .env(CAUSAL_PROOF, proof.display().to_string())
6685            .env(CAUSAL_STOP, stop_file.display().to_string())
6686            // Deterministic identity layout, isolated runtime namespace.
6687            .env("HYPRSTREAM_SECRETS_PROFILE", "shared-directory")
6688            .env("HYPRSTREAM_INSTANCE", child_instance)
6689            .working_dir(working_dir);
6690        child.readiness = hyprstream_service::ProcessReadiness::Notify {
6691            timeout: std::time::Duration::from_secs(30),
6692        };
6693        Ok(supervisor.spawn(child).await)
6694    }
6695
6696    #[tokio::test]
6697    async fn launcher_child_loads_explicit_config_and_provisioned_identity() -> anyhow::Result<()> {
6698        if std::env::var_os(CAUSAL_CHILD).is_some() {
6699            return launcher_child().await;
6700        }
6701
6702        let root = tempfile::tempdir()?;
6703        let config_path = provision_custom_config(&root, KEY_SEED, true)?;
6704        let canonical = std::fs::canonicalize(&config_path)?;
6705        let expected_vk = ed25519_dalek::SigningKey::from_bytes(&[KEY_SEED; 32])
6706            .verifying_key()
6707            .to_bytes();
6708
6709        // The exact invocation the launcher builds for the real binary.
6710        let plan =
6711            direct_child_process_config(SERVICE, true, Some(&canonical), Path::new("hyprstream"))?;
6712        let launch_args = plan.args.clone();
6713        assert!(
6714            !launch_args.iter().any(|arg| arg == "--ipc"),
6715            "required-native child plan must contain no service IPC argument"
6716        );
6717        assert!(
6718            launch_args.windows(2).any(|pair| pair[0] == "--config"
6719                && pair[1] == canonical.to_str().expect("utf-8 canonical path")),
6720            "canonical absolute config path must be forwarded verbatim"
6721        );
6722
6723        // Child runs from a DIFFERENT working directory than the launcher's
6724        // config-relative layout: only the canonical path makes the relative
6725        // selector meaningful across the spawner cwd.
6726        let proof = root.path().join("proof.vk");
6727        let stop_file = root.path().join("stop");
6728        let spawner = hyprstream_service::ProcessSpawner::standalone();
6729        let supervisor = "launcher-causal-happy";
6730        let process = spawn_causal_child(
6731            &spawner,
6732            &launch_args,
6733            "happy",
6734            &proof,
6735            &stop_file,
6736            root.path(),
6737            supervisor,
6738        )
6739        .await?
6740        .expect("ready child proves config load, identity selection, and notify barrier");
6741        let observed = std::fs::read(&proof)?;
6742        assert_eq!(
6743            observed, expected_vk,
6744            "child selected the provisioned signer"
6745        );
6746        // The READY child is genuinely running (its notification was not a
6747        // last-gasp message), with its PID file published under the unique
6748        // supervisor name.
6749        assert!(
6750            spawner.is_running(&process).await?,
6751            "the ready child must still be alive after the spawn reports success"
6752        );
6753        assert!(
6754            hyprstream_rpc::paths::service_pid_file(supervisor).exists(),
6755            "a ready child publishes its PID file"
6756        );
6757        // Controlled shutdown: signal stop, wait for the child's own exit,
6758        // then have the SAME supervisor stop reap it and remove the PID
6759        // artifact — no manual deletion, no second backend.
6760        std::fs::write(&stop_file, b"stop")?;
6761        let stopped = std::time::Instant::now() + std::time::Duration::from_secs(10);
6762        while spawner.is_running(&process).await? {
6763            if std::time::Instant::now() >= stopped {
6764                anyhow::bail!("child did not exit after the controlled stop signal");
6765            }
6766            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
6767        }
6768        spawner.stop(&process).await?;
6769        assert!(!spawner.is_running(&process).await?);
6770        assert!(
6771            !hyprstream_rpc::paths::service_pid_file(supervisor).exists(),
6772            "supervised stop must remove the published PID file"
6773        );
6774
6775        // Negative control: absent provisioned key fails the launch honestly.
6776        let missing_root = tempfile::tempdir()?;
6777        let missing_config = provision_custom_config(&missing_root, KEY_SEED, false)?;
6778        let missing_canonical = std::fs::canonicalize(&missing_config)?;
6779        let missing_plan = direct_child_process_config(
6780            SERVICE,
6781            true,
6782            Some(&missing_canonical),
6783            Path::new("hyprstream"),
6784        )?;
6785        let missing_spawner = hyprstream_service::ProcessSpawner::standalone();
6786        let missing_supervisor = "launcher-causal-missing";
6787        let missing_outcome = spawn_causal_child(
6788            &missing_spawner,
6789            &missing_plan.args,
6790            "missing-key",
6791            &missing_root.path().join("unused.vk"),
6792            &missing_root.path().join("stop"),
6793            missing_root.path(),
6794            missing_supervisor,
6795        )
6796        .await?;
6797        let error =
6798            missing_outcome.expect_err("launch must fail when the provisioned key is absent");
6799        assert!(
6800            error.to_string().contains("exited during startup"),
6801            "child startup failure must propagate, got: {error}"
6802        );
6803        assert!(
6804            !hyprstream_rpc::paths::service_pid_file(missing_supervisor).exists(),
6805            "a never-ready child must not publish a PID file"
6806        );
6807        Ok(())
6808    }
6809}
6810
6811#[cfg(test)]
6812mod config_load_failure_tests {
6813    use super::*;
6814
6815    #[test]
6816    fn invalid_config_never_defaults_account_selection() -> Result<()> {
6817        const CHILD: &str = "HYPRSTREAM_CONFIG_FAILURE_CHILD";
6818        if std::env::var_os(CHILD).is_none() {
6819            let root = tempfile::tempdir()?;
6820            let mut child = std::process::Command::new(std::env::current_exe()?);
6821            for (key, _) in std::env::vars_os() {
6822                if key.to_string_lossy().starts_with("HYPRSTREAM__") {
6823                    child.env_remove(key);
6824                }
6825            }
6826            let status = child
6827                .args([
6828                    "--exact",
6829                    "config_load_failure_tests::invalid_config_never_defaults_account_selection",
6830                    "--nocapture",
6831                ])
6832                .env(CHILD, "1")
6833                .env("XDG_CONFIG_HOME", root.path().join("config"))
6834                .env("XDG_DATA_HOME", root.path().join("data"))
6835                .env("HYPRSTREAM__CREDENTIALS__BACKEND", "postgres")
6836                .env("HYPRSTREAM__OAUTH__PORT", "invalid-port")
6837                .status()?;
6838            anyhow::ensure!(status.success(), "invalid configuration child failed");
6839            return Ok(());
6840        }
6841        let error = match load_config(None) {
6842            Ok(_) => anyhow::bail!("invalid port discarded PostgreSQL selection"),
6843            Err(error) => error,
6844        };
6845        assert!(error
6846            .to_string()
6847            .contains("Failed to load default-location configuration"));
6848        Ok(())
6849    }
6850}
6851
6852#[cfg(test)]
6853mod proof_replay_store_selection_tests {
6854    use super::{select_proof_replay_store, ProofReplayStoreSelection};
6855    use hyprstream_rpc::proof::admission::ReplayDomainGuarantee;
6856
6857    #[cfg(feature = "postgres-replay")]
6858    #[test]
6859    fn dedicated_shared_store_wins_over_a_configured_pds_store() {
6860        assert_eq!(
6861            select_proof_replay_store(ReplayDomainGuarantee::LinearizableSharedStore, true),
6862            ProofReplayStoreSelection::DedicatedPostgresV1,
6863        );
6864    }
6865
6866    #[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
6867    #[test]
6868    fn pds_only_build_retains_legacy_shared_store() {
6869        assert_eq!(
6870            select_proof_replay_store(ReplayDomainGuarantee::LinearizableSharedStore, true),
6871            ProofReplayStoreSelection::LegacyPdsPostgres,
6872        );
6873    }
6874
6875    #[cfg(not(any(feature = "pds-postgres", feature = "postgres-replay")))]
6876    #[test]
6877    fn shared_domain_without_a_backend_fails_closed() {
6878        assert_eq!(
6879            select_proof_replay_store(ReplayDomainGuarantee::LinearizableSharedStore, false),
6880            ProofReplayStoreSelection::Unavailable,
6881        );
6882        assert_eq!(
6883            select_proof_replay_store(ReplayDomainGuarantee::SingleVerifierInstance, false),
6884            ProofReplayStoreSelection::SingleVerifierMemory,
6885        );
6886    }
6887}