1#![allow(clippy::print_stdout, clippy::print_stderr)]
6
7use anyhow::{Context as _, Result};
9use clap::{
10 Arg, Args as ClapArgs, Command as ClapCommand, FromArgMatches, Subcommand as ClapSubcommand,
11};
12use tracing::info;
13
14use hyprstream_core::cli::commands::{
16 ExecutionMode, FlightArgs, ImageCommand, ServiceAction, TrainingAction, TrustCommand,
17 TuiAction, WorkerAction,
18};
19use hyprstream_core::cli::commands::{
20 PolicyCommand, RoleCommand, TokenCommand, UserCommand, UserKeysCommand,
21};
22use hyprstream_core::cli::quick::{QuickCommand, RemoteQuickCommand, WorktreeQuickCommand};
23use hyprstream_core::cli::schema_cli;
24use hyprstream_core::cli::{
25 handle_branch,
26 handle_checkout,
27 handle_clone,
28 handle_images_df,
30 handle_images_list,
31 handle_images_pull,
32 handle_images_rm,
33 handle_infer,
34 handle_info,
35 handle_list,
36 handle_load,
37 handle_notify_command,
38 handle_policy_apply,
39 handle_policy_apply_template,
40 handle_policy_check,
41 handle_policy_diff,
42 handle_policy_edit,
43 handle_policy_history,
44 handle_policy_list_templates,
45 handle_policy_role_add,
46 handle_policy_role_list,
47 handle_policy_role_remove,
48 handle_policy_rollback,
49 handle_policy_show,
50 handle_pull,
51 handle_remote_add,
52 handle_remote_list,
53 handle_remote_remove,
54 handle_remote_rename,
55 handle_remote_set_url,
56 handle_remove,
57 handle_service_install,
59 handle_service_provision_policy_templates,
60 handle_service_start,
61 handle_service_status,
62 handle_service_stop,
63 handle_service_uninstall,
64 handle_sign_challenge,
65 handle_status,
66 handle_token_create,
67 handle_training_batch,
68 handle_training_checkpoint,
69 handle_training_infer,
70 handle_training_init,
71 handle_unload,
72 handle_user_create,
73 handle_user_keys_import,
74 handle_user_keys_list,
75 handle_user_keys_remove,
76 handle_user_list,
77 handle_user_register,
78 handle_user_remove,
79 handle_worker_exec,
80 handle_worker_list,
81 handle_worker_restart,
82 handle_worker_rm,
83 handle_worker_run,
84 handle_worker_start,
85 handle_worker_stats,
86 handle_worker_status,
87 handle_worker_stop,
88 handle_worker_terminal,
89 handle_worktree_add,
90 handle_worktree_info,
91 handle_worktree_list,
92 handle_worktree_remove,
93 load_or_generate_signing_key,
94 parse_filters,
95 parse_status_filter,
96 AppContext,
97 DeviceConfig,
98 DevicePreference,
99 RuntimeConfig,
100};
101
102#[cfg(feature = "experimental")]
103use hyprstream_core::cli::{
104 handle_commit, handle_merge, handle_promote, handle_push, MergeOptions,
105};
106use hyprstream_core::config::HyprConfig;
107use hyprstream_core::storage::{GitRef, ModelRef};
108
109use hyprstream_rpc_std::registry_client::RegistryClient;
111use hyprstream_workers::runtime::WorkerService;
113#[cfg(feature = "oci-image")]
114use hyprstream_workers::ImageConfig;
115use hyprstream_workers::PoolConfig;
116use std::sync::Arc;
117use hyprstream_rpc::events::EventPublisher;
119use hyprstream_rpc::registry::SocketKind;
120use hyprstream_rpc::transport::TransportConfig;
121use hyprstream_rpc::{SigningKey, VerifyingKey};
122use hyprstream_service::{get_factory, InprocManager, ServiceContext, ServiceManager};
123
124fn supports_tui() -> bool {
125 use std::io::IsTerminal;
126 if !std::io::stdout().is_terminal() || !std::io::stdin().is_terminal() {
127 return false;
128 }
129 match std::env::var("TERM").as_deref() {
130 Ok("dumb") | Ok("") | Err(_) => return false,
131 _ => {}
132 }
133 #[cfg(unix)]
134 {
135 let mut ws: libc::winsize = unsafe { std::mem::zeroed() };
136 let ok = unsafe { libc::ioctl(libc::STDOUT_FILENO, libc::TIOCGWINSZ, &mut ws) };
137 if ok == 0 {
138 return ws.ws_col >= 60 && ws.ws_row >= 20;
139 }
140 }
141 true
142}
143#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
145use opentelemetry::trace::TracerProvider;
146#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
147use opentelemetry::KeyValue;
148#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
149use opentelemetry_otlp::{SpanExporter, WithExportConfig};
150#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
151use opentelemetry_sdk::Resource;
152use tracing_appender::rolling::{RollingFileAppender, Rotation};
153#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
154use tracing_opentelemetry::OpenTelemetryLayer;
155#[cfg(any(not(feature = "otel"), target_arch = "wasm32"))]
156use tracing_subscriber::EnvFilter;
157#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
158use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
159
160fn build_cli() -> ClapCommand {
166 let mut app = ClapCommand::new("hyprstream")
167 .version(env!("BUILD_VERSION"))
168 .about("Real-time adaptive ML inference server with dynamic sparse weight adjustments")
169 .arg(
170 Arg::new("config")
171 .long("config")
172 .global(true)
173 .env("HYPRSTREAM_CONFIG")
174 .value_parser(clap::value_parser!(std::path::PathBuf))
175 .help("Path to configuration file (overrides default config locations)"),
176 )
177 .arg(
178 Arg::new("gpu_device")
179 .long("gpu-device")
180 .global(true)
181 .env("HYPRSTREAM_GPU_DEVICE")
182 .value_parser(clap::value_parser!(usize))
183 .help("GPU device ID to use (e.g., 0, 1). Defaults to auto-detect."),
184 );
185
186 app = app.subcommand(schema_cli::build_tool_command());
188
189 app = app.subcommand(<QuickCommand as ClapSubcommand>::augment_subcommands(
191 ClapCommand::new("quick").about("Quick workflows \u{2014} multi-step convenience commands"),
192 ));
193
194 app = app.subcommand(<ServiceAction as ClapSubcommand>::augment_subcommands(
196 ClapCommand::new("service").about("Service management and lifecycle commands"),
197 ));
198
199 app = app.subcommand(<TrustCommand as ClapSubcommand>::augment_subcommands(
201 ClapCommand::new("trust").about("Mint and verify deployment trust artifacts"),
202 ));
203
204 app = app.subcommand(<TuiAction as ClapSubcommand>::augment_subcommands(
206 ClapCommand::new("tui")
207 .about("TUI display server \u{2014} terminal multiplexer with session persistence"),
208 ));
209
210 app = app.subcommand(<FlightArgs as ClapArgs>::augment_args(
212 ClapCommand::new("flight").about("Flight SQL client to query datasets"),
213 ));
214
215 app = app.subcommand(<UserCommand as ClapSubcommand>::augment_subcommands(
217 ClapCommand::new("user").about("Manage local user credentials"),
218 ));
219
220 app = app.subcommand(
222 ClapCommand::new("pds")
223 .about("Attach this host to its home personal data server")
224 .subcommand(
225 ClapCommand::new("init-deployment-store")
226 .about("Initialize an explicitly provisioned checkpoint store")
227 .arg(
228 Arg::new("fresh-genesis")
229 .long("fresh-genesis")
230 .action(clap::ArgAction::SetTrue)
231 .help(
232 "Clear only the configured RDS records KV before first-boot initialization; staging deployment bootstrap only",
233 ),
234 ),
235 )
236 .subcommand(
237 ClapCommand::new("inspect-services")
238 .about("Read existing checkpoint-verified service identities as public JSON without provisioning or renewal")
239 .arg(Arg::new("service").long("service").required(true)
240 .action(clap::ArgAction::Append).value_delimiter(',')),
241 )
242 .subcommand(
243 ClapCommand::new("provision-services")
244 .about("Admit existing local service identities before starting the registry")
245 .arg(Arg::new("service").long("service").required(true)
246 .action(clap::ArgAction::Append).value_delimiter(','))
247 .arg(Arg::new("valid-for-seconds").long("valid-for-seconds")
248 .value_parser(clap::value_parser!(i64)).default_value("86400"))
249 .arg(Arg::new("roster-export").long("roster-export").value_name("PATH")
250 .value_parser(clap::value_parser!(std::path::PathBuf))
251 .help("Opt-in: atomically write a JSON manifest of the verified accepted roster (public fields only; not a trust root) to PATH. Behavior is unchanged when absent.")),
252 )
253 .subcommand(
254 ClapCommand::new("join")
255 .visible_alias("attach")
256 .about("Authorize and attach this host to one home PDS")
257 .arg(
258 Arg::new("url")
259 .required(true)
260 .value_name("PDS_URL")
261 .help("HTTPS origin of the home PDS"),
262 )
263 .arg(
264 Arg::new("scope")
265 .long("scope")
266 .value_name("SCOPE")
267 .help("Space-delimited OAuth scopes requested from the PDS"),
268 ),
269 ),
270 );
271
272 app = app.subcommand(
274 ClapCommand::new("wizard")
275 .about("Interactive setup wizard — configure policies, users, and API tokens")
276 .arg(
277 Arg::new("non_interactive")
278 .long("non-interactive")
279 .short('y')
280 .action(clap::ArgAction::SetTrue)
281 .help("Accept defaults without prompting"),
282 )
283 .arg(
284 Arg::new("start")
285 .long("start")
286 .action(clap::ArgAction::SetTrue)
287 .help("Start services after setup"),
288 )
289 .arg(
290 Arg::new("tui")
291 .long("tui")
292 .action(clap::ArgAction::SetTrue)
293 .help("Force TUI wizard (auto-detected when terminal supports it)"),
294 )
295 .arg(
296 Arg::new("bootstrap_only")
297 .long("bootstrap-only")
298 .action(clap::ArgAction::SetTrue)
299 .help("Run only the trust-root bootstrap (phase 1); skip binary install, policy templates, user creation, token mint, and systemd"),
300 )
301 .arg(
302 Arg::new("enable_federation")
303 .long("enable-federation")
304 .action(clap::ArgAction::SetTrue)
305 .help("Apply the federation-open policy template — accept third-party apps AND remote peer servers from any HTTPS origin (atproto-style federation). Default is disabled; under -y this flag is the only way to enable it."),
306 )
307 .arg(
308 Arg::new("pds_url")
309 .long("pds-url")
310 .value_name("PDS_URL")
311 .help("Attach this host to a home PDS after wizard setup (same as `pds join`)")
312 .conflicts_with("bootstrap_only")
313 .conflicts_with("tui"),
314 )
315 .arg(
316 Arg::new("initial_user_role")
317 .long("initial-user-role")
318 .value_name("ROLE")
319 .default_value("admin")
320 .help("Role to assign the local user under --non-interactive (admin|operator|trainer|viewer). Default is admin; use operator/viewer in tests for least-privilege."),
321 )
322 .arg(
323 Arg::new("deployment_trust")
324 .long("deployment-trust")
325 .action(clap::ArgAction::SetTrue)
326 .help("Run the deployment-trust ceremony after node bootstrap — detects an attached YubiKey and mints the deployment root, delegated signer, and registry credential. Opt-in: node bootstrap alone is unchanged without it."),
327 )
328 .arg(
329 Arg::new("deployment_trust_dir")
330 .long("deployment-trust-dir")
331 .value_name("DIR")
332 .requires("deployment_trust")
333 .help("Absolute ceremony working directory (default: <models-dir>/deployment-ceremony). Destroy it once the artifacts are distributed."),
334 )
335 .arg(
336 Arg::new("deployment_trust_software")
337 .long("deployment-trust-software")
338 .action(clap::ArgAction::SetTrue)
339 .requires("deployment_trust")
340 .help("Mint a DEV-GRADE software deployment root even if a hardware token is attached. Never selected on your behalf."),
341 )
342 .arg(
343 Arg::new("deployment_trust_serial")
344 .long("deployment-trust-serial")
345 .value_name("SERIAL")
346 .requires("deployment_trust")
347 .help("Bind the deployment root to the token with this serial when several are attached."),
348 )
349 .arg(
350 Arg::new("deployment_trust_piv_slot")
351 .long("deployment-trust-piv-slot")
352 .value_name("SLOT")
353 .requires("deployment_trust")
354 .help("PIV slot holding the Ed25519 leg on firmware 5.7.4+ (default: 9c)."),
355 )
356 .arg(
357 Arg::new("deployment_trust_allow_plaintext_break_glass")
358 .long("deployment-trust-allow-plaintext-break-glass")
359 .action(clap::ArgAction::SetTrue)
360 .requires("deployment_trust")
361 .help("Accept an unencrypted break-glass identity. Only for a throwaway root you destroy immediately afterwards."),
362 ),
363 );
364
365 app = app.subcommand(
367 ClapCommand::new("sign-challenge")
368 .about("Sign an Ed25519 challenge for OAuth device or auth code flow")
369 .arg(
370 Arg::new("user_code")
371 .index(1)
372 .required(false)
373 .help("User code from device flow (e.g., ABCD-EFGH)"),
374 )
375 .arg(
376 Arg::new("nonce")
377 .long("nonce")
378 .required(false)
379 .allow_hyphen_values(true)
380 .help("Nonce from the browser authorization challenge form"),
381 )
382 .arg(
383 Arg::new("code_challenge")
384 .long("code-challenge")
385 .required(false)
386 .allow_hyphen_values(true)
387 .help("PKCE code_challenge from the browser authorization URL"),
388 )
389 .arg(
390 Arg::new("server")
391 .long("server")
392 .required(false)
393 .help("OAuth server URL (default: from config or http://localhost:6791)"),
394 )
395 .arg(
396 Arg::new("insecure")
397 .long("insecure")
398 .action(clap::ArgAction::SetTrue)
399 .help("Disable TLS certificate verification (use only against a trusted local dev server). By default the local self-signed dev cert is trusted automatically."),
400 ),
401 );
402
403 app = app.subcommand(
405 ClapCommand::new("update")
406 .about("Check for and install updates")
407 .arg(
408 Arg::new("cleanup")
409 .long("cleanup")
410 .action(clap::ArgAction::SetTrue)
411 .help("Remove old version worktrees"),
412 ),
413 );
414
415 app =
417 app.subcommand(
418 ClapCommand::new("mac")
419 .about("Native MAC (mandatory access control) status and inspection")
420 .subcommand(ClapCommand::new("genesis").about(
421 "Show the MAC genesis coverage report (activation coverage-gate evidence)",
422 )),
423 );
424
425 app
426}
427
428fn load_config(path: Option<&std::path::Path>) -> Result<HyprConfig> {
434 match path {
435 Some(p) => {
436 info!("Loading config from: {}", p.display());
437 HyprConfig::from_file(p)
438 .with_context(|| format!("Failed to load config from {}", p.display()))
439 }
440 None => {
441 info!("Loading config from default locations");
442 HyprConfig::load().context("Failed to load default-location configuration")
446 }
447 }
448}
449
450fn with_runtime<F, Fut>(config: RuntimeConfig, handler: F) -> Result<()>
452where
453 F: FnOnce() -> Fut,
454 Fut: std::future::Future<Output = Result<()>>,
455{
456 if matches!(config.device.preference, DevicePreference::RequireGPU)
458 && std::env::var("CUDA_VISIBLE_DEVICES").is_err()
459 && std::env::var("HIP_VISIBLE_DEVICES").is_err()
460 && !std::path::Path::new("/usr/local/cuda").exists()
461 && !std::path::Path::new("/opt/rocm").exists()
462 {
463 anyhow::bail!("GPU required but no CUDA or ROCm installation detected. Set CUDA_VISIBLE_DEVICES or HIP_VISIBLE_DEVICES, or install CUDA/ROCm.");
464 }
465
466 match config.device.preference {
468 DevicePreference::RequestCPU => {
469 std::env::set_var("HYPRSTREAM_FORCE_CPU", "1");
470 }
471 DevicePreference::RequestGPU | DevicePreference::RequireGPU => {
472 std::env::remove_var("HYPRSTREAM_FORCE_CPU");
473 }
474 }
475
476 if config.multi_threaded {
478 let rt =
479 tokio::runtime::Runtime::new().context("Failed to create multi-threaded runtime")?;
480 rt.block_on(handler())
481 } else {
482 let rt = tokio::runtime::Builder::new_current_thread()
483 .enable_all()
484 .build()
485 .context("Failed to create single-threaded runtime")?;
486 rt.block_on(handler())
487 }
488}
489
490#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
495#[derive(Debug, Clone, Copy)]
497enum TelemetryProvider {
498 Otlp,
500 Stdout,
502}
503
504#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
505fn init_telemetry(provider: TelemetryProvider) -> Result<()> {
507 let service_name =
508 std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "hyprstream".to_owned());
509
510 let resource = Resource::builder()
511 .with_service_name(service_name.clone())
512 .with_attribute(KeyValue::new("service.version", env!("CARGO_PKG_VERSION")))
513 .build();
514
515 let endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")
516 .unwrap_or_else(|_| "http://localhost:4317".to_owned());
517
518 let tracer_provider = match provider {
519 TelemetryProvider::Otlp => {
520 info!("Using OTLP exporter for OpenTelemetry at {}", endpoint);
521
522 let exporter = SpanExporter::builder()
523 .with_tonic()
524 .with_endpoint(endpoint.clone())
525 .build()
526 .context("Failed to create OTLP exporter")?;
527
528 opentelemetry_sdk::trace::SdkTracerProvider::builder()
529 .with_resource(resource.clone())
530 .with_simple_exporter(exporter)
531 .build()
532 }
533 TelemetryProvider::Stdout => {
534 info!("Using stdout exporter for OpenTelemetry (debug)");
535
536 opentelemetry_sdk::trace::SdkTracerProvider::builder()
537 .with_resource(resource.clone())
538 .with_simple_exporter(opentelemetry_stdout::SpanExporter::default())
539 .build()
540 }
541 };
542
543 let meter_provider = match provider {
547 TelemetryProvider::Otlp => {
548 let exporter = opentelemetry_otlp::MetricExporter::builder()
549 .with_tonic()
550 .with_endpoint(endpoint)
551 .build()
552 .context("Failed to create OTLP metric exporter")?;
553
554 opentelemetry_sdk::metrics::SdkMeterProvider::builder()
555 .with_resource(resource.clone())
556 .with_reader(opentelemetry_sdk::metrics::PeriodicReader::builder(exporter).build())
557 .build()
558 }
559 TelemetryProvider::Stdout => opentelemetry_sdk::metrics::SdkMeterProvider::builder()
560 .with_resource(resource)
561 .with_reader(
562 opentelemetry_sdk::metrics::PeriodicReader::builder(
563 opentelemetry_stdout::MetricExporter::default(),
564 )
565 .build(),
566 )
567 .build(),
568 };
569 opentelemetry::global::set_meter_provider(meter_provider);
570
571 let tracer = tracer_provider.tracer("hyprstream");
572 let otel_layer = OpenTelemetryLayer::new(tracer);
573
574 let default_log_level = match provider {
575 TelemetryProvider::Otlp => "hyprstream=info,opentelemetry=info",
576 TelemetryProvider::Stdout => "hyprstream=warn",
577 };
578
579 let filter = EnvFilter::builder()
580 .parse_lossy(std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()));
581
582 let fmt_layer = tracing_subscriber::fmt::layer()
583 .with_target(true)
584 .with_file(true)
585 .with_line_number(true);
586
587 tracing_subscriber::registry()
588 .with(filter)
589 .with(fmt_layer)
590 .with(otel_layer)
591 .init();
592
593 info!("OpenTelemetry initialized with {:?} provider", provider);
594 Ok(())
595}
596
597fn handle_quick_command(
603 cmd: QuickCommand,
604 ctx: AppContext,
605 signing_key: SigningKey,
606 verifying_key: VerifyingKey,
607) -> Result<()> {
608 match cmd {
609 QuickCommand::Infer {
610 model,
611 prompt,
612 image,
613 max_tokens,
614 temperature,
615 top_p,
616 top_k,
617 repeat_penalty,
618 seed,
619 sync,
620 } => {
621 let prompt = match prompt {
623 Some(p) => p,
624 None => {
625 use std::io::Read;
626 let mut buffer = String::new();
627 std::io::stdin().read_to_string(&mut buffer)?;
628 buffer.trim().to_owned()
629 }
630 };
631
632 with_runtime(
633 RuntimeConfig {
634 device: DeviceConfig::request_gpu(),
635 multi_threaded: true,
636 },
637 || async move {
638 handle_infer(
639 &model,
640 &prompt,
641 image,
642 max_tokens,
643 temperature,
644 top_p,
645 top_k,
646 repeat_penalty,
647 seed,
648 sync,
649 signing_key,
650 )
651 .await
652 },
653 )
654 }
655
656 QuickCommand::Clone {
657 repo_url,
658 name,
659 branch,
660 depth,
661 full,
662 quiet,
663 verbose,
664 policy,
665 } => with_runtime(
666 RuntimeConfig {
667 device: DeviceConfig::request_cpu(),
668 multi_threaded: true,
669 },
670 || async move {
671 handle_clone(
672 ctx.registry(),
673 &repo_url,
674 name,
675 branch,
676 depth,
677 full,
678 quiet,
679 verbose,
680 policy,
681 )
682 .await
683 },
684 ),
685
686 QuickCommand::List { filter, status } => with_runtime(
687 RuntimeConfig {
688 device: DeviceConfig::request_cpu(),
689 multi_threaded: true,
690 },
691 || async move {
692 let keys_dir = ctx.models_dir().join(".registry").join("keys");
693 let signing_key = load_or_generate_signing_key(&keys_dir).await?;
694 let model_client = hyprstream_rpc_std::model_client::ModelClient::from_provider(
695 &hyprstream_discovery::ProductionRpcClientProvider,
696 signing_key,
697 None,
698 )?;
699 let filters = parse_filters(&filter)?;
700 let status_filter = parse_status_filter(&status)?;
701 handle_list(ctx.registry(), model_client, &filters, &status_filter).await
702 },
703 ),
704
705 QuickCommand::Info {
706 model,
707 verbose,
708 adapters_only,
709 } => with_runtime(
710 RuntimeConfig {
711 device: DeviceConfig::request_cpu(),
712 multi_threaded: true,
713 },
714 || async move {
715 match handle_info(ctx.registry(), &model, verbose, adapters_only).await {
716 Ok(()) => Ok(()),
717 Err(e) => {
718 eprintln!("Warning: Some operations failed: {}", e);
719 Ok(())
720 }
721 }
722 },
723 ),
724
725 QuickCommand::Status { model, verbose } => with_runtime(
726 RuntimeConfig {
727 device: DeviceConfig::request_cpu(),
728 multi_threaded: true,
729 },
730 || async move { handle_status(ctx.registry(), model, verbose).await },
731 ),
732
733 QuickCommand::Branch {
734 model,
735 name,
736 from,
737 policy,
738 } => with_runtime(
739 RuntimeConfig {
740 device: DeviceConfig::request_cpu(),
741 multi_threaded: true,
742 },
743 || async move { handle_branch(ctx.registry(), &model, &name, from, policy).await },
744 ),
745
746 QuickCommand::Checkout {
747 model,
748 git_ref,
749 create_branch,
750 force,
751 } => with_runtime(
752 RuntimeConfig {
753 device: DeviceConfig::request_cpu(),
754 multi_threaded: true,
755 },
756 || async move {
757 let git_ref_parsed = match git_ref {
758 Some(ref r) => GitRef::parse(r)?,
759 None => GitRef::DefaultBranch,
760 };
761 let model_ref = ModelRef::with_ref(model, git_ref_parsed);
762 handle_checkout(ctx.registry(), &model_ref.to_string(), create_branch, force).await
763 },
764 ),
765
766 QuickCommand::Pull {
767 model,
768 remote,
769 branch,
770 rebase,
771 } => with_runtime(
772 RuntimeConfig {
773 device: DeviceConfig::request_cpu(),
774 multi_threaded: true,
775 },
776 || async move { handle_pull(ctx.registry(), &model, remote, branch, rebase).await },
777 ),
778
779 QuickCommand::Remove {
780 model,
781 force,
782 registry_only,
783 files_only,
784 } => with_runtime(
785 RuntimeConfig {
786 device: DeviceConfig::request_cpu(),
787 multi_threaded: true,
788 },
789 || async move {
790 handle_remove(ctx.registry(), &model, force, registry_only, files_only).await
791 },
792 ),
793
794 QuickCommand::Load {
795 model,
796 max_context,
797 kv_quant,
798 wait,
799 } => with_runtime(
800 RuntimeConfig {
801 device: DeviceConfig::request_gpu(),
802 multi_threaded: true,
803 },
804 || async move { handle_load(&model, max_context, kv_quant.into(), wait, signing_key).await },
805 ),
806
807 QuickCommand::Unload { model } => with_runtime(
808 RuntimeConfig {
809 device: DeviceConfig::request_cpu(),
810 multi_threaded: true,
811 },
812 || async move { handle_unload(&model, signing_key).await },
813 ),
814
815 QuickCommand::Notify { command } => with_runtime(
816 RuntimeConfig {
817 device: DeviceConfig::request_cpu(),
818 multi_threaded: true,
819 },
820 || async move { handle_notify_command(command, signing_key).await },
821 ),
822
823 QuickCommand::Worktree { command } => with_runtime(
824 RuntimeConfig {
825 device: DeviceConfig::request_cpu(),
826 multi_threaded: true,
827 },
828 || async move {
829 match command {
830 WorktreeQuickCommand::Add {
831 model,
832 branch,
833 policy,
834 } => handle_worktree_add(ctx.registry(), &model, &branch, policy).await,
835 WorktreeQuickCommand::List { model, all } => {
836 handle_worktree_list(ctx.registry(), &model, all).await
837 }
838 WorktreeQuickCommand::Info { model, branch } => {
839 handle_worktree_info(ctx.registry(), &model, &branch).await
840 }
841 WorktreeQuickCommand::Remove {
842 model,
843 branch,
844 force,
845 } => handle_worktree_remove(ctx.registry(), &model, &branch, force).await,
846 }
847 },
848 ),
849
850 #[cfg(feature = "experimental")]
851 QuickCommand::Commit {
852 model,
853 message,
854 all,
855 all_untracked,
856 amend,
857 author,
858 author_name,
859 author_email,
860 allow_empty,
861 dry_run,
862 verbose,
863 } => with_runtime(
864 RuntimeConfig {
865 device: DeviceConfig::request_cpu(),
866 multi_threaded: true,
867 },
868 || async move {
869 handle_commit(
870 ctx.registry(),
871 &model,
872 &message,
873 all,
874 all_untracked,
875 amend,
876 author,
877 author_name,
878 author_email,
879 allow_empty,
880 dry_run,
881 verbose,
882 )
883 .await
884 },
885 ),
886
887 #[cfg(feature = "experimental")]
888 QuickCommand::Promote {
889 model,
890 branch,
891 author_name,
892 author_email,
893 } => with_runtime(
894 RuntimeConfig {
895 device: DeviceConfig::request_cpu(),
896 multi_threaded: true,
897 },
898 || async move {
899 handle_promote(ctx.registry(), &model, &branch, author_name, author_email).await
900 },
901 ),
902
903 #[cfg(feature = "experimental")]
904 QuickCommand::Push {
905 model,
906 remote,
907 branch,
908 set_upstream,
909 force,
910 } => with_runtime(
911 RuntimeConfig {
912 device: DeviceConfig::request_cpu(),
913 multi_threaded: true,
914 },
915 || async move {
916 handle_push(ctx.registry(), &model, remote, branch, set_upstream, force).await
917 },
918 ),
919
920 #[cfg(feature = "experimental")]
921 QuickCommand::Merge {
922 target,
923 source,
924 ff,
925 no_ff,
926 ff_only,
927 no_commit,
928 squash,
929 message,
930 abort,
931 continue_merge,
932 quit,
933 no_stat,
934 quiet,
935 verbose,
936 strategy,
937 strategy_option,
938 allow_unrelated_histories,
939 no_verify,
940 } => with_runtime(
941 RuntimeConfig {
942 device: DeviceConfig::request_cpu(),
943 multi_threaded: true,
944 },
945 || async move {
946 let options = MergeOptions {
947 ff,
948 no_ff,
949 ff_only,
950 no_commit,
951 squash,
952 message: message.clone(),
953 abort,
954 continue_merge,
955 quit,
956 no_stat,
957 quiet,
958 verbose,
959 strategy: strategy.clone(),
960 strategy_option: strategy_option.clone(),
961 allow_unrelated_histories,
962 no_verify,
963 };
964 handle_merge(ctx.registry(), &target, &source, options).await
965 },
966 ),
967
968 QuickCommand::Training(training_cmd) => with_runtime(
969 RuntimeConfig {
970 device: DeviceConfig::request_gpu(),
971 multi_threaded: true,
972 },
973 || async move {
974 let registry = ctx.registry();
975 match training_cmd.action {
976 TrainingAction::Init {
977 model,
978 branch,
979 adapter,
980 index,
981 rank,
982 alpha,
983 mode,
984 learning_rate,
985 } => {
986 handle_training_init(
987 registry,
988 &model,
989 branch,
990 adapter,
991 index,
992 rank,
993 alpha,
994 &mode,
995 learning_rate,
996 )
997 .await
998 }
999 TrainingAction::Infer {
1000 model,
1001 prompt,
1002 image,
1003 max_tokens,
1004 temperature,
1005 top_p,
1006 top_k,
1007 repeat_penalty,
1008 sync,
1009 max_context,
1010 kv_quant,
1011 } => {
1012 let prompt_text = match prompt {
1013 Some(p) => p,
1014 None => {
1015 use std::io::Read;
1016 let mut buffer = String::new();
1017 std::io::stdin().read_to_string(&mut buffer)?;
1018 buffer.trim().to_owned()
1019 }
1020 };
1021 handle_training_infer(
1022 registry,
1023 hyprstream_rpc::registry::global().endpoint("policy", SocketKind::Rep),
1024 &model,
1025 &prompt_text,
1026 image,
1027 max_tokens,
1028 temperature,
1029 top_p,
1030 top_k,
1031 repeat_penalty,
1032 sync,
1033 max_context,
1034 kv_quant,
1035 signing_key,
1036 verifying_key,
1037 )
1038 .await
1039 }
1040 TrainingAction::Batch {
1041 model,
1042 input,
1043 input_dir,
1044 pattern,
1045 format,
1046 max_tokens,
1047 chunk_size,
1048 skip,
1049 limit,
1050 progress_interval,
1051 checkpoint_interval,
1052 test_set,
1053 max_context,
1054 kv_quant,
1055 } => {
1056 handle_training_batch(
1057 registry,
1058 hyprstream_rpc::registry::global().endpoint("policy", SocketKind::Rep),
1059 &model,
1060 input,
1061 input_dir,
1062 &pattern,
1063 &format,
1064 max_tokens,
1065 chunk_size,
1066 skip,
1067 limit,
1068 progress_interval,
1069 checkpoint_interval,
1070 test_set,
1071 max_context,
1072 kv_quant,
1073 signing_key,
1074 verifying_key,
1075 )
1076 .await
1077 }
1078 TrainingAction::Checkpoint {
1079 model,
1080 message,
1081 push,
1082 remote,
1083 } => handle_training_checkpoint(registry, &model, message, push, &remote).await,
1084 }
1085 },
1086 ),
1087
1088 QuickCommand::Policy { command } => with_runtime(
1089 RuntimeConfig {
1090 device: DeviceConfig::request_cpu(),
1091 multi_threaded: true,
1092 },
1093 || async move {
1094 let registry_path = ctx.models_dir().join(".registry");
1095 let policies_dir = registry_path.join("policies");
1096 let keys_dir = registry_path.join("keys");
1097
1098 let signing_key = load_or_generate_signing_key(&keys_dir).await?;
1100
1101 tokio::fs::create_dir_all(&policies_dir).await?;
1103 let policy_csv_path = policies_dir.join("policy.csv");
1104
1105 match command {
1106 PolicyCommand::Show { raw } => handle_policy_show(&signing_key, raw).await,
1107 PolicyCommand::History { count, oneline } => {
1108 handle_policy_history(&signing_key, count, oneline).await
1109 }
1110 PolicyCommand::Edit => handle_policy_edit(&signing_key, &policy_csv_path).await,
1111 PolicyCommand::Diff { against } => {
1112 handle_policy_diff(&signing_key, against).await
1113 }
1114 PolicyCommand::Apply { dry_run, message } => {
1115 handle_policy_apply(&signing_key, dry_run, message).await
1116 }
1117 PolicyCommand::Rollback { git_ref, dry_run } => {
1118 handle_policy_rollback(&signing_key, &git_ref, dry_run).await
1119 }
1120 PolicyCommand::Check {
1121 user,
1122 resource,
1123 action,
1124 } => handle_policy_check(&signing_key, &user, &resource, &action).await,
1125 PolicyCommand::Token { command: token_cmd } => match token_cmd {
1126 TokenCommand::Create {
1127 user,
1128 name,
1129 expires,
1130 scope,
1131 } => handle_token_create(&signing_key, &user, name, &expires, scope).await,
1132 TokenCommand::List { user: _ } => {
1133 println!("JWT tokens are stateless and cannot be listed.");
1134 println!();
1135 println!("Tokens are validated by signature and expiry time.");
1136 println!("To see who has access, review the policy:");
1137 println!(" hyprstream quick policy show");
1138 Ok(())
1139 }
1140 TokenCommand::Revoke {
1141 token: _,
1142 name: _,
1143 revoke_user: _,
1144 force: _,
1145 } => {
1146 println!("JWT tokens cannot be revoked individually.");
1147 println!();
1148 println!("Options for token invalidation:");
1149 println!(" 1. Use short expiry times (e.g., --expires 1d)");
1150 println!(" 2. Regenerate the signing key to invalidate all tokens:");
1151 println!(
1152 " rm ~/.local/share/hyprstream/.registry/keys/signing.key"
1153 );
1154 println!(" 3. Remove user permissions via policy:");
1155 println!(" hyprstream quick policy edit");
1156 println!();
1157 println!("A token blocklist feature may be added in the future.");
1158 Ok(())
1159 }
1160 },
1161 PolicyCommand::ApplyTemplate { template, dry_run } => {
1162 handle_policy_apply_template(&signing_key, &template, dry_run).await
1163 }
1164 PolicyCommand::ListTemplates => handle_policy_list_templates().await,
1165 PolicyCommand::Role { command } => match command {
1166 RoleCommand::Add {
1167 user,
1168 role,
1169 dry_run,
1170 } => handle_policy_role_add(&signing_key, &user, &role, dry_run).await,
1171 RoleCommand::Remove { user, role, force } => {
1172 handle_policy_role_remove(&signing_key, &user, &role, force).await
1173 }
1174 RoleCommand::List { user, role } => {
1175 handle_policy_role_list(&signing_key, user.as_deref(), role.as_deref())
1176 .await
1177 }
1178 },
1179 }
1180 },
1181 ),
1182
1183 QuickCommand::Worker { command: action } => {
1184 let signing_key = signing_key.clone();
1185 with_runtime(
1186 RuntimeConfig {
1187 device: DeviceConfig::request_cpu(),
1188 multi_threaded: true,
1189 },
1190 || async move {
1191 let worker_already_running =
1192 ctx.config().services.startup.contains(&"worker".to_owned());
1193
1194 if worker_already_running {
1195 info!(
1196 "WorkerService is managed by systemd, connecting to existing service"
1197 );
1198 } else {
1199 info!("Starting WorkerService for CLI worker commands");
1200 }
1201
1202 let data_dir = dirs::data_local_dir()
1203 .unwrap_or_else(|| std::path::PathBuf::from("."))
1204 .join("hyprstream");
1205 let runtime_dir = dirs::runtime_dir()
1206 .unwrap_or_else(std::env::temp_dir)
1207 .join("hyprstream");
1208
1209 let kata_boot_path = std::env::var("KATA_BOOT_PATH")
1210 .map(std::path::PathBuf::from)
1211 .unwrap_or_else(|_| {
1212 std::path::PathBuf::from("/opt/kata/share/kata-containers")
1213 });
1214
1215 let pool_config = PoolConfig {
1216 warm_pool_size: 0,
1217 runtime_dir: runtime_dir.join("sandboxes"),
1218 kernel_path: kata_boot_path.join("vmlinux.container"),
1219 vm_image: kata_boot_path.join("kata-containers.img"),
1220 cloud_init_dir: data_dir.join("cloud-init"),
1221 ..PoolConfig::default()
1222 };
1223 #[cfg(feature = "oci-image")]
1224 let image_config = ImageConfig {
1225 blobs_dir: data_dir.join("images/blobs"),
1226 bootstrap_dir: data_dir.join("images/bootstrap"),
1227 refs_dir: data_dir.join("images/refs"),
1228 cache_dir: data_dir.join("images/cache"),
1229 runtime_dir: runtime_dir.join("nydus"),
1230 ..ImageConfig::default()
1231 };
1232
1233 let _worker_handle = if !worker_already_running {
1234 use hyprstream_workers::runtime::{
1235 resolve_backend, BackendCtx, SandboxBackend,
1236 };
1237
1238 #[cfg(feature = "oci-image")]
1243 let rafs_store = {
1244 use hyprstream_workers::image::RafsStore;
1245 Arc::new(RafsStore::new(image_config.clone())?)
1246 };
1247
1248 let backend_name: String = ctx
1253 .config()
1254 .worker
1255 .as_ref()
1256 .map(|w| w.backend.clone())
1257 .unwrap_or_else(|| "auto".to_owned());
1258 let ninep_decider = hyprstream_core::mac::production_ninep_decider(
1259 signing_key.clone(),
1260 &ctx.config().oauth,
1261 "ninep-worker",
1262 )
1263 .await
1264 .context("construct worker 9P MAC PEP")?;
1265 let ninep_monitor =
1271 Some(hyprstream_core::mac::enrollment_ninep_reference_monitor(
1272 Arc::clone(&ninep_decider),
1273 ));
1274 let backend_ctx = BackendCtx {
1275 pool_config: pool_config.clone(),
1276 ninep_decider,
1277 ninep_monitor,
1278 #[cfg(feature = "oci-image")]
1279 image_config,
1280 #[cfg(feature = "oci-image")]
1281 rafs_store: Arc::clone(&rafs_store),
1282 };
1283 let backend: Arc<dyn SandboxBackend> =
1284 resolve_backend(&backend_name, &backend_ctx)?;
1285
1286 let worker_transport = TransportConfig::inproc("hyprstream/workers");
1287 let mut worker_service = WorkerService::new(
1288 pool_config,
1289 backend,
1290 #[cfg(any(feature = "kata", feature = "kata-vm"))]
1295 Some(rafs_store),
1296 #[cfg(not(any(feature = "kata", feature = "kata-vm")))]
1297 None,
1298 worker_transport,
1299 signing_key.clone(),
1300 )?;
1301
1302 let worker_policy_client =
1304 hyprstream_core::services::policy_client_for_process(
1305 signing_key.clone(),
1306 resolve_service_vk("policy", Some(ctx.config())).ok_or_else(
1307 || anyhow::anyhow!("Cannot resolve policy pubkey. Run wizard."),
1308 )?,
1309 None,
1310 )?;
1311 worker_service.set_authorize_fn(
1312 hyprstream_core::services::build_authorize_fn(worker_policy_client),
1313 );
1314
1315 install_envelope_verify_config(None, None);
1322
1323 let manager = InprocManager::new();
1324 Some(
1325 manager
1326 .spawn(Box::new(worker_service))
1327 .await
1328 .context("Failed to start worker service")?,
1329 )
1330 } else {
1331 None
1332 };
1333
1334 use hyprstream_rpc_std::worker_client::WorkerClient;
1335 let worker_client = if worker_already_running {
1336 WorkerClient::from_provider(
1337 &hyprstream_discovery::ProductionRpcClientProvider,
1338 signing_key,
1339 None,
1340 )?
1341 } else {
1342 let destination = signing_key.verifying_key();
1343 WorkerClient::for_local_bootstrap(signing_key, destination, None)?
1344 };
1345
1346 match action {
1347 WorkerAction::List {
1348 sandbox,
1349 container,
1350 containers,
1351 sandboxes,
1352 state,
1353 verbose,
1354 } => {
1355 handle_worker_list(
1356 &worker_client,
1357 sandbox,
1358 container,
1359 containers,
1360 sandboxes,
1361 state,
1362 verbose,
1363 )
1364 .await
1365 }
1366 WorkerAction::Run {
1367 image,
1368 command,
1369 sandbox,
1370 name,
1371 env,
1372 workdir,
1373 detach,
1374 rm,
1375 } => {
1376 handle_worker_run(
1377 &worker_client,
1378 &image,
1379 command,
1380 sandbox,
1381 name,
1382 env,
1383 workdir,
1384 detach,
1385 rm,
1386 )
1387 .await
1388 }
1389 WorkerAction::Stop { id, timeout, force } => {
1390 handle_worker_stop(&worker_client, &id, timeout, force).await
1391 }
1392 WorkerAction::Start { container_id } => {
1393 handle_worker_start(&worker_client, &container_id).await
1394 }
1395 WorkerAction::Restart { id, timeout } => {
1396 handle_worker_restart(&worker_client, &id, timeout).await
1397 }
1398 WorkerAction::Rm {
1399 ids,
1400 force,
1401 volumes: _,
1402 } => handle_worker_rm(&worker_client, ids, force).await,
1403 WorkerAction::Status { id, verbose } => {
1404 handle_worker_status(&worker_client, &id, verbose).await
1405 }
1406 WorkerAction::Stats {
1407 ids,
1408 stream: _,
1409 no_header,
1410 } => handle_worker_stats(&worker_client, ids, no_header).await,
1411 WorkerAction::Exec {
1412 container_id,
1413 command,
1414 timeout,
1415 } => {
1416 handle_worker_exec(&worker_client, &container_id, command, timeout)
1417 .await
1418 }
1419 WorkerAction::Terminal {
1420 container_id,
1421 detach_keys,
1422 } => {
1423 handle_worker_terminal(&worker_client, &container_id, &detach_keys)
1424 .await
1425 }
1426 WorkerAction::Images(img_cmd) => match img_cmd {
1427 ImageCommand::List { verbose, filter: _ } => {
1428 handle_images_list(&worker_client, verbose).await
1429 }
1430 ImageCommand::Pull {
1431 image,
1432 username,
1433 password,
1434 } => {
1435 handle_images_pull(&worker_client, &image, username, password).await
1436 }
1437 ImageCommand::Rm { images, force } => {
1438 handle_images_rm(&worker_client, images, force).await
1439 }
1440 ImageCommand::Df => handle_images_df(&worker_client).await,
1441 },
1442 }
1443 },
1444 )
1445 }
1446
1447 QuickCommand::Remote { command } => with_runtime(
1448 RuntimeConfig {
1449 device: DeviceConfig::request_cpu(),
1450 multi_threaded: true,
1451 },
1452 || async move {
1453 match command {
1454 RemoteQuickCommand::Add { model, name, url } => {
1455 handle_remote_add(ctx.registry(), &model, &name, &url).await
1456 }
1457 RemoteQuickCommand::List { model, verbose } => {
1458 handle_remote_list(ctx.registry(), &model, verbose).await
1459 }
1460 RemoteQuickCommand::Remove { model, name } => {
1461 handle_remote_remove(ctx.registry(), &model, &name).await
1462 }
1463 RemoteQuickCommand::SetUrl { model, name, url } => {
1464 handle_remote_set_url(ctx.registry(), &model, &name, &url).await
1465 }
1466 RemoteQuickCommand::Rename {
1467 model,
1468 old_name,
1469 new_name,
1470 } => handle_remote_rename(ctx.registry(), &model, &old_name, &new_name).await,
1471 }
1472 },
1473 ),
1474 }
1475}
1476
1477fn resolve_moq_relay_reach(
1499 relay_uri: &str,
1500) -> Result<hyprstream_rpc::stream_info::TransportConfig> {
1501 use serde_json::json;
1502 let (uri, cert_hashes): (&str, Vec<String>) = match relay_uri.split_once('#') {
1505 Some((u, frag)) => (u, frag.split(',').map(str::to_owned).collect()),
1506 None => (relay_uri, Vec::new()),
1507 };
1508 let webpki = cert_hashes.is_empty();
1509 let entry = json!({
1510 "type": "QuicTransport",
1511 "serviceEndpoint": {
1512 "uri": uri,
1513 "webpki": webpki,
1514 "certHashes": cert_hashes,
1515 "accept": ["moql"],
1516 },
1517 });
1518 let decoded = hyprstream_rpc::service_entry::decode_service_entry(&entry)
1519 .map_err(|e| anyhow::anyhow!("relay URI decode: {e}"))?;
1520 hyprstream_rpc::moq_stream::relay_reach_from_decoded(&decoded.config)
1521 .ok_or_else(|| anyhow::anyhow!("relay URI is not a network-routable transport"))
1522}
1523
1524fn foreground_service_process_names(
1527 matches: &clap::ArgMatches,
1528 config: &HyprConfig,
1529) -> Result<Option<Vec<String>>> {
1530 let Some(("service", service_matches)) = matches.subcommand() else {
1531 return Ok(None);
1532 };
1533 let action = ServiceAction::from_arg_matches(service_matches)
1534 .map_err(|error| anyhow::anyhow!("{error}"))?;
1535 let ServiceAction::Start {
1536 name,
1537 foreground,
1538 standalone,
1539 services,
1540 ..
1541 } = action
1542 else {
1543 return Ok(None);
1544 };
1545 if !foreground && !standalone {
1546 return Ok(None);
1547 }
1548 let names = if standalone {
1549 config.services.startup.clone()
1550 } else {
1551 services.unwrap_or_else(|| name.into_iter().collect())
1552 };
1553 Ok(Some(names))
1554}
1555
1556fn dedicated_service_process_name(
1558 matches: &clap::ArgMatches,
1559 config: &HyprConfig,
1560) -> Result<Option<String>> {
1561 let Some(names) = foreground_service_process_names(matches, config)? else {
1562 return Ok(None);
1563 };
1564 match names.as_slice() {
1565 [service] if get_factory(service).is_some() => Ok(Some(service.clone())),
1566 _ => Ok(None),
1567 }
1568}
1569
1570fn native_service_process_name(
1573 matches: &clap::ArgMatches,
1574 config: &HyprConfig,
1575) -> Result<Option<String>> {
1576 if !config.quic.iroh_required() {
1577 return Ok(None);
1578 }
1579 let Some(names) = foreground_service_process_names(matches, config)? else {
1580 return Ok(None);
1581 };
1582 anyhow::ensure!(names.len() == 1,
1583 "network-iroh-required requires exactly one service per foreground process; launch each provisioned service separately");
1584 let service = names
1585 .into_iter()
1586 .next()
1587 .context("native service name missing")?;
1588 anyhow::ensure!(
1589 hyprstream_service::get_factory(&service).is_some(),
1590 "unknown native service: {service}"
1591 );
1592 Ok(Some(service))
1593}
1594
1595fn process_identity_name<'a>(
1602 native_service_name: Option<&'a str>,
1603 dedicated_service_name: Option<&'a str>,
1604) -> Option<&'a str> {
1605 native_service_name.or(dedicated_service_name)
1606}
1607
1608fn flat_key_policy_resolver_credential(
1617 secrets: &std::path::Path,
1618 profile: hyprstream_core::auth::identity_store::SecretsProfile,
1619 process_key: &SigningKey,
1620) -> Option<String> {
1621 let policy_key = hyprstream_core::auth::identity_store::resolve_service_signing_key(
1622 secrets, "policy", profile,
1623 )
1624 .ok()?;
1625 if policy_key.verifying_key() != process_key.verifying_key() {
1626 return None;
1627 }
1628 hyprstream_core::auth::identity_store::load_service_jwt_for_profile(secrets, "policy", profile)
1629 .ok()
1630 .flatten()
1631}
1632
1633async fn load_process_signing_key(
1637 config: &HyprConfig,
1638 native_service: Option<&str>,
1639) -> Result<SigningKey> {
1640 if let Some(service) = native_service {
1641 let secrets = HyprConfig::resolve_secrets_dir_for(Some(config))?;
1642 return hyprstream_core::auth::identity_store::load_existing_service_signing_key(
1643 &secrets,
1644 service,
1645 hyprstream_core::auth::identity_store::SecretsProfile::from_env()?,
1646 )
1647 .with_context(|| format!("load provisioned native identity for {service}"));
1648 }
1649 let keys_dir = config.models_dir().join(".registry").join("keys");
1650 load_or_generate_signing_key(&keys_dir).await
1651}
1652
1653fn select_single_process_moql_admission_proof<T>(
1658 proofs: impl IntoIterator<Item = (String, Option<T>)>,
1659) -> Result<Option<T>> {
1660 let proofs: Vec<_> = proofs.into_iter().collect();
1661 let authenticated: Vec<_> = proofs
1662 .iter()
1663 .filter_map(|(service, proof)| proof.as_ref().map(|_| service.as_str()))
1664 .collect();
1665
1666 if authenticated.is_empty() {
1667 return Ok(None);
1668 }
1669 if proofs.len() != 1 {
1670 anyhow::bail!(
1671 "native MoQ admission proofs are service-scoped, but this QUIC process contains \
1672 checkpointed service(s) [{}] with {} total services; run each checkpointed service \
1673 in a separate process",
1674 authenticated.join(", "),
1675 proofs.len(),
1676 );
1677 }
1678
1679 Ok(proofs.into_iter().next().and_then(|(_, proof)| proof))
1680}
1681
1682fn select_iroh_moql_admission_proof<T>(
1686 iroh_enabled: bool,
1687 proofs: impl FnOnce() -> Result<Vec<(String, Option<T>)>>,
1688) -> Result<Option<T>> {
1689 if iroh_enabled {
1690 select_single_process_moql_admission_proof(proofs()?)
1691 } else {
1692 Ok(None)
1693 }
1694}
1695
1696fn resolve_service_vk(service_name: &str, config: Option<&HyprConfig>) -> Option<VerifyingKey> {
1697 let trust = hyprstream_service::global_trust_store();
1698 if let Some(vk) = trust.resolve_one(service_name) {
1700 return Some(vk);
1701 }
1702 let Ok(secrets_dir) = HyprConfig::resolve_secrets_dir_for(config) else {
1705 return None;
1706 };
1707 let entries =
1714 match hyprstream_core::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir) {
1715 Ok(entries) => entries,
1716 Err(e) => {
1717 tracing::error!("bootstrap-pubkeys could not be read: {e:#}");
1718 return None;
1719 }
1720 };
1721 if let Err(e) = hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)
1727 {
1728 tracing::error!("{e:#}");
1729 return None;
1730 }
1731 for (name, entry) in &entries {
1732 trust.insert(
1733 entry.ed25519,
1734 hyprstream_service::Attestation {
1735 scopes: std::iter::once(name.clone()).collect(),
1736 subject: None,
1737 jwt: None,
1738 expires_at: 0,
1739 attested_by: None,
1740 },
1741 );
1742 }
1743 trust.resolve_one(service_name)
1744}
1745
1746async fn install_process_production_resolver(
1751 signing_key: &SigningKey,
1752 config: &HyprConfig,
1753 service_token: Option<String>,
1754 ephemeral_outbound_transport: bool,
1755) -> Result<bool> {
1756 let trust_source = hyprstream_discovery::DeploymentTrustSource::from_anchors(
1757 config.cluster_at9p_did.as_deref(),
1758 config.cluster_did_web.as_deref(),
1759 )?;
1760 if let Ok(secrets_dir) = HyprConfig::resolve_secrets_dir_for(Some(config)) {
1770 let entries =
1771 hyprstream_core::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir)?;
1772 hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)?;
1773 if matches!(
1778 trust_source,
1779 hyprstream_discovery::DeploymentTrustSource::OsOwnedFiles
1780 ) {
1781 hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_enrolled(
1782 &secrets_dir,
1783 &entries,
1784 )?;
1785 }
1786 }
1787
1788 let _ = resolve_service_vk("discovery", Some(config));
1793 let trust_source = match (&trust_source, &config.cluster_anchor_root_cert) {
1797 (hyprstream_discovery::DeploymentTrustSource::DidAnchored(anchors), Some(path)) => {
1798 let pem = std::fs::read(path).with_context(|| {
1799 format!(
1800 "cluster_anchor_root_cert is not readable: {}",
1801 path.display()
1802 )
1803 })?;
1804 hyprstream_discovery::DeploymentTrustSource::DidAnchored(
1805 anchors.clone().with_root_cert_pem(pem),
1806 )
1807 }
1808 (hyprstream_discovery::DeploymentTrustSource::OsOwnedFiles, Some(path)) => {
1809 anyhow::bail!(
1810 "cluster_anchor_root_cert ({}) is set but no DID anchors are configured",
1811 path.display()
1812 )
1813 }
1814 _ => trust_source,
1815 };
1816 let is_os_owned_bootstrap = matches!(
1820 trust_source,
1821 hyprstream_discovery::DeploymentTrustSource::OsOwnedFiles
1822 );
1823 if is_os_owned_bootstrap && config.cluster_remote_node {
1831 anyhow::bail!(
1832 "cluster_remote_node is set but no DID anchors are configured; \
1833 remote-node Discovery reach requires cluster_at9p_did and \
1834 cluster_did_web (DidAnchored mode) — the OS-owned trust source \
1835 has no remote-Discovery story and would silently ignore this flag"
1836 );
1837 }
1838 if config.quic.iroh_required() {
1839 hyprstream_rpc::moq_stream::require_native_iroh();
1840 }
1841 let records = config
1846 .rds
1847 .resolved_from_env()
1848 .context("failed to resolve the records-role binding")?;
1849 if ephemeral_outbound_transport {
1850 hyprstream_discovery::bootstrap_deployment_process_with_ephemeral_client_transport(
1851 signing_key.clone(),
1852 service_token,
1853 trust_source,
1854 config.cluster_remote_node,
1855 config.quic.iroh_required(),
1856 &records,
1857 )
1858 .await?;
1859 } else {
1860 hyprstream_discovery::bootstrap_deployment_process(
1861 signing_key.clone(),
1862 service_token,
1863 trust_source,
1864 config.cluster_remote_node,
1865 config.quic.iroh_required(),
1866 &records,
1867 )
1868 .await?;
1869 }
1870 hyprstream_rpc::envelope::install_browser_currentness_verifier(
1871 hyprstream_discovery::production_browser_currentness_verifier()?,
1872 )?;
1873 Ok(is_os_owned_bootstrap)
1874}
1875
1876#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1883enum QuicCheckpointPolicy {
1884 CheckpointedAnnouncements,
1885 DeferForFirstBoot,
1886}
1887
1888fn quic_checkpoint_policy(
1889 is_os_owned_bootstrap: bool,
1890 classify: impl FnOnce() -> Result<hyprstream_core::services::factories::PdsBootState>,
1891) -> Result<QuicCheckpointPolicy> {
1892 if !is_os_owned_bootstrap {
1893 return Ok(QuicCheckpointPolicy::CheckpointedAnnouncements);
1894 }
1895
1896 use hyprstream_core::services::factories::PdsBootState;
1897 match classify()? {
1898 PdsBootState::Populated => Ok(QuicCheckpointPolicy::CheckpointedAnnouncements),
1899 PdsBootState::FirstBoot => Ok(QuicCheckpointPolicy::DeferForFirstBoot),
1900 }
1901}
1902
1903fn install_envelope_verify_config(
1923 oauth: Option<&hyprstream_core::config::OAuthConfig>,
1924 config: Option<&HyprConfig>,
1925) {
1926 use hyprstream_rpc::envelope::{
1927 install_response_verify_config, install_verify_config, mandatory_envelope_policy,
1928 EnvelopeVerifyConfig, KeyedPqTrustStore, PqTrustStore, ResponseVerifyConfig,
1929 };
1930
1931 let policy = mandatory_envelope_policy();
1934
1935 let mut keyed_store = match oauth {
1939 Some(oauth) => hyprstream_core::auth::mesh_trust::build_mesh_pq_trust_store(oauth),
1940 None => KeyedPqTrustStore::new(),
1941 };
1942
1943 if let Ok(secrets_dir) = HyprConfig::resolve_secrets_dir_for(config) {
1950 let anchored = hyprstream_core::auth::mesh_trust::seed_bootstrap_pq_bindings(
1951 &mut keyed_store,
1952 &secrets_dir,
1953 );
1954 tracing::info!("bootstrap PQ trust store seeded with {anchored} service binding(s)");
1955 } else {
1956 tracing::warn!(
1957 "secrets directory unresolvable; bootstrap PQ bindings not seeded. \
1958 Under Hybrid envelope policy local services may be unverifiable."
1959 );
1960 }
1961
1962 tracing::info!(
1963 "mesh PQ trust store installed with {} peer binding(s)",
1964 keyed_store.len()
1965 );
1966
1967 if let Some(oauth) = oauth {
1975 let roster = hyprstream_core::auth::mesh_trust::build_mesh_identity_roster(oauth);
1976 let trust = hyprstream_service::global_trust_store();
1977 for (ed_pubkey, subject) in &roster {
1978 if let Ok(vk) = VerifyingKey::from_bytes(ed_pubkey) {
1979 trust.insert(
1980 vk,
1981 hyprstream_service::Attestation {
1982 scopes: std::iter::once("inference".to_owned()).collect(),
1983 subject: subject.name().map(ToOwned::to_owned),
1984 jwt: None,
1985 expires_at: 0,
1987 attested_by: None,
1988 },
1989 );
1990 }
1991 }
1992 tracing::info!(
1993 "mesh identity roster installed with {} per-host binding(s)",
1994 roster.len()
1995 );
1996 }
1997 let pq_store: std::sync::Arc<dyn PqTrustStore> = std::sync::Arc::new(keyed_store);
2003
2004 let _ = install_response_verify_config(ResponseVerifyConfig {
2009 policy,
2010 pq_store: Some(pq_store.clone()),
2011 });
2012
2013 if install_verify_config(EnvelopeVerifyConfig {
2014 policy,
2015 pq_store: Some(pq_store),
2016 })
2017 .is_ok()
2018 {
2019 tracing::info!(
2020 "envelope verify policy: HYBRID enforced (SNS nested COSE); \
2021 peer ML-DSA bindings required for cross-node traffic"
2022 );
2023 }
2024
2025 install_session_pq_overlay();
2026 install_proof_admission(oauth, config);
2027}
2028
2029#[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
2035struct PostgresProofReplayStore {
2036 kv: hyprstream_pds::pgsql_kv::PgKv,
2037}
2038
2039#[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
2040impl PostgresProofReplayStore {
2041 fn digest(parts: &[&[u8]]) -> [u8; 32] {
2042 use sha2::{Digest as _, Sha256};
2043 let mut digest = Sha256::new();
2044 digest.update(b"hyprstream/proof-replay-admission/v1\0");
2045 for part in parts {
2046 digest.update((part.len() as u64).to_be_bytes());
2047 digest.update(part);
2048 }
2049 digest.finalize().into()
2050 }
2051
2052 fn record(
2053 &self,
2054 parts: &[&[u8]],
2055 expires_at: u64,
2056 ) -> hyprstream_rpc::proof::admission::ProofAdmissionResult {
2057 match self.kv.replay_admit(Self::digest(parts), expires_at) {
2058 Ok(true) => hyprstream_rpc::proof::admission::ProofAdmissionResult::Admitted,
2059 Ok(false) => hyprstream_rpc::proof::admission::ProofAdmissionResult::Replayed,
2060 Err(_) => hyprstream_rpc::proof::admission::ProofAdmissionResult::Failed,
2063 }
2064 }
2065}
2066
2067#[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
2068impl hyprstream_rpc::proof::admission::ProofReplayStore for PostgresProofReplayStore {
2069 fn domain_guarantee(&self) -> hyprstream_rpc::proof::admission::ReplayDomainGuarantee {
2070 hyprstream_rpc::proof::admission::ReplayDomainGuarantee::LinearizableSharedStore
2071 }
2072
2073 fn check_and_insert(
2074 &self,
2075 partition: hyprstream_rpc::proof::ProofDisposition,
2076 key: &hyprstream_rpc::proof::admission::ProofReplayKey,
2077 expires_at: u64,
2078 ) -> hyprstream_rpc::proof::admission::ProofAdmissionResult {
2079 let partition = match partition {
2080 hyprstream_rpc::proof::ProofDisposition::Authenticated => &b"authenticated"[..],
2081 hyprstream_rpc::proof::ProofDisposition::Unattributed => &b"unattributed"[..],
2082 };
2083 self.record(
2084 &[
2085 b"request-proof",
2086 partition,
2087 &key.signer_thumbprint,
2088 &key.request_id,
2089 ],
2090 expires_at,
2091 )
2092 }
2093
2094 fn check_and_insert_mediated(
2095 &self,
2096 key: &hyprstream_rpc::proof::admission::MediatedQueryReplayKey,
2097 expires_at: u64,
2098 ) -> hyprstream_rpc::proof::admission::ProofAdmissionResult {
2099 self.record(
2100 &[
2101 b"mediated-query",
2102 &key.signer_thumbprint,
2103 &key.request_id.to_be_bytes(),
2104 &key.request_nonce,
2105 key.mediator.as_bytes(),
2106 key.resource.as_bytes(),
2107 key.operation.as_bytes(),
2108 ],
2109 expires_at,
2110 )
2111 }
2112
2113 fn check_and_insert_credential(
2114 &self,
2115 key: &hyprstream_rpc::proof::admission::CredentialReplayKey,
2116 expires_at: u64,
2117 ) -> hyprstream_rpc::proof::admission::ProofAdmissionResult {
2118 self.record(
2119 &[
2120 b"source-credential",
2121 &key.issuer_client_namespace,
2122 &key.credential_identity,
2123 ],
2124 expires_at,
2125 )
2126 }
2127}
2128
2129#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2148enum ProofReplayStoreSelection {
2149 #[cfg(feature = "postgres-replay")]
2150 DedicatedPostgresV1,
2151 #[cfg(not(feature = "postgres-replay"))]
2152 LegacyPdsPostgres,
2153 SingleVerifierMemory,
2154 Unavailable,
2155}
2156
2157fn select_proof_replay_store(
2158 guarantee: hyprstream_rpc::proof::admission::ReplayDomainGuarantee,
2159 legacy_pds_store_available: bool,
2160) -> ProofReplayStoreSelection {
2161 use hyprstream_rpc::proof::admission::ReplayDomainGuarantee;
2162
2163 match guarantee {
2164 #[cfg(feature = "postgres-replay")]
2165 ReplayDomainGuarantee::LinearizableSharedStore => {
2166 ProofReplayStoreSelection::DedicatedPostgresV1
2169 }
2170 #[cfg(not(feature = "postgres-replay"))]
2171 ReplayDomainGuarantee::LinearizableSharedStore => {
2172 if legacy_pds_store_available {
2173 ProofReplayStoreSelection::LegacyPdsPostgres
2174 } else {
2175 ProofReplayStoreSelection::Unavailable
2176 }
2177 }
2178 ReplayDomainGuarantee::SingleVerifierInstance if !legacy_pds_store_available => {
2179 ProofReplayStoreSelection::SingleVerifierMemory
2180 }
2181 _ => ProofReplayStoreSelection::Unavailable,
2182 }
2183}
2184
2185fn install_proof_admission(
2186 oauth: Option<&hyprstream_core::config::OAuthConfig>,
2187 config: Option<&HyprConfig>,
2188) {
2189 use hyprstream_rpc::proof::admission::{
2190 set_global_challenge_manager, set_global_proof_replay_store, InMemoryProofReplayStore,
2191 ProofReplayStore, ReplayDomainGuarantee,
2192 };
2193 use hyprstream_rpc::proof::challenge::{
2194 ChallengeManager, DEFAULT_CHALLENGE_OVERLAP_SECS, DEFAULT_CHALLENGE_WINDOW_SECS,
2195 };
2196
2197 #[cfg(feature = "postgres-replay")]
2198 let _ = config;
2199
2200 let now = std::time::SystemTime::now()
2201 .duration_since(std::time::UNIX_EPOCH)
2202 .map(|d| d.as_secs())
2203 .unwrap_or(0);
2204
2205 match hyprstream_rpc::proof::policy::install_generated_method_policy() {
2211 Ok(rows) => {
2212 tracing::info!("generated dispatch method policy installed ({rows} leaf row(s))");
2213 }
2214 Err(e) => tracing::error!(
2215 "generated dispatch method policy failed validation/install; \
2216 proof-bearing dispatch denies: {e:#}"
2217 ),
2218 }
2219
2220 #[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
2226 let shared_store: Option<Box<dyn ProofReplayStore>> = match config {
2227 Some(config) => match config.rds.resolved_from_env() {
2228 Ok(rds) if rds.is_configured() => match rds.connect_replay_admission() {
2229 Ok(kv) => Some(Box::new(PostgresProofReplayStore { kv })),
2230 Err(_) => {
2231 tracing::error!("configured shared Postgres replay store could not open; proof admission denies");
2232 return;
2233 }
2234 },
2235 Ok(_) => None,
2236 Err(_) => {
2237 tracing::error!(
2238 "records-role RDS binding could not be resolved; proof admission denies"
2239 );
2240 return;
2241 }
2242 },
2243 None => None,
2244 };
2245 #[cfg(feature = "postgres-replay")]
2246 let shared_store: Option<Box<dyn ProofReplayStore>> = None;
2247 #[cfg(all(not(feature = "pds-postgres"), not(feature = "postgres-replay")))]
2248 let shared_store: Option<Box<dyn ProofReplayStore>> = {
2249 if let Some(config) = config {
2250 match config.rds.resolved_from_env() {
2251 Ok(rds) if rds.is_configured() => {
2252 tracing::error!("configured shared Postgres replay store requires the pds-postgres feature; proof admission denies");
2253 return;
2254 }
2255 Ok(_) => {}
2256 Err(_) => {
2257 tracing::error!(
2258 "records-role RDS binding could not be resolved; proof admission denies"
2259 );
2260 return;
2261 }
2262 }
2263 }
2264 None
2265 };
2266
2267 let declared_domain = std::env::var("HYPRSTREAM_REPLAY_ADMISSION_DOMAIN").ok();
2278 let guarantee = if shared_store.is_some() {
2279 Some(ReplayDomainGuarantee::LinearizableSharedStore)
2280 } else {
2281 match declared_domain.as_deref() {
2282 Some("single-verifier-instance") => {
2283 let lease_path = std::env::var("HYPRSTREAM_REPLAY_SINGLE_VERIFIER_LEASE").ok();
2290 let Some(lease_path) = lease_path else {
2291 tracing::error!(
2292 "HYPRSTREAM_REPLAY_ADMISSION_DOMAIN=single-verifier-instance requires \
2293 HYPRSTREAM_REPLAY_SINGLE_VERIFIER_LEASE=<path> to an exclusive lease \
2294 file: the env string alone cannot prove sole verification. Proof \
2295 admission denies until the lease is configured."
2296 );
2297 return;
2298 };
2299 match hyprstream_rpc::proof::admission::SingleVerifierLease::acquire(
2300 std::path::Path::new(&lease_path),
2301 ) {
2302 Ok(lease) => {
2303 tracing::info!(
2304 "acquired exclusive single-verifier lease at {} \
2305 (this process is the sole verifier for its domain)",
2306 lease.path().display()
2307 );
2308 if hyprstream_rpc::proof::admission::hold_single_verifier_lease(lease)
2311 .is_err()
2312 {
2313 tracing::error!(
2314 "a single-verifier lease is already held; proof admission denies"
2315 );
2316 return;
2317 }
2318 Some(ReplayDomainGuarantee::SingleVerifierInstance)
2319 }
2320 Err(e) => {
2321 tracing::error!(
2322 "cannot claim sole-verifier lease; another verifier already holds it or \
2323 the path is unusable. Proof admission denies: {e:#}"
2324 );
2325 return;
2326 }
2327 }
2328 }
2329 Some("shared-postgres") => Some(ReplayDomainGuarantee::LinearizableSharedStore),
2330 Some(other) => {
2331 tracing::error!(
2332 "HYPRSTREAM_REPLAY_ADMISSION_DOMAIN='{other}' names a topology this build \
2333 cannot provide in-process: a shared linearizable backend or \
2334 namespace-affine routing must be installed explicitly. Proof \
2335 admission denies."
2336 );
2337 None
2338 }
2339 None => {
2340 tracing::warn!(
2341 "no HYPRSTREAM_REPLAY_ADMISSION_DOMAIN declared; proof replay admission \
2342 is not installed and proof-bearing requests will deny. Declare \
2343 'shared-postgres' for a configured shared replay database, or \
2344 'single-verifier-instance' only when this node is the sole verifier \
2345 for its service domain."
2346 );
2347 None
2348 }
2349 }
2350 };
2351
2352 let Some(guarantee) = guarantee else {
2353 return;
2354 };
2355
2356 match ChallengeManager::rotating_for_domain(
2361 guarantee,
2362 DEFAULT_CHALLENGE_WINDOW_SECS,
2363 DEFAULT_CHALLENGE_OVERLAP_SECS,
2364 now,
2365 ) {
2366 Some(manager) => {
2367 if set_global_challenge_manager(manager).is_ok() {
2368 tracing::info!(
2369 "proof challenge manager installed for {guarantee:?}: \
2370 {DEFAULT_CHALLENGE_WINDOW_SECS}s window, \
2371 {DEFAULT_CHALLENGE_OVERLAP_SECS}s acceptance overlap"
2372 );
2373 }
2374 }
2375 None => tracing::error!(
2376 "replay domain {guarantee:?} needs a domain-wide challenge source; \
2377 unattributed proofs deny"
2378 ),
2379 }
2380
2381 {
2391 let secrets_dir = HyprConfig::resolve_secrets_dir().ok();
2392 let manifest_path = secrets_dir
2393 .as_ref()
2394 .map(|dir| dir.join("proof-enrollment.toml"));
2395 match manifest_path {
2396 Some(path) if path.exists() => {
2397 match hyprstream_core::auth::proof_enrollment::ProofEnrollmentManifest::load(&path)
2398 {
2399 Ok(manifest) => {
2400 let foreign =
2404 hyprstream_core::auth::proof_enrollment::foreign_protocol_keys(
2405 oauth,
2406 secrets_dir.as_deref(),
2407 );
2408 let entries = manifest.entries.len();
2409 let resolver = hyprstream_core::auth::proof_enrollment::build_resolver(
2410 &manifest,
2411 &foreign,
2412 &Default::default(),
2413 now,
2414 );
2415 if hyprstream_rpc::proof::enrollment::set_global_enrollment_resolver(
2416 Box::new(resolver),
2417 )
2418 .is_ok()
2419 {
2420 tracing::info!(
2421 "proof enrollment manifest loaded from {} ({entries} entr(ies))",
2422 path.display()
2423 );
2424 }
2425 }
2426 Err(e) => tracing::error!(
2427 "proof enrollment manifest at {} is unusable; \
2428 authenticated proofs will deny: {e:#}",
2429 path.display()
2430 ),
2431 }
2432 }
2433 _ => tracing::info!(
2434 "no proof enrollment manifest present; authenticated proofs deny \
2435 until one is provisioned"
2436 ),
2437 }
2438 }
2439
2440 const REPLAY_CAPACITY_PER_PARTITION: usize = 100_000;
2443 let selection = select_proof_replay_store(guarantee, shared_store.is_some());
2444 let store: Box<dyn ProofReplayStore> = match selection {
2445 #[cfg(not(feature = "postgres-replay"))]
2446 ProofReplayStoreSelection::LegacyPdsPostgres => match shared_store {
2447 Some(store) => store,
2448 None => {
2449 tracing::error!("configured PDS replay store disappeared; admission denies");
2450 return;
2451 }
2452 },
2453 ProofReplayStoreSelection::SingleVerifierMemory => Box::new(
2454 InMemoryProofReplayStore::single_verifier_instance(REPLAY_CAPACITY_PER_PARTITION),
2455 ),
2456 #[cfg(feature = "postgres-replay")]
2457 ProofReplayStoreSelection::DedicatedPostgresV1 => {
2458 match hyprstream_core::auth::postgres_replay::PostgresProofReplayStore::from_env() {
2459 Ok(store) => Box::new(store),
2460 Err(error) => {
2461 tracing::error!(
2462 "shared Postgres replay admission unavailable; admission denies: {error:#}"
2463 );
2464 return;
2465 }
2466 }
2467 }
2468 ProofReplayStoreSelection::Unavailable => {
2469 tracing::error!("no replay store implementation for {guarantee:?}; admission denies");
2470 return;
2471 }
2472 };
2473 if set_global_proof_replay_store(store).is_ok() {
2474 tracing::info!("proof replay store installed: operator-declared {guarantee:?}");
2475 }
2476}
2477
2478fn install_session_pq_overlay() {
2507 let disabled = std::env::var("HYPRSTREAM_SESSION_PQ_OVERLAY")
2508 .map(|v| matches!(v.trim(), "0" | "false" | "off" | "no"))
2509 .unwrap_or(false);
2510 if disabled {
2511 tracing::info!(
2512 "session PQ overlay disabled by configuration; identities without an \
2513 admin-anchored ML-DSA-65 key are rejected at the envelope layer"
2514 );
2515 return;
2516 }
2517 let overlay = std::sync::Arc::new(hyprstream_rpc::session_pq_overlay::SessionPqOverlay::new(
2518 std::sync::Arc::new(hyprstream_rpc::session_pq_overlay::TracingPqBindingEventSink),
2519 ));
2520 if hyprstream_rpc::session_pq_overlay::install_session_pq_overlay(overlay).is_ok() {
2521 tracing::info!(
2522 "session PQ overlay installed: unenrolled identities bind their ML-DSA-65 key \
2523 at first contact (verifiable, assurance capped at Classical); rebinding an \
2524 established identity is refused and surfaced, never silently applied. \
2525 No approve/promote/revoke surface exists yet: a refused rebinding requires \
2526 a daemon restart to clear."
2527 );
2528 }
2529}
2530
2531#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2537enum NativeAnnouncementRefreshCompletion {
2538 Expired,
2539 Cancelled,
2540}
2541
2542async fn refresh_native_announcement<F, Fut, Error>(
2543 service_name: &str,
2544 socket_kind: &str,
2545 endpoint: &str,
2546 refresh_expires_at_unix_ms: i64,
2547 cancellation: tokio_util::sync::CancellationToken,
2548 mut announce: F,
2549) -> NativeAnnouncementRefreshCompletion
2550where
2551 F: FnMut() -> Fut,
2552 Fut: std::future::Future<Output = std::result::Result<(), Error>>,
2553 Error: std::fmt::Display,
2554{
2555 const ANNOUNCEMENT_REFRESH: std::time::Duration = std::time::Duration::from_secs(25);
2556 const RETRY_INITIAL: std::time::Duration = std::time::Duration::from_secs(5);
2557
2558 let mut delay = RETRY_INITIAL;
2559 loop {
2560 if chrono::Utc::now().timestamp_millis() >= refresh_expires_at_unix_ms {
2561 tracing::warn!(
2562 service = service_name,
2563 socket_kind,
2564 "Stopping native announcement refresh: accepted-state/JWT material expired"
2565 );
2566 return NativeAnnouncementRefreshCompletion::Expired;
2567 }
2568 let result = tokio::select! {
2569 biased;
2570 _ = cancellation.cancelled() => return NativeAnnouncementRefreshCompletion::Cancelled,
2571 result = async { announce().await } => result,
2572 };
2573 let sleep_delay = match result {
2574 Ok(()) => {
2575 tracing::info!(
2576 service = service_name,
2577 socket_kind,
2578 endpoint,
2579 "Announced native endpoint to DiscoveryService"
2580 );
2581 delay = RETRY_INITIAL;
2582 ANNOUNCEMENT_REFRESH
2583 }
2584 Err(error) => {
2585 tracing::warn!(
2586 service = service_name,
2587 socket_kind,
2588 "Failed to announce native endpoint: {error}"
2589 );
2590 let sleep_delay = delay;
2591 delay = delay
2592 .checked_mul(2)
2593 .unwrap_or(ANNOUNCEMENT_REFRESH)
2594 .min(ANNOUNCEMENT_REFRESH)
2595 .max(RETRY_INITIAL);
2596 sleep_delay
2597 }
2598 };
2599 tokio::select! {
2600 biased;
2601 _ = cancellation.cancelled() => return NativeAnnouncementRefreshCompletion::Cancelled,
2602 _ = tokio::time::sleep(sleep_delay) => {}
2603 }
2604 }
2605}
2606
2607type NativeAnnouncementFirstResult = std::result::Result<(), String>;
2609
2610type NativeAnnouncementFirstTx = Option<
2613 std::sync::Arc<
2614 parking_lot::Mutex<Option<tokio::sync::oneshot::Sender<NativeAnnouncementFirstResult>>>,
2615 >,
2616>;
2617
2618fn native_announcement_bearer(
2620 announcement: &hyprstream_rpc_std::discovery_client::ServiceAnnouncement,
2621) -> anyhow::Result<String> {
2622 announcement
2623 .service_jwt
2624 .as_ref()
2625 .filter(|token| !token.trim().is_empty())
2626 .cloned()
2627 .context("Native announcement requires a service JWT as its RPC bearer")
2628}
2629
2630async fn publish_native_announcement_attempt<F, Fut>(
2633 announcement: anyhow::Result<hyprstream_rpc_std::discovery_client::ServiceAnnouncement>,
2634 announce_tx: NativeAnnouncementFirstTx,
2635 publish: F,
2636) -> anyhow::Result<()>
2637where
2638 F: FnOnce(hyprstream_rpc_std::discovery_client::ServiceAnnouncement) -> Fut,
2639 Fut: std::future::Future<Output = anyhow::Result<()>>,
2640{
2641 let result = match announcement {
2642 Ok(announcement) => publish(announcement).await,
2643 Err(error) => Err(error),
2644 };
2645 if let Some(tx) = announce_tx {
2646 if let Some(tx) = tx.lock().take() {
2647 let report = result
2648 .as_ref()
2649 .map(|_| ())
2650 .map_err(std::string::ToString::to_string);
2651 let _ = tx.send(report);
2652 }
2653 }
2654 result
2655}
2656
2657const POLICY_INITIAL_PUBLICATION_BUDGET: std::time::Duration = std::time::Duration::from_secs(120);
2661const POLICY_INITIAL_PUBLICATION_RETRY: std::time::Duration = std::time::Duration::from_millis(500);
2662
2663async fn wait_for_policy_initial_publication<F, Fut>(
2664 cancellation: &tokio_util::sync::CancellationToken,
2665 budget: std::time::Duration,
2666 mut attempt: F,
2667) -> anyhow::Result<()>
2668where
2669 F: FnMut() -> Fut,
2670 Fut: std::future::Future<Output = anyhow::Result<()>>,
2671{
2672 let deadline = tokio::time::Instant::now() + budget;
2673 loop {
2674 let result = tokio::select! {
2675 biased;
2676 _ = cancellation.cancelled() => anyhow::bail!("Policy initial publication cancelled"),
2677 _ = tokio::time::sleep_until(deadline) => anyhow::bail!("Policy initial publication deadline exceeded"),
2678 result = async { attempt().await } => result,
2679 };
2680 match result {
2681 Ok(()) => return Ok(()),
2682 Err(error) if hyprstream_rpc::transport_traits::is_pre_dispatch_transport_error(&error)
2683 && error.chain().any(|cause| cause.downcast_ref::<hyprstream_rpc::transport::lazy_iroh::IrohPeerUnavailable>().is_some()) => {
2684 tracing::info!(%error, "Policy awaiting Discovery initial publication");
2685 }
2686 Err(error) => return Err(error),
2687 }
2688 tokio::select! {
2689 biased;
2690 _ = cancellation.cancelled() => anyhow::bail!("Policy initial publication cancelled"),
2691 _ = tokio::time::sleep_until(deadline) => anyhow::bail!("Policy initial publication deadline exceeded"),
2692 _ = tokio::time::sleep(POLICY_INITIAL_PUBLICATION_RETRY) => {}
2693 }
2694 }
2695}
2696
2697fn spawn_native_announcement_loop<F, Fut>(
2705 require_initial: bool,
2706 cancellation: tokio_util::sync::CancellationToken,
2707 loop_body: F,
2708) -> Option<std::sync::mpsc::Receiver<NativeAnnouncementFirstResult>>
2709where
2710 F: FnOnce(NativeAnnouncementFirstTx) -> Fut + Send + 'static,
2711 Fut: std::future::Future<Output = ()> + Send + 'static,
2712{
2713 let (initial_tx, initial_rx) = if require_initial {
2714 let (tx, rx) = std::sync::mpsc::sync_channel(1);
2715 (Some(tx), Some(rx))
2716 } else {
2717 (None, None)
2718 };
2719
2720 std::thread::spawn(move || {
2721 let mut initial_tx = initial_tx;
2722 let runtime = match tokio::runtime::Builder::new_current_thread()
2723 .enable_all()
2724 .build()
2725 {
2726 Ok(runtime) => runtime,
2727 Err(error) => {
2728 tracing::warn!("Failed to create announcement runtime: {error}");
2729 if let Some(tx) = initial_tx.take() {
2730 let _ = tx.send(Err(error.to_string()));
2731 }
2732 return;
2733 }
2734 };
2735
2736 runtime.block_on(async move {
2737 let (announce_tx, announce_rx) = if require_initial {
2738 let (tx, rx) = tokio::sync::oneshot::channel();
2739 (
2740 Some(std::sync::Arc::new(parking_lot::Mutex::new(Some(tx)))),
2741 Some(rx),
2742 )
2743 } else {
2744 (None, None)
2745 };
2746
2747 let mut task = tokio::spawn(loop_body(announce_tx));
2748 if let Some(rx) = announce_rx {
2749 let first = tokio::select! {
2750 biased;
2751 _ = cancellation.cancelled() => Ok(Err(
2752 "native announcement cancelled before first publication".to_owned(),
2753 )),
2754 result = rx => result,
2755 };
2756 match first {
2757 Ok(Ok(())) => {
2758 if let Some(tx) = initial_tx.take() {
2759 let result = if cancellation.is_cancelled() {
2760 Err("native announcement cancelled before readiness".to_owned())
2761 } else {
2762 Ok(())
2763 };
2764 let _ = tx.send(result);
2765 }
2766 }
2767 Ok(Err(error)) => {
2768 if let Some(tx) = initial_tx.take() {
2769 let _ = tx.send(Err(error.clone()));
2770 }
2771 task.abort();
2772 }
2773 Err(_) => {
2774 if let Some(tx) = initial_tx.take() {
2775 let _ = tx
2776 .send(Err("native announcement task exited before first result"
2777 .to_owned()));
2778 }
2779 }
2780 }
2781 }
2782 tokio::select! {
2783 biased;
2784 _ = cancellation.cancelled() => {
2785 task.abort();
2786 let _ = task.await;
2787 }
2788 _ = &mut task => {}
2789 }
2790 });
2791 });
2792
2793 initial_rx
2794}
2795
2796fn start_native_announcement_publisher<F, Fut>(
2799 reach: hyprstream_service::NativeAnnouncementReach,
2800 network_required: bool,
2801 cancellation: tokio_util::sync::CancellationToken,
2802 loop_body: F,
2803) -> anyhow::Result<()>
2804where
2805 F: FnOnce(NativeAnnouncementFirstTx) -> Fut + Send + 'static,
2806 Fut: std::future::Future<Output = ()> + Send + 'static,
2807{
2808 let require_initial = network_required
2809 && matches!(
2810 reach,
2811 hyprstream_service::NativeAnnouncementReach::Iroh { .. }
2812 );
2813 if let Some(rx) = spawn_native_announcement_loop(require_initial, cancellation, loop_body) {
2814 match rx.recv() {
2815 Ok(Ok(())) => {}
2816 Ok(Err(error)) => anyhow::bail!("initial Iroh announcement failed: {error}"),
2817 Err(_) => anyhow::bail!("initial Iroh announcement thread exited"),
2818 }
2819 }
2820 Ok(())
2821}
2822
2823fn main() -> Result<()> {
2824 if std::env::var("HIP_VISIBLE_DEVICES").is_ok() {
2828 if std::env::var("PYTORCH_HIP_ALLOC_CONF").is_err() {
2829 unsafe {
2831 std::env::set_var(
2832 "PYTORCH_HIP_ALLOC_CONF",
2833 "expandable_segments:True,garbage_collection_threshold:0.8",
2834 );
2835 }
2836 }
2837 if std::env::var("TORCH_BLAS_PREFER_HIPBLASLT").is_err() {
2838 unsafe {
2839 std::env::set_var("TORCH_BLAS_PREFER_HIPBLASLT", "1");
2840 }
2841 }
2842 if std::env::var("HIPBLASLT_FORCE_SPLIT_K").is_err() {
2843 unsafe {
2844 std::env::set_var("HIPBLASLT_FORCE_SPLIT_K", "1");
2845 }
2846 }
2847 }
2848
2849 if std::env::var("HIP_VISIBLE_DEVICES").is_err()
2853 && std::env::var("PYTORCH_CUDA_ALLOC_CONF").is_err()
2854 {
2855 unsafe {
2857 std::env::set_var(
2858 "PYTORCH_CUDA_ALLOC_CONF",
2859 "expandable_segments:True,garbage_collection_threshold:0.8",
2860 );
2861 }
2862 }
2863
2864 std::panic::set_hook(Box::new(|info| {
2866 eprintln!("\u{1f6a8} PANIC occurred:");
2867 if let Some(loc) = info.location() {
2868 eprintln!(" Location: {}:{}", loc.file(), loc.line());
2869 }
2870 if let Some(msg) = info.payload().downcast_ref::<&str>() {
2871 eprintln!(" Message: {}", msg);
2872 } else if let Some(msg) = info.payload().downcast_ref::<String>() {
2873 eprintln!(" Message: {}", msg);
2874 }
2875 eprintln!("\nThis was likely caused by a threading issue or memory corruption.");
2876 eprintln!("Please check for unsafe RefCell usage or race conditions.");
2877 }));
2878
2879 {
2883 let data_dir = dirs::data_local_dir()
2884 .unwrap_or_else(|| std::path::PathBuf::from("."))
2885 .join("hyprstream");
2886 if let Some((variant_id, version)) =
2887 hyprstream_core::cli::update_handlers::check_should_reexec(&data_dir)
2888 {
2889 hyprstream_core::cli::update_handlers::re_exec_variant(
2890 &data_dir,
2891 &variant_id,
2892 &version,
2893 );
2894 }
2896 }
2897
2898 let matches = build_cli().get_matches();
2900
2901 if let Some(("trust", sub_m)) = matches.subcommand() {
2905 let command =
2906 TrustCommand::from_arg_matches(sub_m).map_err(|error| anyhow::anyhow!("{error}"))?;
2907 hyprstream_core::cli::trust::handle_trust_command(command)?;
2908 return Ok(());
2909 }
2910
2911 let config_path = matches
2913 .get_one::<std::path::PathBuf>("config")
2914 .map(std::path::PathBuf::as_path);
2915
2916 let explicit_config_path: Option<std::path::PathBuf> =
2923 match config_path {
2924 Some(path) => Some(std::fs::canonicalize(path).with_context(|| {
2925 format!("canonicalizing explicit config path {}", path.display())
2926 })?),
2927 None => None,
2928 };
2929 if let Some(canonical) = &explicit_config_path {
2932 let _ = hyprstream_core::config::install_explicit_config_path(canonical.clone());
2933 }
2934
2935 let config = load_config(explicit_config_path.as_deref())?;
2937
2938 config
2940 .validate()
2941 .context("Configuration validation failed")?;
2942
2943 let _ = hyprstream_core::config::install_pinned_config(config.clone());
2949
2950 let explicit_config: Option<&std::path::Path> = explicit_config_path.as_deref();
2952 let iroh_required = config.quic.iroh_required();
2953
2954 if let Some(("pds", pds)) = matches.subcommand() {
2958 #[cfg(feature = "rocksdb")]
2959 if let Some(("inspect-services", inspect)) = pds.subcommand() {
2960 let services = inspect
2961 .get_many::<String>("service")
2962 .context("service roster is required")?
2963 .cloned()
2964 .collect::<Vec<_>>();
2965 let bytes =
2970 hyprstream_core::cli::deployment_bootstrap::inspect_services(&config, &services)
2971 .map_err(|_| anyhow::anyhow!("read-only service roster inspection failed"))?;
2972 std::io::Write::write_all(&mut std::io::stdout().lock(), &bytes)?;
2973 return Ok(());
2974 }
2975 }
2976
2977 install_envelope_verify_config(Some(&config.oauth), Some(&config));
2983
2984 let _ = hyprstream_rpc::streaming::install_max_concurrent_streams_per_service(
2988 config.server.max_concurrent_streams_per_service,
2989 );
2990
2991 let is_service_command = matches.subcommand_name() == Some("service");
2993 let _log_guard: Option<tracing_appender::non_blocking::WorkerGuard>;
2994
2995 #[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
2996 {
2997 let telemetry_provider = if is_service_command {
2998 TelemetryProvider::Otlp
2999 } else {
3000 TelemetryProvider::Stdout
3001 };
3002
3003 let otel_enabled = std::env::var("HYPRSTREAM_OTEL_ENABLE")
3004 .unwrap_or_else(|_| "false".to_owned())
3005 .parse::<bool>()
3006 .unwrap_or(false);
3007
3008 if otel_enabled {
3009 init_telemetry(telemetry_provider)?;
3010 } else {
3011 let default_log_level = match telemetry_provider {
3012 TelemetryProvider::Otlp => "hyprstream=info,hyprstream_core=info",
3013 TelemetryProvider::Stdout => "hyprstream=warn,hyprstream_core=warn",
3014 };
3015
3016 if let Ok(log_dir) = std::env::var("HYPRSTREAM_LOG_DIR") {
3017 let file_appender =
3018 RollingFileAppender::new(Rotation::DAILY, &log_dir, "hyprstream.log");
3019 let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
3020 _log_guard = Some(guard);
3021
3022 tracing_subscriber::fmt()
3023 .with_env_filter(EnvFilter::builder().parse_lossy(
3024 std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()),
3025 ))
3026 .with_target(true)
3027 .with_file(true)
3028 .with_line_number(true)
3029 .with_writer(non_blocking)
3030 .init();
3031
3032 info!("File logging enabled to {}/hyprstream.log", log_dir);
3033 } else {
3034 _log_guard = None;
3035 tracing_subscriber::fmt()
3036 .with_env_filter(EnvFilter::builder().parse_lossy(
3037 std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()),
3038 ))
3039 .with_target(true)
3040 .with_file(true)
3041 .with_line_number(true)
3042 .init();
3043 }
3044 }
3045 }
3046
3047 #[cfg(any(not(feature = "otel"), target_arch = "wasm32"))]
3048 {
3049 let default_log_level = if is_service_command {
3050 "hyprstream=info,hyprstream_core=info"
3051 } else {
3052 "hyprstream=warn,hyprstream_core=warn"
3053 };
3054
3055 if let Ok(log_dir) = std::env::var("HYPRSTREAM_LOG_DIR") {
3056 let file_appender =
3057 RollingFileAppender::new(Rotation::DAILY, &log_dir, "hyprstream.log");
3058 let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
3059 _log_guard = Some(guard);
3060
3061 tracing_subscriber::fmt()
3062 .with_env_filter(EnvFilter::builder().parse_lossy(
3063 std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()),
3064 ))
3065 .with_target(true)
3066 .with_file(true)
3067 .with_line_number(true)
3068 .with_writer(non_blocking)
3069 .init();
3070
3071 info!("File logging enabled to {}/hyprstream.log", log_dir);
3072 } else {
3073 _log_guard = None;
3074 tracing_subscriber::fmt()
3075 .with_env_filter(EnvFilter::builder().parse_lossy(
3076 std::env::var("RUST_LOG").unwrap_or_else(|_| default_log_level.to_owned()),
3077 ))
3078 .with_target(true)
3079 .with_file(true)
3080 .with_line_number(true)
3081 .init();
3082 }
3083 }
3084
3085 info!("Hyprstream v{} starting up", env!("CARGO_PKG_VERSION"));
3086 let execution_mode = ExecutionMode::detect();
3090 info!("Execution mode: {}", execution_mode);
3091
3092 {
3094 use hyprstream_rpc::registry::init as init_registry;
3095
3096 let mode = execution_mode.endpoint_mode();
3097 let runtime_dir = if execution_mode.uses_ipc() {
3098 Some(hyprstream_rpc::paths::runtime_dir())
3099 } else {
3100 None
3101 };
3102
3103 init_registry(mode, runtime_dir);
3104 }
3105 {
3112 let models_dir = config.models_dir().clone();
3113 let is_wizard = matches!(matches.subcommand_name(), Some("wizard"));
3114 let is_first_run = is_wizard
3115 || (matches.subcommand_name().is_none()
3116 && hyprstream_core::cli::bootstrap_manager::is_first_run(&models_dir));
3117
3118 if is_first_run {
3119 let services = config.services.startup.clone();
3120 if let Some(("wizard", sub_m)) = matches.subcommand() {
3121 let tui_mode = sub_m.get_flag("tui");
3122 let non_interactive = sub_m.get_flag("non_interactive");
3123 let start_services = sub_m.get_flag("start");
3124 let bootstrap_only = sub_m.get_flag("bootstrap_only");
3125 let enable_federation = sub_m.get_flag("enable_federation");
3126 let pds_url = sub_m.get_one::<String>("pds_url").cloned();
3127 let initial_user_role = sub_m
3128 .get_one::<String>("initial_user_role")
3129 .cloned()
3130 .unwrap_or_else(|| "admin".to_owned());
3131 let deployment_trust =
3132 hyprstream_core::cli::wizard_handlers::DeploymentTrustOptions {
3133 enabled: sub_m.get_flag("deployment_trust"),
3134 dir: sub_m
3135 .get_one::<String>("deployment_trust_dir")
3136 .map(std::path::PathBuf::from),
3137 force_software: sub_m.get_flag("deployment_trust_software"),
3138 serial: sub_m.get_one::<String>("deployment_trust_serial").cloned(),
3139 piv_slot: sub_m
3140 .get_one::<String>("deployment_trust_piv_slot")
3141 .cloned(),
3142 allow_plaintext_break_glass: sub_m
3143 .get_flag("deployment_trust_allow_plaintext_break_glass"),
3144 };
3145 let wizard_options = hyprstream_core::cli::WizardOptions {
3146 non_interactive,
3147 start_services,
3148 bootstrap_only,
3149 enable_federation,
3150 initial_user_role,
3151 deployment_trust,
3152 };
3153 let use_tui = tui_mode
3156 || (pds_url.is_none()
3157 && !non_interactive
3158 && !bootstrap_only
3159 && !wizard_options.deployment_trust.enabled
3160 && supports_tui());
3161 let config = config.clone();
3162 return with_runtime(
3163 RuntimeConfig {
3164 device: DeviceConfig::request_cpu(),
3165 multi_threaded: true,
3166 },
3167 || async move {
3168 if use_tui {
3169 hyprstream_core::cli::handle_wizard_tui(&models_dir, &services).await
3170 } else {
3171 hyprstream_core::cli::handle_wizard(
3172 &models_dir,
3173 &services,
3174 wizard_options,
3175 )
3176 .await?;
3177 if let Some(pds_url) = pds_url {
3178 hyprstream_core::cli::pds_handlers::handle_pds_join(
3179 &config, &pds_url, None,
3180 )
3181 .await?;
3182 }
3183 Ok(())
3184 }
3185 },
3186 );
3187 }
3188 return with_runtime(
3190 RuntimeConfig {
3191 device: DeviceConfig::request_cpu(),
3192 multi_threaded: true,
3193 },
3194 || async move {
3195 if supports_tui() {
3196 hyprstream_core::cli::handle_wizard_tui(&models_dir, &services).await
3197 } else {
3198 hyprstream_core::cli::handle_wizard(
3201 &models_dir,
3202 &services,
3203 hyprstream_core::cli::WizardOptions::first_run(),
3204 )
3205 .await
3206 }
3207 },
3208 );
3209 }
3210 }
3211
3212 if let Some(("mac", sub_m)) = matches.subcommand() {
3218 match sub_m.subcommand() {
3219 Some(("genesis", _)) => {
3220 let gate = hyprstream_core::mac::GenesisGate::production();
3223 print!("{}", gate.render_report());
3224 }
3225 _ => {
3226 eprintln!("usage: hyprstream mac genesis");
3227 }
3228 }
3229 return Ok(());
3230 }
3231
3232 if let Some(("pds", sub_m)) = matches.subcommand() {
3236 match sub_m.subcommand() {
3237 #[cfg(feature = "rocksdb")]
3238 Some(("init-deployment-store", init_m)) => {
3239 if init_m.get_flag("fresh-genesis") {
3240 hyprstream_core::cli::deployment_bootstrap::init_checkpoint_store_for_fresh_genesis(
3241 &config,
3242 )?;
3243 println!("fresh-genesis deployment checkpoint store initialized");
3244 } else {
3245 match hyprstream_core::cli::deployment_bootstrap::init_checkpoint_store(&config)? {
3246 hyprstream_core::cli::deployment_bootstrap::DeploymentStoreInit::Created => {
3247 println!("initialized empty deployment checkpoint store");
3248 }
3249 hyprstream_core::cli::deployment_bootstrap::DeploymentStoreInit::Reused => {
3250 println!("reused healthy retained deployment checkpoint store");
3251 }
3252 }
3253 }
3254 return Ok(());
3255 }
3256 #[cfg(feature = "rocksdb")]
3257 Some(("provision-services", provision_m)) => {
3258 let services = provision_m.get_many::<String>("service")
3259 .context("service roster is required")?.cloned().collect::<Vec<_>>();
3260 let lifetime = *provision_m.get_one::<i64>("valid-for-seconds")
3261 .context("service identity lifetime is required")?;
3262 let roster_export = provision_m.get_one::<std::path::PathBuf>("roster-export");
3263 hyprstream_core::cli::deployment_bootstrap::provision_services(
3264 &config,
3265 &services,
3266 lifetime,
3267 roster_export.map(std::path::PathBuf::as_path),
3268 )?;
3269 if let Some(path) = roster_export {
3270 println!("verified service roster manifest written to {}", path.display());
3271 }
3272 println!("checkpoint-accepted service roster ready ({} services)", services.len());
3273 return Ok(());
3274 }
3275 Some(("join", join_m)) => {
3276 let pds_url = join_m
3277 .get_one::<String>("url")
3278 .ok_or_else(|| anyhow::anyhow!("PDS URL is required"))?;
3279 let scope = join_m.get_one::<String>("scope").map(String::as_str);
3280 return with_runtime(
3281 RuntimeConfig { device: DeviceConfig::request_cpu(), multi_threaded: true },
3282 || hyprstream_core::cli::pds_handlers::handle_pds_join(&config, pds_url, scope),
3283 );
3284 }
3285 _ => anyhow::bail!("usage: hyprstream pds init-deployment-store | pds provision-services --service <NAMES> | pds join <PDS_URL> [--scope <SCOPE>]"),
3286 }
3287 }
3288
3289 if let Some(("service", sub_m)) = matches.subcommand() {
3295 if let Some(("repair", _)) = sub_m.subcommand() {
3296 let verbose = sub_m
3297 .subcommand_matches("repair")
3298 .map(|m| m.get_flag("verbose"))
3299 .unwrap_or(false);
3300 let models_dir = config.models_dir().clone();
3301 return with_runtime(
3302 RuntimeConfig {
3303 device: DeviceConfig::request_cpu(),
3304 multi_threaded: true,
3305 },
3306 || async move {
3307 hyprstream_core::cli::service_handlers::run_repair_checks(&models_dir, verbose)
3308 .await
3309 },
3310 );
3311 }
3312 }
3313
3314 if let Some(("service", sub_m)) = matches.subcommand() {
3319 if let Some(("provision-policy-templates", provision_m)) = sub_m.subcommand() {
3320 let templates = provision_m
3321 .get_many::<String>("template")
3322 .context("at least one policy template is required")?
3323 .cloned()
3324 .collect::<Vec<_>>();
3325 let models_dir = config.models_dir().clone();
3326 return with_runtime(
3327 RuntimeConfig {
3328 device: DeviceConfig::request_cpu(),
3329 multi_threaded: true,
3330 },
3331 || async move {
3332 handle_service_provision_policy_templates(&models_dir, &templates).await
3333 },
3334 );
3335 }
3336 }
3337
3338 if let Some(("service", sub_m)) = matches.subcommand() {
3340 if sub_m.subcommand_name() == Some("reconcile-enrollment") {
3341 return hyprstream_core::cli::service_handlers::handle_service_reconcile_enrollment(
3342 &config,
3343 );
3344 }
3345 if sub_m.subcommand_name() == Some("ensure-node-authority") {
3346 let secrets =
3347 hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3348 return hyprstream_core::auth::service_enrollment::ensure_node_authority(&secrets);
3349 }
3350 }
3351
3352 if let Some(("service", sub_m)) = matches.subcommand() {
3357 if let Some(("ensure-key", ek_m)) = sub_m.subcommand() {
3358 let name = ek_m
3361 .get_one::<String>("name")
3362 .cloned()
3363 .ok_or_else(|| anyhow::anyhow!("service ensure-key: missing required <name>"))?;
3364 return hyprstream_core::cli::service_handlers::handle_service_ensure_key(
3365 Some(&config),
3366 &name,
3367 );
3368 }
3369 }
3370
3371 if let Some(("service", sub_m)) = matches.subcommand() {
3375 if let Some(("readback-local-canary", readback_m)) = sub_m.subcommand() {
3376 let path = readback_m
3377 .get_one::<std::path::PathBuf>("path")
3378 .cloned()
3379 .context("service readback-local-canary requires --path")?;
3380 let expected_head = readback_m
3381 .get_one::<String>("expected_head")
3382 .cloned()
3383 .context("service readback-local-canary requires --expected-head")?;
3384 return with_runtime(
3385 RuntimeConfig {
3386 device: DeviceConfig::request_cpu(),
3387 multi_threaded: true,
3388 },
3389 || async move {
3390 let secrets = HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3391 let profile =
3392 hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3393 let key =
3394 hyprstream_core::auth::identity_store::load_existing_service_signing_key(
3395 &secrets, "registry", profile,
3396 )?;
3397 let jwt = hyprstream_core::auth::identity_store::load_service_jwt_for_profile(
3398 &secrets, "registry", profile,
3399 )?
3400 .context("enrolled Registry service credential is missing")?;
3401 hyprstream_core::cli::registry_readback::require_registry_service_subject(&jwt)
3402 .context("Registry service credential preflight failed")?;
3403 install_process_production_resolver(&key, &config, Some(jwt.clone()), true)
3404 .await?;
3405 let provider = &hyprstream_discovery::ProductionRpcClientProvider;
3406 let policy = hyprstream_rpc_std::policy_client::PolicyClient::from_provider(
3407 provider,
3408 key.clone(),
3409 None,
3410 )?;
3411 let registry = RegistryClient::from_provider(provider, key, Some(jwt.clone()))?;
3412 hyprstream_core::cli::registry_readback::readback_local_canary(
3413 &policy,
3414 ®istry,
3415 &jwt,
3416 config.models_dir(),
3417 &path,
3418 &expected_head,
3419 )
3420 .await
3421 },
3422 )
3423 .map_err(|error| {
3424 anyhow::anyhow!(
3425 "Registry canary readback inconclusive: {}",
3426 hyprstream_core::cli::registry_readback::error_category(&error)
3427 )
3428 });
3429 }
3430 }
3431
3432 if let Some(("service", sub_m)) = matches.subcommand() {
3436 if let Some(("register-local-canary", register_m)) = sub_m.subcommand() {
3437 let path = register_m
3438 .get_one::<std::path::PathBuf>("path")
3439 .cloned()
3440 .context("service register-local-canary requires --path")?;
3441 let expected_head = register_m
3442 .get_one::<String>("expected_head")
3443 .cloned()
3444 .context("service register-local-canary requires --expected-head")?;
3445 return with_runtime(
3446 RuntimeConfig {
3447 device: DeviceConfig::request_cpu(),
3448 multi_threaded: true,
3449 },
3450 || async move {
3451 let secrets = HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3452 let profile =
3453 hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3454 let key =
3455 hyprstream_core::auth::identity_store::load_existing_service_signing_key(
3456 &secrets, "registry", profile,
3457 )?;
3458 let jwt = hyprstream_core::auth::identity_store::load_service_jwt_for_profile(
3459 &secrets, "registry", profile,
3460 )?
3461 .context("enrolled Registry service credential is missing")?;
3462 hyprstream_core::cli::registry_readback::require_registry_service_subject(&jwt)
3463 .context("Registry service credential preflight failed")?;
3464 install_process_production_resolver(&key, &config, Some(jwt.clone()), true)
3465 .await?;
3466 let provider = &hyprstream_discovery::ProductionRpcClientProvider;
3467 let policy = hyprstream_rpc_std::policy_client::PolicyClient::from_provider(
3468 provider,
3469 key.clone(),
3470 None,
3471 )?;
3472 let registry = RegistryClient::from_provider(provider, key, Some(jwt.clone()))?;
3473 hyprstream_core::cli::registry_readback::register_local_canary(
3474 &policy,
3475 ®istry,
3476 &jwt,
3477 config.models_dir(),
3478 &path,
3479 &expected_head,
3480 )
3481 .await
3482 },
3483 )
3484 .map_err(|error| {
3485 anyhow::anyhow!(
3486 "Registry canary registration inconclusive: {}",
3487 hyprstream_core::cli::registry_readback::registration_error_category(&error)
3488 )
3489 });
3490 }
3491 }
3492
3493 let native_service_name = native_service_process_name(&matches, &config)?;
3494 let dedicated_service_name = dedicated_service_process_name(&matches, &config)?;
3495
3496 let _registry_runtime = tokio::runtime::Builder::new_multi_thread()
3498 .enable_all()
3499 .build()
3500 .context("Failed to create registry runtime")?;
3501
3502 let (registry_client, signing_key, verifying_key, is_os_owned_bootstrap): (
3504 RegistryClient,
3505 SigningKey,
3506 VerifyingKey,
3507 bool,
3508 ) = _registry_runtime
3509 .block_on(async {
3510 let process_identity = process_identity_name(
3522 native_service_name.as_deref(),
3523 dedicated_service_name.as_deref(),
3524 );
3525 let signing_key = load_process_signing_key(&config, process_identity).await?;
3526 let verifying_key = signing_key.verifying_key();
3527
3528 let service_token = match process_identity {
3533 Some(name) => {
3534 let secrets = HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3535 let profile =
3536 hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3537 let token =
3538 hyprstream_core::auth::identity_store::load_service_jwt_for_profile(
3539 &secrets, name, profile,
3540 )?
3541 .with_context(|| {
3542 format!("missing enrolled service credential for {name}")
3543 })?;
3544 Some(token)
3545 }
3546 None => {
3547 let secrets = HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3548 let profile =
3549 hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3550 flat_key_policy_resolver_credential(&secrets, profile, &signing_key)
3551 }
3552 };
3553
3554 let is_os_owned_bootstrap = install_process_production_resolver(
3555 &signing_key,
3556 &config,
3557 service_token.clone(),
3558 false,
3559 )
3560 .await
3561 .context("Failed to install checkpoint-backed production resolver")?;
3562 let client = hyprstream_rpc_std::registry_client::RegistryClient::from_provider(
3563 &hyprstream_discovery::ProductionRpcClientProvider,
3564 signing_key.clone(),
3565 service_token,
3566 )?;
3567
3568 Ok::<_, anyhow::Error>((client, signing_key, verifying_key, is_os_owned_bootstrap))
3569 })
3570 .context("Failed to connect to services")?;
3571
3572 let config_for_service = config.clone();
3574 let ctx = AppContext::with_client(config, Clone::clone(®istry_client));
3575
3576 match matches.subcommand() {
3578 Some(("tool", sub_m)) => {
3580 with_runtime(
3581 RuntimeConfig {
3582 device: DeviceConfig::request_cpu(),
3583 multi_threaded: true,
3584 },
3585 || async move { schema_cli::handle_tool_command(sub_m, signing_key).await },
3586 )?;
3587 }
3588
3589 Some(("quick", sub_m)) => {
3591 let cmd =
3592 QuickCommand::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
3593 handle_quick_command(cmd, ctx, signing_key, verifying_key)?;
3594 }
3595
3596 Some(("service", sub_m)) => {
3598 let action =
3599 ServiceAction::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
3600
3601 let services = config_for_service.services.startup.clone();
3602
3603 match action {
3604 ServiceAction::Install {
3605 services: filter,
3606 start,
3607 enable,
3608 system,
3609 systemd_session,
3610 verbose,
3611 } => {
3612 let models_dir = config_for_service.models_dir().clone();
3613 let target = if system {
3614 hyprstream_service::ServiceTarget::System
3615 } else if systemd_session {
3616 hyprstream_service::ServiceTarget::UserSession
3617 } else {
3618 hyprstream_service::ServiceTarget::User
3619 };
3620 with_runtime(
3621 RuntimeConfig {
3622 device: DeviceConfig::request_cpu(),
3623 multi_threaded: true,
3624 },
3625 || async move {
3626 handle_service_install(
3627 &models_dir,
3628 &services,
3629 filter,
3630 start,
3631 enable,
3632 target,
3633 verbose,
3634 explicit_config,
3635 iroh_required,
3636 )
3637 .await
3638 },
3639 )?;
3640 }
3641
3642 ServiceAction::Uninstall { services: filter } => {
3643 with_runtime(
3644 RuntimeConfig {
3645 device: DeviceConfig::request_cpu(),
3646 multi_threaded: false,
3647 },
3648 || async move { handle_service_uninstall(&services, filter).await },
3649 )?;
3650 }
3651
3652 ServiceAction::Start {
3653 name,
3654 foreground,
3655 daemon,
3656 ipc,
3657 services: multi_services,
3658 quic_bind,
3659 print_cert_hash,
3660 standalone,
3661 } => {
3662 if let Some(artifacts) = &config_for_service.policy.boot_policy {
3666 let secrets_dir =
3667 HyprConfig::resolve_secrets_dir_for(Some(&config_for_service))?;
3668 let policy =
3669 hyprstream_core::mac::bootload::install_configured_boot_policy(
3670 artifacts,
3671 &secrets_dir,
3672 is_os_owned_bootstrap,
3673 )
3674 .context("configured MAC boot policy rejected")?;
3675 tracing::info!(
3676 generation = policy.generation,
3677 "Verified configured MAC boot policy installed"
3678 );
3679 }
3680 let mac_gate = hyprstream_core::mac::GenesisGate::production();
3688 mac_gate.log_report();
3689 hyprstream_core::mac::install_production_rpc_dispatch_pep()?;
3692
3693 if foreground || standalone {
3694 let (name, multi_services) = if standalone {
3697 (String::from("standalone"), Some(services.clone()))
3698 } else {
3699 match (name, &multi_services) {
3700 (Some(n), _) => (n, multi_services),
3701 (None, Some(_)) => (String::from("multi"), multi_services),
3702 (None, None) => {
3703 return Err(anyhow::anyhow!(
3704 "--foreground requires a service name or --services list"
3705 ))
3706 }
3707 }
3708 };
3709
3710 let name = native_service_name.clone().unwrap_or(name);
3713
3714 let rpc_mode = if ipc {
3716 hyprstream_rpc::registry::EndpointMode::Ipc
3717 } else {
3718 hyprstream_rpc::registry::EndpointMode::Inproc
3719 };
3720
3721 let config = config_for_service;
3722
3723 with_runtime(
3724 RuntimeConfig {
3725 device: DeviceConfig::request_cpu(),
3726 multi_threaded: true,
3727 },
3728 || async move {
3729 let (shutdown_tx, shutdown_rx) =
3730 tokio::sync::oneshot::channel::<()>();
3731
3732 tokio::spawn(async move {
3733 let _ = tokio::signal::ctrl_c().await;
3734 info!("Received shutdown signal");
3735 let _ = shutdown_tx.send(());
3736 });
3737
3738 {
3739 use hyprstream_rpc::registry::init as init_registry;
3740 let runtime_dir = if ipc {
3741 Some(hyprstream_rpc::paths::runtime_dir())
3742 } else {
3743 None
3744 };
3745 init_registry(rpc_mode, runtime_dir);
3746 }
3747
3748 let models_dir = config.models_dir();
3749 let keys_dir = models_dir.join(".registry").join("keys");
3750 let signing_key = if process_identity_name(
3751 native_service_name.as_deref(),
3752 dedicated_service_name.as_deref(),
3753 )
3754 .is_some()
3755 {
3756 signing_key.clone()
3758 } else {
3759 load_or_generate_signing_key(&keys_dir).await?
3760 };
3761 let verifying_key = signing_key.verifying_key();
3762
3763 let fed_src: Arc<dyn hyprstream_rpc::auth::FederationKeySource> =
3764 Arc::new(hyprstream_core::auth::FederationKeyResolver::new(
3765 &config.oauth.trusted_issuers,
3766 ));
3767 let mut ctx = ServiceContext::new(
3768 signing_key.clone(),
3769 verifying_key,
3770 ipc,
3771 models_dir.clone(),
3772 )
3773 .with_oauth_issuer(config.oauth.issuer_url())
3774 .with_federation_key_source(fed_src)
3775 .with_secrets_dir(
3782 hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(
3783 Some(&config),
3784 )?,
3785 );
3786
3787 if let Some(service) = &dedicated_service_name {
3790 ctx = ctx.with_dedicated_process_service(service.clone());
3791 }
3792
3793 let mut quic_cfg = if let Some(ref bind_addr) = quic_bind {
3795 let mut qc = hyprstream_core::config::QuicConfig::default();
3796 qc.enabled = true;
3797 qc.bind_addr = bind_addr.clone();
3798 qc.iroh = config.quic.iroh;
3799 qc.native_network_profile = config.quic.native_network_profile;
3800 qc
3801 } else {
3802 config.quic.clone()
3803 };
3804 quic_cfg
3805 .validate_native_network_profile()
3806 .context("invalid native network profile")?;
3807
3808 let service_names: Vec<String> =
3810 if let Some(ref svc_list) = multi_services {
3811 svc_list.clone()
3812 } else {
3813 vec![name.clone()]
3814 };
3815
3816 if ipc || native_service_name.is_some() {
3817 let secrets_dir = hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3831 ctx = ctx.with_ca_verifying_key(
3832 hyprstream_core::auth::identity_store::load_ca_verifying_key(
3833 &secrets_dir,
3834 )?,
3835 );
3836 match hyprstream_core::auth::identity_store::load_ca_ml_dsa_verifying_key(&secrets_dir) {
3840 Ok(pq_vk) => ctx = ctx.with_ca_ml_dsa_verifying_key(pq_vk),
3841 Err(e) => tracing::warn!(
3842 "ca-mldsa-pubkey unavailable — hybrid service JWTs will not verify \
3843 until the wizard re-provisions credentials: {e}"
3844 ),
3845 }
3846
3847 if let Ok(entries) = hyprstream_core::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir) {
3851 hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)?;
3852 let pubkeys: std::collections::HashMap<String, VerifyingKey> =
3853 entries.into_iter().map(|(n, e)| (n, e.ed25519)).collect();
3854 for (svc_name, vk) in &pubkeys {
3855 hyprstream_service::global_trust_store().insert(
3856 *vk,
3857 hyprstream_service::Attestation {
3858 scopes: std::iter::once(svc_name.clone()).collect(),
3859 subject: None,
3860 jwt: None,
3861 expires_at: 0,
3862 attested_by: None,
3863 },
3864 );
3865 }
3866 }
3867
3868 let secrets_profile =
3873 hyprstream_core::auth::identity_store::SecretsProfile::from_env()?;
3874 let own_key = if native_service_name.is_some() {
3875 signing_key.clone()
3876 } else {
3877 hyprstream_core::auth::identity_store::resolve_service_signing_key(
3878 &secrets_dir, &name, secrets_profile,
3879 )?
3880 };
3881
3882 if name == "policy" {
3883 ctx = ctx.with_service_key(&name, own_key.clone());
3885 } else {
3886 ctx = ctx.swap_signing_key(own_key.clone());
3889 ctx = ctx.with_service_key(&name, own_key.clone());
3890 }
3891 hyprstream_core::auth::identity_store::seed_service_jwt_into_trust_store(
3896 &name,
3897 &own_key,
3898 &secrets_dir,
3899 secrets_profile,
3900 );
3901 } else {
3902 let secrets_dir = hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
3908 ctx = ctx.with_ca_verifying_key(
3909 hyprstream_core::auth::identity_store::load_ca_verifying_key(
3910 &secrets_dir,
3911 )?,
3912 );
3913 match hyprstream_core::auth::identity_store::load_ca_ml_dsa_verifying_key(&secrets_dir) {
3917 Ok(pq_vk) => ctx = ctx.with_ca_ml_dsa_verifying_key(pq_vk),
3918 Err(e) => tracing::warn!(
3919 "ca-mldsa-pubkey unavailable — hybrid service JWTs will not verify \
3920 until the wizard re-provisions credentials: {e}"
3921 ),
3922 }
3923
3924 let entries = hyprstream_core::auth::identity_store::load_bootstrap_pubkeys_hybrid(&secrets_dir)
3928 .context("Bootstrap pubkeys not found — run 'hyprstream wizard' first")?;
3929 if entries.is_empty() {
3930 anyhow::bail!("Bootstrap pubkeys file is empty — run 'hyprstream wizard' first");
3931 }
3932 hyprstream_core::auth::identity_store::ensure_bootstrap_pubkeys_hybrid(&entries)?;
3933 let pubkeys: std::collections::HashMap<String, VerifyingKey> =
3934 entries.into_iter().map(|(n, e)| (n, e.ed25519)).collect();
3935 for (svc_name, vk) in &pubkeys {
3936 hyprstream_service::global_trust_store().insert(
3937 *vk,
3938 hyprstream_service::Attestation {
3939 scopes: std::iter::once(svc_name.clone()).collect(),
3940 subject: None,
3941 jwt: None,
3942 expires_at: 0,
3943 attested_by: None,
3944 },
3945 );
3946 }
3947 tracing::debug!(
3948 "Loaded {} bootstrap pubkeys from {}",
3949 pubkeys.len(),
3950 secrets_dir.display()
3951 );
3952
3953 for svc_name in &service_names {
3962 let svc_key = hyprstream_core::auth::identity_store::resolve_service_signing_key(
3963 &secrets_dir,
3964 svc_name,
3965 hyprstream_core::auth::identity_store::SecretsProfile::SharedDirectory,
3966 )
3967 .with_context(|| format!("Failed to load signing key for service '{}'", svc_name))?;
3968 ctx = ctx.with_service_key(svc_name, svc_key.clone());
3969
3970 if svc_name != "policy" {
3971 if let Ok(Some(jwt_str)) = hyprstream_core::auth::identity_store::load_service_jwt(&secrets_dir, svc_name) {
3972 let exp = hyprstream_core::auth::identity_store::decode_jwt_exp_raw(&jwt_str).unwrap_or(0);
3973 hyprstream_service::global_trust_store().insert(
3974 svc_key.verifying_key(),
3975 hyprstream_service::Attestation {
3976 scopes: std::iter::once(svc_name.clone()).collect(),
3977 subject: None,
3978 jwt: Some(jwt_str),
3979 expires_at: exp,
3980 attested_by: None,
3981 },
3982 );
3983 tracing::info!(service = %svc_name, "Seeded trust store with own service-jwt from credential dir");
3984 }
3985 }
3986 }
3987 }
3988
3989 {
3998 let secrets_dir =
3999 hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
4000 match hyprstream_core::auth::service_enrollment::ServiceEnrollmentManifest::load_and_validate(&secrets_dir)
4001 .context("service enrollment manifest validation failed")?
4002 {
4003 Some(manifest) => {
4004 let count = manifest.services.len();
4005 hyprstream_core::auth::service_enrollment::set_global_service_enrollment(std::sync::Arc::new(manifest))
4006 .map_err(|e| anyhow::anyhow!("{e}"))?;
4007 tracing::info!(services = count, "Service enrollment manifest installed");
4008 }
4009 None => {
4010 tracing::warn!(
4011 "No service-enrollment manifest — legacy mode: service credentials carry no manifest-backed clearance"
4012 );
4013 }
4014 }
4015 }
4016
4017 if quic_cfg.enabled {
4018 match quic_checkpoint_policy(is_os_owned_bootstrap, || {
4028 hyprstream_core::services::factories::classify_pds_store_for_quic(&ctx)
4029 })
4030 .context(
4031 "PDS accepted-state store read failed; \
4032 refusing to start QUIC (fail-closed)",
4033 )? {
4034 QuicCheckpointPolicy::CheckpointedAnnouncements => {
4035 ctx = hyprstream_core::services::factories::with_checkpointed_native_announcements(
4036 ctx,
4037 &service_names,
4038 )?;
4039 }
4040 QuicCheckpointPolicy::DeferForFirstBoot => {
4041 if quic_cfg.iroh_required() {
4042 anyhow::bail!(
4043 "network-iroh-required requires a checkpoint-verified accepted state and initial Iroh announcement; first-boot local/QUIC deferral is forbidden"
4044 );
4045 }
4046 if quic_bind.is_some() {
4050 anyhow::bail!(
4051 "QUIC requested via --quic-bind but the PDS \
4052 accepted-state store is at first boot (no \
4053 accepted states written yet). Complete \
4054 discovery/registry provisioning, then restart \
4055 to activate QUIC."
4056 );
4057 }
4058 tracing::warn!(
4064 "QUIC disabled for this run: the PDS \
4065 accepted-state store is at first boot (no \
4066 accepted states written yet). Services will \
4067 bind via inproc/IPC and serve HTTP. Restart \
4068 after the discovery/registry pipeline writes \
4069 accepted states to activate QUIC. No runtime \
4070 auto-activation is implemented."
4071 );
4072 quic_cfg.enabled = false;
4073 }
4074 }
4075 }
4076
4077 if quic_cfg.enabled {
4079 let qc = quic_cfg;
4080 let (cert_chain, key_der) = qc
4081 .load_tls_materials()
4082 .context("Failed to load QUIC TLS materials")?;
4083 let certificate_hashes = if qc.use_self_signed() {
4087 use sha2::Digest as _;
4088 vec![sha2::Sha256::digest(
4089 cert_chain
4090 .first()
4091 .context("QUIC TLS chain is empty")?,
4092 )
4093 .into()]
4094 } else {
4095 Vec::new()
4096 };
4097
4098 if print_cert_hash {
4100 let hash = hyprstream_rpc::transport::zmtp_quic::cert_hash(
4101 &cert_chain[0],
4102 );
4103 println!("QUIC/WebTransport cert hash: {}", hash);
4104 }
4105
4106 let moq_relay = if qc.relay.trim().is_empty() {
4112 None
4113 } else {
4114 match resolve_moq_relay_reach(&qc.relay) {
4115 Ok(reach) => Some(reach),
4116 Err(e) => {
4117 tracing::warn!(
4118 relay = %qc.relay,
4119 "ignoring invalid [quic] relay; continuing direct-only: {e}"
4120 );
4121 None
4122 }
4123 }
4124 };
4125 let moq_admission_proof =
4131 select_iroh_moql_admission_proof(qc.iroh, || {
4132 service_names
4133 .iter()
4134 .map(|service_name| {
4135 ctx.moql_admission_proof(service_name)
4136 .map(|proof| (service_name.clone(), proof))
4137 })
4138 .collect::<Result<Vec<_>>>()
4139 })?;
4140 if let Some(proof) = moq_admission_proof.clone() {
4143 let _ = hyprstream_rpc::moq_stream::init_global_moq_admission_proof(proof);
4144 }
4145 let discovery_transport =
4146 ctx.transport("discovery", SocketKind::Rep);
4147 let shared = hyprstream_service::QuicSharedConfig {
4148 cert_chain,
4149 certificate_hashes,
4150 key_der,
4151 base_ip: qc.socket_addr()?.ip(),
4152 server_name: qc.server_name.clone(),
4153 oauth_issuer_url: Some(config.oauth.issuer_url()),
4154 jwt_verifying_key: Some(ctx.jwt_verifying_key()),
4155 iroh_enabled: qc.iroh,
4157 iroh_required: qc.iroh_required(),
4158 moq_relay,
4160 moq_relay_server_identity: None,
4164 moq_admission: if moq_admission_proof.is_some() {
4165 Some(hyprstream_core::services::stream_network::production_stream_admission()?)
4166 } else { None },
4167 moq_ingress_authorizer: if moq_admission_proof.is_some() {
4173 let roster = hyprstream_discovery::production_deployment_roster()?;
4180 let _ = roster.roster();
4181 Some(hyprstream_core::services::stream_network::stream_ingress_authorizer(
4182 &qc,
4183 roster,
4184 ))
4185 } else { None },
4186 moq_admission_proof,
4187 native_announcement_publisher: Some(std::sync::Arc::new(
4188 move |request: hyprstream_service::NativeAnnouncementRequest| {
4189 let discovery_transport = discovery_transport.clone();
4190 start_native_announcement_publisher(
4191 request.reach.clone(),
4192 hyprstream_discovery::native_network_required(),
4193 request.cancellation.clone(),
4194 move |announce_tx| async move {
4195 let mut request = request;
4196 let socket_kind = request.reach.socket_kind().to_owned();
4197 let endpoint = request.reach.endpoint();
4198 let service_name = request.service_name.clone();
4199 let signing_key = request.signing_key.clone();
4200 let discovery_verifying_key = request.discovery_verifying_key;
4201 let make_client = |announcement: &hyprstream_rpc_std::discovery_client::ServiceAnnouncement| {
4206 let token = native_announcement_bearer(announcement)?;
4207 if hyprstream_discovery::native_network_required() {
4208 hyprstream_rpc_std::discovery_client::DiscoveryClient::from_provider(
4209 &hyprstream_discovery::ProductionRpcClientProvider,
4210 signing_key.clone(), Some(token),
4211 )
4212 } else {
4213 hyprstream_rpc_std::discovery_client::DiscoveryClient::for_local_transport_bootstrap(
4214 &discovery_transport, signing_key.clone(),
4215 discovery_verifying_key, Some(token),
4216 )
4217 }
4218 };
4219 if service_name == "policy"
4220 && hyprstream_discovery::native_network_required()
4221 && matches!(request.reach, hyprstream_service::NativeAnnouncementReach::Iroh { .. })
4222 {
4223 let cancellation = request.cancellation.clone();
4224 let initial = wait_for_policy_initial_publication(
4225 &cancellation,
4226 POLICY_INITIAL_PUBLICATION_BUDGET,
4227 || {
4228 let announcement = hyprstream_core::services::factories::current_native_announcement(&mut request);
4231 let client = announcement.as_ref().map_err(|e| anyhow::anyhow!("{e}")).and_then(&make_client);
4232 async move {
4233 let announcement = announcement?;
4234 let client = client?;
4235 let jwt_expiry = announcement.service_jwt.as_deref()
4236 .and_then(hyprstream_core::auth::identity_store::decode_jwt_exp_raw)
4237 .and_then(|seconds| seconds.checked_mul(1_000))
4238 .context("Policy initial announcement requires a bounded service JWT")?;
4239 let expiry = announcement.expires_at_unix_ms.min(jwt_expiry);
4240 let remaining = expiry.saturating_sub(chrono::Utc::now().timestamp_millis());
4241 anyhow::ensure!(remaining > 0, "Policy initial announcement authority expired");
4242 let duration = std::time::Duration::from_millis(u64::try_from(remaining)?);
4243 tokio::time::timeout(duration, client.announce(&announcement)).await
4244 .context("Policy initial announcement authority expired during publication")??;
4245 anyhow::ensure!(chrono::Utc::now().timestamp_millis() < expiry,
4246 "Policy initial announcement authority expired before readiness");
4247 Ok(())
4248 }
4249 },
4250 ).await;
4251 if let Some(tx) = announce_tx.as_ref() {
4255 if let Some(tx) = tx.lock().take() {
4256 let _ = tx.send(initial.as_ref().map(|_| ()).map_err(ToString::to_string));
4257 }
4258 }
4259 if initial.is_err() { return; }
4260 tokio::select! {
4261 biased;
4262 _ = cancellation.cancelled() => return,
4263 _ = tokio::time::sleep(std::time::Duration::from_secs(25)) => {}
4264 }
4265 }
4266 let _completion = refresh_native_announcement(
4267 &service_name,
4268 &socket_kind,
4269 &endpoint,
4270 i64::MAX,
4273 request.cancellation.clone(),
4274 || {
4275 let announcement = hyprstream_core::services::factories::current_native_announcement(&mut request);
4276 let announce_tx = announce_tx.as_ref().map(std::sync::Arc::clone);
4277 let client = announcement.as_ref().map_err(|e| anyhow::anyhow!("{e}")).and_then(&make_client);
4278 publish_native_announcement_attempt(announcement, announce_tx, move |announcement| async move {
4279 client?.announce(&announcement).await.map(|_| ())
4280 })
4281 },
4282 )
4283 .await;
4284 },
4285 )
4286 },
4287 )),
4288 };
4289 ctx = ctx.with_quic(shared);
4290 }
4291
4292 {
4294 let issuer_url = config.oauth.issuer_url();
4295 let jwks_url =
4296 format!("{}/oauth/jwks", issuer_url.trim_end_matches('/'));
4297 let fetcher =
4298 hyprstream_core::auth::jwks_fetcher::default_jwks_fetcher();
4299 ctx.set_jwks_fetcher(fetcher);
4300 tracing::debug!(
4301 "JWKS-backed key source configured: {}",
4302 jwks_url
4303 );
4304 }
4305
4306 {
4308 let secrets_dir = hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(Some(&config))?;
4309 let ml_dsa_store = hyprstream_core::auth::key_rotation::global_ml_dsa_key_store(
4310 &secrets_dir,
4311 &config.oauth,
4312 );
4313 let vks: Vec<_> = tokio::task::block_in_place(|| {
4314 let rt = tokio::runtime::Handle::current();
4315 rt.block_on(ml_dsa_store.all_slots_snapshot())
4316 })
4317 .iter()
4318 .map(hyprstream_core::auth::MlDsaKeySlot::verifying_key)
4319 .collect();
4320 let shared_vks = hyprstream_core::auth::key_rotation::global_ml_dsa_verifying_keys();
4321 let _ = shared_vks.write().map(|mut guard| *guard = vks);
4322 ctx.set_ml_dsa_verifying_keys(shared_vks);
4323 let ed_store = hyprstream_core::auth::key_rotation::global_ed25519_key_store(
4324 &secrets_dir,
4325 &config.oauth,
4326 );
4327 let restore_result = tokio::task::block_in_place(|| {
4328 let rt = tokio::runtime::Handle::current();
4329 rt.block_on(hyprstream_core::auth::key_rotation::restore_composite_verifying_key_set(
4330 &secrets_dir,
4331 &ed_store,
4332 &ml_dsa_store,
4333 ctx.jwt_verifying_key(),
4334 ))
4335 });
4336 if let Err(error) = restore_result {
4337 tracing::warn!("exact composite pair ledger unavailable in verifier process: {error}");
4338 }
4339 tracing::info!("PQ-hybrid: ML-DSA-65 verifying keys loaded for JWT verification");
4340
4341 if config.policy.boot_policy.is_none() {
4344 let active_pq = tokio::task::block_in_place(|| {
4363 let rt = tokio::runtime::Handle::current();
4364 rt.block_on(ml_dsa_store.active_key())
4365 });
4366 match active_pq {
4367 Some(pq_sk) => match hyprstream_core::mac::install_baseline_boot_policy(
4368 &signing_key,
4369 &pq_sk,
4370 ) {
4371 Ok((policy, true)) => tracing::info!(
4372 "MAC S4: baseline compiled policy installed \
4373 (generation {}, 9P enforcement active)",
4374 policy.generation
4375 ),
4376 Ok((_policy, false)) => tracing::warn!(
4381 "MAC S4: a compiled policy was already installed; \
4382 baseline NOT installed (write-once seam unchanged)"
4383 ),
4384 Err(e) => tracing::error!(
4385 "MAC S4: baseline policy bootload failed \
4386 (grant-path resolver stays fail-closed): {e}"
4387 ),
4388 },
4389 None => tracing::warn!(
4390 "MAC S4: no active ML-DSA signing key; baseline policy \
4391 NOT installed (fail-closed — grant-path resolver denies)"
4392 ),
4393 }
4394 }
4395 }
4396
4397 install_envelope_verify_config(Some(&config.oauth), Some(&config));
4420
4421 if !hyprstream_rpc::events::event_authz_installed() {
4429 let audit_stream =
4430 format!("moq-event-{}", service_names.join("-"));
4431 let authz = hyprstream_core::services::native_event_policy::production_event_authz(
4432 signing_key.clone(),
4433 &config,
4434 &audit_stream,
4435 )
4436 .await
4437 .context("construct process-wide MoQ/event MAC PEP")?;
4438 hyprstream_rpc::events::install_event_authz(authz)
4439 .map_err(anyhow::Error::msg)?;
4440 }
4441
4442 let manager = InprocManager::new();
4443 let mut handles = Vec::new();
4444
4445 let stages = hyprstream_service::service::ordering::startup_stages_for_profile(
4447 &service_names, ctx.iroh_required(),
4448 );
4449
4450 let defer_policy_probe = if ctx.iroh_required()
4461 && service_names.len() == 1
4462 {
4463 let configured_stages =
4464 hyprstream_service::service::ordering::startup_stages_for_profile(
4465 &services,
4466 true,
4467 );
4468 let policy_position =
4469 configured_stages.iter().enumerate().find_map(
4470 |(stage_index, stage)| {
4471 stage.iter().position(|name| name == "policy").map(
4472 |service_index| (stage_index, service_index),
4473 )
4474 },
4475 );
4476 let service_position = configured_stages
4477 .iter()
4478 .enumerate()
4479 .find_map(|(stage_index, stage)| {
4480 stage
4481 .iter()
4482 .position(|name| name == &service_names[0])
4483 .map(|service_index| (stage_index, service_index))
4484 });
4485 match (service_position, policy_position) {
4486 (Some(service), Some(policy)) => service < policy,
4487 _ => false,
4488 }
4489 } else {
4490 false
4491 };
4492
4493 hyprstream_core::services::revocation::init_process_authority_stores(
4503 &ctx,
4504 ctx.signing_key(),
4505 service_names.iter().any(|n| n == "policy"),
4506 defer_policy_probe,
4507 )
4508 .await
4509 .context("revocation/session authority initialization failed")?;
4510
4511 let hosts_event_service = stages
4521 .iter()
4522 .any(|stage| stage.iter().any(|s| s == "event"));
4523 if !hosts_event_service {
4524 hyprstream_core::services::event_network::ensure_event_origin_for_profile()?;
4525 }
4526
4527 for stage in &stages {
4528 for svc_name in stage {
4529 let factory = get_factory(svc_name).ok_or_else(|| {
4530 let available: Vec<_> =
4531 hyprstream_service::list_factories()
4532 .map(|f| f.name)
4533 .collect();
4534 anyhow::anyhow!(
4535 "Unknown service: '{}'. Available services: {}",
4536 svc_name,
4537 available.join(", ")
4538 )
4539 })?;
4540
4541 info!(
4542 "Starting {} service in standalone mode (IPC: {})",
4543 svc_name, ipc
4544 );
4545
4546 let spawnable = (factory.factory)(&ctx).context(
4547 format!("Failed to create {} service", svc_name),
4548 )?;
4549 let handle = manager.spawn(spawnable).await.context(
4550 format!("Failed to start {} service", svc_name),
4551 )?;
4552 handles.push((svc_name.clone(), handle));
4553 }
4554
4555 for svc_name in stage {
4557 if let Ok(publisher) = EventPublisher::new("system") {
4558 let _ = publisher
4559 .publish_raw(
4560 &format!("system.{}.ready", svc_name),
4561 b"",
4562 )
4563 .await;
4564 }
4565 }
4566 }
4567
4568 if handles.is_empty() {
4569 return Err(anyhow::anyhow!("No services to start"));
4570 }
4571
4572 let svc_names: Vec<_> =
4573 handles.iter().map(|(n, _)| n.as_str()).collect();
4574 info!(
4575 "Services ready: [{}], waiting for shutdown signal",
4576 svc_names.join(", ")
4577 );
4578
4579 let _ = shutdown_rx.await;
4580
4581 for (svc_name, mut handle) in handles {
4583 if let Ok(publisher) = EventPublisher::new("system") {
4584 let _ = publisher
4585 .publish_raw(
4586 &format!("system.{}.stopping", svc_name),
4587 b"",
4588 )
4589 .await;
4590 }
4591 let _ = handle.stop().await;
4592 info!("{} service stopped", svc_name);
4593 }
4594
4595 Ok(())
4596 },
4597 )?;
4598 } else {
4599 with_runtime(
4600 RuntimeConfig {
4601 device: DeviceConfig::request_cpu(),
4602 multi_threaded: false,
4603 },
4604 || async move {
4605 handle_service_start(
4606 &services,
4607 name,
4608 daemon,
4609 explicit_config,
4610 iroh_required,
4611 )
4612 .await
4613 },
4614 )?;
4615 }
4616 }
4617
4618 ServiceAction::Stop { name } => {
4619 with_runtime(
4620 RuntimeConfig {
4621 device: DeviceConfig::request_cpu(),
4622 multi_threaded: false,
4623 },
4624 || async move { handle_service_stop(&services, name).await },
4625 )?;
4626 }
4627
4628 ServiceAction::Status { verbose } => {
4629 with_runtime(
4630 RuntimeConfig {
4631 device: DeviceConfig::request_cpu(),
4632 multi_threaded: false,
4633 },
4634 || async move { handle_service_status(&services, verbose).await },
4635 )?;
4636 }
4637
4638 ServiceAction::Repair { verbose } => {
4639 let models_dir = config_for_service.models_dir().clone();
4644 with_runtime(
4645 RuntimeConfig {
4646 device: DeviceConfig::request_cpu(),
4647 multi_threaded: true,
4648 },
4649 || async move {
4650 hyprstream_core::cli::service_handlers::run_repair_checks(
4651 &models_dir,
4652 verbose,
4653 )
4654 .await
4655 },
4656 )?;
4657 }
4658
4659 ServiceAction::EnsureKey { name } => {
4660 hyprstream_core::cli::service_handlers::handle_service_ensure_key(
4664 Some(ctx.config()),
4665 &name,
4666 )?;
4667 }
4668 ServiceAction::ReconcileEnrollment => {
4669 hyprstream_core::cli::service_handlers::handle_service_reconcile_enrollment(
4670 ctx.config(),
4671 )?;
4672 }
4673 ServiceAction::EnsureNodeAuthority => {
4674 let secrets = hyprstream_core::config::HyprConfig::resolve_secrets_dir_for(
4675 Some(ctx.config()),
4676 )?;
4677 hyprstream_core::auth::service_enrollment::ensure_node_authority(&secrets)?;
4678 }
4679 ServiceAction::ProvisionPolicyTemplates { template } => {
4680 let models_dir = config_for_service.models_dir().clone();
4681 with_runtime(
4682 RuntimeConfig {
4683 device: DeviceConfig::request_cpu(),
4684 multi_threaded: true,
4685 },
4686 || async move {
4687 handle_service_provision_policy_templates(&models_dir, &template).await
4688 },
4689 )?;
4690 }
4691 ServiceAction::ReadbackLocalCanary { .. } => {
4692 anyhow::bail!("readback-local-canary must use its enrolled early dispatch")
4693 }
4694 ServiceAction::RegisterLocalCanary { .. } => {
4695 anyhow::bail!("register-local-canary must use its enrolled early dispatch")
4696 }
4697 }
4698 }
4699
4700 Some(("user", sub_m)) => {
4702 let cmd = UserCommand::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
4703 let credentials_dir =
4708 hyprstream_core::auth::identity_store::credentials_dir_for_config(Some(
4709 ctx.config(),
4710 ))?;
4711 let rt = tokio::runtime::Builder::new_current_thread()
4713 .enable_all()
4714 .build()
4715 .context("Failed to create runtime for user command")?;
4716 rt.block_on(async {
4717 match cmd {
4718 UserCommand::Create {
4719 username,
4720 role,
4721 generate,
4722 key,
4723 ssh,
4724 no_role,
4725 } => {
4726 handle_user_create(
4727 &credentials_dir,
4728 &username,
4729 &role,
4730 no_role,
4731 generate,
4732 key.as_deref(),
4733 ssh.as_deref(),
4734 )
4735 .await?;
4736 }
4737 UserCommand::Register { username } => {
4738 handle_user_register(&credentials_dir, &username).await?;
4739 }
4740 UserCommand::List => {
4741 handle_user_list(&credentials_dir).await?;
4742 }
4743 UserCommand::Remove { username, force } => {
4744 handle_user_remove(&credentials_dir, &username, force).await?;
4745 }
4746 UserCommand::Keys { command } => match command {
4747 UserKeysCommand::List { username } => {
4748 handle_user_keys_list(&credentials_dir, &username).await?;
4749 }
4750 UserKeysCommand::Import { username, format } => {
4751 handle_user_keys_import(&credentials_dir, &username, &format).await?;
4752 }
4753 UserKeysCommand::Remove {
4754 username,
4755 fingerprint,
4756 } => {
4757 handle_user_keys_remove(&credentials_dir, &username, &fingerprint)
4758 .await?;
4759 }
4760 },
4761 }
4762 Ok::<_, anyhow::Error>(())
4763 })?;
4764 }
4765
4766 Some(("tui", sub_m)) => {
4768 let action =
4769 TuiAction::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
4770
4771 with_runtime(
4772 RuntimeConfig {
4773 device: DeviceConfig::request_cpu(),
4774 multi_threaded: true,
4775 },
4776 || async move {
4777 use hyprstream_core::cli::tui_handlers;
4778
4779 let data_dir = dirs::data_local_dir()
4781 .unwrap_or_else(|| std::path::PathBuf::from("."))
4782 .join("hyprstream");
4783 let keys_dir = data_dir.join("keys");
4784 let signing_key = load_or_generate_signing_key(&keys_dir).await?;
4785
4786 match action {
4787 TuiAction::Attach { session } => {
4788 let sid = if session == 0 { None } else { Some(session) };
4789 tui_handlers::handle_tui_attach(&signing_key, sid).await
4790 }
4791 TuiAction::New => tui_handlers::handle_tui_new(&signing_key).await,
4792 TuiAction::List => tui_handlers::handle_tui_list(&signing_key).await,
4793 TuiAction::Detach => tui_handlers::handle_tui_detach().await,
4794 TuiAction::Play {
4795 cast_file,
4796 session,
4797 loop_playback,
4798 } => {
4799 tui_handlers::handle_tui_play(
4800 &signing_key,
4801 &cast_file,
4802 session,
4803 loop_playback,
4804 )
4805 .await
4806 }
4807 TuiAction::Shell { session } => {
4808 let models_dir = config_for_service.models_dir().clone();
4809 hyprstream_core::cli::shell_handlers::handle_tui_shell(
4810 &signing_key,
4811 &models_dir,
4812 session,
4813 )
4814 .await
4815 }
4816 }
4817 },
4818 )?;
4819 }
4820
4821 Some(("flight", sub_m)) => {
4823 #[cfg(not(feature = "metrics"))]
4824 {
4825 let _ = sub_m;
4826 return Err(anyhow::anyhow!(
4827 "Flight SQL client requires the 'metrics' feature"
4828 ));
4829 }
4830 #[cfg(feature = "metrics")]
4831 {
4832 let args =
4833 FlightArgs::from_arg_matches(sub_m).map_err(|e| anyhow::anyhow!("{}", e))?;
4834
4835 with_runtime(
4836 RuntimeConfig {
4837 device: DeviceConfig::request_cpu(),
4838 multi_threaded: true,
4839 },
4840 || async move {
4841 use hyprstream_flight::client::{
4842 format_batches_as_csv, format_batches_as_json, format_batches_as_table,
4843 FlightClient,
4844 };
4845
4846 let mut client = FlightClient::connect(&args.addr)
4847 .await
4848 .map_err(|e| anyhow::anyhow!("Failed to connect: {}", e))?;
4849
4850 if let Some(sql) = args.query {
4851 let batches = client
4852 .query(&sql)
4853 .await
4854 .map_err(|e| anyhow::anyhow!("Query failed: {}", e))?;
4855
4856 let output = match args.format.as_str() {
4857 "csv" => format_batches_as_csv(&batches),
4858 "json" => format_batches_as_json(&batches)
4859 .map_err(|e| anyhow::anyhow!("JSON format error: {}", e))?,
4860 _ => format_batches_as_table(&batches),
4861 };
4862 print!("{}", output);
4863 } else {
4864 println!("Connected to Flight SQL server at {}", args.addr);
4865 println!(
4866 "Interactive mode not yet implemented. Use --query to execute SQL."
4867 );
4868 }
4869 Ok(())
4870 },
4871 )?;
4872 }
4873 }
4874
4875 Some(("sign-challenge", sub_m)) => {
4877 let user_code = sub_m.get_one::<String>("user_code").cloned();
4878 let nonce = sub_m.get_one::<String>("nonce").cloned();
4879 let code_challenge = sub_m.get_one::<String>("code_challenge").cloned();
4880 let server = sub_m.get_one::<String>("server").cloned();
4881 let insecure = sub_m.get_flag("insecure");
4882 let sign_cfg = config_for_service.clone();
4888 with_runtime(
4889 RuntimeConfig {
4890 device: DeviceConfig::request_cpu(),
4891 multi_threaded: false,
4892 },
4893 || async move {
4894 handle_sign_challenge(
4895 user_code,
4896 nonce,
4897 code_challenge,
4898 server,
4899 insecure,
4900 Some(&sign_cfg),
4901 )
4902 .await
4903 },
4904 )?;
4905 }
4906
4907 Some(("wizard", _)) => {
4909 unreachable!("wizard command is dispatched before registry init")
4910 }
4911
4912 Some(("update", sub_m)) => {
4914 let cleanup = sub_m.get_flag("cleanup");
4915 let models_dir = config_for_service.models_dir().clone();
4916 with_runtime(
4917 RuntimeConfig {
4918 device: DeviceConfig::request_cpu(),
4919 multi_threaded: true,
4920 },
4921 || async move {
4922 hyprstream_core::cli::update_handlers::handle_update(&models_dir, cleanup).await
4923 },
4924 )?;
4925 }
4926
4927 Some(("mac", _)) => {
4931 unreachable!("mac command is dispatched before registry init")
4932 }
4933
4934 _ => {
4936 let models_dir = config_for_service.models_dir().clone();
4937 if hyprstream_core::cli::bootstrap_manager::is_first_run(&models_dir) {
4938 let services = config_for_service.services.startup.clone();
4939 with_runtime(
4940 RuntimeConfig {
4941 device: DeviceConfig::request_cpu(),
4942 multi_threaded: true,
4943 },
4944 || async move {
4945 if supports_tui() {
4946 hyprstream_core::cli::handle_wizard_tui(&models_dir, &services).await
4947 } else {
4948 hyprstream_core::cli::handle_wizard(
4950 &models_dir,
4951 &services,
4952 hyprstream_core::cli::WizardOptions::first_run(),
4953 )
4954 .await
4955 }
4956 },
4957 )?;
4958 } else {
4959 with_runtime(
4960 RuntimeConfig {
4961 device: DeviceConfig::request_cpu(),
4962 multi_threaded: true,
4963 },
4964 || {
4965 let md = models_dir.clone();
4966 async move {
4967 let keys_dir = md.join(".registry").join("keys");
4968 let sk = load_or_generate_signing_key(&keys_dir).await?;
4969 hyprstream_core::cli::shell_handlers::handle_shell_tui(&sk, &md).await
4970 }
4971 },
4972 )?;
4973 }
4974 }
4975 };
4976
4977 Ok(())
4978}
4979
4980#[cfg(test)]
4981mod process_identity_tests {
4982 use hyprstream_rpc::crypto::SigningKey;
4983
4984 #[test]
4985 fn dedicated_service_process_uses_service_identity_without_native_profile() {
4986 assert_eq!(
4987 super::process_identity_name(None, Some("oauth")),
4988 Some("oauth"),
4989 "a dedicated compatibility process must own its service identity"
4990 );
4991 }
4992
4993 #[test]
4994 fn native_selection_takes_precedence_over_dedicated_name() {
4995 assert_eq!(
4996 super::process_identity_name(Some("oauth"), Some("oauth")),
4997 Some("oauth")
4998 );
4999 assert_eq!(
5000 super::process_identity_name(Some("oauth"), None),
5001 Some("oauth")
5002 );
5003 }
5004
5005 #[test]
5006 fn plain_cli_process_has_no_service_identity() {
5007 assert_eq!(super::process_identity_name(None, None), None);
5008 }
5009
5010 #[test]
5011 fn flat_key_policy_credential_only_for_the_flat_key_with_a_jwt() -> anyhow::Result<()> {
5012 let dir = tempfile::tempdir()?;
5013 let profile = hyprstream_core::auth::identity_store::SecretsProfile::SharedDirectory;
5014 let policy_key = hyprstream_core::auth::identity_store::resolve_service_signing_key(
5016 dir.path(),
5017 "policy",
5018 profile,
5019 )?;
5020 assert!(
5022 super::flat_key_policy_resolver_credential(dir.path(), profile, &policy_key).is_none()
5023 );
5024 let other = SigningKey::from_bytes(&[0x41; 32]);
5026 let now = chrono::Utc::now().timestamp();
5027 let claims = hyprstream_rpc::auth::Claims::new("service:policy".to_owned(), now, now + 600)
5028 .with_cnf_jwk(policy_key.verifying_key().as_bytes());
5029 let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &policy_key);
5030 hyprstream_core::auth::identity_store::write_service_jwt(dir.path(), "policy", &jwt)?;
5031 assert!(
5032 super::flat_key_policy_resolver_credential(dir.path(), profile, &other).is_none(),
5033 "split-key deployments never present policy's credential"
5034 );
5035 assert_eq!(
5036 super::flat_key_policy_resolver_credential(dir.path(), profile, &policy_key),
5037 Some(jwt),
5038 "the flat-key process presents policy's enrolled credential"
5039 );
5040 Ok(())
5041 }
5042}
5043
5044#[cfg(test)]
5045#[allow(clippy::expect_used)]
5046mod resolver_startup_controls {
5047 use clap::FromArgMatches as _;
5048
5049 #[test]
5050 fn canary_registration_cli_requires_exact_path_and_head_arguments() {
5051 let matches = super::build_cli()
5052 .try_get_matches_from([
5053 "hyprstream",
5054 "service",
5055 "register-local-canary",
5056 "--path",
5057 "/var/lib/hyprstream/models/canary",
5058 "--expected-head",
5059 "3f1d3a63258e132cd06314b736445a8bab13b9bd",
5060 ])
5061 .expect("canary registration CLI");
5062 let service = matches.subcommand_matches("service").expect("service");
5063 let action = hyprstream_core::cli::commands::ServiceAction::from_arg_matches(service)
5064 .expect("service action");
5065 assert!(matches!(
5066 action,
5067 hyprstream_core::cli::commands::ServiceAction::RegisterLocalCanary { .. }
5068 ));
5069 assert!(super::build_cli()
5070 .try_get_matches_from(["hyprstream", "service", "register-local-canary"])
5071 .is_err());
5072 }
5073
5074 #[test]
5075 fn offline_policy_provision_cli_requires_and_preserves_templates() {
5076 let matches = super::build_cli()
5077 .try_get_matches_from([
5078 "hyprstream",
5079 "service",
5080 "provision-policy-templates",
5081 "--template",
5082 "public-inference",
5083 "--template",
5084 "public-read",
5085 ])
5086 .expect("offline policy CLI");
5087 let service = matches.subcommand_matches("service").expect("service");
5088 let action = hyprstream_core::cli::commands::ServiceAction::from_arg_matches(service)
5089 .expect("service action");
5090 let hyprstream_core::cli::commands::ServiceAction::ProvisionPolicyTemplates { template } =
5091 action
5092 else {
5093 panic!("wrong service action");
5094 };
5095 assert_eq!(template, ["public-inference", "public-read"]);
5096 assert!(super::build_cli()
5097 .try_get_matches_from(["hyprstream", "service", "provision-policy-templates",])
5098 .is_err());
5099 }
5100
5101 #[test]
5102 fn native_service_identity_loads_existing_key_without_ipc() -> anyhow::Result<()> {
5103 const CHILD: &str = "HYPRSTREAM_NATIVE_SERVICE_IDENTITY_TEST";
5104 let Ok(profile) = std::env::var(CHILD) else {
5105 for profile in ["shared-directory", "per-service-scoped"] {
5106 let status = std::process::Command::new(std::env::current_exe()?)
5107 .args(["--exact", "resolver_startup_controls::native_service_identity_loads_existing_key_without_ipc", "--nocapture"])
5108 .env(CHILD, profile)
5109 .env("HYPRSTREAM_SECRETS_PROFILE", profile)
5110 .env_remove("HYPRSTREAM__SECRETS__PATH")
5111 .env_remove("HYPRSTREAM__SIGNING_KEY")
5112 .status()?;
5113 anyhow::ensure!(
5114 status.success(),
5115 "isolated {profile} identity regression failed"
5116 );
5117 }
5118 return Ok(());
5119 };
5120 let root = tempfile::tempdir()?;
5121 let runtime = tokio::runtime::Builder::new_current_thread()
5122 .enable_all()
5123 .build()?;
5124 for (service, seed) in [("policy", 31u8), ("model", 32u8), ("registry", 33u8)] {
5125 let secrets = root.path().join(format!("{service}-credentials"));
5126 std::fs::create_dir_all(secrets.join(service))?;
5127 let key_path = if service == "policy" || profile == "per-service-scoped" {
5128 std::fs::write(secrets.join(service).join("signing-key"), [99u8; 32])?;
5131 secrets.join("signing-key")
5132 } else {
5133 std::fs::write(secrets.join("signing-key"), [31u8; 32])?;
5136 secrets.join(service).join("signing-key")
5137 };
5138 std::fs::write(&key_path, [seed; 32])?;
5139 let mut configured = super::HyprConfig::default();
5140 configured.quic.native_network_profile =
5141 hyprstream_core::config::NativeNetworkProfile::NetworkIrohRequired;
5142 configured.secrets.path = Some(secrets.clone());
5143 configured.storage.models_dir = root.path().join(format!("{service}-models"));
5144 let config_path = root.path().join(format!("{service}.toml"));
5145 configured.to_file(&config_path)?;
5146 let matches = super::build_cli().try_get_matches_from([
5147 "hyprstream",
5148 "--config",
5149 config_path.to_str().expect("temporary UTF-8 path"),
5150 "service",
5151 "start",
5152 service,
5153 "--foreground",
5154 ])?;
5155 let config = super::load_config(
5156 matches
5157 .get_one::<std::path::PathBuf>("config")
5158 .map(std::path::PathBuf::as_path),
5159 )?;
5160 let selected = super::native_service_process_name(&matches, &config)?;
5161 assert_eq!(selected.as_deref(), Some(service));
5162 let key = runtime.block_on(super::load_process_signing_key(
5163 &config,
5164 selected.as_deref(),
5165 ))?;
5166 assert_eq!(key.to_bytes(), [seed; 32]);
5167 let ctx = super::ServiceContext::new(
5168 key.clone(),
5169 key.verifying_key(),
5170 false,
5171 config.models_dir().clone(),
5172 )
5173 .with_service_key(service, key.clone());
5174 assert_eq!(ctx.signing_key().verifying_key(), key.verifying_key());
5175 assert_eq!(
5176 ctx.service_signing_key(service).verifying_key(),
5177 key.verifying_key()
5178 );
5179 assert!(
5180 !config.models_dir().join(".registry/keys").exists(),
5181 "native startup must not materialize a CLI/root key"
5182 );
5183 std::fs::remove_file(&key_path)?;
5184 assert!(runtime
5185 .block_on(super::load_process_signing_key(
5186 &config,
5187 selected.as_deref()
5188 ))
5189 .is_err());
5190 assert!(
5191 !key_path.exists(),
5192 "missing provisioned key must not be regenerated"
5193 );
5194 std::fs::write(&key_path, [seed; 31])?;
5195 assert!(runtime
5196 .block_on(super::load_process_signing_key(
5197 &config,
5198 selected.as_deref()
5199 ))
5200 .is_err());
5201 assert_eq!(
5202 std::fs::read(&key_path)?,
5203 vec![seed; 31],
5204 "malformed key must not be repaired during startup"
5205 );
5206 }
5207 Ok(())
5208 }
5209
5210 #[test]
5211 fn dedicated_service_containment_is_independent_of_network_profile() -> anyhow::Result<()> {
5212 use hyprstream_core::config::NativeNetworkProfile;
5213 let mut config = super::HyprConfig::default();
5214 config.account.http = Some(hyprstream_core::account::AccountHttpConfig {
5215 host: "::1".to_owned(),
5216 port: 8443,
5217 tls_cert: "unused.pem".into(),
5218 tls_key: "unused.key".into(),
5219 });
5220 for profile in [
5221 NativeNetworkProfile::Compatibility,
5222 NativeNetworkProfile::NetworkIrohRequired,
5223 ] {
5224 config.quic.native_network_profile = profile;
5225 for argv in [
5226 vec!["hyprstream", "service", "start", "oauth", "--foreground"],
5227 vec![
5228 "hyprstream",
5229 "service",
5230 "start",
5231 "--foreground",
5232 "--services",
5233 "oauth",
5234 ],
5235 vec!["hyprstream", "service", "start", "--standalone"],
5236 ] {
5237 config.services.startup = vec!["oauth".into()];
5238 let matches = super::build_cli().try_get_matches_from(argv)?;
5239 let service = super::dedicated_service_process_name(&matches, &config)?
5240 .expect("a single OAuth foreground service has process containment");
5241 assert_eq!(service, "oauth");
5242 let key = super::SigningKey::from_bytes(&[9; 32]);
5243 let context = super::ServiceContext::new(
5244 key.clone(),
5245 key.verifying_key(),
5246 false,
5247 "/unused".into(),
5248 )
5249 .with_dedicated_process_service(service);
5250 assert!(context.is_dedicated_process_for("oauth"));
5251 assert!(!context.is_dedicated_process_for("registry"));
5252 assert_eq!(
5253 super::native_service_process_name(&matches, &config)?.is_some(),
5254 config.quic.iroh_required(),
5255 "containment must not change compatibility signing-key selection",
5256 );
5257 }
5258 config.services.startup = vec!["oauth".into(), "registry".into()];
5259 for argv in [
5260 vec!["hyprstream", "service", "start", "oauth"],
5261 vec!["hyprstream", "service", "start", "--standalone"],
5262 vec![
5263 "hyprstream",
5264 "service",
5265 "start",
5266 "--foreground",
5267 "--services",
5268 "oauth,registry",
5269 ],
5270 vec!["hyprstream", "service", "status"],
5271 ] {
5272 let matches = super::build_cli().try_get_matches_from(argv)?;
5273 assert_eq!(
5274 super::dedicated_service_process_name(&matches, &config)?,
5275 None
5276 );
5277 }
5278 }
5279 Ok(())
5280 }
5281
5282 #[test]
5283 fn native_service_identity_rejects_multi_and_selects_single_service_list() -> anyhow::Result<()>
5284 {
5285 let mut config = super::HyprConfig::default();
5286 config.quic.native_network_profile =
5287 hyprstream_core::config::NativeNetworkProfile::NetworkIrohRequired;
5288 let single = super::build_cli().try_get_matches_from([
5289 "hyprstream",
5290 "service",
5291 "start",
5292 "--foreground",
5293 "--services",
5294 "model",
5295 ])?;
5296 assert_eq!(
5297 super::native_service_process_name(&single, &config)?.as_deref(),
5298 Some("model")
5299 );
5300 let multi = super::build_cli().try_get_matches_from([
5301 "hyprstream",
5302 "service",
5303 "start",
5304 "--foreground",
5305 "--services",
5306 "model,registry",
5307 ])?;
5308 assert!(super::native_service_process_name(&multi, &config).is_err());
5309 let standalone = super::build_cli().try_get_matches_from([
5310 "hyprstream",
5311 "service",
5312 "start",
5313 "--standalone",
5314 ])?;
5315 config.services.startup = vec!["model".into(), "registry".into()];
5316 assert!(super::native_service_process_name(&standalone, &config).is_err());
5317 config.quic.native_network_profile =
5318 hyprstream_core::config::NativeNetworkProfile::Compatibility;
5319 assert_eq!(super::native_service_process_name(&multi, &config)?, None);
5320 assert_eq!(super::native_service_process_name(&single, &config)?, None);
5321 Ok(())
5322 }
5323
5324 #[test]
5325 fn native_service_identity_seeds_discovery_from_loaded_config() -> anyhow::Result<()> {
5326 const CHILD: &str = "HYPRSTREAM_NATIVE_DISCOVERY_SEED_TEST";
5327 if std::env::var_os(CHILD).is_none() {
5328 let status = std::process::Command::new(std::env::current_exe()?)
5329 .args([
5330 "--exact",
5331 "resolver_startup_controls::native_service_identity_seeds_discovery_from_loaded_config",
5332 "--nocapture",
5333 ])
5334 .env(CHILD, "1")
5335 .env_remove("HYPRSTREAM__SECRETS__PATH")
5336 .env_remove("HYPRSTREAM_SECRETS_PROFILE")
5337 .status()?;
5338 anyhow::ensure!(
5339 status.success(),
5340 "isolated discovery seeding regression failed"
5341 );
5342 return Ok(());
5343 }
5344
5345 let root = tempfile::tempdir()?;
5346 let xdg_config_home = root.path().join("xdg-config");
5347 std::env::set_var("XDG_CONFIG_HOME", &xdg_config_home);
5348
5349 let default_secrets = xdg_config_home.join("hyprstream/credentials");
5352 let default_key = ed25519_dalek::SigningKey::from_bytes(&[91u8; 32]);
5353 let mut default_entries = std::collections::HashMap::new();
5354 default_entries.insert(
5355 "discovery".to_owned(),
5356 hyprstream_core::auth::identity_store::BootstrapPubkey::for_service_key(&default_key)?,
5357 );
5358 hyprstream_core::auth::identity_store::write_bootstrap_pubkeys_hybrid(
5359 &default_secrets,
5360 &default_entries,
5361 )?;
5362
5363 let custom_secrets = root.path().join("custom-credentials");
5364 let custom_key = ed25519_dalek::SigningKey::from_bytes(&[92u8; 32]);
5365 let mut custom_entries = std::collections::HashMap::new();
5366 custom_entries.insert(
5367 "discovery".to_owned(),
5368 hyprstream_core::auth::identity_store::BootstrapPubkey::for_service_key(&custom_key)?,
5369 );
5370 hyprstream_core::auth::identity_store::write_bootstrap_pubkeys_hybrid(
5371 &custom_secrets,
5372 &custom_entries,
5373 )?;
5374
5375 let mut config = super::HyprConfig::default();
5376 config.secrets.path = Some(custom_secrets);
5377 let selected = super::resolve_service_vk("discovery", Some(&config))
5378 .expect("configured bootstrap-pubkeys must seed discovery");
5379 assert_eq!(selected, custom_key.verifying_key());
5380 assert_ne!(selected, default_key.verifying_key());
5381 assert_eq!(
5382 hyprstream_service::global_trust_store().resolve_one("discovery"),
5383 Some(custom_key.verifying_key()),
5384 );
5385 Ok(())
5386 }
5387
5388 #[test]
5389 fn readonly_roster_cli_requires_services_and_has_no_mutation_options() {
5390 let matches = super::build_cli()
5391 .try_get_matches_from([
5392 "hyprstream",
5393 "pds",
5394 "inspect-services",
5395 "--service",
5396 "model,event",
5397 ])
5398 .expect("read-only roster CLI");
5399 let inspect = matches
5400 .subcommand_matches("pds")
5401 .expect("pds")
5402 .subcommand_matches("inspect-services")
5403 .expect("inspect");
5404 assert_eq!(
5405 inspect
5406 .get_many::<String>("service")
5407 .expect("services")
5408 .map(String::as_str)
5409 .collect::<Vec<_>>(),
5410 vec!["model", "event"]
5411 );
5412 for args in [
5413 vec!["hyprstream", "pds", "inspect-services"],
5414 vec![
5415 "hyprstream",
5416 "pds",
5417 "inspect-services",
5418 "--service",
5419 "model",
5420 "--valid-for-seconds",
5421 "86400",
5422 ],
5423 vec![
5424 "hyprstream",
5425 "pds",
5426 "inspect-services",
5427 "--service",
5428 "model",
5429 "--roster-export",
5430 "out.json",
5431 ],
5432 ] {
5433 assert!(super::build_cli().try_get_matches_from(args).is_err());
5434 }
5435 }
5436
5437 fn unavailable_policy_publication() -> anyhow::Error {
5438 hyprstream_rpc::transport_traits::PreDispatchTransportError::new(
5439 hyprstream_rpc::transport::lazy_iroh::IrohPeerUnavailable::ConnectTimeout.into(),
5440 )
5441 .into()
5442 }
5443
5444 #[tokio::test(start_paused = true)]
5445 async fn policy_initial_publication_retries_only_typed_availability_and_reprojects() {
5446 let cancellation = tokio_util::sync::CancellationToken::new();
5447 let mut projections = 0;
5448 let start = tokio::time::Instant::now();
5449 super::wait_for_policy_initial_publication(
5450 &cancellation,
5451 std::time::Duration::from_secs(120),
5452 || {
5453 projections += 1;
5454 let generation = projections;
5455 async move {
5456 if generation < 3 {
5457 Err(unavailable_policy_publication())
5458 } else {
5459 Ok(())
5460 }
5461 }
5462 },
5463 )
5464 .await
5465 .expect("late Discovery should allow Policy to publish");
5466 assert_eq!(
5467 projections, 3,
5468 "every retry must request fresh authority material"
5469 );
5470 assert_eq!(start.elapsed(), std::time::Duration::from_secs(1));
5471 }
5472
5473 #[tokio::test(start_paused = true)]
5474 async fn policy_initial_publication_permanent_error_after_availability_is_terminal() {
5475 let cancellation = tokio_util::sync::CancellationToken::new();
5476 for terminal in [
5479 "expired current authority",
5480 "revoked service JWT",
5481 "iroh connect timed out",
5482 ] {
5483 let mut attempts = 0;
5484 let error = super::wait_for_policy_initial_publication(
5485 &cancellation,
5486 std::time::Duration::from_secs(120),
5487 || {
5488 attempts += 1;
5489 let first = attempts == 1;
5490 async move {
5491 if first {
5492 Err(unavailable_policy_publication())
5493 } else {
5494 anyhow::bail!(terminal)
5495 }
5496 }
5497 },
5498 )
5499 .await
5500 .expect_err("security failures must be terminal");
5501 assert_eq!(error.to_string(), terminal);
5502 assert_eq!(attempts, 2);
5503 }
5504 let mut attempts = 0;
5505 let error = super::wait_for_policy_initial_publication(
5506 &cancellation,
5507 std::time::Duration::from_secs(120),
5508 || {
5509 attempts += 1;
5510 async {
5511 Err(
5512 hyprstream_rpc::transport_traits::PreDispatchTransportError::new(
5513 anyhow::anyhow!("missing Iroh client endpoint"),
5514 )
5515 .into(),
5516 )
5517 }
5518 },
5519 )
5520 .await
5521 .expect_err("pre-dispatch alone is not an availability proof");
5522 assert_eq!(attempts, 1);
5523 assert!(error.to_string().contains("missing Iroh client endpoint"));
5524 }
5525
5526 #[tokio::test(start_paused = true)]
5527 async fn policy_initial_publication_deadline_bounds_retries_and_inflight_call() {
5528 let cancellation = tokio_util::sync::CancellationToken::new();
5529 let start = tokio::time::Instant::now();
5530 let error = super::wait_for_policy_initial_publication(
5531 &cancellation,
5532 std::time::Duration::from_secs(2),
5533 || async { Err(unavailable_policy_publication()) },
5534 )
5535 .await
5536 .expect_err("absent Discovery cannot keep startup pending forever");
5537 assert!(error.to_string().contains("deadline exceeded"));
5538 assert_eq!(start.elapsed(), std::time::Duration::from_secs(2));
5539 let start = tokio::time::Instant::now();
5540 let error = super::wait_for_policy_initial_publication(
5541 &cancellation,
5542 std::time::Duration::from_secs(2),
5543 std::future::pending,
5544 )
5545 .await
5546 .expect_err("deadline also interrupts a stalled RPC");
5547 assert!(error.to_string().contains("deadline exceeded"));
5548 assert_eq!(start.elapsed(), std::time::Duration::from_secs(2));
5549 }
5550
5551 #[tokio::test(start_paused = true)]
5552 async fn policy_initial_publication_cancellation_precedes_success() {
5553 let cancellation = tokio_util::sync::CancellationToken::new();
5554 cancellation.cancel();
5555 let mut called = false;
5556 super::wait_for_policy_initial_publication(
5557 &cancellation,
5558 std::time::Duration::from_secs(120),
5559 || {
5560 called = true;
5561 async { Ok(()) }
5562 },
5563 )
5564 .await
5565 .expect_err("cancelled startup cannot publish or become ready");
5566 assert!(!called);
5567 let cancellation = tokio_util::sync::CancellationToken::new();
5568 let cancel = cancellation.clone();
5569 let cancel_task = tokio::spawn(async move {
5570 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
5571 cancel.cancel();
5572 });
5573 super::wait_for_policy_initial_publication(
5574 &cancellation,
5575 std::time::Duration::from_secs(120),
5576 std::future::pending,
5577 )
5578 .await
5579 .expect_err("shutdown interrupts an in-flight publication");
5580 cancel_task.await.expect("canceller");
5581 }
5582
5583 #[test]
5584 fn deployment_bootstrap_cli_requires_roster_and_parses_lifetime() {
5585 let matches = super::build_cli()
5586 .try_get_matches_from([
5587 "hyprstream",
5588 "pds",
5589 "provision-services",
5590 "--service",
5591 "model,event",
5592 "--valid-for-seconds",
5593 "3600",
5594 ])
5595 .expect("bootstrap CLI");
5596 let pds = matches.subcommand_matches("pds").expect("pds");
5597 let provision = pds
5598 .subcommand_matches("provision-services")
5599 .expect("provision");
5600 assert_eq!(
5601 provision
5602 .get_many::<String>("service")
5603 .expect("roster")
5604 .map(String::as_str)
5605 .collect::<Vec<_>>(),
5606 vec!["model", "event"]
5607 );
5608 assert_eq!(provision.get_one::<i64>("valid-for-seconds"), Some(&3600));
5609 assert!(provision
5610 .get_one::<std::path::PathBuf>("roster-export")
5611 .is_none());
5612 assert!(super::build_cli()
5613 .try_get_matches_from(["hyprstream", "pds", "provision-services",])
5614 .is_err());
5615 let matches = super::build_cli()
5616 .try_get_matches_from([
5617 "hyprstream",
5618 "pds",
5619 "provision-services",
5620 "--service",
5621 "model",
5622 "--roster-export",
5623 "/tmp/roster.json",
5624 ])
5625 .expect("bootstrap CLI with roster export");
5626 let provision = matches
5627 .subcommand_matches("pds")
5628 .expect("pds")
5629 .subcommand_matches("provision-services")
5630 .expect("provision");
5631 assert_eq!(
5632 provision
5633 .get_one::<std::path::PathBuf>("roster-export")
5634 .map(std::path::PathBuf::as_path),
5635 Some(std::path::Path::new("/tmp/roster.json"))
5636 );
5637 }
5638 const REFRESH_SCHEDULER_TURNS: usize = 32;
5639
5640 async fn assert_publication_ready(
5641 published_rx: &mut tokio::sync::mpsc::UnboundedReceiver<usize>,
5642 expected: usize,
5643 label: &str,
5644 ) {
5645 for _ in 0..REFRESH_SCHEDULER_TURNS {
5646 match published_rx.try_recv() {
5647 Ok(publication) => {
5648 assert_eq!(publication, expected, "{label}");
5649 return;
5650 }
5651 Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
5652 tokio::task::yield_now().await;
5653 }
5654 Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
5655 panic!("{label}: production refresh task exited before publishing");
5656 }
5657 }
5658 }
5659 panic!("{label}: publication was not ready after bounded scheduler turns");
5660 }
5661
5662 async fn publication_ready(
5663 published_rx: &mut tokio::sync::mpsc::UnboundedReceiver<usize>,
5664 label: &str,
5665 ) -> Option<usize> {
5666 for _ in 0..REFRESH_SCHEDULER_TURNS {
5667 match published_rx.try_recv() {
5668 Ok(publication) => return Some(publication),
5669 Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
5670 tokio::task::yield_now().await;
5671 }
5672 Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
5673 panic!("{label}: production refresh task exited before publishing");
5674 }
5675 }
5676 }
5677 None
5678 }
5679
5680 async fn assert_no_publication_ready(
5681 published_rx: &mut tokio::sync::mpsc::UnboundedReceiver<usize>,
5682 label: &str,
5683 ) {
5684 for _ in 0..REFRESH_SCHEDULER_TURNS {
5685 match published_rx.try_recv() {
5686 Ok(publication) => panic!("{label}: unexpected publication {publication}"),
5687 Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
5688 tokio::task::yield_now().await;
5689 }
5690 Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
5691 panic!("{label}: production refresh task exited unexpectedly");
5692 }
5693 }
5694 }
5695 }
5696
5697 #[test]
5698 fn checkpointed_moq_services_cannot_share_a_process_dial_proof() {
5699 let err = super::select_single_process_moql_admission_proof(vec![
5700 ("inference".to_owned(), Some(1_u8)),
5701 ("registry".to_owned(), Some(2_u8)),
5702 ])
5703 .expect_err("different checkpointed services need separate dialers");
5704 assert!(err.to_string().contains("separate process"));
5705
5706 let err = super::select_single_process_moql_admission_proof(vec![
5707 ("inference".to_owned(), Some(1_u8)),
5708 ("metrics".to_owned(), None),
5709 ])
5710 .expect_err("a proof cannot be inherited by an unauthenticated service");
5711 assert!(err.to_string().contains("service-scoped"));
5712
5713 assert_eq!(
5714 super::select_single_process_moql_admission_proof(vec![(
5715 "inference".to_owned(),
5716 Some(7_u8)
5717 ),])
5718 .expect("one service has one scoped proof"),
5719 Some(7),
5720 );
5721 }
5722
5723 #[test]
5724 fn quinn_only_multi_service_startup_never_selects_an_iroh_proof() {
5725 let selected = super::select_iroh_moql_admission_proof::<u8>(false, || {
5726 panic!("Quinn-only startup must not inspect Iroh admission proofs")
5727 })
5728 .expect("Quinn-only profile does not need an Iroh proof");
5729 assert_eq!(selected, None);
5730 }
5731
5732 #[test]
5733 fn command_and_service_processes_install_before_consumers() {
5734 let source = include_str!("main.rs");
5735 let production = source
5736 .split("mod resolver_startup_controls {")
5737 .next()
5738 .expect("production binary source")
5739 .split_whitespace()
5740 .collect::<String>();
5741 let startup = production
5742 .find("install_process_production_resolver(&signing_key,&config,service_token.clone(),false,).await")
5743 .expect("trusted startup boundary");
5744 let install = production[startup..]
5745 .find("install_process_production_resolver(&signing_key,&config,service_token.clone(),false,).await")
5746 .expect("process resolver install");
5747 let first_generated = production[startup..]
5748 .find(
5749 "RegistryClient::from_provider(&hyprstream_discovery::ProductionRpcClientProvider,",
5750 )
5751 .expect("first generated client");
5752 let command_dispatch = production[startup..]
5753 .find("matchmatches.subcommand()")
5754 .expect("top-level command dispatch");
5755 assert!(install < first_generated);
5756 assert!(first_generated < command_dispatch);
5757 assert!(production.contains("load_service_jwt_for_profile(&secrets,name,profile,"));
5758 assert!(production.contains("signing_key.clone(),service_token,"));
5759 assert_eq!(
5760 production
5761 .matches("install_process_production_resolver(&signing_key,&config,service_token.clone(),false,).await")
5762 .count(),
5763 1
5764 );
5765 assert!(!production.contains("install_process_production_resolver(&mut ctx)"));
5766 assert!(production.contains(
5767 "install_process_production_resolver(&key,&config,Some(jwt.clone()),true).await"
5768 ));
5769 assert!(
5770 production.contains("RegistryClient::from_provider(provider,key,Some(jwt.clone()))")
5771 );
5772
5773 let announcements = production
5774 .find("with_checkpointed_native_announcements(")
5775 .expect("authenticated announcements");
5776 let first_factory = production[announcements..]
5777 .find("get_factory(")
5778 .expect("first inventory factory invocation");
5779 assert!(first_factory > 0);
5780
5781 for entry in [
5785 "Some((\"tool\"",
5786 "Some((\"quick\"",
5787 "Some((\"tui\"",
5788 "handle_shell_tui",
5789 "handle_training_batch",
5790 ] {
5791 assert!(production.contains(entry), "missing {entry}");
5792 }
5793 }
5794
5795 #[test]
5796 fn configured_mac_boot_precedes_all_start_paths_and_never_falls_back() {
5797 let source = include_str!("main.rs");
5798 let production = source
5799 .split("mod resolver_startup_controls {")
5800 .next()
5801 .expect("production source");
5802 let configured = production
5803 .find("if let Some(artifacts) = &config_for_service.policy.boot_policy")
5804 .expect("configured boot branch");
5805 let start = &production[configured..];
5806 let install = start
5807 .find("install_configured_boot_policy(")
5808 .expect("configured install");
5809 let fatal = start
5810 .find(".context(\"configured MAC boot policy rejected\")?")
5811 .expect("fatal configured failure");
5812 let branch = start
5813 .find("if foreground || standalone {")
5814 .expect("startup branch");
5815 assert!(install < fatal && fatal < branch);
5816 let fallback = start
5817 .find("if config.policy.boot_policy.is_none() {")
5818 .expect("absent-only fallback");
5819 let baseline = start
5820 .find("install_baseline_boot_policy(")
5821 .expect("baseline install");
5822 assert!(branch < fallback && fallback < baseline);
5823 assert_eq!(
5824 production
5825 .matches("install_configured_boot_policy(")
5826 .count(),
5827 1
5828 );
5829 }
5830
5831 #[test]
5836 fn first_boot_quic_deferral_is_scoped_to_os_owned_bootstrap() {
5837 use std::cell::Cell;
5838
5839 use hyprstream_core::services::factories::PdsBootState;
5840
5841 let os_owned_calls = Cell::new(0);
5842 assert_eq!(
5843 super::quic_checkpoint_policy(true, || {
5844 os_owned_calls.set(os_owned_calls.get() + 1);
5845 Ok(PdsBootState::FirstBoot)
5846 })
5847 .expect("OS-owned classification succeeds"),
5848 super::QuicCheckpointPolicy::DeferForFirstBoot,
5849 );
5850 assert_eq!(os_owned_calls.get(), 1, "OS-owned input classifies once");
5851
5852 let did_anchored_calls = Cell::new(0);
5853 assert_eq!(
5854 super::quic_checkpoint_policy(false, || {
5855 did_anchored_calls.set(did_anchored_calls.get() + 1);
5856 anyhow::bail!("DID-anchored startup must never classify lifecycle state")
5857 })
5858 .expect("DID-anchored startup bypasses the classifier"),
5859 super::QuicCheckpointPolicy::CheckpointedAnnouncements,
5860 );
5861 assert_eq!(
5862 did_anchored_calls.get(),
5863 0,
5864 "DID-anchored input must not invoke the classifier"
5865 );
5866 }
5867
5868 #[test]
5869 fn command_local_spawns_never_use_the_production_resolver() {
5870 let source = include_str!("main.rs");
5871 let worker = source
5872 .find("let worker_client = if worker_already_running")
5873 .expect("worker client selection");
5874 let worker_selection = source[worker..worker + 500]
5875 .split_whitespace()
5876 .collect::<String>();
5877 assert!(worker_selection.contains(
5878 "WorkerClient::from_provider(&hyprstream_discovery::ProductionRpcClientProvider"
5879 ));
5880 assert!(worker_selection.contains("WorkerClient::for_local_bootstrap"));
5881
5882 let training = include_str!("../cli/training_handlers.rs");
5883 assert_eq!(
5884 training.matches("InferenceClient::from_provider").count(),
5885 0
5886 );
5887 assert_eq!(
5888 training
5889 .matches("InferenceClient::for_local_bootstrap")
5890 .count(),
5891 2
5892 );
5893 }
5894
5895 #[tokio::test]
5896 async fn native_announcement_refresh_loop_publishes_immediately_and_after_two_intervals() {
5897 use std::sync::{
5898 atomic::{AtomicUsize, Ordering},
5899 Arc,
5900 };
5901 use std::time::Duration;
5902
5903 tokio::time::pause();
5904 tokio::time::sleep(Duration::ZERO).await;
5908 let initial_time = tokio::time::Instant::now();
5909 let just_before_refresh = Duration::from_secs(24) + Duration::from_millis(999);
5910 let timer_tick = Duration::from_millis(1);
5911 let publications = Arc::new(AtomicUsize::new(0));
5912 let (published_tx, mut published_rx) = tokio::sync::mpsc::unbounded_channel();
5913 let observed_publications = Arc::clone(&publications);
5914 let task = tokio::spawn(async move {
5915 super::refresh_native_announcement(
5916 "model",
5917 "iroh",
5918 "iroh://test-node",
5919 chrono::Utc::now().timestamp_millis() + 60_000,
5920 tokio_util::sync::CancellationToken::new(),
5921 move || {
5922 let publication = observed_publications.fetch_add(1, Ordering::SeqCst) + 1;
5923 published_tx
5924 .send(publication)
5925 .expect("test receiver remains live");
5926 async { Ok::<(), std::convert::Infallible>(()) }
5927 },
5928 )
5929 .await
5930 });
5931
5932 assert_publication_ready(&mut published_rx, 1, "initial production publish").await;
5933 assert_no_publication_ready(&mut published_rx, "duplicate immediate publish").await;
5934 assert_eq!(tokio::time::Instant::now(), initial_time);
5935
5936 tokio::time::advance(just_before_refresh).await;
5937 assert_eq!(
5938 tokio::time::Instant::now(),
5939 initial_time + just_before_refresh
5940 );
5941 assert_no_publication_ready(&mut published_rx, "refresh before 25 virtual seconds").await;
5942 assert_eq!(
5943 tokio::time::Instant::now(),
5944 initial_time + just_before_refresh
5945 );
5946 tokio::time::advance(timer_tick).await;
5947 let first_nominal_wake = initial_time + Duration::from_secs(25);
5948 let second_wake =
5949 match publication_ready(&mut published_rx, "25-second production refresh").await {
5950 Some(publication) => {
5951 assert_eq!(publication, 2, "25-second production refresh");
5952 tokio::time::Instant::now()
5953 }
5954 None => {
5955 tokio::time::advance(timer_tick).await;
5956 assert_publication_ready(&mut published_rx, 2, "25-second production refresh")
5957 .await;
5958 tokio::time::Instant::now()
5959 }
5960 };
5961 assert!(
5962 second_wake >= first_nominal_wake && second_wake <= first_nominal_wake + timer_tick,
5963 "production refresh must occur at its 25-second cadence or within one Tokio timer tick"
5964 );
5965 assert_no_publication_ready(&mut published_rx, "duplicate 25-second refresh").await;
5966 assert_eq!(tokio::time::Instant::now(), second_wake);
5967
5968 tokio::time::advance(just_before_refresh).await;
5969 assert_eq!(
5970 tokio::time::Instant::now(),
5971 second_wake + just_before_refresh
5972 );
5973 assert_no_publication_ready(&mut published_rx, "refresh before 50 virtual seconds").await;
5974 assert_eq!(
5975 tokio::time::Instant::now(),
5976 second_wake + just_before_refresh
5977 );
5978 tokio::time::advance(timer_tick).await;
5979 let second_nominal_wake = second_wake + Duration::from_secs(25);
5980 let third_wake =
5981 match publication_ready(&mut published_rx, "50-second production refresh").await {
5982 Some(publication) => {
5983 assert_eq!(publication, 3, "50-second production refresh");
5984 tokio::time::Instant::now()
5985 }
5986 None => {
5987 tokio::time::advance(timer_tick).await;
5988 assert_publication_ready(&mut published_rx, 3, "50-second production refresh")
5989 .await;
5990 tokio::time::Instant::now()
5991 }
5992 };
5993 assert!(
5994 third_wake >= second_nominal_wake && third_wake <= second_nominal_wake + timer_tick,
5995 "next production refresh must use the prior actual wake and stay within one Tokio timer tick"
5996 );
5997 assert_eq!(tokio::time::Instant::now(), third_wake);
5998 assert_eq!(publications.load(Ordering::SeqCst), 3);
5999
6000 task.abort();
6001 assert!(task
6002 .await
6003 .expect_err("refresh task is cancelled")
6004 .is_cancelled());
6005 }
6006
6007 async fn expired_announcement_must_not_run() -> Result<(), std::convert::Infallible> {
6008 panic!("expired production refresh loop attempted an announcement");
6009 }
6010
6011 #[tokio::test(start_paused = true)]
6012 async fn native_announcement_refresh_loop_shutdown_cannot_overwrite_restart() {
6013 use parking_lot::Mutex;
6014 use std::sync::Arc;
6015 let publications = Arc::new(Mutex::new(Vec::new()));
6016 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
6017 let old_cancel = tokio_util::sync::CancellationToken::new();
6018 let new_cancel = tokio_util::sync::CancellationToken::new();
6019 let spawn = |endpoint: &'static str, cancellation| {
6020 let publications = Arc::clone(&publications);
6021 let tx = tx.clone();
6022 tokio::spawn(async move {
6023 super::refresh_native_announcement(
6024 "model",
6025 "quic",
6026 endpoint,
6027 i64::MAX,
6028 cancellation,
6029 move || {
6030 publications.lock().push(endpoint);
6031 tx.send(endpoint).expect("publication receiver");
6032 async { Ok::<(), std::convert::Infallible>(()) }
6033 },
6034 )
6035 .await
6036 })
6037 };
6038 let old = spawn("quic://old:10001", old_cancel.clone());
6039 assert_eq!(rx.recv().await, Some("quic://old:10001"));
6040 old_cancel.cancel();
6041 assert_eq!(
6042 old.await.expect("old task"),
6043 super::NativeAnnouncementRefreshCompletion::Cancelled
6044 );
6045 let new = spawn("quic://new:10002", new_cancel.clone());
6046 assert_eq!(rx.recv().await, Some("quic://new:10002"));
6047 tokio::time::advance(std::time::Duration::from_secs(26)).await;
6048 assert_eq!(rx.recv().await, Some("quic://new:10002"));
6049 new_cancel.cancel();
6050 assert_eq!(
6051 new.await.expect("new task"),
6052 super::NativeAnnouncementRefreshCompletion::Cancelled
6053 );
6054 assert_eq!(
6055 *publications.lock(),
6056 vec!["quic://old:10001", "quic://new:10002", "quic://new:10002"],
6057 );
6058 }
6059
6060 #[tokio::test]
6061 async fn native_announcement_refresh_loop_cancels_in_flight_publish() {
6062 use std::sync::{
6063 atomic::{AtomicBool, Ordering},
6064 Arc,
6065 };
6066 let cancellation = tokio_util::sync::CancellationToken::new();
6067 let completed = Arc::new(AtomicBool::new(false));
6068 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
6069 let task = tokio::spawn({
6070 let cancellation = cancellation.clone();
6071 let completed = Arc::clone(&completed);
6072 let mut started_tx = Some(started_tx);
6073 async move {
6074 super::refresh_native_announcement(
6075 "model",
6076 "iroh",
6077 "iroh://test",
6078 i64::MAX,
6079 cancellation,
6080 move || {
6081 let started = started_tx.take();
6082 let completed = Arc::clone(&completed);
6083 async move {
6084 if let Some(started) = started {
6085 let _ = started.send(());
6086 }
6087 std::future::pending::<()>().await;
6088 completed.store(true, Ordering::SeqCst);
6089 Ok::<(), std::convert::Infallible>(())
6090 }
6091 },
6092 )
6093 .await
6094 }
6095 });
6096 started_rx.await.expect("publish started");
6097 cancellation.cancel();
6098 let completion = tokio::time::timeout(std::time::Duration::from_secs(1), task)
6099 .await
6100 .expect("cancellation must not wait for RPC completion")
6101 .expect("task");
6102 assert_eq!(
6103 completion,
6104 super::NativeAnnouncementRefreshCompletion::Cancelled
6105 );
6106 assert!(!completed.load(Ordering::SeqCst));
6107 }
6108
6109 #[tokio::test(start_paused = true)]
6110 async fn native_announcement_refresh_loop_backoff_starts_at_five_and_resets_after_success() {
6111 let cancellation = tokio_util::sync::CancellationToken::new();
6112 let (sent, mut received) = tokio::sync::mpsc::unbounded_channel();
6113 let task = tokio::spawn({
6114 let cancellation = cancellation.clone();
6115 let mut attempt = 0;
6116 async move {
6117 super::refresh_native_announcement(
6118 "model",
6119 "iroh",
6120 "iroh://test",
6121 i64::MAX,
6122 cancellation,
6123 move || {
6124 attempt += 1;
6125 sent.send(tokio::time::Instant::now())
6126 .expect("receiver live");
6127 let success = attempt == 5;
6128 async move {
6129 if success {
6130 Ok(())
6131 } else {
6132 Err("retry fixture")
6133 }
6134 }
6135 },
6136 )
6137 .await
6138 }
6139 });
6140 let mut previous = received.recv().await.expect("initial attempt");
6141 for seconds in [5, 10, 20, 25, 25, 5, 10] {
6142 let current = received.recv().await.expect("next attempt");
6143 let elapsed = current.duration_since(previous);
6144 let expected = std::time::Duration::from_secs(seconds);
6145 assert!(
6146 elapsed >= expected && elapsed <= expected + std::time::Duration::from_millis(1),
6147 "expected {expected:?} retry interval, got {elapsed:?}"
6148 );
6149 previous = current;
6150 }
6151 cancellation.cancel();
6152 assert_eq!(
6153 task.await.expect("refresh task"),
6154 super::NativeAnnouncementRefreshCompletion::Cancelled
6155 );
6156 }
6157
6158 #[tokio::test]
6159 async fn native_announcement_refresh_loop_stops_before_announce_after_absolute_expiry() {
6160 let completion = super::refresh_native_announcement(
6161 "model",
6162 "iroh",
6163 "iroh://test-node",
6164 chrono::Utc::now().timestamp_millis() - 1,
6165 tokio_util::sync::CancellationToken::new(),
6166 expired_announcement_must_not_run,
6167 )
6168 .await;
6169
6170 assert_eq!(
6171 completion,
6172 super::NativeAnnouncementRefreshCompletion::Expired
6173 );
6174 }
6175}
6176
6177#[cfg(test)]
6178#[allow(clippy::unwrap_used, clippy::expect_used)]
6179mod native_announcement_wiring {
6180 use std::time::Duration;
6181
6182 #[test]
6183 fn production_publisher_profile_controls_first_failure_and_retry() {
6184 use hyprstream_service::NativeAnnouncementReach;
6185 let source = include_str!("main.rs");
6189 let production = source
6190 .split("mod native_announcement_wiring")
6191 .next()
6192 .unwrap();
6193 let compact: String = production.split_whitespace().collect();
6194 assert!(compact.contains("start_native_announcement_publisher(request.reach.clone(),hyprstream_discovery::native_network_required(),request.cancellation.clone(),"));
6195
6196 for network_required in [false, true] {
6197 for reach in [
6198 NativeAnnouncementReach::Iroh { node_id: [7; 32] },
6199 NativeAnnouncementReach::Quic {
6200 address: "127.0.0.1:12345".parse().unwrap(),
6201 server_name: "fixture".to_owned(),
6202 certificate_hashes: Vec::new(),
6203 },
6204 ] {
6205 let fatal =
6206 network_required && matches!(reach, NativeAnnouncementReach::Iroh { .. });
6207 let cancellation = tokio_util::sync::CancellationToken::new();
6208 let loop_cancellation = cancellation.clone();
6209 let (retry_tx, retry_rx) = std::sync::mpsc::sync_channel(2);
6210 let (dropped_tx, dropped_rx) = std::sync::mpsc::sync_channel(1);
6211 struct Dropped(std::sync::mpsc::SyncSender<()>);
6212 impl Drop for Dropped {
6213 fn drop(&mut self) {
6214 let _ = self.0.send(());
6215 }
6216 }
6217 let result = super::start_native_announcement_publisher(
6218 reach,
6219 network_required,
6220 cancellation.clone(),
6221 move |announce_tx| async move {
6222 let _dropped = Dropped(dropped_tx);
6223 let mut attempts = 0;
6224 super::refresh_native_announcement(
6225 "policy",
6226 "iroh",
6227 "iroh://fixture",
6228 i64::MAX,
6229 loop_cancellation,
6230 || {
6231 attempts += 1;
6232 let attempt = attempts;
6233 let tx = announce_tx.clone();
6234 let retry_tx = retry_tx.clone();
6235 async move {
6236 let outcome = if attempt == 1 {
6237 Err(anyhow::anyhow!("local Discovery is not bound yet"))
6238 } else {
6239 retry_tx.send(attempt).unwrap();
6240 Ok(())
6241 };
6242 send_first_result(
6243 &tx,
6244 outcome.as_ref().map(|_| ()).map_err(ToString::to_string),
6245 );
6246 outcome
6247 }
6248 },
6249 )
6250 .await;
6251 },
6252 );
6253 if fatal {
6254 assert!(result
6255 .unwrap_err()
6256 .to_string()
6257 .contains("local Discovery is not bound yet"));
6258 dropped_rx
6259 .recv_timeout(Duration::from_secs(2))
6260 .expect("failed required startup aborts the publisher");
6261 assert!(
6262 retry_rx.try_recv().is_err(),
6263 "required startup cannot silently retry its first failure"
6264 );
6265 } else {
6266 result.expect("Compatibility/non-Iroh startup is nonfatal");
6267 assert_eq!(retry_rx.recv_timeout(Duration::from_secs(8)).expect("publication retries at the production five-second cadence when Discovery becomes ready"), 2);
6268 cancellation.cancel();
6269 dropped_rx
6270 .recv_timeout(Duration::from_secs(2))
6271 .expect("service cancellation still owns the retry task");
6272 }
6273 }
6274 }
6275 }
6276
6277 fn send_first_result(
6278 announce_tx: &super::NativeAnnouncementFirstTx,
6279 result: super::NativeAnnouncementFirstResult,
6280 ) {
6281 if let Some(tx) = announce_tx {
6282 if let Some(tx) = tx.lock().take() {
6283 let _ = tx.send(result);
6284 }
6285 }
6286 }
6287
6288 #[test]
6289 fn required_path_handshakes_and_runs_multiple_cycles() {
6290 let (cycle_tx, cycle_rx) = std::sync::mpsc::sync_channel(10);
6291 let initial_rx = super::spawn_native_announcement_loop(
6292 true,
6293 tokio_util::sync::CancellationToken::new(),
6294 move |announce_tx| {
6295 let cycle_tx = cycle_tx.clone();
6296 async move {
6297 for i in 0..3 {
6298 let announcement =
6299 hyprstream_rpc_std::discovery_client::ServiceAnnouncement {
6300 service_name: "model".to_owned(),
6301 socket_kind: "iroh".to_owned(),
6302 endpoint: "iroh://bound-node".to_owned(),
6303 service_jwt: Some(format!("jwt-{i}")),
6304 service_did: "did:at9p:fixture".into(),
6305 capabilities: vec!["model".to_owned()],
6306 accepted_state_digest: vec![i as u8; 64],
6307 accepted_state_epoch: i,
6308 response_key_id: "#response".to_owned(),
6309 request_kem_key_id: "#kem".to_owned(),
6310 request_kem_recipient: vec![i as u8],
6311 expires_at_unix_ms: 1000 + i as i64,
6312 };
6313 let cycle_tx = cycle_tx.clone();
6314 super::publish_native_announcement_attempt(
6315 Ok(announcement),
6316 announce_tx.as_ref().map(std::sync::Arc::clone),
6317 move |announcement| async move {
6318 assert_eq!(announcement.service_jwt, Some(format!("jwt-{i}")));
6319 assert_eq!(
6320 super::native_announcement_bearer(&announcement).unwrap(),
6321 format!("jwt-{i}")
6322 );
6323 let mut missing = announcement.clone();
6324 missing.service_jwt = None;
6325 assert!(super::native_announcement_bearer(&missing).is_err());
6326 missing.service_jwt = Some(" ".into());
6327 assert!(super::native_announcement_bearer(&missing).is_err());
6328 assert_eq!(announcement.expires_at_unix_ms, 1000 + i as i64);
6329 cycle_tx
6330 .send(announcement.accepted_state_epoch)
6331 .expect("test receiver is live");
6332 Ok(())
6333 },
6334 )
6335 .await
6336 .expect("publication succeeds");
6337 tokio::task::yield_now().await;
6338 }
6339 }
6340 },
6341 );
6342
6343 let result = initial_rx
6344 .expect("required path has a handshake receiver")
6345 .recv_timeout(Duration::from_secs(2))
6346 .expect("handshake completes");
6347 assert!(result.is_ok(), "first announcement should succeed");
6348
6349 let mut seen = Vec::with_capacity(3);
6350 for _ in 0..3 {
6351 seen.push(
6352 cycle_rx
6353 .recv_timeout(Duration::from_secs(2))
6354 .expect("loop continues producing cycles"),
6355 );
6356 }
6357 assert_eq!(
6358 seen,
6359 vec![0, 1, 2],
6360 "observed three full publication cycles"
6361 );
6362 }
6363
6364 #[test]
6365 fn compat_path_publishes_without_handshake() {
6366 let (cycle_tx, cycle_rx) = std::sync::mpsc::sync_channel(10);
6367 let initial_rx = super::spawn_native_announcement_loop(
6368 false,
6369 tokio_util::sync::CancellationToken::new(),
6370 move |_announce_tx| {
6371 let cycle_tx = cycle_tx.clone();
6372 async move {
6373 cycle_tx.send(0).expect("test receiver is live");
6374 }
6375 },
6376 );
6377
6378 assert!(
6379 initial_rx.is_none(),
6380 "compat path has no handshake receiver"
6381 );
6382 let got = cycle_rx
6383 .recv_timeout(Duration::from_secs(2))
6384 .expect("compat path publishes");
6385 assert_eq!(got, 0);
6386 }
6387
6388 #[test]
6389 fn required_path_cancels_pending_first_publication() {
6390 let cancellation = tokio_util::sync::CancellationToken::new();
6391 let loop_cancellation = cancellation.clone();
6392 let (started_tx, started_rx) = std::sync::mpsc::sync_channel(1);
6393 let (dropped_tx, dropped_rx) = std::sync::mpsc::sync_channel(1);
6394 struct Dropped(std::sync::mpsc::SyncSender<()>);
6395 impl Drop for Dropped {
6396 fn drop(&mut self) {
6397 let _ = self.0.send(());
6398 }
6399 }
6400 let initial = super::spawn_native_announcement_loop(
6401 true,
6402 cancellation.clone(),
6403 move |announce_tx| async move {
6404 let _dropped = Dropped(dropped_tx);
6405 super::refresh_native_announcement(
6406 "model",
6407 "iroh",
6408 "iroh://test",
6409 i64::MAX,
6410 loop_cancellation,
6411 || {
6412 started_tx.send(()).expect("publication started");
6413 std::future::pending::<Result<(), std::io::Error>>()
6414 },
6415 )
6416 .await;
6417 drop(announce_tx);
6419 },
6420 )
6421 .expect("required handshake");
6422 started_rx
6423 .recv_timeout(Duration::from_secs(2))
6424 .expect("first publication entered");
6425 cancellation.cancel();
6426 assert!(initial
6427 .recv_timeout(Duration::from_secs(2))
6428 .expect("cancelled handshake must finish")
6429 .is_err());
6430 dropped_rx
6431 .recv_timeout(Duration::from_secs(2))
6432 .expect("publication task must be dropped");
6433 }
6434
6435 #[test]
6436 fn required_path_rejects_success_when_already_cancelled() {
6437 let cancellation = tokio_util::sync::CancellationToken::new();
6438 cancellation.cancel();
6439 let initial =
6440 super::spawn_native_announcement_loop(true, cancellation, |announce_tx| async move {
6441 send_first_result(&announce_tx, Ok(()));
6442 })
6443 .expect("required handshake");
6444 assert!(initial
6445 .recv_timeout(Duration::from_secs(2))
6446 .expect("cancelled handshake must finish")
6447 .is_err());
6448 }
6449
6450 #[test]
6451 fn required_path_aborts_on_first_failure() {
6452 let (cycle_tx, cycle_rx) = std::sync::mpsc::sync_channel(10);
6453 let initial_rx = super::spawn_native_announcement_loop(
6454 true,
6455 tokio_util::sync::CancellationToken::new(),
6456 move |announce_tx| {
6457 let cycle_tx = cycle_tx.clone();
6458 async move {
6459 send_first_result(&announce_tx, Err("injected".to_owned()));
6460 loop {
6461 tokio::task::yield_now().await;
6462 cycle_tx.send(1).expect("test receiver is live");
6463 }
6464 }
6465 },
6466 );
6467
6468 let result = initial_rx
6469 .expect("required path has a handshake receiver")
6470 .recv_timeout(Duration::from_secs(2))
6471 .expect("handshake completes");
6472 assert!(result.is_err(), "first failure must be reported");
6473
6474 assert!(
6475 cycle_rx.recv_timeout(Duration::from_millis(200)).is_err(),
6476 "loop must abort after first failure; extra cycle observed"
6477 );
6478 }
6479
6480 #[test]
6481 fn required_path_reports_first_authority_failure_without_publication() {
6482 let published = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
6483 let observed = published.clone();
6484 let cancellation = tokio_util::sync::CancellationToken::new();
6485 let initial = super::spawn_native_announcement_loop(
6486 true,
6487 cancellation.clone(),
6488 move |announce_tx| async move {
6489 super::refresh_native_announcement(
6490 "model",
6491 "iroh",
6492 "iroh://test",
6493 i64::MAX,
6494 cancellation,
6495 || {
6496 let tx = announce_tx.as_ref().map(std::sync::Arc::clone);
6497 let published = published.clone();
6498 super::publish_native_announcement_attempt(
6499 Err(anyhow::anyhow!("fresh checkpoint unavailable")),
6500 tx,
6501 move |_| async move {
6502 published.store(true, std::sync::atomic::Ordering::SeqCst);
6503 Ok(())
6504 },
6505 )
6506 },
6507 )
6508 .await;
6509 },
6510 )
6511 .expect("required handshake");
6512 let error = initial
6513 .recv_timeout(Duration::from_secs(2))
6514 .expect("authority failure must complete first handshake")
6515 .expect_err("startup must fail");
6516 assert!(error.contains("fresh checkpoint unavailable"));
6517 assert!(
6518 !observed.load(std::sync::atomic::Ordering::SeqCst),
6519 "failed authority projection must never publish stale reach"
6520 );
6521 }
6522}
6523
6524#[cfg(test)]
6525#[allow(clippy::expect_used)]
6526mod native_launcher {
6527 use super::*;
6534 use hyprstream_core::cli::service_handlers::direct_child_process_config;
6535 use std::path::Path;
6536
6537 const CAUSAL_CHILD: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_CHILD";
6538 const CAUSAL_MODE: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_MODE";
6539 const CAUSAL_ARGV: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_ARGV";
6540 const CAUSAL_PROOF: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_PROOF";
6541 const CAUSAL_STOP: &str = "HYPRSTREAM_LAUNCHER_CAUSAL_STOP";
6542 const SERVICE: &str = "model";
6543 const KEY_SEED: u8 = 0x5A;
6544
6545 #[cfg(unix)]
6546 #[test]
6547 fn cli_parser_loads_non_utf8_config_path_without_changing_bytes() -> anyhow::Result<()> {
6548 use std::os::unix::ffi::{OsStrExt, OsStringExt};
6549
6550 let root = tempfile::tempdir()?;
6551 let config_path = root
6552 .path()
6553 .join(std::ffi::OsString::from_vec(b"parent-\xff.toml".to_vec()));
6554 HyprConfig::default().to_file(&config_path)?;
6555 let canonical = std::fs::canonicalize(&config_path)?;
6556 assert!(
6557 canonical.to_str().is_none(),
6558 "fixture path must be non-UTF-8"
6559 );
6560
6561 let matches = build_cli().try_get_matches_from([
6562 std::ffi::OsString::from("hyprstream"),
6563 std::ffi::OsString::from("--config"),
6564 canonical.as_os_str().to_owned(),
6565 ])?;
6566 let parsed = matches
6567 .get_one::<std::path::PathBuf>("config")
6568 .context("config PathBuf parsed by clap")?;
6569 assert_eq!(
6570 parsed.as_os_str().as_bytes(),
6571 canonical.as_os_str().as_bytes(),
6572 "clap must preserve the explicit selector byte for byte"
6573 );
6574 let config = load_config(Some(parsed))?;
6575 config.validate()?;
6576 Ok(())
6577 }
6578
6579 fn provision_custom_config(
6580 root: &tempfile::TempDir,
6581 seed: u8,
6582 with_key: bool,
6583 ) -> anyhow::Result<std::path::PathBuf> {
6584 let secrets = root.path().join("custom credentials");
6585 if with_key {
6586 std::fs::create_dir_all(secrets.join(SERVICE))?;
6587 std::fs::write(secrets.join(SERVICE).join("signing-key"), [seed; 32])?;
6588 }
6589 let mut configured = HyprConfig::default();
6590 configured.quic.native_network_profile =
6591 hyprstream_core::config::NativeNetworkProfile::NetworkIrohRequired;
6592 configured.secrets.path = Some(secrets);
6593 configured.storage.models_dir = root.path().join("custom-models");
6594 let launch_dir = root.path().join("launch dir");
6596 let config_dir = launch_dir.join("my configs");
6597 std::fs::create_dir_all(&config_dir)?;
6598 let config_path = config_dir.join("custom.toml");
6599 configured.to_file(&config_path)?;
6600 Ok(config_path)
6601 }
6602
6603 async fn launcher_child() -> anyhow::Result<()> {
6604 let mode = std::env::var(CAUSAL_MODE).unwrap_or_default();
6605 let args: Vec<String> = serde_json::from_str(&std::env::var(CAUSAL_ARGV)?)?;
6607 let matches = build_cli()
6608 .try_get_matches_from(std::iter::once("hyprstream".to_owned()).chain(args))?;
6609 let config = load_config(
6610 matches
6611 .get_one::<std::path::PathBuf>("config")
6612 .map(std::path::PathBuf::as_path),
6613 )?;
6614 config
6615 .validate()
6616 .context("child configuration validation")?;
6617 let _ = hyprstream_core::config::install_pinned_config(config.clone());
6618
6619 let selected = native_service_process_name(&matches, &config)?;
6620 anyhow::ensure!(
6621 selected.as_deref() == Some(SERVICE),
6622 "child selected {selected:?} instead of the configured native service"
6623 );
6624 if mode == "missing-key" {
6625 let outcome = load_process_signing_key(&config, selected.as_deref()).await;
6628 anyhow::ensure!(
6629 outcome.is_err(),
6630 "missing provisioned key must fail required-native startup"
6631 );
6632 anyhow::bail!("missing-key control refused startup as required");
6633 }
6634
6635 let key = load_process_signing_key(&config, selected.as_deref()).await?;
6636 let proof = std::env::var(CAUSAL_PROOF)?;
6638 std::fs::write(&proof, key.verifying_key().to_bytes())?;
6639 hyprstream_rpc::notify::ready()?;
6641 let stop_file = std::path::PathBuf::from(std::env::var(CAUSAL_STOP)?);
6645 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
6646 while !stop_file.exists() {
6647 if std::time::Instant::now() >= deadline {
6648 anyhow::bail!("supervisor stop signal never arrived");
6649 }
6650 std::thread::sleep(std::time::Duration::from_millis(50));
6651 }
6652 Ok(())
6653 }
6654
6655 async fn spawn_causal_child(
6656 supervisor: &hyprstream_service::ProcessSpawner,
6657 launch_args: &[std::ffi::OsString],
6658 mode: &str,
6659 proof: &Path,
6660 stop_file: &Path,
6661 working_dir: &Path,
6662 child_instance: &str,
6663 ) -> anyhow::Result<Result<hyprstream_service::SpawnedProcess, hyprstream_rpc::error::RpcError>>
6664 {
6665 let exe = std::env::current_exe()?;
6666 let serialized_args: Vec<&str> = launch_args
6669 .iter()
6670 .map(|arg| {
6671 arg.to_str()
6672 .context("UTF-8 fixture argument for launcher causal-test envelope")
6673 })
6674 .collect::<anyhow::Result<_>>()?;
6675 let mut child = hyprstream_service::ProcessConfig::new(child_instance, &exe)
6676 .args([
6677 "--exact",
6678 "native_launcher::launcher_child_loads_explicit_config_and_provisioned_identity",
6679 "--nocapture",
6680 ])
6681 .env(CAUSAL_CHILD, "1")
6682 .env(CAUSAL_MODE, mode)
6683 .env(CAUSAL_ARGV, serde_json::to_string(&serialized_args)?)
6684 .env(CAUSAL_PROOF, proof.display().to_string())
6685 .env(CAUSAL_STOP, stop_file.display().to_string())
6686 .env("HYPRSTREAM_SECRETS_PROFILE", "shared-directory")
6688 .env("HYPRSTREAM_INSTANCE", child_instance)
6689 .working_dir(working_dir);
6690 child.readiness = hyprstream_service::ProcessReadiness::Notify {
6691 timeout: std::time::Duration::from_secs(30),
6692 };
6693 Ok(supervisor.spawn(child).await)
6694 }
6695
6696 #[tokio::test]
6697 async fn launcher_child_loads_explicit_config_and_provisioned_identity() -> anyhow::Result<()> {
6698 if std::env::var_os(CAUSAL_CHILD).is_some() {
6699 return launcher_child().await;
6700 }
6701
6702 let root = tempfile::tempdir()?;
6703 let config_path = provision_custom_config(&root, KEY_SEED, true)?;
6704 let canonical = std::fs::canonicalize(&config_path)?;
6705 let expected_vk = ed25519_dalek::SigningKey::from_bytes(&[KEY_SEED; 32])
6706 .verifying_key()
6707 .to_bytes();
6708
6709 let plan =
6711 direct_child_process_config(SERVICE, true, Some(&canonical), Path::new("hyprstream"))?;
6712 let launch_args = plan.args.clone();
6713 assert!(
6714 !launch_args.iter().any(|arg| arg == "--ipc"),
6715 "required-native child plan must contain no service IPC argument"
6716 );
6717 assert!(
6718 launch_args.windows(2).any(|pair| pair[0] == "--config"
6719 && pair[1] == canonical.to_str().expect("utf-8 canonical path")),
6720 "canonical absolute config path must be forwarded verbatim"
6721 );
6722
6723 let proof = root.path().join("proof.vk");
6727 let stop_file = root.path().join("stop");
6728 let spawner = hyprstream_service::ProcessSpawner::standalone();
6729 let supervisor = "launcher-causal-happy";
6730 let process = spawn_causal_child(
6731 &spawner,
6732 &launch_args,
6733 "happy",
6734 &proof,
6735 &stop_file,
6736 root.path(),
6737 supervisor,
6738 )
6739 .await?
6740 .expect("ready child proves config load, identity selection, and notify barrier");
6741 let observed = std::fs::read(&proof)?;
6742 assert_eq!(
6743 observed, expected_vk,
6744 "child selected the provisioned signer"
6745 );
6746 assert!(
6750 spawner.is_running(&process).await?,
6751 "the ready child must still be alive after the spawn reports success"
6752 );
6753 assert!(
6754 hyprstream_rpc::paths::service_pid_file(supervisor).exists(),
6755 "a ready child publishes its PID file"
6756 );
6757 std::fs::write(&stop_file, b"stop")?;
6761 let stopped = std::time::Instant::now() + std::time::Duration::from_secs(10);
6762 while spawner.is_running(&process).await? {
6763 if std::time::Instant::now() >= stopped {
6764 anyhow::bail!("child did not exit after the controlled stop signal");
6765 }
6766 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
6767 }
6768 spawner.stop(&process).await?;
6769 assert!(!spawner.is_running(&process).await?);
6770 assert!(
6771 !hyprstream_rpc::paths::service_pid_file(supervisor).exists(),
6772 "supervised stop must remove the published PID file"
6773 );
6774
6775 let missing_root = tempfile::tempdir()?;
6777 let missing_config = provision_custom_config(&missing_root, KEY_SEED, false)?;
6778 let missing_canonical = std::fs::canonicalize(&missing_config)?;
6779 let missing_plan = direct_child_process_config(
6780 SERVICE,
6781 true,
6782 Some(&missing_canonical),
6783 Path::new("hyprstream"),
6784 )?;
6785 let missing_spawner = hyprstream_service::ProcessSpawner::standalone();
6786 let missing_supervisor = "launcher-causal-missing";
6787 let missing_outcome = spawn_causal_child(
6788 &missing_spawner,
6789 &missing_plan.args,
6790 "missing-key",
6791 &missing_root.path().join("unused.vk"),
6792 &missing_root.path().join("stop"),
6793 missing_root.path(),
6794 missing_supervisor,
6795 )
6796 .await?;
6797 let error =
6798 missing_outcome.expect_err("launch must fail when the provisioned key is absent");
6799 assert!(
6800 error.to_string().contains("exited during startup"),
6801 "child startup failure must propagate, got: {error}"
6802 );
6803 assert!(
6804 !hyprstream_rpc::paths::service_pid_file(missing_supervisor).exists(),
6805 "a never-ready child must not publish a PID file"
6806 );
6807 Ok(())
6808 }
6809}
6810
6811#[cfg(test)]
6812mod config_load_failure_tests {
6813 use super::*;
6814
6815 #[test]
6816 fn invalid_config_never_defaults_account_selection() -> Result<()> {
6817 const CHILD: &str = "HYPRSTREAM_CONFIG_FAILURE_CHILD";
6818 if std::env::var_os(CHILD).is_none() {
6819 let root = tempfile::tempdir()?;
6820 let mut child = std::process::Command::new(std::env::current_exe()?);
6821 for (key, _) in std::env::vars_os() {
6822 if key.to_string_lossy().starts_with("HYPRSTREAM__") {
6823 child.env_remove(key);
6824 }
6825 }
6826 let status = child
6827 .args([
6828 "--exact",
6829 "config_load_failure_tests::invalid_config_never_defaults_account_selection",
6830 "--nocapture",
6831 ])
6832 .env(CHILD, "1")
6833 .env("XDG_CONFIG_HOME", root.path().join("config"))
6834 .env("XDG_DATA_HOME", root.path().join("data"))
6835 .env("HYPRSTREAM__CREDENTIALS__BACKEND", "postgres")
6836 .env("HYPRSTREAM__OAUTH__PORT", "invalid-port")
6837 .status()?;
6838 anyhow::ensure!(status.success(), "invalid configuration child failed");
6839 return Ok(());
6840 }
6841 let error = match load_config(None) {
6842 Ok(_) => anyhow::bail!("invalid port discarded PostgreSQL selection"),
6843 Err(error) => error,
6844 };
6845 assert!(error
6846 .to_string()
6847 .contains("Failed to load default-location configuration"));
6848 Ok(())
6849 }
6850}
6851
6852#[cfg(test)]
6853mod proof_replay_store_selection_tests {
6854 use super::{select_proof_replay_store, ProofReplayStoreSelection};
6855 use hyprstream_rpc::proof::admission::ReplayDomainGuarantee;
6856
6857 #[cfg(feature = "postgres-replay")]
6858 #[test]
6859 fn dedicated_shared_store_wins_over_a_configured_pds_store() {
6860 assert_eq!(
6861 select_proof_replay_store(ReplayDomainGuarantee::LinearizableSharedStore, true),
6862 ProofReplayStoreSelection::DedicatedPostgresV1,
6863 );
6864 }
6865
6866 #[cfg(all(feature = "pds-postgres", not(feature = "postgres-replay")))]
6867 #[test]
6868 fn pds_only_build_retains_legacy_shared_store() {
6869 assert_eq!(
6870 select_proof_replay_store(ReplayDomainGuarantee::LinearizableSharedStore, true),
6871 ProofReplayStoreSelection::LegacyPdsPostgres,
6872 );
6873 }
6874
6875 #[cfg(not(any(feature = "pds-postgres", feature = "postgres-replay")))]
6876 #[test]
6877 fn shared_domain_without_a_backend_fails_closed() {
6878 assert_eq!(
6879 select_proof_replay_store(ReplayDomainGuarantee::LinearizableSharedStore, false),
6880 ProofReplayStoreSelection::Unavailable,
6881 );
6882 assert_eq!(
6883 select_proof_replay_store(ReplayDomainGuarantee::SingleVerifierInstance, false),
6884 ProofReplayStoreSelection::SingleVerifierMemory,
6885 );
6886 }
6887}