1use std::path::PathBuf;
7use std::sync::Arc;
8use std::thread::{self, JoinHandle};
9
10use tokio::sync::{watch, Notify};
11
12use super::{ProcessConfig, ProcessSpawner, SpawnedProcess};
13use hyprstream_rpc::error::Result;
14use hyprstream_rpc::registry::{ServiceRegistration, SocketKind};
15use hyprstream_rpc::service::RequestService;
16use hyprstream_rpc::transport::TransportConfig;
17
18use anyhow::anyhow;
20
21fn production_moq_authz(
26 admission: Arc<hyprstream_rpc::transport::moql_admission::MoqlAdmissionAuthenticator>,
27 ingress_authorizer: Option<hyprstream_rpc::transport::iroh_moq::SharedIngressAuthorizer>,
28) -> hyprstream_rpc::transport::iroh_moq::MoqAuthzConfig {
29 hyprstream_rpc::transport::iroh_moq::MoqAuthzConfig::default()
30 .with_admission(admission)
31 .with_ingress_authorizer_option(ingress_authorizer)
32}
33
34fn producer_moq_origin(
36 required: bool,
37 relay: Option<hyprstream_rpc::moq_stream::MoqStreamOrigin>,
38 has_producer: bool,
39) -> Option<hyprstream_rpc::moq_stream::MoqStreamOrigin> {
40 relay.or_else(|| {
41 if required {
42 has_producer.then(|| {
43 hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone()
44 .with_prefix(hyprstream_rpc::moq_stream::DEFAULT_PREFIX)
45 .build()
46 })
47 } else {
48 hyprstream_rpc::moq_stream::global_moq_origin().cloned()
49 }
50 })
51}
52
53pub use hyprstream_rpc::service::Spawnable;
56
57pub struct UnifiedServiceConfig<S: RequestService + Send + 'static> {
69 service: S,
70 quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>,
71}
72
73impl<S: RequestService + Send + 'static> UnifiedServiceConfig<S> {
74 pub fn new(service: S, quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>) -> Self {
76 Self {
77 service,
78 quic_config,
79 }
80 }
81}
82
83fn validate_required_iroh_enabled(config: &hyprstream_rpc::service::QuicLoopConfig) -> Result<()> {
85 if config.iroh_required && !config.iroh_enabled {
86 return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
87 "network-iroh-required rejects iroh = false".to_owned(),
88 ));
89 }
90 Ok(())
91}
92
93fn announce_iroh_bound(
97 callback: Option<Box<dyn FnOnce(String, [u8; 32]) -> anyhow::Result<()> + Send>>,
98 required: bool,
99 service_name: &str,
100 node_id: [u8; 32],
101) -> Result<()> {
102 match callback {
103 Some(callback) => match callback(service_name.to_owned(), node_id) {
104 Ok(()) => Ok(()),
105 Err(error) if required => Err(hyprstream_rpc::error::RpcError::SpawnFailed(format!(
106 "network-iroh-required initial Iroh announcement failed: {error}"
107 ))),
108 Err(error) => {
109 tracing::warn!(service = %service_name, "Iroh announcement failed; continuing compatibility profile: {error}");
110 Ok(())
111 }
112 },
113 None if required => Err(hyprstream_rpc::error::RpcError::SpawnFailed(
114 "network-iroh-required has no initial Iroh announcement callback".to_owned(),
115 )),
116 None => Ok(()),
117 }
118}
119
120async fn announce_iroh_bound_until_shutdown(
124 config: &mut hyprstream_rpc::service::QuicLoopConfig,
125 service_name: &str,
126 node_id: [u8; 32],
127 shutdown: &Notify,
128) -> Result<()> {
129 let callback = config.on_iroh_bound.take();
130 let required = config.iroh_required;
131 let cancellation = config.announcement_cancellation.clone();
132 let service_name = service_name.to_owned();
133 let publication = tokio::task::spawn_blocking(move || {
134 announce_iroh_bound(callback, required, &service_name, node_id)
135 });
136 tokio::select! {
137 biased;
138 _ = shutdown.notified() => {
139 cancellation.cancel();
140 Err(hyprstream_rpc::error::RpcError::SpawnFailed(
141 "service stopped before initial Iroh announcement completed".to_owned(),
142 ))
143 }
144 _ = cancellation.cancelled() => Err(hyprstream_rpc::error::RpcError::SpawnFailed(
145 "initial Iroh announcement cancelled before readiness".to_owned(),
146 )),
147 result = publication => result.map_err(|error| {
148 hyprstream_rpc::error::RpcError::SpawnFailed(format!("initial Iroh announcement task: {error}"))
149 })?,
150 }
151}
152
153impl<S: RequestService + Send + Sync + 'static> Spawnable for UnifiedServiceConfig<S> {
154 fn name(&self) -> &str {
155 RequestService::name(&self.service)
156 }
157
158 fn registrations(&self) -> Vec<(SocketKind, TransportConfig)> {
159 vec![(
160 SocketKind::Rep,
161 RequestService::transport(&self.service).clone(),
162 )]
163 }
164
165 fn run(
166 self: Box<Self>,
167 shutdown: Arc<Notify>,
168 on_ready: Option<tokio::sync::oneshot::Sender<()>>,
169 ) -> Result<()> {
170 use hyprstream_rpc::transport::rpc_session::IrohRequestProcessor;
171
172 let UnifiedServiceConfig {
173 service,
174 quic_config,
175 } = *self;
176 let transport = RequestService::transport(&service).clone();
177 let signing_key = RequestService::signing_key(&service);
178 let server_pubkey = signing_key.verifying_key();
179 let service_name = RequestService::name(&service).to_owned();
180 let reach_config_handle = service.producer_reach_config_handle();
181 let moq_origin_handle = service.moq_origin_handle();
182
183 let rt = tokio::runtime::Builder::new_current_thread()
184 .enable_all()
185 .build()
186 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("runtime: {e}")))?;
187
188 let local = tokio::task::LocalSet::new();
191 local.block_on(&rt, async move {
192 let _ = server_pubkey; let nonce_cache = Arc::new(hyprstream_rpc::envelope::InMemoryNonceCache::new());
194 let bridge = hyprstream_rpc::transport::iroh_rpc::LocalServiceBridge::spawn(
195 service, Arc::clone(&nonce_cache), 0,
196 ).map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("bridge: {e}")))?;
197 let bridge = Arc::new(bridge);
198 let processor: Arc<dyn IrohRequestProcessor> = bridge.clone();
199
200 let result = async {
201 if let Some(mut qc) = quic_config {
202 let announcement_cancellation = qc.announcement_cancellation.clone();
203 let _announcement_guard = announcement_cancellation.clone().drop_guard();
204 validate_required_iroh_enabled(&qc)?;
205 let moq_server_identity = qc
210 .moq_admission_proof
211 .as_ref()
212 .map(hyprstream_rpc::transport::moql_admission::MoqlServerIdentityProof::from_local_admission_proof)
213 .transpose()
214 .map_err(|error| {
215 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
216 "MoQ server confirmation identity: {error}"
217 ))
218 })?;
219 hyprstream_rpc::transport::install_pq_crypto_provider().map_err(|error| {
224 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
225 "QUIC crypto provider: {error}"
226 ))
227 })?;
228 let _ = &qc.protected_resource_json; let chain: Vec<rustls::pki_types::CertificateDer<'static>> = qc
233 .cert_chain
234 .iter()
235 .map(|d| rustls::pki_types::CertificateDer::from(d.clone()))
236 .collect();
237 let key = rustls::pki_types::PrivateKeyDer::try_from((*qc.key_der).clone())
238 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC key: {e}")))?;
239 let wt_server = web_transport_quinn::ServerBuilder::new()
240 .with_addr(qc.bind_addr)
241 .with_certificate(chain, key)
242 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC bind: {e}")))?;
243 let actual_addr = wt_server.local_addr()
244 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC local_addr: {e}")))?;
245 let advertise_addr = if actual_addr.ip().is_unspecified() {
251 match actual_addr {
252 std::net::SocketAddr::V4(_) => {
253 std::net::SocketAddr::from((std::net::Ipv4Addr::LOCALHOST, actual_addr.port()))
254 }
255 std::net::SocketAddr::V6(_) => {
256 std::net::SocketAddr::from((std::net::Ipv6Addr::LOCALHOST, actual_addr.port()))
257 }
258 }
259 } else {
260 actual_addr
261 };
262 let pin = hyprstream_rpc::transport::quinn_transport::cert_sha256(&qc.cert_chain[0]);
263 let relay_origin = qc.moq_relay.as_ref().map(|_| {
264 hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone()
265 .with_prefix(hyprstream_rpc::moq_stream::DEFAULT_PREFIX)
266 .build()
267 });
268 let moq_origin = producer_moq_origin(qc.iroh_required, relay_origin.clone(), moq_origin_handle.is_some());
270 if let Some(handle) = &moq_origin_handle {
271 *handle.write() = moq_origin.clone();
272 }
273
274 let mut rpc_server = hyprstream_rpc::transport::quinn_transport::QuinnRpcServer::with_capacity(
275 wt_server,
276 Arc::clone(&processor),
277 signing_key.clone(),
278 hyprstream_rpc::transport::rpc_session::DEFAULT_STREAM_LIMIT,
279 );
280 if let Some(origin) = &moq_origin {
283 rpc_server = rpc_server.with_moq_consumer(origin.consumer().clone());
284 let _moq_authz_default =
306 hyprstream_rpc::transport::iroh_moq::MoqAuthzConfig::default();
307
308 }
322
323 if let Some(reg) = hyprstream_rpc::registry::try_global() {
328 reg.register(
329 &service_name,
330 hyprstream_rpc::registry::SocketKind::Quic,
331 hyprstream_rpc::transport::TransportConfig::quic_pinned(
332 advertise_addr,
333 &qc.server_name,
334 pin,
335 ),
336 None,
337 );
338 }
339 if let Some(handle) = &reach_config_handle {
340 let moql_server_identity = qc
341 .moq_admission_proof
342 .as_ref()
343 .map(|proof| proof.expected_server.clone());
344 *handle.write() = hyprstream_rpc::moq_stream::ProducerReachConfig {
345 moql_server_identity,
346 relay_moql_server_identity: qc.moq_relay_server_identity.clone(),
347 iroh_node_id: None,
348 quic_reach: Some(hyprstream_rpc::moq_stream::NodeStreamReach {
349 addr: advertise_addr,
350 server_name: qc.server_name.clone(),
351 cert_hashes: vec![pin],
352 }),
353 relay: qc.moq_relay.clone(),
354 };
355 }
356 if let Some(cb) = qc.on_quic_bound.take() {
357 cb(service_name.clone(), advertise_addr, qc.server_name.clone());
358 }
359
360 let relay_admission_proof = qc.moq_admission_proof.clone();
364 if let Some(proof) = relay_admission_proof.clone() {
365 let _ = hyprstream_rpc::moq_stream::init_global_moq_admission_proof(proof);
366 }
367
368 if let Some(relay) = qc.moq_relay.take() {
371 if qc.iroh_required && !matches!(relay, hyprstream_rpc::stream_info::TransportConfig::Iroh(_)) {
372 return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
373 "network-iroh-required rejects non-Iroh stream relay".into(),
374 ));
375 }
376 if let Some(origin) = relay_origin {
377 hyprstream_rpc::moq_stream::serve_origin_to_relay_background(
378 origin.producer().clone(),
379 relay,
380 relay_admission_proof,
381 qc.moq_relay_server_identity.take(),
382 );
383 tracing::info!(
384 service = %service_name,
385 "moq relay rendezvous enabled (announcing origin UP to relay)"
386 );
387 }
388 }
389
390 let _iroh_substrate_guard = if qc.iroh_enabled {
403 let iroh_transport_key = hyprstream_rpc::node_identity::derive_purpose_key(
404 &signing_key,
405 "hyprstream-iroh-transport-v1",
406 );
407 let iroh_secret = iroh_transport_key.to_bytes();
408 let node_id: [u8; 32] = iroh_transport_key.verifying_key().to_bytes();
409 let moq_handler = match moq_origin.as_ref() {
412 Some(origin) => {
413 let shared = hyprstream_rpc::transport::iroh_moq::OriginShared::from_pair(
414 origin.producer().clone(),
415 origin.consumer().clone(),
416 );
417 hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler::with_origin(shared)
418 }
419 None => hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler::new(),
420 };
421 let moq_ingress_authorizer = qc.moq_ingress_authorizer.take();
428 let moq_handler = match qc.moq_admission.take() {
429 Some(admission) => {
430 let server_identity = moq_server_identity.ok_or_else(|| {
431 hyprstream_rpc::error::RpcError::SpawnFailed(
432 "MoQ admission enabled without checkpointed server confirmation identity"
433 .to_owned(),
434 )
435 })?;
436 admission
437 .install_server_identity(server_identity, node_id)
438 .map_err(|error| {
439 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
440 "MoQ server confirmation identity: {error}"
441 ))
442 })?;
443 moq_handler.with_authz(
444 production_moq_authz(admission, moq_ingress_authorizer),
445 )
446 }
447 None => moq_handler,
448 };
449 let rpc_handler =
451 hyprstream_rpc::transport::iroh_rpc::IrohRpcProtocolHandler::with_stream_limit(
452 Arc::clone(&processor),
453 signing_key.clone(),
454 hyprstream_rpc::transport::rpc_session::DEFAULT_STREAM_LIMIT,
455 );
456 match hyprstream_rpc::transport::iroh_substrate::IrohSubstrate::new(
457 iroh_secret, moq_handler, rpc_handler,
458 )
459 .await
460 {
461 Ok(substrate) => {
462 let _ = hyprstream_rpc::transport::lazy_iroh::install_iroh_client_endpoint(
465 substrate.owned_client_endpoint(),
466 );
467 if let Some(handle) = &reach_config_handle {
468 handle.write().iroh_node_id = Some(node_id);
469 }
470 announce_iroh_bound_until_shutdown(&mut qc, &service_name, node_id, &shutdown).await?;
473 tracing::info!(
474 service = %service_name,
475 node_id = %hex_short(&node_id),
476 "iroh substrate bound (ALPNs hyprstream-rpc/1 + moql)"
477 );
478 Some(substrate)
479 }
480 Err(e) => {
481 if qc.iroh_required {
482 return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
483 format!("network-iroh-required Iroh bind failed: {e}"),
484 ));
485 }
486 tracing::warn!(
489 service = %service_name,
490 "iroh substrate bind failed; continuing quinn-only: {e}"
491 );
492 None
493 }
494 }
495 } else {
496 validate_required_iroh_enabled(&qc)?;
497 None
498 };
499 let drain_limit = rpc_server.stream_limit();
504 let drain_capacity = rpc_server.capacity();
505 let drain_token = rpc_server.shutdown_token();
506 let local_shutdown = Arc::new(tokio::sync::Notify::new());
507 let rep_fut = async {
508 if qc.iroh_required {
509 if !announcement_cancellation.is_cancelled() {
514 let _ = hyprstream_rpc::notify::ready();
515 if let Some(ready) = on_ready { let _ = ready.send(()); }
516 }
517 local_shutdown.notified().await;
518 Ok(())
519 } else {
520 hyprstream_rpc::service::serve::serve_bridged(
521 &transport, Arc::clone(&processor), signing_key.clone(),
522 Arc::clone(&local_shutdown), on_ready,
523 ).await
524 }
525 };
526 let quic_fut = rpc_server.run();
527 tokio::pin!(rep_fut, quic_fut);
528 let completed_rep = tokio::select! {
529 biased;
530 _ = shutdown.notified() => None,
531 _ = announcement_cancellation.cancelled() => None,
532 result = &mut rep_fut => Some(result),
533 result = &mut quic_fut => {
534 if let Err(error) = result {
535 tracing::warn!("QUIC server loop ended with error: {error}");
536 }
537 None
538 }
539 _ = async {
540 loop {
541 match _iroh_substrate_guard.as_ref() {
542 Some(substrate) if substrate.router().is_shutdown()
543 || substrate.endpoint().is_closed() => break,
544 Some(_) => tokio::time::sleep(std::time::Duration::from_secs(1)).await,
545 None => std::future::pending::<()>().await,
546 }
547 }
548 } => None,
549 };
550 announcement_cancellation.cancel();
551 let deadline = bridge.begin_shutdown(
555 tokio::time::Instant::now() + hyprstream_rpc::transport::rpc_session::DRAIN_TIMEOUT,
556 );
557 local_shutdown.notify_one();
558 drain_token.cancel();
559 let substrate_drain = async {
560 if let Some(substrate) = _iroh_substrate_guard {
561 if let Err(error) = substrate.shutdown().await {
562 tracing::warn!("iroh substrate shutdown error: {error}");
563 }
564 }
565 };
566 let quinn_drain = hyprstream_rpc::transport::quinn_transport::QuinnRpcServer::shutdown_until(
567 &drain_limit, drain_capacity, &drain_token, deadline,
568 );
569 let rep_drain = async {
570 match completed_rep {
571 Some(result) => result,
572 None => rep_fut.await,
573 }
574 };
575 let (_, (), rep_result) = tokio::join!(substrate_drain, quinn_drain, rep_drain);
576 rep_result.map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(e.to_string()))
577 } else {
578 hyprstream_rpc::service::serve::serve_bridged(
579 &transport, processor, signing_key, shutdown, on_ready,
580 ).await.map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(e.to_string()))
581 }
582 }.await;
583 let teardown = bridge.shutdown().await.map_err(|error| {
584 tracing::error!(%error, "bridge shutdown failed");
585 hyprstream_rpc::error::RpcError::SpawnFailed(error.to_string())
586 });
587 result.and(teardown)
589 })
590 }
591}
592
593fn hex_short(id: &[u8; 32]) -> String {
595 format!("{:02x}{:02x}{:02x}{:02x}…", id[0], id[1], id[2], id[3])
596}
597
598#[derive(Debug, Clone)]
600pub enum ServiceMode {
601 Tokio,
603
604 Thread,
607
608 Subprocess {
610 binary: PathBuf,
612 },
613}
614
615pub struct ServiceSpawner {
640 mode: ServiceMode,
641 process_spawner: Option<ProcessSpawner>,
642}
643
644impl ServiceSpawner {
645 pub fn tokio() -> Self {
647 Self {
648 mode: ServiceMode::Tokio,
649 process_spawner: None,
650 }
651 }
652
653 pub fn threaded() -> Self {
658 Self {
659 mode: ServiceMode::Thread,
660 process_spawner: None,
661 }
662 }
663
664 pub fn subprocess(binary: PathBuf) -> Self {
668 Self {
669 mode: ServiceMode::Subprocess { binary },
670 process_spawner: Some(ProcessSpawner::new()),
671 }
672 }
673
674 pub fn subprocess_with(binary: PathBuf, process_spawner: ProcessSpawner) -> Self {
676 Self {
677 mode: ServiceMode::Subprocess { binary },
678 process_spawner: Some(process_spawner),
679 }
680 }
681
682 pub fn mode(&self) -> &ServiceMode {
684 &self.mode
685 }
686
687 pub async fn spawn<S: Spawnable>(&self, service: S) -> Result<SpawnedService> {
708 let registrations = service.registrations();
710 let registration = if !registrations.is_empty() {
711 ServiceRegistration::multi(service.name(), registrations, None).ok()
712 } else {
713 None
714 };
715
716 match &self.mode {
718 ServiceMode::Tokio => self.spawn_tokio(service, registration).await,
719 ServiceMode::Thread => self.spawn_thread(service, registration).await,
720 ServiceMode::Subprocess { binary } => {
721 self.spawn_subprocess(service, binary.clone(), registration)
722 .await
723 }
724 }
725 }
726
727 async fn spawn_tokio<S: Spawnable>(
729 &self,
730 service: S,
731 registration: Option<ServiceRegistration>,
732 ) -> Result<SpawnedService> {
733 let name = service.name().to_owned();
734 let name_for_spawn = name.clone();
735 let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::<()>();
736 let shutdown = Arc::new(Notify::new());
737 let shutdown_clone = shutdown.clone();
738
739 let blocking_handle = tokio::task::spawn_blocking(move || {
743 if let Err(e) = Box::new(service).run(shutdown_clone, Some(ready_tx)) {
744 tracing::error!("Service {} failed: {}", name_for_spawn, e);
745 }
746 });
747
748 let join_handle = tokio::spawn(async move {
750 if let Err(e) = blocking_handle.await {
751 tracing::error!("Service blocking task panicked: {}", e);
752 }
753 });
754
755 if ready_rx.await.is_err() {
757 return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
758 "service task exited before ready".to_owned(),
759 ));
760 }
761
762 let handle = hyprstream_rpc::service::ServiceHandle::from_task(join_handle, shutdown);
764
765 Ok(SpawnedService {
766 id: format!("{name}-tokio"),
767 kind: ServiceKind::TokioTask {
768 handle: Some(handle),
769 },
770 _registration: registration,
771 })
772 }
773
774 async fn spawn_thread<S: Spawnable>(
776 &self,
777 service: S,
778 registration: Option<ServiceRegistration>,
779 ) -> Result<SpawnedService> {
780 let name = service.name().to_owned();
781 let name_for_thread = name.clone();
782 let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
783 let shutdown = Arc::new(Notify::new());
784 let shutdown_clone = shutdown.clone();
785
786 let thread_handle = thread::Builder::new()
788 .name(format!("{}-service", name))
789 .spawn(move || {
790 if let Err(e) = Box::new(service).run(shutdown_clone, Some(ready_tx)) {
792 tracing::error!("Service {} failed: {}", name_for_thread, e);
793 }
794 })
795 .map_err(|e| {
796 hyprstream_rpc::error::RpcError::SpawnFailed(format!("thread spawn: {e}"))
797 })?;
798
799 if ready_rx.await.is_err() {
801 return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
802 "service thread exited before ready".to_owned(),
803 ));
804 }
805
806 Ok(SpawnedService {
807 id: format!("{name}-thread"),
808 kind: ServiceKind::Thread {
809 handle: Some(thread_handle),
810 shutdown,
811 completion: None,
812 },
813 _registration: registration,
814 })
815 }
816
817 async fn spawn_subprocess<S: Spawnable>(
819 &self,
820 service: S,
821 binary: PathBuf,
822 registration: Option<ServiceRegistration>,
823 ) -> Result<SpawnedService> {
824 let name = service.name().to_owned();
825 let spawner = match self.process_spawner.as_ref() {
826 Some(s) => s,
827 None => {
828 return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
829 "subprocess mode requires process_spawner".to_owned(),
830 ));
831 }
832 };
833
834 let process_config = ProcessConfig::new(&name, binary).args(["service", &name]);
835
836 let process = spawner.spawn(process_config).await?;
837
838 let pid_file = hyprstream_rpc::paths::service_pid_file(&name);
840 if let Some(pid) = process.pid() {
841 if let Some(parent) = pid_file.parent() {
843 let _ = std::fs::create_dir_all(parent);
844 }
845 if let Err(e) = std::fs::write(&pid_file, pid.to_string()) {
846 tracing::warn!("Failed to write PID file {:?}: {}", pid_file, e);
847 } else {
848 tracing::debug!("Wrote PID {} to {:?}", pid, pid_file);
849 }
850 }
851
852 Ok(SpawnedService {
853 id: process.id.clone(),
854 kind: ServiceKind::Subprocess { process, pid_file },
855 _registration: registration,
856 })
857 }
858}
859
860impl Default for ServiceSpawner {
861 fn default() -> Self {
862 Self::tokio()
863 }
864}
865
866pub enum ServiceKind {
868 TokioTask {
870 handle: Option<hyprstream_rpc::service::ServiceHandle>,
871 },
872
873 Thread {
875 handle: Option<JoinHandle<()>>,
876 shutdown: Arc<Notify>,
877 completion: Option<Arc<ThreadStopCompletion>>,
881 },
882
883 Subprocess {
885 process: SpawnedProcess,
886 pid_file: PathBuf,
888 },
889}
890
891#[derive(Clone)]
892enum ThreadStopTerminal {
893 Stopped,
894 Failed(String),
895}
896
897#[doc(hidden)]
898pub struct ThreadStopCompletion {
899 terminal: watch::Sender<Option<ThreadStopTerminal>>,
900 join_handle: parking_lot::Mutex<Option<JoinHandle<()>>>,
904 join_started: std::sync::atomic::AtomicBool,
905 registration: parking_lot::Mutex<Option<ServiceRegistration>>,
909}
910
911impl ThreadStopCompletion {
912 fn new(registration: Option<ServiceRegistration>, join_handle: JoinHandle<()>) -> Self {
913 let (terminal, _receiver) = watch::channel(None);
914 Self {
915 terminal,
916 join_handle: parking_lot::Mutex::new(Some(join_handle)),
917 join_started: std::sync::atomic::AtomicBool::new(false),
918 registration: parking_lot::Mutex::new(registration),
919 }
920 }
921
922 fn publish(&self, terminal: ThreadStopTerminal) {
923 self.terminal.send_replace(Some(terminal));
926 }
927
928 fn release_registration(&self) {
929 self.registration.lock().take();
930 }
931
932 fn ensure_join_started(self: &Arc<Self>, service_id: &str) -> Result<()> {
933 use std::sync::atomic::Ordering;
934
935 if self.terminal.borrow().is_some()
936 || self
937 .join_started
938 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
939 .is_err()
940 {
941 return Ok(());
942 }
943
944 let completion = Arc::clone(self);
945 let joiner_name = format!("{service_id}-join");
946 if let Err(error) = thread::Builder::new().name(joiner_name).spawn(move || {
947 let handle = completion.join_handle.lock().take();
948 let terminal = match handle {
949 Some(handle) => match handle.join() {
950 Ok(()) => ThreadStopTerminal::Stopped,
951 Err(_) => ThreadStopTerminal::Failed(
952 "service thread panicked during shutdown".to_owned(),
953 ),
954 },
955 None => ThreadStopTerminal::Failed(
956 "service thread join owner started without a worker handle".to_owned(),
957 ),
958 };
959 completion.release_registration();
962 completion.publish(terminal);
963 }) {
964 self.join_started.store(false, Ordering::Release);
965 return Err(hyprstream_rpc::error::RpcError::SpawnFailed(format!(
966 "service thread {service_id} join supervisor failed to start: {error}"
967 )));
968 }
969 Ok(())
970 }
971
972 async fn wait(&self) -> Result<()> {
973 let mut receiver = self.terminal.subscribe();
974 loop {
975 if let Some(terminal) = receiver.borrow().clone() {
976 return match terminal {
977 ThreadStopTerminal::Stopped => Ok(()),
978 ThreadStopTerminal::Failed(error) => {
979 Err(hyprstream_rpc::error::RpcError::SpawnFailed(error))
980 }
981 };
982 }
983 receiver.changed().await.map_err(|_| {
984 hyprstream_rpc::error::RpcError::SpawnFailed(
985 "service thread stop completion owner exited without a terminal result"
986 .to_owned(),
987 )
988 })?;
989 }
990 }
991}
992
993pub struct SpawnedService {
995 id: String,
997
998 kind: ServiceKind,
1000
1001 _registration: Option<ServiceRegistration>,
1004}
1005
1006impl SpawnedService {
1007 pub fn dummy() -> Self {
1010 Self {
1011 id: "dummy".to_owned(),
1012 kind: ServiceKind::TokioTask { handle: None },
1013 _registration: None,
1014 }
1015 }
1016
1017 pub fn subprocess(id: String, process: SpawnedProcess, pid_file: PathBuf) -> Self {
1019 Self {
1020 id,
1021 kind: ServiceKind::Subprocess { process, pid_file },
1022 _registration: None,
1023 }
1024 }
1025
1026 pub fn thread(
1028 id: String,
1029 handle: Option<JoinHandle<()>>,
1030 shutdown: Arc<Notify>,
1031 registration: Option<ServiceRegistration>,
1032 ) -> Self {
1033 Self {
1034 id,
1035 kind: ServiceKind::Thread {
1036 handle,
1037 shutdown,
1038 completion: None,
1039 },
1040 _registration: registration,
1041 }
1042 }
1043
1044 pub fn id(&self) -> &str {
1046 &self.id
1047 }
1048
1049 pub fn is_running(&self) -> bool {
1051 match &self.kind {
1052 ServiceKind::TokioTask { handle } => handle
1053 .as_ref()
1054 .map(hyprstream_rpc::service::ServiceHandle::is_running)
1055 .unwrap_or(false),
1056 ServiceKind::Thread {
1057 handle, completion, ..
1058 } => completion
1059 .as_ref()
1060 .map(|completion| completion.terminal.borrow().is_none())
1061 .unwrap_or_else(|| handle.as_ref().map(|h| !h.is_finished()).unwrap_or(false)),
1062 ServiceKind::Subprocess { pid_file, .. } => {
1063 if let Ok(pid_str) = std::fs::read_to_string(pid_file) {
1065 if let Ok(pid) = pid_str.trim().parse::<i32>() {
1066 return nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), None)
1068 .is_ok();
1069 }
1070 }
1071 false
1072 }
1073 }
1074 }
1075
1076 fn request_thread_shutdown(shutdown: &Notify) {
1086 shutdown.notify_waiters();
1087 shutdown.notify_one();
1088 }
1089
1090 pub async fn stop(&mut self) -> Result<()> {
1094 let (kind, registration) = (&mut self.kind, &mut self._registration);
1095 match kind {
1096 ServiceKind::TokioTask { handle } => {
1097 if let Some(mut h) = handle.take() {
1098 h.stop().await;
1099 }
1100 }
1101 ServiceKind::Thread {
1102 handle,
1103 shutdown,
1104 completion,
1105 } => {
1106 Self::request_thread_shutdown(shutdown);
1110
1111 if let Some(existing) = completion.as_ref() {
1112 existing.ensure_join_started(&self.id)?;
1116 existing.wait().await?;
1117 } else if let Some(h) = handle.take() {
1118 let service_id = self.id.clone();
1122 let owned_completion =
1123 Arc::new(ThreadStopCompletion::new(registration.take(), h));
1124 *completion = Some(Arc::clone(&owned_completion));
1125 owned_completion.ensure_join_started(&service_id)?;
1126 owned_completion.wait().await?;
1127 }
1128 }
1129 ServiceKind::Subprocess { process, pid_file } => {
1130 if let Ok(pid_str) = std::fs::read_to_string(&pid_file) {
1132 if let Ok(pid) = pid_str.trim().parse::<i32>() {
1133 tracing::info!(
1134 "Sending SIGTERM to subprocess {} (PID {})",
1135 process.id,
1136 pid
1137 );
1138 if let Err(e) = nix::sys::signal::kill(
1139 nix::unistd::Pid::from_raw(pid),
1140 nix::sys::signal::Signal::SIGTERM,
1141 ) {
1142 tracing::warn!("Failed to send SIGTERM to PID {}: {}", pid, e);
1143 }
1144 }
1145 }
1146 if let Err(e) = std::fs::remove_file(&pid_file) {
1148 tracing::debug!("Failed to remove PID file {:?}: {}", pid_file, e);
1149 }
1150 }
1151 }
1152
1153 tracing::info!("Service {} stopped", self.id);
1154 Ok(())
1155 }
1156}
1157
1158impl Drop for SpawnedService {
1159 fn drop(&mut self) {
1160 if let ServiceKind::Thread {
1161 completion: Some(completion),
1162 ..
1163 } = &self.kind
1164 {
1165 if let Err(error) = completion.ensure_join_started(&self.id) {
1166 tracing::error!(
1167 service = %self.id,
1168 error = %error,
1169 "retaining thread teardown state after join-supervisor start failure"
1170 );
1171 let _ = Arc::into_raw(Arc::clone(completion));
1175 }
1176 }
1177 }
1178}
1179
1180use crate::service::manager::ServiceManager;
1185use async_trait::async_trait;
1186
1187pub struct InprocManager {
1191 #[allow(dead_code)]
1192 spawner: ServiceSpawner,
1193}
1194
1195impl InprocManager {
1196 pub fn new() -> Self {
1198 Self {
1199 spawner: ServiceSpawner::threaded(),
1200 }
1201 }
1202
1203 pub fn with_spawner(spawner: ServiceSpawner) -> Self {
1205 Self { spawner }
1206 }
1207}
1208
1209impl Default for InprocManager {
1210 fn default() -> Self {
1211 Self::new()
1212 }
1213}
1214
1215#[async_trait]
1216impl ServiceManager for InprocManager {
1217 async fn install(&self, _service: &str) -> anyhow::Result<()> {
1218 Ok(())
1220 }
1221
1222 async fn uninstall(&self, _service: &str) -> anyhow::Result<()> {
1223 Ok(())
1224 }
1225
1226 async fn start(&self, _service: &str) -> anyhow::Result<()> {
1227 Ok(())
1229 }
1230
1231 async fn stop(&self, _service: &str) -> anyhow::Result<()> {
1232 Ok(())
1234 }
1235
1236 async fn is_active(&self, _service: &str) -> anyhow::Result<bool> {
1237 Ok(true)
1239 }
1240
1241 async fn reload(&self) -> anyhow::Result<()> {
1242 Ok(())
1243 }
1244
1245 async fn spawn(&self, service: Box<dyn Spawnable>) -> anyhow::Result<SpawnedService> {
1246 let name = service.name().to_owned();
1249 let registrations = service.registrations();
1250
1251 let _registration = if !registrations.is_empty() {
1253 ServiceRegistration::multi(&name, registrations, None).ok()
1254 } else {
1255 None
1256 };
1257
1258 let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::<()>();
1261 let (startup_error_tx, startup_error_rx) = tokio::sync::oneshot::channel::<String>();
1262 let shutdown = Arc::new(Notify::new());
1263 let shutdown_clone = shutdown.clone();
1264 let name_clone = name.clone(); let thread_handle = thread::Builder::new()
1267 .name(format!("{}-service", name))
1268 .spawn(move || {
1269 if let Err(e) = service.run(shutdown_clone, Some(ready_tx)) {
1270 let _ = startup_error_tx.send(e.to_string());
1274 tracing::error!("Service {} failed: {}", name_clone, e);
1275 }
1276 })
1277 .map_err(|e| anyhow!("thread spawn: {}", e))?;
1278
1279 if ready_rx.await.is_err() {
1282 let (deadline_tx, deadline_rx) = tokio::sync::oneshot::channel::<()>();
1286 let _deadline_thread = thread::Builder::new()
1287 .name("service-startup-deadline".to_owned())
1288 .spawn(move || {
1289 thread::sleep(std::time::Duration::from_secs(10));
1290 let _ = deadline_tx.send(());
1291 })
1292 .map_err(|e| anyhow!("startup deadline thread spawn: {e}"))?;
1293 let cause = tokio::select! {
1294 result = startup_error_rx => result.ok(),
1295 _ = deadline_rx => None,
1296 };
1297 return Err(match cause {
1298 Some(cause) => anyhow!("service thread exited before ready: {cause}"),
1299 None => anyhow!("service thread exited before ready"),
1300 });
1301 }
1302
1303 Ok(SpawnedService::thread(
1304 format!("{name}-thread"),
1305 Some(thread_handle),
1306 shutdown,
1307 _registration,
1308 ))
1309 }
1310}
1311
1312pub struct DualSpawnable {
1332 primary: Box<dyn Spawnable>,
1333 secondary: Box<dyn Spawnable>,
1334}
1335
1336impl DualSpawnable {
1337 pub fn new(primary: Box<dyn Spawnable>, secondary: Box<dyn Spawnable>) -> Self {
1342 Self { primary, secondary }
1343 }
1344}
1345
1346impl Spawnable for DualSpawnable {
1347 fn name(&self) -> &str {
1348 self.primary.name()
1350 }
1351
1352 fn registrations(&self) -> Vec<(SocketKind, TransportConfig)> {
1353 let mut regs = self.primary.registrations();
1355 regs.extend(self.secondary.registrations());
1356 regs
1357 }
1358
1359 fn run(
1360 self: Box<Self>,
1361 shutdown: Arc<Notify>,
1362 on_ready: Option<tokio::sync::oneshot::Sender<()>>,
1363 ) -> Result<()> {
1364 let shutdown2 = shutdown.clone();
1365 let secondary = self.secondary;
1366 let secondary_name = secondary.name().to_owned();
1367
1368 let handle = thread::Builder::new()
1370 .name(format!("{}-quic", secondary_name))
1371 .spawn(move || {
1372 if let Err(e) = secondary.run(shutdown2, None) {
1373 tracing::error!("Secondary service {} failed: {}", secondary_name, e);
1374 }
1375 })
1376 .map_err(|e| hyprstream_rpc::error::RpcError::Other(format!("thread spawn: {}", e)))?;
1377
1378 let result = self.primary.run(shutdown, on_ready);
1380
1381 let _ = handle.join();
1383
1384 result
1385 }
1386}
1387
1388#[cfg(test)]
1389mod tests {
1390 use super::*;
1391 use anyhow::Result as AnyhowResult;
1392 use hyprstream_rpc::crypto::generate_signing_keypair;
1393 use hyprstream_rpc::prelude::SigningKey;
1394 use hyprstream_rpc::service::RequestService;
1395
1396 #[tokio::test]
1397 async fn native_producer_origins_do_not_expose_sibling_tracks() -> AnyhowResult<()> {
1398 let first = producer_moq_origin(true, None, true)
1399 .ok_or_else(|| anyhow!("native producer origin missing"))?;
1400 let second = producer_moq_origin(true, None, true)
1401 .ok_or_else(|| anyhow!("native producer origin missing"))?;
1402 let _broadcast = first
1403 .producer()
1404 .create_broadcast("local/streams/first")
1405 .ok_or_else(|| anyhow!("first broadcast missing"))?;
1406 assert!(first
1407 .consumer()
1408 .announced_broadcast("local/streams/first")
1409 .await
1410 .is_some());
1411 assert!(tokio::time::timeout(
1412 std::time::Duration::from_millis(100),
1413 second.consumer().announced_broadcast("local/streams/first")
1414 )
1415 .await
1416 .is_err());
1417 assert!(producer_moq_origin(true, None, false).is_none());
1418 let relayed = producer_moq_origin(true, Some(first), true)
1419 .ok_or_else(|| anyhow!("relay origin missing"))?;
1420 assert!(relayed
1421 .consumer()
1422 .announced_broadcast("local/streams/first")
1423 .await
1424 .is_some());
1425 Ok(())
1426 }
1427
1428 #[test]
1429 fn production_moq_handler_preserves_explicit_ingress_and_defaults_to_denial() {
1430 let admission = Arc::new(
1431 hyprstream_rpc::transport::moql_admission::MoqlAdmissionAuthenticator::new(
1432 Arc::new(|_: &str| None),
1433 Arc::new(|_: &hyprstream_rpc::moq_authz::PeerIdentity| None),
1434 ),
1435 );
1436 let producer = hyprstream_rpc::moq_authz::PeerIdentity::authenticated("did:at9p:producer");
1437 let authorizer = Arc::new(
1438 |peer: &hyprstream_rpc::moq_authz::PeerIdentity, tenant: &str| {
1439 peer.subject.as_deref() == Some("did:at9p:producer") && tenant == "local"
1440 },
1441 );
1442
1443 let granted =
1444 hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler::new().with_authz(
1445 production_moq_authz(Arc::clone(&admission), Some(authorizer)),
1446 );
1447 assert!(granted.authorizes_ingress(&producer, "local"));
1448 assert!(!granted.authorizes_ingress(&producer, "other"));
1449
1450 let absent = hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler::new()
1451 .with_authz(production_moq_authz(admission, None));
1452 assert!(
1453 !absent.authorizes_ingress(&producer, "local"),
1454 "an admission-only production handler must remain read-only"
1455 );
1456 }
1457
1458 struct EchoService {
1460 transport: TransportConfig,
1461 signing_key: SigningKey,
1462 }
1463
1464 impl EchoService {
1465 fn new(transport: TransportConfig, signing_key: SigningKey) -> Self {
1466 Self {
1467 transport,
1468 signing_key,
1469 }
1470 }
1471 }
1472
1473 fn spawner_test_hybrid_identity(
1478 ) -> (SigningKey, Arc<hyprstream_rpc::envelope::KeyedPqTrustStore>) {
1479 use hyprstream_rpc::envelope::{EnvelopeVerifyConfig, KeyedPqTrustStore};
1480
1481 let key = SigningKey::from_bytes(&[0xD3; 32]);
1482 let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&key);
1483 let mut store = KeyedPqTrustStore::new();
1484 store.bind(
1485 key.verifying_key().to_bytes(),
1486 &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk(&pq),
1487 );
1488 let store = Arc::new(store);
1489 let _ = hyprstream_rpc::envelope::install_verify_config(EnvelopeVerifyConfig {
1490 policy: hyprstream_rpc::crypto::CryptoPolicy::Hybrid,
1491 pq_store: Some(store.clone()),
1492 });
1493 (key, store)
1494 }
1495
1496 #[async_trait::async_trait(?Send)]
1497 impl RequestService for EchoService {
1498 fn decode_request_body(
1499 &self,
1500 signed_body: &[u8],
1501 ) -> AnyhowResult<hyprstream_rpc::service::DecodedRequestBody> {
1502 Ok(hyprstream_rpc::service::DecodedRequestBody::opaque(
1503 signed_body.to_vec(),
1504 ))
1505 }
1506
1507 async fn handle_request(
1508 &self,
1509 _ctx: &hyprstream_rpc::service::EnvelopeContext,
1510 body: &hyprstream_rpc::service::DecodedRequestBody,
1511 ) -> AnyhowResult<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
1512 Ok((body.bytes().to_vec(), None))
1513 }
1514
1515 fn name(&self) -> &str {
1516 "echo"
1517 }
1518
1519 fn transport(&self) -> &TransportConfig {
1520 &self.transport
1521 }
1522
1523 fn signing_key(&self) -> SigningKey {
1524 self.signing_key.clone()
1525 }
1526 }
1527
1528 struct GatedService {
1529 echo: EchoService,
1530 entered: Arc<Notify>,
1531 release: Arc<Notify>,
1532 dropped: Arc<std::sync::atomic::AtomicBool>,
1533 panic_on_drop: bool,
1534 }
1535
1536 impl Drop for GatedService {
1537 fn drop(&mut self) {
1538 self.dropped
1539 .store(true, std::sync::atomic::Ordering::SeqCst);
1540 assert!(!self.panic_on_drop, "causal service destructor panic");
1541 }
1542 }
1543
1544 #[async_trait::async_trait(?Send)]
1545 impl RequestService for GatedService {
1546 fn decode_request_body(
1547 &self,
1548 signed_body: &[u8],
1549 ) -> AnyhowResult<hyprstream_rpc::service::DecodedRequestBody> {
1550 self.echo.decode_request_body(signed_body)
1551 }
1552 async fn handle_request(
1553 &self,
1554 _ctx: &hyprstream_rpc::service::EnvelopeContext,
1555 body: &hyprstream_rpc::service::DecodedRequestBody,
1556 ) -> AnyhowResult<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
1557 anyhow::bail!("denied request reached handler: {}", body.bytes().len())
1558 }
1559 async fn verify_claims(
1560 &self,
1561 _ctx: &mut hyprstream_rpc::service::EnvelopeContext,
1562 ) -> AnyhowResult<()> {
1563 self.entered.notify_one();
1564 self.release.notified().await;
1565 anyhow::bail!("gated claims denial")
1566 }
1567 fn build_error_payload(&self, _request_id: u64, error: &str) -> Vec<u8> {
1568 error.as_bytes().to_vec()
1569 }
1570 fn name(&self) -> &str {
1571 "echo"
1572 }
1573 fn transport(&self) -> &TransportConfig {
1574 &self.echo.transport
1575 }
1576 fn signing_key(&self) -> SigningKey {
1577 self.echo.signing_key.clone()
1578 }
1579 }
1580
1581 struct TwoWaiterInprocService {
1586 service: GatedService,
1587 lifecycle_observed: Arc<std::sync::atomic::AtomicBool>,
1588 }
1589
1590 impl Spawnable for TwoWaiterInprocService {
1591 fn name(&self) -> &str {
1592 RequestService::name(&self.service)
1593 }
1594
1595 fn registrations(&self) -> Vec<(SocketKind, TransportConfig)> {
1596 vec![(SocketKind::Rep, self.service.transport().clone())]
1597 }
1598
1599 fn run(
1600 self: Box<Self>,
1601 shutdown: Arc<Notify>,
1602 on_ready: Option<tokio::sync::oneshot::Sender<()>>,
1603 ) -> Result<()> {
1604 let TwoWaiterInprocService {
1605 service,
1606 lifecycle_observed,
1607 } = *self;
1608 let transport = service.transport().clone();
1609 let signing_key = service.signing_key();
1610 let runtime = tokio::runtime::Builder::new_multi_thread()
1611 .enable_all()
1612 .build()
1613 .map_err(|error| {
1614 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1615 "two-waiter test runtime: {error}"
1616 ))
1617 })?;
1618
1619 runtime.block_on(async move {
1620 let nonce_cache = Arc::new(hyprstream_rpc::envelope::InMemoryNonceCache::new());
1621 let bridge = Arc::new(
1622 hyprstream_rpc::transport::iroh_rpc::LocalServiceBridge::spawn(
1623 service,
1624 nonce_cache,
1625 0,
1626 )
1627 .map_err(|error| {
1628 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1629 "two-waiter test bridge: {error}"
1630 ))
1631 })?,
1632 );
1633 let processor: Arc<
1634 dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor,
1635 > = bridge.clone();
1636
1637 let (lifecycle_armed_tx, lifecycle_armed_rx) = tokio::sync::oneshot::channel();
1638 let lifecycle_shutdown = Arc::clone(&shutdown);
1639 let lifecycle = tokio::spawn(async move {
1640 let notified = lifecycle_shutdown.notified();
1641 tokio::pin!(notified);
1642 notified.as_mut().enable();
1643 let _ = lifecycle_armed_tx.send(());
1644 notified.await;
1645 lifecycle_observed.store(true, std::sync::atomic::Ordering::Release);
1646 });
1647 lifecycle_armed_rx.await.map_err(|_| {
1648 hyprstream_rpc::error::RpcError::SpawnFailed(
1649 "two-waiter lifecycle exited before arming".to_owned(),
1650 )
1651 })?;
1652
1653 let (serve_armed_tx, serve_armed_rx) = tokio::sync::oneshot::channel();
1654 let serve_shutdown = Arc::clone(&shutdown);
1655 let serve = tokio::spawn(async move {
1656 hyprstream_rpc::service::serve::serve_bridged_with_shutdown_armed_silent(
1657 &transport,
1658 processor,
1659 signing_key,
1660 serve_shutdown,
1661 None,
1662 Some(serve_armed_tx),
1663 )
1664 .await
1665 });
1666 serve_armed_rx.await.map_err(|_| {
1667 hyprstream_rpc::error::RpcError::SpawnFailed(
1668 "two-waiter inproc server exited before arming".to_owned(),
1669 )
1670 })?;
1671 if let Some(ready) = on_ready {
1672 let _ = ready.send(());
1673 }
1674
1675 let serve_result = serve
1676 .await
1677 .map_err(|error| {
1678 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1679 "two-waiter inproc server task failed: {error}"
1680 ))
1681 })?
1682 .map_err(|error| {
1683 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1684 "two-waiter inproc server failed: {error}"
1685 ))
1686 });
1687 let lifecycle_result = lifecycle.await.map_err(|error| {
1688 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1689 "two-waiter lifecycle task failed: {error}"
1690 ))
1691 });
1692 let bridge_result = bridge.shutdown().await.map_err(|error| {
1693 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1694 "two-waiter bridge shutdown failed: {error}"
1695 ))
1696 });
1697 serve_result.and(lifecycle_result).and(bridge_result)
1698 })
1699 }
1700 }
1701
1702 #[tokio::test]
1703 async fn threaded_stop_broadcasts_to_lifecycle_and_inproc_with_retained_client(
1704 ) -> AnyhowResult<()> {
1705 use hyprstream_rpc::rpc_client::RpcClientImpl;
1706 use hyprstream_rpc::signer::LocalSigner;
1707 use hyprstream_rpc::transport::in_memory::InMemoryTransport;
1708
1709 let (key, store) = spawner_test_hybrid_identity();
1710
1711 let endpoint = "threaded-two-waiter-retained-client";
1712 let entered = Arc::new(Notify::new());
1713 let release = Arc::new(Notify::new());
1714 let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
1715 let lifecycle_observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
1716 let service = TwoWaiterInprocService {
1717 service: GatedService {
1718 echo: EchoService::new(TransportConfig::inproc(endpoint), key.clone()),
1719 entered: entered.clone(),
1720 release: release.clone(),
1721 dropped: dropped.clone(),
1722 panic_on_drop: false,
1723 },
1724 lifecycle_observed: lifecycle_observed.clone(),
1725 };
1726 let mut spawned = ServiceSpawner::threaded().spawn(service).await?;
1727 let processor = hyprstream_rpc::dial::lookup_inproc(endpoint)
1728 .ok_or_else(|| anyhow!("two-waiter inproc endpoint was not registered"))?;
1729 let client = Arc::new(
1730 RpcClientImpl::new(
1731 LocalSigner::new(key.clone()),
1732 InMemoryTransport::new(processor),
1733 Some(key.verifying_key()),
1734 )
1735 .with_response_pq_store(store),
1736 );
1737 let caller = client.clone();
1738 let response = tokio::spawn(async move { caller.call(b"accepted".to_vec()).await });
1739 tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified()).await?;
1740
1741 let stop = tokio::spawn(async move { spawned.stop().await });
1742 tokio::time::timeout(std::time::Duration::from_secs(5), async {
1743 while hyprstream_rpc::dial::lookup_inproc(endpoint).is_some() {
1744 tokio::task::yield_now().await;
1745 }
1746 })
1747 .await?;
1748 assert!(
1749 lifecycle_observed.load(std::sync::atomic::Ordering::Acquire),
1750 "broadcast shutdown must wake the lifecycle waiter"
1751 );
1752 assert!(
1753 !stop.is_finished(),
1754 "thread owner returned before the accepted request drained"
1755 );
1756 assert!(
1757 tokio::time::timeout(
1758 std::time::Duration::from_secs(5),
1759 client.call(b"late".to_vec())
1760 )
1761 .await?
1762 .is_err(),
1763 "retained client must lose admission after endpoint unregister"
1764 );
1765
1766 release.notify_one();
1767 assert_eq!(
1768 tokio::time::timeout(std::time::Duration::from_secs(5), response).await???,
1769 hyprstream_rpc::service::dispatch::DISPATCH_DENIED.as_bytes()
1770 );
1771 tokio::time::timeout(std::time::Duration::from_secs(5), stop).await???;
1772 assert!(
1773 dropped.load(std::sync::atomic::Ordering::SeqCst),
1774 "thread owner must join the bridge after drain"
1775 );
1776 assert!(client.call(b"late".to_vec()).await.is_err());
1777 Ok(())
1778 }
1779
1780 #[tokio::test]
1781 async fn cancelled_direct_thread_stop_waits_for_the_same_terminal_join() -> AnyhowResult<()> {
1782 use hyprstream_rpc::rpc_client::RpcClientImpl;
1783 use hyprstream_rpc::signer::LocalSigner;
1784 use hyprstream_rpc::transport::in_memory::InMemoryTransport;
1785
1786 let (key, store) = spawner_test_hybrid_identity();
1787
1788 let endpoint = "threaded-stop-cancellation-retained-client";
1789 let entered = Arc::new(Notify::new());
1790 let release = Arc::new(Notify::new());
1791 let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
1792 let lifecycle_observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
1793 let service = TwoWaiterInprocService {
1794 service: GatedService {
1795 echo: EchoService::new(TransportConfig::inproc(endpoint), key.clone()),
1796 entered: entered.clone(),
1797 release: release.clone(),
1798 dropped: dropped.clone(),
1799 panic_on_drop: false,
1800 },
1801 lifecycle_observed: lifecycle_observed.clone(),
1802 };
1803 let mut spawned = ServiceSpawner::threaded().spawn(service).await?;
1804 let processor = hyprstream_rpc::dial::lookup_inproc(endpoint)
1805 .ok_or_else(|| anyhow!("direct-stop endpoint was not registered"))?;
1806 let client = Arc::new(
1807 RpcClientImpl::new(
1808 LocalSigner::new(key.clone()),
1809 InMemoryTransport::new(processor),
1810 Some(key.verifying_key()),
1811 )
1812 .with_response_pq_store(store),
1813 );
1814 let caller = client.clone();
1815 let response = tokio::spawn(async move { caller.call(b"accepted".to_vec()).await });
1816 tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified()).await?;
1817
1818 let mut first_stop = Box::pin(spawned.stop());
1819 let endpoint_unregistered = async {
1820 tokio::time::timeout(std::time::Duration::from_secs(5), async {
1821 while hyprstream_rpc::dial::lookup_inproc(endpoint).is_some() {
1822 tokio::task::yield_now().await;
1823 }
1824 })
1825 .await
1826 .map_err(|_| anyhow!("first stop did not unregister endpoint before cancellation"))
1827 };
1828 tokio::select! {
1829 result = &mut first_stop => anyhow::bail!(
1830 "first stop returned before the accepted request drained: {result:?}"
1831 ),
1832 result = endpoint_unregistered => result?,
1833 }
1834 drop(first_stop);
1835
1836 assert!(
1837 lifecycle_observed.load(std::sync::atomic::Ordering::Acquire),
1838 "first stop must broadcast to the lifecycle waiter before cancellation"
1839 );
1840 assert!(
1841 tokio::time::timeout(
1842 std::time::Duration::from_secs(5),
1843 client.call(b"late".to_vec())
1844 )
1845 .await?
1846 .is_err(),
1847 "endpoint removal must reject retained-client late admission"
1848 );
1849
1850 let mut second_stop = Box::pin(spawned.stop());
1851 tokio::select! {
1852 result = &mut second_stop => anyhow::bail!(
1853 "second stop returned before the original join completed: {result:?}"
1854 ),
1855 _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {},
1856 }
1857 assert!(
1858 !dropped.load(std::sync::atomic::Ordering::SeqCst),
1859 "bridge/service destruction must wait for the accepted request"
1860 );
1861
1862 release.notify_one();
1863 assert_eq!(
1864 tokio::time::timeout(std::time::Duration::from_secs(5), response).await???,
1865 hyprstream_rpc::service::dispatch::DISPATCH_DENIED.as_bytes()
1866 );
1867 tokio::time::timeout(std::time::Duration::from_secs(5), &mut second_stop).await??;
1868 assert!(
1869 dropped.load(std::sync::atomic::Ordering::SeqCst),
1870 "second stop must await bridge/service destruction from the original join"
1871 );
1872 assert!(client.call(b"late".to_vec()).await.is_err());
1873 Ok(())
1874 }
1875
1876 fn blocking_panicking_thread_for_stop_test(
1877 armed: std::sync::mpsc::Sender<()>,
1878 shutdown_observed: tokio::sync::oneshot::Sender<()>,
1879 release: Arc<Notify>,
1880 ) -> SpawnedService {
1881 let shutdown = Arc::new(Notify::new());
1882 let worker_shutdown = Arc::clone(&shutdown);
1883 let handle = std::thread::spawn(move || {
1884 let runtime = tokio::runtime::Builder::new_current_thread()
1885 .enable_all()
1886 .build()
1887 .unwrap_or_else(|error| panic!("build stop-error test runtime: {error}"));
1888 runtime.block_on(async move {
1889 let notified = worker_shutdown.notified();
1890 tokio::pin!(notified);
1891 notified.as_mut().enable();
1892 armed
1893 .send(())
1894 .unwrap_or_else(|error| panic!("signal stop-error worker readiness: {error}"));
1895 notified.await;
1896 let _ = shutdown_observed.send(());
1897 release.notified().await;
1898 panic!("deliberate direct-stop terminal error")
1899 });
1900 });
1901 SpawnedService::thread(
1902 "direct-stop-terminal-error".to_owned(),
1903 Some(handle),
1904 shutdown,
1905 None,
1906 )
1907 }
1908
1909 fn ready_before_waiter_thread_for_stop_test(
1910 ready: std::sync::mpsc::Sender<()>,
1911 arm_waiter: std::sync::mpsc::Receiver<()>,
1912 shutdown_observed: tokio::sync::oneshot::Sender<()>,
1913 ) -> (SpawnedService, Arc<ThreadStopCompletion>) {
1914 let shutdown = Arc::new(Notify::new());
1915 let worker_shutdown = Arc::clone(&shutdown);
1916 let handle = std::thread::spawn(move || {
1917 ready
1918 .send(())
1919 .unwrap_or_else(|error| panic!("signal ready-before-waiter readiness: {error}"));
1920 arm_waiter
1921 .recv()
1922 .unwrap_or_else(|error| panic!("release ready-before-waiter arm: {error}"));
1923 let runtime = tokio::runtime::Builder::new_current_thread()
1924 .enable_all()
1925 .build()
1926 .unwrap_or_else(|error| panic!("build ready-before-waiter runtime: {error}"));
1927 runtime.block_on(async move {
1928 worker_shutdown.notified().await;
1929 let _ = shutdown_observed.send(());
1930 });
1931 });
1932 let completion = Arc::new(ThreadStopCompletion::new(None, handle));
1933 (
1934 SpawnedService {
1935 id: "ready-before-waiter".to_owned(),
1936 kind: ServiceKind::Thread {
1937 handle: None,
1938 shutdown,
1939 completion: Some(Arc::clone(&completion)),
1940 },
1941 _registration: None,
1942 },
1943 completion,
1944 )
1945 }
1946
1947 #[tokio::test]
1948 async fn threaded_stop_retains_shutdown_for_waiter_armed_after_readiness() -> AnyhowResult<()> {
1949 let (ready_tx, ready_rx) = std::sync::mpsc::channel();
1950 let (arm_waiter_tx, arm_waiter_rx) = std::sync::mpsc::channel();
1951 let (shutdown_observed_tx, shutdown_observed_rx) = tokio::sync::oneshot::channel();
1952 let (mut spawned, completion) =
1953 ready_before_waiter_thread_for_stop_test(ready_tx, arm_waiter_rx, shutdown_observed_tx);
1954
1955 tokio::task::spawn_blocking(move || {
1956 ready_rx
1957 .recv_timeout(std::time::Duration::from_secs(5))
1958 .map_err(|error| anyhow!("thread did not report readiness: {error}"))
1959 })
1960 .await??;
1961
1962 let mut stop = Box::pin(spawned.stop());
1963 tokio::select! {
1964 result = &mut stop => anyhow::bail!(
1965 "stop returned before the thread installed its post-readiness waiter: {result:?}"
1966 ),
1967 _ = async {
1968 while !completion.join_started.load(std::sync::atomic::Ordering::Acquire) {
1969 tokio::task::yield_now().await;
1970 }
1971 } => {},
1972 }
1973 arm_waiter_tx
1976 .send(())
1977 .map_err(|error| anyhow!("release post-readiness waiter: {error}"))?;
1978 tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_observed_rx)
1979 .await
1980 .map_err(|_| anyhow!("late shutdown waiter did not observe retained stop"))?
1981 .map_err(|error| anyhow!("late shutdown observer dropped: {error}"))?;
1982 tokio::time::timeout(std::time::Duration::from_secs(5), &mut stop)
1983 .await
1984 .map_err(|_| anyhow!("stop did not join the late waiter thread"))??;
1985 Ok(())
1986 }
1987
1988 #[tokio::test]
1989 async fn cancelled_direct_thread_stop_replays_the_same_terminal_error() -> AnyhowResult<()> {
1990 let (armed_tx, armed_rx) = std::sync::mpsc::channel();
1991 let (shutdown_observed_tx, shutdown_observed_rx) = tokio::sync::oneshot::channel();
1992 let release = Arc::new(Notify::new());
1993 let mut spawned = blocking_panicking_thread_for_stop_test(
1994 armed_tx,
1995 shutdown_observed_tx,
1996 Arc::clone(&release),
1997 );
1998 tokio::task::spawn_blocking(move || {
1999 armed_rx
2000 .recv_timeout(std::time::Duration::from_secs(5))
2001 .map_err(|error| anyhow!("stop-error worker did not arm: {error}"))
2002 })
2003 .await??;
2004
2005 let mut first_stop = Box::pin(spawned.stop());
2006 tokio::select! {
2007 result = &mut first_stop => anyhow::bail!(
2008 "first stop returned before the controlled thread terminal: {result:?}"
2009 ),
2010 result = shutdown_observed_rx => result.map_err(|error| anyhow!(
2011 "stop-error worker dropped shutdown observation: {error}"
2012 ))?,
2013 }
2014 drop(first_stop);
2015
2016 let mut second_stop = Box::pin(spawned.stop());
2017 tokio::select! {
2018 result = &mut second_stop => anyhow::bail!(
2019 "second stop returned before the original terminal error: {result:?}"
2020 ),
2021 _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {},
2022 }
2023 release.notify_one();
2024 let second_error = match tokio::time::timeout(
2025 std::time::Duration::from_secs(5),
2026 &mut second_stop,
2027 )
2028 .await?
2029 {
2030 Err(error) => error.to_string(),
2031 Ok(()) => anyhow::bail!("joined panicking worker unexpectedly stopped successfully"),
2032 };
2033 drop(second_stop);
2034 let third_error = match spawned.stop().await {
2035 Err(error) => error.to_string(),
2036 Ok(()) => anyhow::bail!("later stop unexpectedly hid the terminal join error"),
2037 };
2038 assert_eq!(
2039 second_error, third_error,
2040 "every later stop must receive the same terminal error"
2041 );
2042 assert!(
2043 second_error.contains("panicked during shutdown"),
2044 "terminal error must remain truthful: {second_error}"
2045 );
2046 Ok(())
2047 }
2048
2049 #[tokio::test]
2050 async fn cancelled_direct_thread_stop_retains_terminal_after_all_waiters_drop(
2051 ) -> AnyhowResult<()> {
2052 let (armed_tx, armed_rx) = std::sync::mpsc::channel();
2053 let (shutdown_observed_tx, shutdown_observed_rx) = tokio::sync::oneshot::channel();
2054 let release = Arc::new(Notify::new());
2055 let mut spawned = blocking_panicking_thread_for_stop_test(
2056 armed_tx,
2057 shutdown_observed_tx,
2058 Arc::clone(&release),
2059 );
2060 tokio::task::spawn_blocking(move || {
2061 armed_rx
2062 .recv_timeout(std::time::Duration::from_secs(5))
2063 .map_err(|error| anyhow!("no-waiter worker did not arm: {error}"))
2064 })
2065 .await??;
2066
2067 let mut first_stop = Box::pin(spawned.stop());
2068 tokio::select! {
2069 result = &mut first_stop => anyhow::bail!(
2070 "first stop returned before the controlled thread terminal: {result:?}"
2071 ),
2072 result = shutdown_observed_rx => result.map_err(|error| anyhow!(
2073 "no-waiter worker dropped shutdown observation: {error}"
2074 ))?,
2075 }
2076 drop(first_stop);
2077 release.notify_one();
2078
2079 tokio::time::timeout(std::time::Duration::from_secs(5), async {
2080 while spawned.is_running() {
2081 tokio::task::yield_now().await;
2082 }
2083 })
2084 .await
2085 .map_err(|_| anyhow!("joined worker never published a terminal result"))?;
2086 let second_error = match spawned.stop().await {
2087 Err(error) => error.to_string(),
2088 Ok(()) => anyhow::bail!("late stop unexpectedly hid the retained terminal error"),
2089 };
2090 assert!(
2091 second_error.contains("panicked during shutdown"),
2092 "late terminal error must remain truthful: {second_error}"
2093 );
2094 Ok(())
2095 }
2096
2097 #[tokio::test]
2098 async fn both_inproc_owners_drain_entered_request_with_retained_client() -> AnyhowResult<()> {
2099 use hyprstream_rpc::rpc_client::RpcClientImpl;
2100 use hyprstream_rpc::signer::LocalSigner;
2101 use hyprstream_rpc::transport::in_memory::InMemoryTransport;
2102 let (key, store) = spawner_test_hybrid_identity();
2103 for unified in [false, true] {
2104 let endpoint = format!("bridge-drain-owner-{unified}");
2105 let transport = TransportConfig::inproc(&endpoint);
2106 let entered = Arc::new(Notify::new());
2107 let release = Arc::new(Notify::new());
2108 let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
2109 let service = GatedService {
2110 echo: EchoService::new(transport, key.clone()),
2111 entered: entered.clone(),
2112 release: release.clone(),
2113 dropped: dropped.clone(),
2114 panic_on_drop: false,
2115 };
2116 let owner: Box<dyn Spawnable> = if unified {
2117 Box::new(UnifiedServiceConfig::new(service, None))
2118 } else {
2119 Box::new(service)
2120 };
2121 let shutdown = Arc::new(Notify::new());
2122 struct Cleanup(Arc<Notify>);
2123 impl Drop for Cleanup {
2124 fn drop(&mut self) {
2125 self.0.notify_one();
2126 }
2127 }
2128 let _cleanup = Cleanup(shutdown.clone());
2129 let stopped = shutdown.clone();
2130 let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
2131 let task = tokio::task::spawn_blocking(move || owner.run(stopped, Some(ready_tx)));
2132 tokio::time::timeout(std::time::Duration::from_secs(5), ready_rx).await??;
2133 let processor = hyprstream_rpc::dial::lookup_inproc(&endpoint)
2134 .ok_or_else(|| anyhow::anyhow!("missing ready processor"))?;
2135 let client = Arc::new(
2136 RpcClientImpl::new(
2137 LocalSigner::new(key.clone()),
2138 InMemoryTransport::new(processor),
2139 Some(key.verifying_key()),
2140 )
2141 .with_response_pq_store(store.clone()),
2142 );
2143 let caller = client.clone();
2144 let response = tokio::spawn(async move { caller.call(b"accepted".to_vec()).await });
2145 tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified()).await?;
2146 shutdown.notify_one();
2147 tokio::time::timeout(std::time::Duration::from_secs(5), async {
2148 while hyprstream_rpc::dial::lookup_inproc(&endpoint).is_some() {
2149 tokio::task::yield_now().await;
2150 }
2151 })
2152 .await?;
2153 assert!(
2154 !task.is_finished(),
2155 "owner returned while accepted request is gated"
2156 );
2157 assert!(!dropped.load(std::sync::atomic::Ordering::SeqCst));
2158 assert!(tokio::time::timeout(
2161 std::time::Duration::from_secs(5),
2162 client.call(b"late".to_vec())
2163 )
2164 .await?
2165 .is_err());
2166 release.notify_one();
2167 assert_eq!(
2168 tokio::time::timeout(std::time::Duration::from_secs(5), response).await???,
2169 hyprstream_rpc::service::dispatch::DISPATCH_DENIED.as_bytes()
2170 );
2171 tokio::time::timeout(std::time::Duration::from_secs(5), task).await???;
2172 assert!(
2173 dropped.load(std::sync::atomic::Ordering::SeqCst),
2174 "retained client pinned service"
2175 );
2176 assert!(client.call(b"late".to_vec()).await.is_err());
2177 }
2178 Ok(())
2179 }
2180
2181 #[tokio::test]
2182 async fn both_inproc_owners_propagate_destructor_panic() -> AnyhowResult<()> {
2183 for unified in [false, true] {
2184 let (key, _) = generate_signing_keypair();
2185 let service = GatedService {
2186 echo: EchoService::new(
2187 TransportConfig::inproc(format!("bridge-panic-owner-{unified}")),
2188 key,
2189 ),
2190 entered: Arc::new(Notify::new()),
2191 release: Arc::new(Notify::new()),
2192 dropped: Arc::new(std::sync::atomic::AtomicBool::new(false)),
2193 panic_on_drop: true,
2194 };
2195 let owner: Box<dyn Spawnable> = if unified {
2196 Box::new(UnifiedServiceConfig::new(service, None))
2197 } else {
2198 Box::new(service)
2199 };
2200 let shutdown = Arc::new(Notify::new());
2201 struct Cleanup(Arc<Notify>);
2202 impl Drop for Cleanup {
2203 fn drop(&mut self) {
2204 self.0.notify_one();
2205 }
2206 }
2207 let _cleanup = Cleanup(shutdown.clone());
2208 let stopped = shutdown.clone();
2209 let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
2210 let task = tokio::task::spawn_blocking(move || owner.run(stopped, Some(ready_tx)));
2211 tokio::time::timeout(std::time::Duration::from_secs(5), ready_rx).await??;
2212 shutdown.notify_one();
2213 let result = tokio::time::timeout(std::time::Duration::from_secs(5), task).await??;
2214 assert!(result.is_err(), "owner hid bridge destructor panic");
2215 }
2216 Ok(())
2217 }
2218
2219 fn loop_config(
2220 iroh_enabled: bool,
2221 iroh_required: bool,
2222 on_iroh_bound: Option<Box<dyn FnOnce(String, [u8; 32]) -> anyhow::Result<()> + Send>>,
2223 ) -> hyprstream_rpc::service::QuicLoopConfig {
2224 hyprstream_rpc::service::QuicLoopConfig {
2225 announcement_cancellation: tokio_util::sync::CancellationToken::new(),
2226 cert_chain: Vec::new(),
2227 key_der: zeroize::Zeroizing::new(Vec::new()),
2228 bind_addr: "127.0.0.1:0"
2229 .parse()
2230 .unwrap_or_else(|error| panic!("test socket address: {error}")),
2231 server_name: "service.test".to_owned(),
2232 protected_resource_json: None,
2233 on_quic_bound: None,
2234 iroh_enabled,
2235 iroh_required,
2236 on_iroh_bound,
2237 moq_relay: None,
2238 moq_relay_server_identity: None,
2239 moq_admission: None,
2240 moq_ingress_authorizer: None,
2241 moq_admission_proof: None,
2242 }
2243 }
2244
2245 #[tokio::test]
2246 async fn test_tokio_spawner() -> hyprstream_rpc::Result<()> {
2247 let (signing_key, _verifying_key) = generate_signing_keypair();
2248 let transport = TransportConfig::inproc("test-spawner-tokio");
2249
2250 let service = EchoService::new(transport, signing_key);
2252
2253 let spawner = ServiceSpawner::tokio();
2254 let mut spawned = spawner.spawn(service).await?;
2255
2256 assert!(spawned.is_running());
2257
2258 spawned.stop().await?;
2259 Ok(())
2260 }
2261
2262 #[tokio::test]
2263 async fn test_thread_spawner() -> hyprstream_rpc::Result<()> {
2264 let (signing_key, _verifying_key) = generate_signing_keypair();
2265 let transport = TransportConfig::inproc("test-spawner-thread");
2266
2267 let service = EchoService::new(transport, signing_key);
2269
2270 let spawner = ServiceSpawner::threaded();
2271 let mut spawned = spawner.spawn(service).await?;
2272
2273 assert!(spawned.is_running());
2274
2275 spawned.stop().await?;
2276 Ok(())
2277 }
2278
2279 #[test]
2280 fn required_profile_rejects_disabled_iroh_at_spawner_seam() {
2281 let config = loop_config(false, true, None);
2282 let error = match validate_required_iroh_enabled(&config) {
2283 Ok(()) => panic!("required profile accepted a disabled Iroh carrier"),
2284 Err(error) => error,
2285 };
2286 assert!(error.to_string().contains("rejects iroh = false"));
2287 }
2288
2289 #[cfg(all(unix, feature = "systemd"))]
2292 #[test]
2293 fn required_profile_systemd_ready_follows_bind_and_publication() -> AnyhowResult<()> {
2294 use std::os::unix::net::UnixDatagram;
2295 use std::time::Duration;
2296 const CHILD: &str = "HYPRSTREAM_REQUIRED_NOTIFY_TEST";
2297 let Ok(outcome) = std::env::var(CHILD) else {
2298 for outcome in ["success", "failure", "cancel"] {
2299 let status = std::process::Command::new(std::env::current_exe()?)
2300 .args(["--exact", "service::spawner::service::tests::required_profile_systemd_ready_follows_bind_and_publication", "--nocapture"])
2301 .env(CHILD, outcome).status()?;
2302 anyhow::ensure!(status.success(), "notification case {outcome} failed");
2303 }
2304 return Ok(());
2305 };
2306 let dir = tempfile::tempdir()?;
2307 let socket_path = dir.path().join("notify.sock");
2308 let socket = UnixDatagram::bind(&socket_path)?;
2309 socket.set_read_timeout(Some(Duration::from_millis(100)))?;
2310 std::env::set_var("NOTIFY_SOCKET", &socket_path);
2312 let rpc_path = dir.path().join("echo.sock");
2313 let (signing_key, _) = generate_signing_keypair();
2314 let service = EchoService::new(TransportConfig::ipc(&rpc_path), signing_key);
2315 let mut config = loop_config(true, true, None);
2316 let certified = rcgen::generate_simple_self_signed(vec!["service.test".to_owned()])?;
2317 config.cert_chain = vec![certified.cert.der().to_vec()];
2318 config.key_der = zeroize::Zeroizing::new(certified.key_pair.serialize_der());
2319 let cancellation = config.announcement_cancellation.clone();
2320 let callback_cancellation = cancellation.clone();
2321 let (bound_tx, bound_rx) = std::sync::mpsc::sync_channel(1);
2322 let (publication_tx, publication_rx) = std::sync::mpsc::sync_channel(1);
2323 config.on_iroh_bound = Some(Box::new(move |name, node| {
2324 anyhow::ensure!(name == "echo" && node != [0; 32], "actual bound carrier");
2325 bound_tx.send(())?;
2326 loop {
2327 if callback_cancellation.is_cancelled() {
2328 anyhow::bail!("publication cancelled");
2329 }
2330 match publication_rx.recv_timeout(Duration::from_millis(10)) {
2331 Ok(true) => return Ok(()),
2332 Ok(false) => anyhow::bail!("publication rejected"),
2333 Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue,
2334 Err(error) => return Err(error.into()),
2335 }
2336 }
2337 }));
2338 let shutdown = Arc::new(Notify::new());
2339 let shutdown_service = Arc::clone(&shutdown);
2340 let (ready_tx, mut ready_rx) = tokio::sync::oneshot::channel();
2341 let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1);
2342 let worker = std::thread::spawn(move || {
2343 let result = Box::new(UnifiedServiceConfig::new(service, Some(config)))
2344 .run(shutdown_service, Some(ready_tx));
2345 let _ = done_tx.send(result);
2346 });
2347 bound_rx.recv_timeout(Duration::from_secs(20))?;
2348 let mut message = [0; 128];
2349 assert!(
2350 socket.recv(&mut message).is_err(),
2351 "no READY while publication pending"
2352 );
2353 assert!(matches!(
2354 ready_rx.try_recv(),
2355 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2356 ));
2357 assert!(!rpc_path.exists(), "required RPC never binds IPC");
2358 match outcome.as_str() {
2359 "success" => {
2360 publication_tx.send(true)?;
2361 socket.set_read_timeout(Some(Duration::from_secs(5)))?;
2362 let size = socket.recv(&mut message)?;
2363 assert_eq!(&message[..size], b"READY=1");
2364 ready_rx.blocking_recv()?;
2365 shutdown.notify_one();
2366 }
2367 "failure" => publication_tx.send(false)?,
2368 "cancel" => shutdown.notify_one(),
2369 _ => anyhow::bail!("unknown case"),
2370 }
2371 let result = done_rx.recv_timeout(Duration::from_secs(20))?;
2372 worker
2373 .join()
2374 .map_err(|_| anyhow::anyhow!("service thread panicked"))?;
2375 assert_eq!(result.is_ok(), outcome == "success");
2376 assert!(cancellation.is_cancelled());
2377 assert!(!rpc_path.exists());
2378 socket.set_read_timeout(Some(Duration::from_millis(100)))?;
2379 assert!(
2380 socket.recv(&mut message).is_err(),
2381 "failure/cancellation cannot emit READY"
2382 );
2383 Ok(())
2384 }
2385
2386 #[test]
2387 fn required_profile_requires_announcement_while_compatibility_continues() {
2388 let mut required = loop_config(true, true, None);
2389 let error = match announce_iroh_bound(
2390 required.on_iroh_bound.take(),
2391 required.iroh_required,
2392 "echo",
2393 [0x11; 32],
2394 ) {
2395 Ok(()) => panic!("required profile accepted a missing announcement callback"),
2396 Err(error) => error,
2397 };
2398 assert!(error
2399 .to_string()
2400 .contains("no initial Iroh announcement callback"));
2401
2402 let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2403 let mut compatibility = loop_config(
2404 true,
2405 false,
2406 Some(Box::new({
2407 let calls = Arc::clone(&calls);
2408 move |_, _| {
2409 calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2410 anyhow::bail!("announcement unavailable")
2411 }
2412 })),
2413 );
2414 if let Err(error) = announce_iroh_bound(
2415 compatibility.on_iroh_bound.take(),
2416 compatibility.iroh_required,
2417 "echo",
2418 [0x22; 32],
2419 ) {
2420 panic!("compatibility profile did not continue after announcement failure: {error}");
2421 }
2422 assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
2423 }
2424
2425 #[tokio::test]
2426 async fn required_profile_stop_cancels_pending_first_publication() {
2427 let mut config = loop_config(true, true, None);
2428 let cancellation = config.announcement_cancellation.clone();
2429 let callback_cancellation = cancellation.clone();
2430 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
2431 config.on_iroh_bound = Some(Box::new(move |_, _| {
2432 let _ = started_tx.send(());
2433 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
2434 while !callback_cancellation.is_cancelled() && std::time::Instant::now() < deadline {
2435 std::thread::sleep(std::time::Duration::from_millis(5));
2436 }
2437 anyhow::bail!("first publication cancelled")
2438 }));
2439 let shutdown = Notify::new();
2440 let (result, ()) = tokio::join!(
2441 announce_iroh_bound_until_shutdown(&mut config, "echo", [0x11; 32], &shutdown),
2442 async {
2443 assert!(started_rx.await.is_ok(), "first callback entered");
2444 shutdown.notify_one();
2445 },
2446 );
2447 assert!(
2448 result.is_err(),
2449 "cancelled first publication cannot permit readiness"
2450 );
2451 assert!(
2452 cancellation.is_cancelled(),
2453 "shutdown must reach the blocked publication"
2454 );
2455 }
2456}