Skip to main content

hyprstream_service/service/spawner/
service.rs

1//! Unified service spawner for RequestService hosting.
2//!
3//! Provides a single API for spawning RequestService implementations with different
4//! execution modes (Tokio task, dedicated thread, or subprocess).
5
6use std::path::PathBuf;
7use std::sync::Arc;
8use std::thread::{self, JoinHandle};
9
10use tokio::sync::{watch, Notify};
11
12use super::{ProcessConfig, ProcessSpawner, SpawnedProcess};
13use hyprstream_rpc::error::Result;
14use hyprstream_rpc::registry::{ServiceRegistration, SocketKind};
15use hyprstream_rpc::service::RequestService;
16use hyprstream_rpc::transport::TransportConfig;
17
18// Import anyhow! macro for error creation in ServiceManager impl
19use anyhow::anyhow;
20
21/// Build the exact MoQL authorization configuration installed on the Iroh
22/// handler. Keeping this at the production seam makes it impossible for a
23/// shared-config field to be silently dropped between service setup and the
24/// handler; absence of a service-owned ingress grant remains deny-by-default.
25fn production_moq_authz(
26    admission: Arc<hyprstream_rpc::transport::moql_admission::MoqlAdmissionAuthenticator>,
27    ingress_authorizer: Option<hyprstream_rpc::transport::iroh_moq::SharedIngressAuthorizer>,
28) -> hyprstream_rpc::transport::iroh_moq::MoqAuthzConfig {
29    hyprstream_rpc::transport::iroh_moq::MoqAuthzConfig::default()
30        .with_admission(admission)
31        .with_ingress_authorizer_option(ingress_authorizer)
32}
33
34/// Native producer reach and channel must refer to the same service-owned tree.
35fn producer_moq_origin(
36    required: bool,
37    relay: Option<hyprstream_rpc::moq_stream::MoqStreamOrigin>,
38    has_producer: bool,
39) -> Option<hyprstream_rpc::moq_stream::MoqStreamOrigin> {
40    relay.or_else(|| {
41        if required {
42            has_producer.then(|| {
43                hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone()
44                    .with_prefix(hyprstream_rpc::moq_stream::DEFAULT_PREFIX)
45                    .build()
46            })
47        } else {
48            hyprstream_rpc::moq_stream::global_moq_origin().cloned()
49        }
50    })
51}
52
53// Re-export Spawnable trait from hyprstream-rpc (where it's defined so
54// types in that crate can implement it without circular deps).
55pub use hyprstream_rpc::service::Spawnable;
56
57// ============================================================================
58// QuicServiceLoop — Spawnable wrapper for QUIC-only service loop
59// ============================================================================
60
61/// Spawnable wrapper that runs a RequestService with explicit QUIC configuration.
62///
63/// This is used when a service factory wants to pass a `QuicLoopConfig` to
64/// `RequestLoop::with_quic()` without relying on the blanket `Spawnable` impl.
65///
66/// The blanket `impl Spawnable for S: RequestService` creates a plain `RequestLoop`;
67/// this wrapper additionally enables QUIC when `quic_config` is `Some`.
68pub struct UnifiedServiceConfig<S: RequestService + Send + 'static> {
69    service: S,
70    quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>,
71}
72
73impl<S: RequestService + Send + 'static> UnifiedServiceConfig<S> {
74    /// Create a unified service config with optional QUIC.
75    pub fn new(service: S, quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>) -> Self {
76        Self {
77            service,
78            quic_config,
79        }
80    }
81}
82
83/// Reject a required native profile before any compatibility carrier can start.
84fn validate_required_iroh_enabled(config: &hyprstream_rpc::service::QuicLoopConfig) -> Result<()> {
85    if config.iroh_required && !config.iroh_enabled {
86        return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
87            "network-iroh-required rejects iroh = false".to_owned(),
88        ));
89    }
90    Ok(())
91}
92
93/// Complete the native bind seam: required deployments must publish their
94/// first Iroh reach, while the compatibility profile deliberately logs and
95/// continues on publication failure.
96fn announce_iroh_bound(
97    callback: Option<Box<dyn FnOnce(String, [u8; 32]) -> anyhow::Result<()> + Send>>,
98    required: bool,
99    service_name: &str,
100    node_id: [u8; 32],
101) -> Result<()> {
102    match callback {
103        Some(callback) => match callback(service_name.to_owned(), node_id) {
104            Ok(()) => Ok(()),
105            Err(error) if required => Err(hyprstream_rpc::error::RpcError::SpawnFailed(format!(
106                "network-iroh-required initial Iroh announcement failed: {error}"
107            ))),
108            Err(error) => {
109                tracing::warn!(service = %service_name, "Iroh announcement failed; continuing compatibility profile: {error}");
110                Ok(())
111            }
112        },
113        None if required => Err(hyprstream_rpc::error::RpcError::SpawnFailed(
114            "network-iroh-required has no initial Iroh announcement callback".to_owned(),
115        )),
116        None => Ok(()),
117    }
118}
119
120/// The publisher's first-result handshake is synchronous. Keep it off this
121/// service's current-thread runtime so shutdown can cancel an in-flight first
122/// publication instead of waiting for the readiness barrier to finish.
123async fn announce_iroh_bound_until_shutdown(
124    config: &mut hyprstream_rpc::service::QuicLoopConfig,
125    service_name: &str,
126    node_id: [u8; 32],
127    shutdown: &Notify,
128) -> Result<()> {
129    let callback = config.on_iroh_bound.take();
130    let required = config.iroh_required;
131    let cancellation = config.announcement_cancellation.clone();
132    let service_name = service_name.to_owned();
133    let publication = tokio::task::spawn_blocking(move || {
134        announce_iroh_bound(callback, required, &service_name, node_id)
135    });
136    tokio::select! {
137        biased;
138        _ = shutdown.notified() => {
139            cancellation.cancel();
140            Err(hyprstream_rpc::error::RpcError::SpawnFailed(
141                "service stopped before initial Iroh announcement completed".to_owned(),
142            ))
143        }
144        _ = cancellation.cancelled() => Err(hyprstream_rpc::error::RpcError::SpawnFailed(
145            "initial Iroh announcement cancelled before readiness".to_owned(),
146        )),
147        result = publication => result.map_err(|error| {
148            hyprstream_rpc::error::RpcError::SpawnFailed(format!("initial Iroh announcement task: {error}"))
149        })?,
150    }
151}
152
153impl<S: RequestService + Send + Sync + 'static> Spawnable for UnifiedServiceConfig<S> {
154    fn name(&self) -> &str {
155        RequestService::name(&self.service)
156    }
157
158    fn registrations(&self) -> Vec<(SocketKind, TransportConfig)> {
159        vec![(
160            SocketKind::Rep,
161            RequestService::transport(&self.service).clone(),
162        )]
163    }
164
165    fn run(
166        self: Box<Self>,
167        shutdown: Arc<Notify>,
168        on_ready: Option<tokio::sync::oneshot::Sender<()>>,
169    ) -> Result<()> {
170        use hyprstream_rpc::transport::rpc_session::IrohRequestProcessor;
171
172        let UnifiedServiceConfig {
173            service,
174            quic_config,
175        } = *self;
176        let transport = RequestService::transport(&service).clone();
177        let signing_key = RequestService::signing_key(&service);
178        let server_pubkey = signing_key.verifying_key();
179        let service_name = RequestService::name(&service).to_owned();
180        let reach_config_handle = service.producer_reach_config_handle();
181        let moq_origin_handle = service.moq_origin_handle();
182
183        let rt = tokio::runtime::Builder::new_current_thread()
184            .enable_all()
185            .build()
186            .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("runtime: {e}")))?;
187
188        // A LocalSet is retained for any `!Send` per-service internals reachable
189        // through the bridge; the quinn accept loop itself is fully `Send`.
190        let local = tokio::task::LocalSet::new();
191        local.block_on(&rt, async move {
192            let _ = server_pubkey; // identity is verified app-layer via the bridge
193            let nonce_cache = Arc::new(hyprstream_rpc::envelope::InMemoryNonceCache::new());
194            let bridge = hyprstream_rpc::transport::iroh_rpc::LocalServiceBridge::spawn(
195                service, Arc::clone(&nonce_cache), 0,
196            ).map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("bridge: {e}")))?;
197            let bridge = Arc::new(bridge);
198            let processor: Arc<dyn IrohRequestProcessor> = bridge.clone();
199
200            let result = async {
201            if let Some(mut qc) = quic_config {
202                let announcement_cancellation = qc.announcement_cancellation.clone();
203                let _announcement_guard = announcement_cancellation.clone().drop_guard();
204                validate_required_iroh_enabled(&qc)?;
205                // The native client proof is projected from the same
206                // checkpoint-verified local accepted state that StreamInfo
207                // advertises. Reuse its private halves for the server's mutual
208                // confirmation; never manufacture a separate MoQ identity.
209                let moq_server_identity = qc
210                    .moq_admission_proof
211                    .as_ref()
212                    .map(hyprstream_rpc::transport::moql_admission::MoqlServerIdentityProof::from_local_admission_proof)
213                    .transpose()
214                    .map_err(|error| {
215                        hyprstream_rpc::error::RpcError::SpawnFailed(format!(
216                            "MoQ server confirmation identity: {error}"
217                        ))
218                    })?;
219                // web-transport-quinn has no per-builder provider hook and
220                // resolves rustls's process default. Install and validate it at
221                // the actual bind seam so task/thread/subprocess startup cannot
222                // silently inherit a non-PQ first-installed provider (#557).
223                hyprstream_rpc::transport::install_pq_crypto_provider().map_err(|error| {
224                    hyprstream_rpc::error::RpcError::SpawnFailed(format!(
225                        "QUIC crypto provider: {error}"
226                    ))
227                })?;
228                // #274: serve the RPC plane over `web_transport_quinn` (replacing
229                // the bespoke h3 WebTransportServer) and multiplex the moq
230                // streaming plane on the same endpoint via `/moq` path-dispatch.
231                let _ = &qc.protected_resource_json; // RFC 9728 metadata served by axum (:6790)
232                let chain: Vec<rustls::pki_types::CertificateDer<'static>> = qc
233                    .cert_chain
234                    .iter()
235                    .map(|d| rustls::pki_types::CertificateDer::from(d.clone()))
236                    .collect();
237                let key = rustls::pki_types::PrivateKeyDer::try_from((*qc.key_der).clone())
238                    .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC key: {e}")))?;
239                let wt_server = web_transport_quinn::ServerBuilder::new()
240                    .with_addr(qc.bind_addr)
241                    .with_certificate(chain, key)
242                    .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC bind: {e}")))?;
243                let actual_addr = wt_server.local_addr()
244                    .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC local_addr: {e}")))?;
245                // The bound addr is unspecified (0.0.0.0 / ::) when binding all
246                // interfaces — that is NOT a dialable destination, so advertising
247                // it makes dial_stream's QUIC connect time out. Advertise loopback
248                // for same-host (the single-node default); cross-host routable-IP
249                // advertisement is the wire-reach follow-up (#131/#282).
250                let advertise_addr = if actual_addr.ip().is_unspecified() {
251                    match actual_addr {
252                        std::net::SocketAddr::V4(_) => {
253                            std::net::SocketAddr::from((std::net::Ipv4Addr::LOCALHOST, actual_addr.port()))
254                        }
255                        std::net::SocketAddr::V6(_) => {
256                            std::net::SocketAddr::from((std::net::Ipv6Addr::LOCALHOST, actual_addr.port()))
257                        }
258                    }
259                } else {
260                    actual_addr
261                };
262                let pin = hyprstream_rpc::transport::quinn_transport::cert_sha256(&qc.cert_chain[0]);
263                let relay_origin = qc.moq_relay.as_ref().map(|_| {
264                    hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone()
265                        .with_prefix(hyprstream_rpc::moq_stream::DEFAULT_PREFIX)
266                        .build()
267                });
268                // Native producers serve only their own stream channel origin.
269                let moq_origin = producer_moq_origin(qc.iroh_required, relay_origin.clone(), moq_origin_handle.is_some());
270                if let Some(handle) = &moq_origin_handle {
271                    *handle.write() = moq_origin.clone();
272                }
273
274                let mut rpc_server = hyprstream_rpc::transport::quinn_transport::QuinnRpcServer::with_capacity(
275                    wt_server,
276                    Arc::clone(&processor),
277                    signing_key.clone(),
278                    hyprstream_rpc::transport::rpc_session::DEFAULT_STREAM_LIMIT,
279                );
280                // Relay-configured services get a distinct origin; other services
281                // retain the process-global origin and existing behavior.
282                if let Some(origin) = &moq_origin {
283                    rpc_server = rpc_server.with_moq_consumer(origin.consumer().clone());
284                    // #276: subscribe-authz + per-tenant announce scoping. The
285                    // default config is permissive (no resolver, no authorizer)
286                    // so the working open same-tenant subscribe model is
287                    // preserved. A deployment opts into per-tenant scoping /
288                    // private-stream gating by building a
289                    // `hyprstream_rpc::transport::iroh_moq::MoqAuthzConfig` with
290                    // a `tenant_resolver` and a `DefaultAuthorizer` whose policy
291                    // gate calls into the Casbin `PolicyManager`
292                    // (`hyprstream::auth::policy_manager::global_policy_manager()`
293                    // -> `check_with_domain(subject, tenant, broadcast, "subscribe")`).
294                    // That gate is constructed in the `hyprstream` crate (which
295                    // owns PolicyManager) and threaded down via the service
296                    // factory; wiring it here would create a `hyprstream-service`
297                    // -> `hyprstream` dependency cycle, so it is left as a seam.
298                    // TODO(#276): thread a `MoqAuthzConfig` through the service
299                    // factory (built in the `hyprstream` crate) and call
300                    // `.with_moq_authz(cfg)` here. NOTE: on this quinn `/moq`
301                    // path the peer is anonymous (no mutual TLS), so the
302                    // resolver/gate sees `PeerIdentity::anonymous()` — full
303                    // per-peer enforcement requires the iroh `moql` path, where
304                    // `with_authz` is already honoured.
305                    let _moq_authz_default =
306                        hyprstream_rpc::transport::iroh_moq::MoqAuthzConfig::default();
307
308                    // #1153: the `/moq` CONNECT authenticator is resolved
309                    // per-CONNECT inside `QuinnRpcServer` (builder override OR
310                    // `global_moq_connect_authz()`), NOT snapshotted here at
311                    // construction — so registration order is not load-bearing.
312                    // The `hyprstream` crate registers an authenticator over its
313                    // authoritative subject→tenant provisioning map
314                    // (PolicyManager) via `set_global_moq_connect_authz` at
315                    // startup; until that wiring lands a served `/moq` plane
316                    // logs a one-time fail-open warning (the #1145 shape must be
317                    // detectable, not silent). See follow-up #1198 (client
318                    // credential threading) — flipping the boundary on before
319                    // clients can present a credential would outage every
320                    // legitimate Quinn subscriber/relay publisher.
321                }
322
323                // Register this endpoint for RPC resolution and, once, as the
324                // node's network reach for StreamInfo producers (#274) — the
325                // SAME (addr, server_name, cert-hash) the DID-doc #quic entry
326                // advertises, built from one source so no per-site drift.
327                if let Some(reg) = hyprstream_rpc::registry::try_global() {
328                    reg.register(
329                        &service_name,
330                        hyprstream_rpc::registry::SocketKind::Quic,
331                        hyprstream_rpc::transport::TransportConfig::quic_pinned(
332                            advertise_addr,
333                            &qc.server_name,
334                            pin,
335                        ),
336                        None,
337                    );
338                }
339                if let Some(handle) = &reach_config_handle {
340                    let moql_server_identity = qc
341                        .moq_admission_proof
342                        .as_ref()
343                        .map(|proof| proof.expected_server.clone());
344                    *handle.write() = hyprstream_rpc::moq_stream::ProducerReachConfig {
345                        moql_server_identity,
346                        relay_moql_server_identity: qc.moq_relay_server_identity.clone(),
347                        iroh_node_id: None,
348                        quic_reach: Some(hyprstream_rpc::moq_stream::NodeStreamReach {
349                        addr: advertise_addr,
350                        server_name: qc.server_name.clone(),
351                        cert_hashes: vec![pin],
352                        }),
353                        relay: qc.moq_relay.clone(),
354                    };
355                }
356                if let Some(cb) = qc.on_quic_bound.take() {
357                    cb(service_name.clone(), advertise_addr, qc.server_name.clone());
358                }
359
360                // Install an accepted-state-bound proof before any native
361                // subscriber or relay dial. It is absent for browser/local
362                // profiles, where no Iroh admission is attempted.
363                let relay_admission_proof = qc.moq_admission_proof.clone();
364                if let Some(proof) = relay_admission_proof.clone() {
365                    let _ = hyprstream_rpc::moq_stream::init_global_moq_admission_proof(proof);
366                }
367
368                // Link a relay only to this service's scoped origin. The shared
369                // process origin would leak other services' broadcasts into it.
370                if let Some(relay) = qc.moq_relay.take() {
371                    if qc.iroh_required && !matches!(relay, hyprstream_rpc::stream_info::TransportConfig::Iroh(_)) {
372                        return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
373                            "network-iroh-required rejects non-Iroh stream relay".into(),
374                        ));
375                    }
376                    if let Some(origin) = relay_origin {
377                        hyprstream_rpc::moq_stream::serve_origin_to_relay_background(
378                            origin.producer().clone(),
379                            relay,
380                            relay_admission_proof,
381                            qc.moq_relay_server_identity.take(),
382                        );
383                        tracing::info!(
384                            service = %service_name,
385                            "moq relay rendezvous enabled (announcing origin UP to relay)"
386                        );
387                    }
388                }
389
390                // #410/#282: bind an iroh substrate as the PRIMARY production
391                // endpoint (on by default; `[quic] iroh = false` opts out), serving
392                // BOTH ALPNs (`hyprstream-rpc/1` + `moql`) with the SAME request
393                // processor + moq origin, in parallel to the quinn endpoint (kept
394                // for back-compat). The iroh endpoint uses a domain-separated
395                // transport key, so its NodeId is not even byte-equal to the
396                // service signer. It remains only a carrier address, never a DID
397                // or admission input (#1031).
398                // (`presets::N0`), so this node is dial-by-node_id-discoverable.
399                //
400                // Kept alive in this scope via `_iroh_substrate`; on shutdown the
401                // spawned task calls `IrohSubstrate::shutdown` to drain handlers.
402                let _iroh_substrate_guard = if qc.iroh_enabled {
403                    let iroh_transport_key = hyprstream_rpc::node_identity::derive_purpose_key(
404                        &signing_key,
405                        "hyprstream-iroh-transport-v1",
406                    );
407                    let iroh_secret = iroh_transport_key.to_bytes();
408                    let node_id: [u8; 32] = iroh_transport_key.verifying_key().to_bytes();
409                    // Use the same service-specific origin as the quinn `/moq`
410                    // path, so relay-scoped broadcasts remain isolated on iroh.
411                    let moq_handler = match moq_origin.as_ref() {
412                        Some(origin) => {
413                            let shared = hyprstream_rpc::transport::iroh_moq::OriginShared::from_pair(
414                                origin.producer().clone(),
415                                origin.consumer().clone(),
416                            );
417                            hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler::with_origin(shared)
418                        }
419                        None => hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler::new(),
420                    };
421                    // #1027: when the daemon threaded an admission authenticator
422                    // through `QuicLoopConfig`, install it so every accepted
423                    // `moql` connection must prove an accepted current
424                    // Ed25519 + ML-DSA-65 identity inside the carrier before the
425                    // moq handshake. Without it the accept path stays in its
426                    // fail-closed anonymous posture.
427                    let moq_ingress_authorizer = qc.moq_ingress_authorizer.take();
428                    let moq_handler = match qc.moq_admission.take() {
429                        Some(admission) => {
430                            let server_identity = moq_server_identity.ok_or_else(|| {
431                                hyprstream_rpc::error::RpcError::SpawnFailed(
432                                    "MoQ admission enabled without checkpointed server confirmation identity"
433                                        .to_owned(),
434                                )
435                            })?;
436                            admission
437                                .install_server_identity(server_identity, node_id)
438                                .map_err(|error| {
439                                    hyprstream_rpc::error::RpcError::SpawnFailed(format!(
440                                        "MoQ server confirmation identity: {error}"
441                                    ))
442                            })?;
443                            moq_handler.with_authz(
444                                production_moq_authz(admission, moq_ingress_authorizer),
445                            )
446                        }
447                        None => moq_handler,
448                    };
449                    // RPC plane: same processor + signing key as the quinn path.
450                    let rpc_handler =
451                        hyprstream_rpc::transport::iroh_rpc::IrohRpcProtocolHandler::with_stream_limit(
452                            Arc::clone(&processor),
453                            signing_key.clone(),
454                            hyprstream_rpc::transport::rpc_session::DEFAULT_STREAM_LIMIT,
455                        );
456                    match hyprstream_rpc::transport::iroh_substrate::IrohSubstrate::new(
457                        iroh_secret, moq_handler, rpc_handler,
458                    )
459                    .await
460                    {
461                        Ok(substrate) => {
462                            // Install the shared client endpoint (install-once) so
463                            // outbound iroh RPC/stream dials reuse this carrier.
464                            let _ = hyprstream_rpc::transport::lazy_iroh::install_iroh_client_endpoint(
465                                substrate.owned_client_endpoint(),
466                            );
467                            if let Some(handle) = &reach_config_handle {
468                                handle.write().iroh_node_id = Some(node_id);
469                            }
470                            // Advertise iroh reachability only now that the carrier
471                            // is bound; EndpointId is never application authority.
472                            announce_iroh_bound_until_shutdown(&mut qc, &service_name, node_id, &shutdown).await?;
473                            tracing::info!(
474                                service = %service_name,
475                                node_id = %hex_short(&node_id),
476                                "iroh substrate bound (ALPNs hyprstream-rpc/1 + moql)"
477                            );
478                            Some(substrate)
479                        }
480                        Err(e) => {
481                            if qc.iroh_required {
482                                return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
483                                    format!("network-iroh-required Iroh bind failed: {e}"),
484                                ));
485                            }
486                            // Fail soft: an iroh bind failure must not take down the
487                            // working quinn plane. Log and continue quinn-only.
488                            tracing::warn!(
489                                service = %service_name,
490                                "iroh substrate bind failed; continuing quinn-only: {e}"
491                            );
492                            None
493                        }
494                    }
495                } else {
496                    validate_required_iroh_enabled(&qc)?;
497                    None
498                };
499                // One owner consumes the service shutdown notification. Multiple
500                // independent Notify waiters would race for notify_one's permit.
501                // Cancel publication before draining either carrier, including
502                // when a serving loop exits without an explicit stop request.
503                let drain_limit = rpc_server.stream_limit();
504                let drain_capacity = rpc_server.capacity();
505                let drain_token = rpc_server.shutdown_token();
506                let local_shutdown = Arc::new(tokio::sync::Notify::new());
507                let rep_fut = async {
508                    if qc.iroh_required {
509                        // Iroh bind and first publication have completed. Local
510                        // sockets do not gate readiness or serve required RPCs.
511                        // This future may first be polled while draining after
512                        // shutdown won the outer select. Never notify in that case.
513                        if !announcement_cancellation.is_cancelled() {
514                            let _ = hyprstream_rpc::notify::ready();
515                            if let Some(ready) = on_ready { let _ = ready.send(()); }
516                        }
517                        local_shutdown.notified().await;
518                        Ok(())
519                    } else {
520                        hyprstream_rpc::service::serve::serve_bridged(
521                            &transport, Arc::clone(&processor), signing_key.clone(),
522                            Arc::clone(&local_shutdown), on_ready,
523                        ).await
524                    }
525                };
526                let quic_fut = rpc_server.run();
527                tokio::pin!(rep_fut, quic_fut);
528                let completed_rep = tokio::select! {
529                    biased;
530                    _ = shutdown.notified() => None,
531                    _ = announcement_cancellation.cancelled() => None,
532                    result = &mut rep_fut => Some(result),
533                    result = &mut quic_fut => {
534                        if let Err(error) = result {
535                            tracing::warn!("QUIC server loop ended with error: {error}");
536                        }
537                        None
538                    }
539                    _ = async {
540                        loop {
541                            match _iroh_substrate_guard.as_ref() {
542                                Some(substrate) if substrate.router().is_shutdown()
543                                    || substrate.endpoint().is_closed() => break,
544                                Some(_) => tokio::time::sleep(std::time::Duration::from_secs(1)).await,
545                                None => std::future::pending::<()>().await,
546                            }
547                        }
548                    } => None,
549                };
550                announcement_cancellation.cancel();
551                // All carriers and the independent bridge consume one grace.
552                // Publishing the deadline starts the bridge drain immediately;
553                // the final shutdown below still joins its owned thread/result.
554                let deadline = bridge.begin_shutdown(
555                    tokio::time::Instant::now() + hyprstream_rpc::transport::rpc_session::DRAIN_TIMEOUT,
556                );
557                local_shutdown.notify_one();
558                drain_token.cancel();
559                let substrate_drain = async {
560                    if let Some(substrate) = _iroh_substrate_guard {
561                        if let Err(error) = substrate.shutdown().await {
562                            tracing::warn!("iroh substrate shutdown error: {error}");
563                        }
564                    }
565                };
566                let quinn_drain = hyprstream_rpc::transport::quinn_transport::QuinnRpcServer::shutdown_until(
567                    &drain_limit, drain_capacity, &drain_token, deadline,
568                );
569                let rep_drain = async {
570                    match completed_rep {
571                        Some(result) => result,
572                        None => rep_fut.await,
573                    }
574                };
575                let (_, (), rep_result) = tokio::join!(substrate_drain, quinn_drain, rep_drain);
576                rep_result.map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(e.to_string()))
577            } else {
578                hyprstream_rpc::service::serve::serve_bridged(
579                    &transport, processor, signing_key, shutdown, on_ready,
580                ).await.map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(e.to_string()))
581            }
582            }.await;
583            let teardown = bridge.shutdown().await.map_err(|error| {
584                tracing::error!(%error, "bridge shutdown failed");
585                hyprstream_rpc::error::RpcError::SpawnFailed(error.to_string())
586            });
587            // Always join, while retaining the earlier serving error if both fail.
588            result.and(teardown)
589        })
590    }
591}
592
593/// Short hex fingerprint of a 32-byte node_id for logs (never the full key).
594fn hex_short(id: &[u8; 32]) -> String {
595    format!("{:02x}{:02x}{:02x}{:02x}…", id[0], id[1], id[2], id[3])
596}
597
598/// Mode for spawning services.
599#[derive(Debug, Clone)]
600pub enum ServiceMode {
601    /// Spawn as a tokio task in the current runtime.
602    Tokio,
603
604    /// Spawn on a dedicated thread with its own tokio runtime.
605    /// Useful for services with !Send types (like tch-rs tensors).
606    Thread,
607
608    /// Spawn as a subprocess.
609    Subprocess {
610        /// Path to the binary.
611        binary: PathBuf,
612    },
613}
614
615/// Unified service spawner.
616///
617/// Spawns `Spawnable` services with consistent lifecycle management,
618/// regardless of execution mode.
619///
620/// # Example
621///
622/// ```ignore
623/// use hyprstream_rpc::service::spawner::{ServiceSpawner, ProxyService};
624/// use hyprstream_rpc::transport::TransportConfig;
625///
626/// // Spawn a REQ/REP service (RequestService implementations are directly Spawnable)
627/// let service = MyRequestService::new(ctx, transport, verifying_key);
628/// let spawner = ServiceSpawner::tokio();
629/// let spawned = spawner.spawn(service).await?;
630///
631/// // Spawn an XSUB/XPUB proxy on dedicated thread
632/// let proxy = ProxyService::new("events", ctx, pub_transport, sub_transport);
633/// let spawner = ServiceSpawner::threaded();
634/// let spawned = spawner.spawn(proxy).await?;
635///
636/// // Stop the service
637/// spawned.stop().await?;
638/// ```
639pub struct ServiceSpawner {
640    mode: ServiceMode,
641    process_spawner: Option<ProcessSpawner>,
642}
643
644impl ServiceSpawner {
645    /// Create a spawner that runs services as tokio tasks.
646    pub fn tokio() -> Self {
647        Self {
648            mode: ServiceMode::Tokio,
649            process_spawner: None,
650        }
651    }
652
653    /// Create a spawner that runs services on dedicated threads.
654    ///
655    /// Each service gets its own thread with a single-threaded tokio runtime.
656    /// Use this for services with !Send types (like tch-rs tensors).
657    pub fn threaded() -> Self {
658        Self {
659            mode: ServiceMode::Thread,
660            process_spawner: None,
661        }
662    }
663
664    /// Create a spawner that runs services as subprocesses.
665    ///
666    /// The binary should be a `hyprstream service <name>` command.
667    pub fn subprocess(binary: PathBuf) -> Self {
668        Self {
669            mode: ServiceMode::Subprocess { binary },
670            process_spawner: Some(ProcessSpawner::new()),
671        }
672    }
673
674    /// Create a subprocess spawner with a custom process spawner.
675    pub fn subprocess_with(binary: PathBuf, process_spawner: ProcessSpawner) -> Self {
676        Self {
677            mode: ServiceMode::Subprocess { binary },
678            process_spawner: Some(process_spawner),
679        }
680    }
681
682    /// Get the spawning mode.
683    pub fn mode(&self) -> &ServiceMode {
684        &self.mode
685    }
686
687    /// Spawn any Spawnable service with registry integration.
688    ///
689    /// This is the unified spawning API that works with both:
690    /// - Any `S: RequestService` - REQ/REP services (directly Spawnable via blanket impl)
691    /// - `ProxyService` - XSUB/XPUB proxies
692    ///
693    /// The service is automatically registered with the EndpointRegistry and
694    /// unregistered when the returned handle is dropped.
695    ///
696    /// # Example
697    ///
698    /// ```ignore
699    /// // Spawn a REQ/REP service (RequestService implementations are directly Spawnable)
700    /// let service = MyRequestService::new(ctx, transport, verifying_key);
701    /// let spawned = spawner.spawn(service).await?;
702    ///
703    /// // Spawn a proxy
704    /// let proxy = ProxyService::new("events", ctx, pub_transport, sub_transport);
705    /// let spawned = spawner.spawn(proxy).await?;
706    /// ```
707    pub async fn spawn<S: Spawnable>(&self, service: S) -> Result<SpawnedService> {
708        // 1. Register with EndpointRegistry (if initialized)
709        let registrations = service.registrations();
710        let registration = if !registrations.is_empty() {
711            ServiceRegistration::multi(service.name(), registrations, None).ok()
712        } else {
713            None
714        };
715
716        // 2. Spawn based on mode
717        match &self.mode {
718            ServiceMode::Tokio => self.spawn_tokio(service, registration).await,
719            ServiceMode::Thread => self.spawn_thread(service, registration).await,
720            ServiceMode::Subprocess { binary } => {
721                self.spawn_subprocess(service, binary.clone(), registration)
722                    .await
723            }
724        }
725    }
726
727    /// Spawn a Spawnable as a tokio task.
728    async fn spawn_tokio<S: Spawnable>(
729        &self,
730        service: S,
731        registration: Option<ServiceRegistration>,
732    ) -> Result<SpawnedService> {
733        let name = service.name().to_owned();
734        let name_for_spawn = name.clone();
735        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::<()>();
736        let shutdown = Arc::new(Notify::new());
737        let shutdown_clone = shutdown.clone();
738
739        // Spawn on the blocking thread pool.
740        // Spawnable::run() creates a new_current_thread runtime and calls block_on(),
741        // which would block a worker thread (or panic) if called directly in tokio::spawn.
742        let blocking_handle = tokio::task::spawn_blocking(move || {
743            if let Err(e) = Box::new(service).run(shutdown_clone, Some(ready_tx)) {
744                tracing::error!("Service {} failed: {}", name_for_spawn, e);
745            }
746        });
747
748        // Wrap in tokio::spawn so ServiceHandle gets a JoinHandle<()> (not JoinHandle<Result<(), JoinError>>)
749        let join_handle = tokio::spawn(async move {
750            if let Err(e) = blocking_handle.await {
751                tracing::error!("Service blocking task panicked: {}", e);
752            }
753        });
754
755        // Wait for socket to bind before returning
756        if ready_rx.await.is_err() {
757            return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
758                "service task exited before ready".to_owned(),
759            ));
760        }
761
762        // Create a handle wrapper
763        let handle = hyprstream_rpc::service::ServiceHandle::from_task(join_handle, shutdown);
764
765        Ok(SpawnedService {
766            id: format!("{name}-tokio"),
767            kind: ServiceKind::TokioTask {
768                handle: Some(handle),
769            },
770            _registration: registration,
771        })
772    }
773
774    /// Spawn a Spawnable on a dedicated thread.
775    async fn spawn_thread<S: Spawnable>(
776        &self,
777        service: S,
778        registration: Option<ServiceRegistration>,
779    ) -> Result<SpawnedService> {
780        let name = service.name().to_owned();
781        let name_for_thread = name.clone();
782        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
783        let shutdown = Arc::new(Notify::new());
784        let shutdown_clone = shutdown.clone();
785
786        // Spawn thread
787        let thread_handle = thread::Builder::new()
788            .name(format!("{}-service", name))
789            .spawn(move || {
790                // Run the service - it will signal ready after socket binds
791                if let Err(e) = Box::new(service).run(shutdown_clone, Some(ready_tx)) {
792                    tracing::error!("Service {} failed: {}", name_for_thread, e);
793                }
794            })
795            .map_err(|e| {
796                hyprstream_rpc::error::RpcError::SpawnFailed(format!("thread spawn: {e}"))
797            })?;
798
799        // Wait for ready signal (sent by service after socket binds)
800        if ready_rx.await.is_err() {
801            return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
802                "service thread exited before ready".to_owned(),
803            ));
804        }
805
806        Ok(SpawnedService {
807            id: format!("{name}-thread"),
808            kind: ServiceKind::Thread {
809                handle: Some(thread_handle),
810                shutdown,
811                completion: None,
812            },
813            _registration: registration,
814        })
815    }
816
817    /// Spawn a Spawnable as a subprocess with PID file tracking.
818    async fn spawn_subprocess<S: Spawnable>(
819        &self,
820        service: S,
821        binary: PathBuf,
822        registration: Option<ServiceRegistration>,
823    ) -> Result<SpawnedService> {
824        let name = service.name().to_owned();
825        let spawner = match self.process_spawner.as_ref() {
826            Some(s) => s,
827            None => {
828                return Err(hyprstream_rpc::error::RpcError::SpawnFailed(
829                    "subprocess mode requires process_spawner".to_owned(),
830                ));
831            }
832        };
833
834        let process_config = ProcessConfig::new(&name, binary).args(["service", &name]);
835
836        let process = spawner.spawn(process_config).await?;
837
838        // Write PID file for lifecycle management
839        let pid_file = hyprstream_rpc::paths::service_pid_file(&name);
840        if let Some(pid) = process.pid() {
841            // Ensure runtime directory exists
842            if let Some(parent) = pid_file.parent() {
843                let _ = std::fs::create_dir_all(parent);
844            }
845            if let Err(e) = std::fs::write(&pid_file, pid.to_string()) {
846                tracing::warn!("Failed to write PID file {:?}: {}", pid_file, e);
847            } else {
848                tracing::debug!("Wrote PID {} to {:?}", pid, pid_file);
849            }
850        }
851
852        Ok(SpawnedService {
853            id: process.id.clone(),
854            kind: ServiceKind::Subprocess { process, pid_file },
855            _registration: registration,
856        })
857    }
858}
859
860impl Default for ServiceSpawner {
861    fn default() -> Self {
862        Self::tokio()
863    }
864}
865
866/// Kind of spawned service (determines cleanup behavior).
867pub enum ServiceKind {
868    /// Running as a tokio task.
869    TokioTask {
870        handle: Option<hyprstream_rpc::service::ServiceHandle>,
871    },
872
873    /// Running on a dedicated thread.
874    Thread {
875        handle: Option<JoinHandle<()>>,
876        shutdown: Arc<Notify>,
877        /// Shared completion for a join started by an earlier `stop()` call.
878        /// A cancelled waiter must not make a later stop report success before
879        /// the owner thread has actually reached a terminal result.
880        completion: Option<Arc<ThreadStopCompletion>>,
881    },
882
883    /// Running as a subprocess with PID file tracking.
884    Subprocess {
885        process: SpawnedProcess,
886        /// PID file path (XDG-compliant) for lifecycle management.
887        pid_file: PathBuf,
888    },
889}
890
891#[derive(Clone)]
892enum ThreadStopTerminal {
893    Stopped,
894    Failed(String),
895}
896
897#[doc(hidden)]
898pub struct ThreadStopCompletion {
899    terminal: watch::Sender<Option<ThreadStopTerminal>>,
900    /// The original worker handle stays here until the dedicated join owner
901    /// has taken and joined it. This avoids an ambiguous `spawn_blocking`
902    /// cancellation path that could otherwise detach the worker.
903    join_handle: parking_lot::Mutex<Option<JoinHandle<()>>>,
904    join_started: std::sync::atomic::AtomicBool,
905    /// The joining task owns registration while teardown is in progress, so a
906    /// cancelled stop waiter (or a dropped outer owner) cannot unregister the
907    /// service before its thread and bridge have stopped.
908    registration: parking_lot::Mutex<Option<ServiceRegistration>>,
909}
910
911impl ThreadStopCompletion {
912    fn new(registration: Option<ServiceRegistration>, join_handle: JoinHandle<()>) -> Self {
913        let (terminal, _receiver) = watch::channel(None);
914        Self {
915            terminal,
916            join_handle: parking_lot::Mutex::new(Some(join_handle)),
917            join_started: std::sync::atomic::AtomicBool::new(false),
918            registration: parking_lot::Mutex::new(registration),
919        }
920    }
921
922    fn publish(&self, terminal: ThreadStopTerminal) {
923        // `send` drops the value when a cancelled first stop waiter was the
924        // final receiver. Retain the terminal result for a later stop caller.
925        self.terminal.send_replace(Some(terminal));
926    }
927
928    fn release_registration(&self) {
929        self.registration.lock().take();
930    }
931
932    fn ensure_join_started(self: &Arc<Self>, service_id: &str) -> Result<()> {
933        use std::sync::atomic::Ordering;
934
935        if self.terminal.borrow().is_some()
936            || self
937                .join_started
938                .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
939                .is_err()
940        {
941            return Ok(());
942        }
943
944        let completion = Arc::clone(self);
945        let joiner_name = format!("{service_id}-join");
946        if let Err(error) = thread::Builder::new().name(joiner_name).spawn(move || {
947            let handle = completion.join_handle.lock().take();
948            let terminal = match handle {
949                Some(handle) => match handle.join() {
950                    Ok(()) => ThreadStopTerminal::Stopped,
951                    Err(_) => ThreadStopTerminal::Failed(
952                        "service thread panicked during shutdown".to_owned(),
953                    ),
954                },
955                None => ThreadStopTerminal::Failed(
956                    "service thread join owner started without a worker handle".to_owned(),
957                ),
958            };
959            // Only this dedicated owner consumes the actual JoinHandle, so
960            // registration cannot be released before a definitive join result.
961            completion.release_registration();
962            completion.publish(terminal);
963        }) {
964            self.join_started.store(false, Ordering::Release);
965            return Err(hyprstream_rpc::error::RpcError::SpawnFailed(format!(
966                "service thread {service_id} join supervisor failed to start: {error}"
967            )));
968        }
969        Ok(())
970    }
971
972    async fn wait(&self) -> Result<()> {
973        let mut receiver = self.terminal.subscribe();
974        loop {
975            if let Some(terminal) = receiver.borrow().clone() {
976                return match terminal {
977                    ThreadStopTerminal::Stopped => Ok(()),
978                    ThreadStopTerminal::Failed(error) => {
979                        Err(hyprstream_rpc::error::RpcError::SpawnFailed(error))
980                    }
981                };
982            }
983            receiver.changed().await.map_err(|_| {
984                hyprstream_rpc::error::RpcError::SpawnFailed(
985                    "service thread stop completion owner exited without a terminal result"
986                        .to_owned(),
987                )
988            })?;
989        }
990    }
991}
992
993/// Handle for a spawned service.
994pub struct SpawnedService {
995    /// Unique identifier.
996    id: String,
997
998    /// Service kind (determines cleanup behavior).
999    kind: ServiceKind,
1000
1001    /// Registry registration (RAII cleanup on drop).
1002    /// Stored as Option to allow Drop to consume it.
1003    _registration: Option<ServiceRegistration>,
1004}
1005
1006impl SpawnedService {
1007    /// Create a dummy handle for services that manage their own lifecycle
1008    /// (e.g., systemd-managed services)
1009    pub fn dummy() -> Self {
1010        Self {
1011            id: "dummy".to_owned(),
1012            kind: ServiceKind::TokioTask { handle: None },
1013            _registration: None,
1014        }
1015    }
1016
1017    /// Create a subprocess handle
1018    pub fn subprocess(id: String, process: SpawnedProcess, pid_file: PathBuf) -> Self {
1019        Self {
1020            id,
1021            kind: ServiceKind::Subprocess { process, pid_file },
1022            _registration: None,
1023        }
1024    }
1025
1026    /// Create a thread handle
1027    pub fn thread(
1028        id: String,
1029        handle: Option<JoinHandle<()>>,
1030        shutdown: Arc<Notify>,
1031        registration: Option<ServiceRegistration>,
1032    ) -> Self {
1033        Self {
1034            id,
1035            kind: ServiceKind::Thread {
1036                handle,
1037                shutdown,
1038                completion: None,
1039            },
1040            _registration: registration,
1041        }
1042    }
1043
1044    /// Get the service ID.
1045    pub fn id(&self) -> &str {
1046        &self.id
1047    }
1048
1049    /// Check if the service is running.
1050    pub fn is_running(&self) -> bool {
1051        match &self.kind {
1052            ServiceKind::TokioTask { handle } => handle
1053                .as_ref()
1054                .map(hyprstream_rpc::service::ServiceHandle::is_running)
1055                .unwrap_or(false),
1056            ServiceKind::Thread {
1057                handle, completion, ..
1058            } => completion
1059                .as_ref()
1060                .map(|completion| completion.terminal.borrow().is_none())
1061                .unwrap_or_else(|| handle.as_ref().map(|h| !h.is_finished()).unwrap_or(false)),
1062            ServiceKind::Subprocess { pid_file, .. } => {
1063                // Check if PID file exists and process is alive (signal 0)
1064                if let Ok(pid_str) = std::fs::read_to_string(pid_file) {
1065                    if let Ok(pid) = pid_str.trim().parse::<i32>() {
1066                        // Signal 0 checks if process exists without sending a signal
1067                        return nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), None)
1068                            .is_ok();
1069                    }
1070                }
1071                false
1072            }
1073        }
1074    }
1075
1076    /// Broadcast threaded shutdown to waiters that are already armed and retain
1077    /// one notification for the lifecycle waiter that is created immediately
1078    /// after a service reports readiness.
1079    ///
1080    /// `Notify::notify_waiters()` intentionally has no stored permit. Threaded
1081    /// services that create their runtime after their ready handshake therefore
1082    /// need the retained `notify_one()` permit to observe a stop requested in
1083    /// that narrow interval. Services with multiple independent waiters must
1084    /// still arm those waiters before reporting ready.
1085    fn request_thread_shutdown(shutdown: &Notify) {
1086        shutdown.notify_waiters();
1087        shutdown.notify_one();
1088    }
1089
1090    /// Stop the service.
1091    ///
1092    /// Idempotent: subsequent calls share the first terminal teardown result.
1093    pub async fn stop(&mut self) -> Result<()> {
1094        let (kind, registration) = (&mut self.kind, &mut self._registration);
1095        match kind {
1096            ServiceKind::TokioTask { handle } => {
1097                if let Some(mut h) = handle.take() {
1098                    h.stop().await;
1099                }
1100            }
1101            ServiceKind::Thread {
1102                handle,
1103                shutdown,
1104                completion,
1105            } => {
1106                // A threaded service may have independent lifecycle and serving
1107                // waiters. Wake armed waiters and retain shutdown for the
1108                // lifecycle waiter that may be installed just after readiness.
1109                Self::request_thread_shutdown(shutdown);
1110
1111                if let Some(existing) = completion.as_ref() {
1112                    // An earlier caller may have been cancelled while the join
1113                    // was pending. Its teardown owner remains live, and every
1114                    // later caller must observe that same terminal result.
1115                    existing.ensure_join_started(&self.id)?;
1116                    existing.wait().await?;
1117                } else if let Some(h) = handle.take() {
1118                    // The cancellable caller does not own the join. A dedicated
1119                    // native join owner retains both the actual JoinHandle and
1120                    // registry registration until the worker is terminal.
1121                    let service_id = self.id.clone();
1122                    let owned_completion =
1123                        Arc::new(ThreadStopCompletion::new(registration.take(), h));
1124                    *completion = Some(Arc::clone(&owned_completion));
1125                    owned_completion.ensure_join_started(&service_id)?;
1126                    owned_completion.wait().await?;
1127                }
1128            }
1129            ServiceKind::Subprocess { process, pid_file } => {
1130                // Read PID from file and send SIGTERM
1131                if let Ok(pid_str) = std::fs::read_to_string(&pid_file) {
1132                    if let Ok(pid) = pid_str.trim().parse::<i32>() {
1133                        tracing::info!(
1134                            "Sending SIGTERM to subprocess {} (PID {})",
1135                            process.id,
1136                            pid
1137                        );
1138                        if let Err(e) = nix::sys::signal::kill(
1139                            nix::unistd::Pid::from_raw(pid),
1140                            nix::sys::signal::Signal::SIGTERM,
1141                        ) {
1142                            tracing::warn!("Failed to send SIGTERM to PID {}: {}", pid, e);
1143                        }
1144                    }
1145                }
1146                // Clean up PID file
1147                if let Err(e) = std::fs::remove_file(&pid_file) {
1148                    tracing::debug!("Failed to remove PID file {:?}: {}", pid_file, e);
1149                }
1150            }
1151        }
1152
1153        tracing::info!("Service {} stopped", self.id);
1154        Ok(())
1155    }
1156}
1157
1158impl Drop for SpawnedService {
1159    fn drop(&mut self) {
1160        if let ServiceKind::Thread {
1161            completion: Some(completion),
1162            ..
1163        } = &self.kind
1164        {
1165            if let Err(error) = completion.ensure_join_started(&self.id) {
1166                tracing::error!(
1167                    service = %self.id,
1168                    error = %error,
1169                    "retaining thread teardown state after join-supervisor start failure"
1170                );
1171                // Releasing registration or the JoinHandle would make an
1172                // uncertain worker invisible. Leak only this exceptional
1173                // completion owner until process-level recovery instead.
1174                let _ = Arc::into_raw(Arc::clone(completion));
1175            }
1176        }
1177    }
1178}
1179
1180// ============================================================================
1181// InprocManager - ServiceManager for in-process spawning
1182// ============================================================================
1183
1184use crate::service::manager::ServiceManager;
1185use async_trait::async_trait;
1186
1187/// In-process service manager
1188///
1189/// Spawns services in the current process using ServiceSpawner.
1190pub struct InprocManager {
1191    #[allow(dead_code)]
1192    spawner: ServiceSpawner,
1193}
1194
1195impl InprocManager {
1196    /// Create a new InprocManager with threaded spawner
1197    pub fn new() -> Self {
1198        Self {
1199            spawner: ServiceSpawner::threaded(),
1200        }
1201    }
1202
1203    /// Create with a custom spawner mode
1204    pub fn with_spawner(spawner: ServiceSpawner) -> Self {
1205        Self { spawner }
1206    }
1207}
1208
1209impl Default for InprocManager {
1210    fn default() -> Self {
1211        Self::new()
1212    }
1213}
1214
1215#[async_trait]
1216impl ServiceManager for InprocManager {
1217    async fn install(&self, _service: &str) -> anyhow::Result<()> {
1218        // No-op for inproc
1219        Ok(())
1220    }
1221
1222    async fn uninstall(&self, _service: &str) -> anyhow::Result<()> {
1223        Ok(())
1224    }
1225
1226    async fn start(&self, _service: &str) -> anyhow::Result<()> {
1227        // Services spawned via spawn(), not start()
1228        Ok(())
1229    }
1230
1231    async fn stop(&self, _service: &str) -> anyhow::Result<()> {
1232        // Services managed via SpawnedService handles
1233        Ok(())
1234    }
1235
1236    async fn is_active(&self, _service: &str) -> anyhow::Result<bool> {
1237        // Always true for inproc (services managed by handles)
1238        Ok(true)
1239    }
1240
1241    async fn reload(&self) -> anyhow::Result<()> {
1242        Ok(())
1243    }
1244
1245    async fn spawn(&self, service: Box<dyn Spawnable>) -> anyhow::Result<SpawnedService> {
1246        // Can't call spawn(*service) because trait objects aren't Sized
1247        // Need to spawn inline instead
1248        let name = service.name().to_owned();
1249        let registrations = service.registrations();
1250
1251        // Register with EndpointRegistry
1252        let _registration = if !registrations.is_empty() {
1253            ServiceRegistration::multi(&name, registrations, None).ok()
1254        } else {
1255            None
1256        };
1257
1258        // Spawn on dedicated thread
1259        // The ready signal is sent by the service after the socket binds
1260        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::<()>();
1261        let (startup_error_tx, startup_error_rx) = tokio::sync::oneshot::channel::<String>();
1262        let shutdown = Arc::new(Notify::new());
1263        let shutdown_clone = shutdown.clone();
1264        let name_clone = name.clone(); // Clone for closure use
1265
1266        let thread_handle = thread::Builder::new()
1267            .name(format!("{}-service", name))
1268            .spawn(move || {
1269                if let Err(e) = service.run(shutdown_clone, Some(ready_tx)) {
1270                    // Deliver the concrete startup error before the manager
1271                    // reports a closed READY channel. The foreground binary
1272                    // can otherwise exit before this thread's log is flushed.
1273                    let _ = startup_error_tx.send(e.to_string());
1274                    tracing::error!("Service {} failed: {}", name_clone, e);
1275                }
1276            })
1277            .map_err(|e| anyhow!("thread spawn: {}", e))?;
1278
1279        // Wait for service to be ready (socket bound)
1280        // Note: systemd notification is handled inside run() after socket binds
1281        if ready_rx.await.is_err() {
1282            // A service may drop READY before its teardown completes. Bound
1283            // the error wait without requiring the caller's Tokio runtime to
1284            // have a time driver (ServiceManager::spawn does not require one).
1285            let (deadline_tx, deadline_rx) = tokio::sync::oneshot::channel::<()>();
1286            let _deadline_thread = thread::Builder::new()
1287                .name("service-startup-deadline".to_owned())
1288                .spawn(move || {
1289                    thread::sleep(std::time::Duration::from_secs(10));
1290                    let _ = deadline_tx.send(());
1291                })
1292                .map_err(|e| anyhow!("startup deadline thread spawn: {e}"))?;
1293            let cause = tokio::select! {
1294                result = startup_error_rx => result.ok(),
1295                _ = deadline_rx => None,
1296            };
1297            return Err(match cause {
1298                Some(cause) => anyhow!("service thread exited before ready: {cause}"),
1299                None => anyhow!("service thread exited before ready"),
1300            });
1301        }
1302
1303        Ok(SpawnedService::thread(
1304            format!("{name}-thread"),
1305            Some(thread_handle),
1306            shutdown,
1307            _registration,
1308        ))
1309    }
1310}
1311
1312// ============================================================================
1313// DualSpawnable - Run two Spawnables concurrently
1314// ============================================================================
1315
1316/// Wrapper that runs two Spawnables: primary on calling thread, secondary on a sub-thread.
1317///
1318/// Used when a service needs to listen on two transports simultaneously
1319/// (e.g., ZMQ and QUIC). Since `Spawnable::run()` is blocking, we can't run
1320/// both on the same thread.
1321///
1322/// # Example
1323///
1324/// ```ignore
1325/// let zmq_loop = create_zmq_loop(&ctx)?;
1326/// let quic_loop = QuicServiceLoop::new(quic_rep, service);
1327///
1328/// let dual = DualSpawnable::new(zmq_loop, quic_loop);
1329/// Ok(Box::new(dual))
1330/// ```
1331pub struct DualSpawnable {
1332    primary: Box<dyn Spawnable>,
1333    secondary: Box<dyn Spawnable>,
1334}
1335
1336impl DualSpawnable {
1337    /// Create a new DualSpawnable.
1338    ///
1339    /// The primary spawnable runs on the calling thread.
1340    /// The secondary spawnable runs on a dedicated sub-thread.
1341    pub fn new(primary: Box<dyn Spawnable>, secondary: Box<dyn Spawnable>) -> Self {
1342        Self { primary, secondary }
1343    }
1344}
1345
1346impl Spawnable for DualSpawnable {
1347    fn name(&self) -> &str {
1348        // Use primary's name as the service name
1349        self.primary.name()
1350    }
1351
1352    fn registrations(&self) -> Vec<(SocketKind, TransportConfig)> {
1353        // Merge registrations from both
1354        let mut regs = self.primary.registrations();
1355        regs.extend(self.secondary.registrations());
1356        regs
1357    }
1358
1359    fn run(
1360        self: Box<Self>,
1361        shutdown: Arc<Notify>,
1362        on_ready: Option<tokio::sync::oneshot::Sender<()>>,
1363    ) -> Result<()> {
1364        let shutdown2 = shutdown.clone();
1365        let secondary = self.secondary;
1366        let secondary_name = secondary.name().to_owned();
1367
1368        // Spawn secondary on a sub-thread
1369        let handle = thread::Builder::new()
1370            .name(format!("{}-quic", secondary_name))
1371            .spawn(move || {
1372                if let Err(e) = secondary.run(shutdown2, None) {
1373                    tracing::error!("Secondary service {} failed: {}", secondary_name, e);
1374                }
1375            })
1376            .map_err(|e| hyprstream_rpc::error::RpcError::Other(format!("thread spawn: {}", e)))?;
1377
1378        // Run primary on current thread (blocks until shutdown)
1379        let result = self.primary.run(shutdown, on_ready);
1380
1381        // Wait for secondary thread to finish
1382        let _ = handle.join();
1383
1384        result
1385    }
1386}
1387
1388#[cfg(test)]
1389mod tests {
1390    use super::*;
1391    use anyhow::Result as AnyhowResult;
1392    use hyprstream_rpc::crypto::generate_signing_keypair;
1393    use hyprstream_rpc::prelude::SigningKey;
1394    use hyprstream_rpc::service::RequestService;
1395
1396    #[tokio::test]
1397    async fn native_producer_origins_do_not_expose_sibling_tracks() -> AnyhowResult<()> {
1398        let first = producer_moq_origin(true, None, true)
1399            .ok_or_else(|| anyhow!("native producer origin missing"))?;
1400        let second = producer_moq_origin(true, None, true)
1401            .ok_or_else(|| anyhow!("native producer origin missing"))?;
1402        let _broadcast = first
1403            .producer()
1404            .create_broadcast("local/streams/first")
1405            .ok_or_else(|| anyhow!("first broadcast missing"))?;
1406        assert!(first
1407            .consumer()
1408            .announced_broadcast("local/streams/first")
1409            .await
1410            .is_some());
1411        assert!(tokio::time::timeout(
1412            std::time::Duration::from_millis(100),
1413            second.consumer().announced_broadcast("local/streams/first")
1414        )
1415        .await
1416        .is_err());
1417        assert!(producer_moq_origin(true, None, false).is_none());
1418        let relayed = producer_moq_origin(true, Some(first), true)
1419            .ok_or_else(|| anyhow!("relay origin missing"))?;
1420        assert!(relayed
1421            .consumer()
1422            .announced_broadcast("local/streams/first")
1423            .await
1424            .is_some());
1425        Ok(())
1426    }
1427
1428    #[test]
1429    fn production_moq_handler_preserves_explicit_ingress_and_defaults_to_denial() {
1430        let admission = Arc::new(
1431            hyprstream_rpc::transport::moql_admission::MoqlAdmissionAuthenticator::new(
1432                Arc::new(|_: &str| None),
1433                Arc::new(|_: &hyprstream_rpc::moq_authz::PeerIdentity| None),
1434            ),
1435        );
1436        let producer = hyprstream_rpc::moq_authz::PeerIdentity::authenticated("did:at9p:producer");
1437        let authorizer = Arc::new(
1438            |peer: &hyprstream_rpc::moq_authz::PeerIdentity, tenant: &str| {
1439                peer.subject.as_deref() == Some("did:at9p:producer") && tenant == "local"
1440            },
1441        );
1442
1443        let granted =
1444            hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler::new().with_authz(
1445                production_moq_authz(Arc::clone(&admission), Some(authorizer)),
1446            );
1447        assert!(granted.authorizes_ingress(&producer, "local"));
1448        assert!(!granted.authorizes_ingress(&producer, "other"));
1449
1450        let absent = hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler::new()
1451            .with_authz(production_moq_authz(admission, None));
1452        assert!(
1453            !absent.authorizes_ingress(&producer, "local"),
1454            "an admission-only production handler must remain read-only"
1455        );
1456    }
1457
1458    /// Test service that includes infrastructure (new pattern)
1459    struct EchoService {
1460        transport: TransportConfig,
1461        signing_key: SigningKey,
1462    }
1463
1464    impl EchoService {
1465        fn new(transport: TransportConfig, signing_key: SigningKey) -> Self {
1466            Self {
1467                transport,
1468                signing_key,
1469            }
1470        }
1471    }
1472
1473    /// The envelope verifier is intentionally process-global and first-write
1474    /// wins. Every inproc spawner test in this binary therefore uses this same
1475    /// deterministic identity and equivalent trust store, whether it performs
1476    /// the initial install or observes an already-installed test configuration.
1477    fn spawner_test_hybrid_identity(
1478    ) -> (SigningKey, Arc<hyprstream_rpc::envelope::KeyedPqTrustStore>) {
1479        use hyprstream_rpc::envelope::{EnvelopeVerifyConfig, KeyedPqTrustStore};
1480
1481        let key = SigningKey::from_bytes(&[0xD3; 32]);
1482        let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&key);
1483        let mut store = KeyedPqTrustStore::new();
1484        store.bind(
1485            key.verifying_key().to_bytes(),
1486            &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk(&pq),
1487        );
1488        let store = Arc::new(store);
1489        let _ = hyprstream_rpc::envelope::install_verify_config(EnvelopeVerifyConfig {
1490            policy: hyprstream_rpc::crypto::CryptoPolicy::Hybrid,
1491            pq_store: Some(store.clone()),
1492        });
1493        (key, store)
1494    }
1495
1496    #[async_trait::async_trait(?Send)]
1497    impl RequestService for EchoService {
1498        fn decode_request_body(
1499            &self,
1500            signed_body: &[u8],
1501        ) -> AnyhowResult<hyprstream_rpc::service::DecodedRequestBody> {
1502            Ok(hyprstream_rpc::service::DecodedRequestBody::opaque(
1503                signed_body.to_vec(),
1504            ))
1505        }
1506
1507        async fn handle_request(
1508            &self,
1509            _ctx: &hyprstream_rpc::service::EnvelopeContext,
1510            body: &hyprstream_rpc::service::DecodedRequestBody,
1511        ) -> AnyhowResult<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
1512            Ok((body.bytes().to_vec(), None))
1513        }
1514
1515        fn name(&self) -> &str {
1516            "echo"
1517        }
1518
1519        fn transport(&self) -> &TransportConfig {
1520            &self.transport
1521        }
1522
1523        fn signing_key(&self) -> SigningKey {
1524            self.signing_key.clone()
1525        }
1526    }
1527
1528    struct GatedService {
1529        echo: EchoService,
1530        entered: Arc<Notify>,
1531        release: Arc<Notify>,
1532        dropped: Arc<std::sync::atomic::AtomicBool>,
1533        panic_on_drop: bool,
1534    }
1535
1536    impl Drop for GatedService {
1537        fn drop(&mut self) {
1538            self.dropped
1539                .store(true, std::sync::atomic::Ordering::SeqCst);
1540            assert!(!self.panic_on_drop, "causal service destructor panic");
1541        }
1542    }
1543
1544    #[async_trait::async_trait(?Send)]
1545    impl RequestService for GatedService {
1546        fn decode_request_body(
1547            &self,
1548            signed_body: &[u8],
1549        ) -> AnyhowResult<hyprstream_rpc::service::DecodedRequestBody> {
1550            self.echo.decode_request_body(signed_body)
1551        }
1552        async fn handle_request(
1553            &self,
1554            _ctx: &hyprstream_rpc::service::EnvelopeContext,
1555            body: &hyprstream_rpc::service::DecodedRequestBody,
1556        ) -> AnyhowResult<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
1557            anyhow::bail!("denied request reached handler: {}", body.bytes().len())
1558        }
1559        async fn verify_claims(
1560            &self,
1561            _ctx: &mut hyprstream_rpc::service::EnvelopeContext,
1562        ) -> AnyhowResult<()> {
1563            self.entered.notify_one();
1564            self.release.notified().await;
1565            anyhow::bail!("gated claims denial")
1566        }
1567        fn build_error_payload(&self, _request_id: u64, error: &str) -> Vec<u8> {
1568            error.as_bytes().to_vec()
1569        }
1570        fn name(&self) -> &str {
1571            "echo"
1572        }
1573        fn transport(&self) -> &TransportConfig {
1574            &self.echo.transport
1575        }
1576        fn signing_key(&self) -> SigningKey {
1577            self.echo.signing_key.clone()
1578        }
1579    }
1580
1581    /// Test-only threaded owner with two independently armed shutdown waiters:
1582    /// a lifecycle observer and the inproc serving loop. Readiness is withheld
1583    /// until both waiters are armed so this test exercises broadcast shutdown,
1584    /// not Notify registration timing.
1585    struct TwoWaiterInprocService {
1586        service: GatedService,
1587        lifecycle_observed: Arc<std::sync::atomic::AtomicBool>,
1588    }
1589
1590    impl Spawnable for TwoWaiterInprocService {
1591        fn name(&self) -> &str {
1592            RequestService::name(&self.service)
1593        }
1594
1595        fn registrations(&self) -> Vec<(SocketKind, TransportConfig)> {
1596            vec![(SocketKind::Rep, self.service.transport().clone())]
1597        }
1598
1599        fn run(
1600            self: Box<Self>,
1601            shutdown: Arc<Notify>,
1602            on_ready: Option<tokio::sync::oneshot::Sender<()>>,
1603        ) -> Result<()> {
1604            let TwoWaiterInprocService {
1605                service,
1606                lifecycle_observed,
1607            } = *self;
1608            let transport = service.transport().clone();
1609            let signing_key = service.signing_key();
1610            let runtime = tokio::runtime::Builder::new_multi_thread()
1611                .enable_all()
1612                .build()
1613                .map_err(|error| {
1614                    hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1615                        "two-waiter test runtime: {error}"
1616                    ))
1617                })?;
1618
1619            runtime.block_on(async move {
1620                let nonce_cache = Arc::new(hyprstream_rpc::envelope::InMemoryNonceCache::new());
1621                let bridge = Arc::new(
1622                    hyprstream_rpc::transport::iroh_rpc::LocalServiceBridge::spawn(
1623                        service,
1624                        nonce_cache,
1625                        0,
1626                    )
1627                    .map_err(|error| {
1628                        hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1629                            "two-waiter test bridge: {error}"
1630                        ))
1631                    })?,
1632                );
1633                let processor: Arc<
1634                    dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor,
1635                > = bridge.clone();
1636
1637                let (lifecycle_armed_tx, lifecycle_armed_rx) = tokio::sync::oneshot::channel();
1638                let lifecycle_shutdown = Arc::clone(&shutdown);
1639                let lifecycle = tokio::spawn(async move {
1640                    let notified = lifecycle_shutdown.notified();
1641                    tokio::pin!(notified);
1642                    notified.as_mut().enable();
1643                    let _ = lifecycle_armed_tx.send(());
1644                    notified.await;
1645                    lifecycle_observed.store(true, std::sync::atomic::Ordering::Release);
1646                });
1647                lifecycle_armed_rx.await.map_err(|_| {
1648                    hyprstream_rpc::error::RpcError::SpawnFailed(
1649                        "two-waiter lifecycle exited before arming".to_owned(),
1650                    )
1651                })?;
1652
1653                let (serve_armed_tx, serve_armed_rx) = tokio::sync::oneshot::channel();
1654                let serve_shutdown = Arc::clone(&shutdown);
1655                let serve = tokio::spawn(async move {
1656                    hyprstream_rpc::service::serve::serve_bridged_with_shutdown_armed_silent(
1657                        &transport,
1658                        processor,
1659                        signing_key,
1660                        serve_shutdown,
1661                        None,
1662                        Some(serve_armed_tx),
1663                    )
1664                    .await
1665                });
1666                serve_armed_rx.await.map_err(|_| {
1667                    hyprstream_rpc::error::RpcError::SpawnFailed(
1668                        "two-waiter inproc server exited before arming".to_owned(),
1669                    )
1670                })?;
1671                if let Some(ready) = on_ready {
1672                    let _ = ready.send(());
1673                }
1674
1675                let serve_result = serve
1676                    .await
1677                    .map_err(|error| {
1678                        hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1679                            "two-waiter inproc server task failed: {error}"
1680                        ))
1681                    })?
1682                    .map_err(|error| {
1683                        hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1684                            "two-waiter inproc server failed: {error}"
1685                        ))
1686                    });
1687                let lifecycle_result = lifecycle.await.map_err(|error| {
1688                    hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1689                        "two-waiter lifecycle task failed: {error}"
1690                    ))
1691                });
1692                let bridge_result = bridge.shutdown().await.map_err(|error| {
1693                    hyprstream_rpc::error::RpcError::SpawnFailed(format!(
1694                        "two-waiter bridge shutdown failed: {error}"
1695                    ))
1696                });
1697                serve_result.and(lifecycle_result).and(bridge_result)
1698            })
1699        }
1700    }
1701
1702    #[tokio::test]
1703    async fn threaded_stop_broadcasts_to_lifecycle_and_inproc_with_retained_client(
1704    ) -> AnyhowResult<()> {
1705        use hyprstream_rpc::rpc_client::RpcClientImpl;
1706        use hyprstream_rpc::signer::LocalSigner;
1707        use hyprstream_rpc::transport::in_memory::InMemoryTransport;
1708
1709        let (key, store) = spawner_test_hybrid_identity();
1710
1711        let endpoint = "threaded-two-waiter-retained-client";
1712        let entered = Arc::new(Notify::new());
1713        let release = Arc::new(Notify::new());
1714        let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
1715        let lifecycle_observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
1716        let service = TwoWaiterInprocService {
1717            service: GatedService {
1718                echo: EchoService::new(TransportConfig::inproc(endpoint), key.clone()),
1719                entered: entered.clone(),
1720                release: release.clone(),
1721                dropped: dropped.clone(),
1722                panic_on_drop: false,
1723            },
1724            lifecycle_observed: lifecycle_observed.clone(),
1725        };
1726        let mut spawned = ServiceSpawner::threaded().spawn(service).await?;
1727        let processor = hyprstream_rpc::dial::lookup_inproc(endpoint)
1728            .ok_or_else(|| anyhow!("two-waiter inproc endpoint was not registered"))?;
1729        let client = Arc::new(
1730            RpcClientImpl::new(
1731                LocalSigner::new(key.clone()),
1732                InMemoryTransport::new(processor),
1733                Some(key.verifying_key()),
1734            )
1735            .with_response_pq_store(store),
1736        );
1737        let caller = client.clone();
1738        let response = tokio::spawn(async move { caller.call(b"accepted".to_vec()).await });
1739        tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified()).await?;
1740
1741        let stop = tokio::spawn(async move { spawned.stop().await });
1742        tokio::time::timeout(std::time::Duration::from_secs(5), async {
1743            while hyprstream_rpc::dial::lookup_inproc(endpoint).is_some() {
1744                tokio::task::yield_now().await;
1745            }
1746        })
1747        .await?;
1748        assert!(
1749            lifecycle_observed.load(std::sync::atomic::Ordering::Acquire),
1750            "broadcast shutdown must wake the lifecycle waiter"
1751        );
1752        assert!(
1753            !stop.is_finished(),
1754            "thread owner returned before the accepted request drained"
1755        );
1756        assert!(
1757            tokio::time::timeout(
1758                std::time::Duration::from_secs(5),
1759                client.call(b"late".to_vec())
1760            )
1761            .await?
1762            .is_err(),
1763            "retained client must lose admission after endpoint unregister"
1764        );
1765
1766        release.notify_one();
1767        assert_eq!(
1768            tokio::time::timeout(std::time::Duration::from_secs(5), response).await???,
1769            hyprstream_rpc::service::dispatch::DISPATCH_DENIED.as_bytes()
1770        );
1771        tokio::time::timeout(std::time::Duration::from_secs(5), stop).await???;
1772        assert!(
1773            dropped.load(std::sync::atomic::Ordering::SeqCst),
1774            "thread owner must join the bridge after drain"
1775        );
1776        assert!(client.call(b"late".to_vec()).await.is_err());
1777        Ok(())
1778    }
1779
1780    #[tokio::test]
1781    async fn cancelled_direct_thread_stop_waits_for_the_same_terminal_join() -> AnyhowResult<()> {
1782        use hyprstream_rpc::rpc_client::RpcClientImpl;
1783        use hyprstream_rpc::signer::LocalSigner;
1784        use hyprstream_rpc::transport::in_memory::InMemoryTransport;
1785
1786        let (key, store) = spawner_test_hybrid_identity();
1787
1788        let endpoint = "threaded-stop-cancellation-retained-client";
1789        let entered = Arc::new(Notify::new());
1790        let release = Arc::new(Notify::new());
1791        let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
1792        let lifecycle_observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
1793        let service = TwoWaiterInprocService {
1794            service: GatedService {
1795                echo: EchoService::new(TransportConfig::inproc(endpoint), key.clone()),
1796                entered: entered.clone(),
1797                release: release.clone(),
1798                dropped: dropped.clone(),
1799                panic_on_drop: false,
1800            },
1801            lifecycle_observed: lifecycle_observed.clone(),
1802        };
1803        let mut spawned = ServiceSpawner::threaded().spawn(service).await?;
1804        let processor = hyprstream_rpc::dial::lookup_inproc(endpoint)
1805            .ok_or_else(|| anyhow!("direct-stop endpoint was not registered"))?;
1806        let client = Arc::new(
1807            RpcClientImpl::new(
1808                LocalSigner::new(key.clone()),
1809                InMemoryTransport::new(processor),
1810                Some(key.verifying_key()),
1811            )
1812            .with_response_pq_store(store),
1813        );
1814        let caller = client.clone();
1815        let response = tokio::spawn(async move { caller.call(b"accepted".to_vec()).await });
1816        tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified()).await?;
1817
1818        let mut first_stop = Box::pin(spawned.stop());
1819        let endpoint_unregistered = async {
1820            tokio::time::timeout(std::time::Duration::from_secs(5), async {
1821                while hyprstream_rpc::dial::lookup_inproc(endpoint).is_some() {
1822                    tokio::task::yield_now().await;
1823                }
1824            })
1825            .await
1826            .map_err(|_| anyhow!("first stop did not unregister endpoint before cancellation"))
1827        };
1828        tokio::select! {
1829            result = &mut first_stop => anyhow::bail!(
1830                "first stop returned before the accepted request drained: {result:?}"
1831            ),
1832            result = endpoint_unregistered => result?,
1833        }
1834        drop(first_stop);
1835
1836        assert!(
1837            lifecycle_observed.load(std::sync::atomic::Ordering::Acquire),
1838            "first stop must broadcast to the lifecycle waiter before cancellation"
1839        );
1840        assert!(
1841            tokio::time::timeout(
1842                std::time::Duration::from_secs(5),
1843                client.call(b"late".to_vec())
1844            )
1845            .await?
1846            .is_err(),
1847            "endpoint removal must reject retained-client late admission"
1848        );
1849
1850        let mut second_stop = Box::pin(spawned.stop());
1851        tokio::select! {
1852            result = &mut second_stop => anyhow::bail!(
1853                "second stop returned before the original join completed: {result:?}"
1854            ),
1855            _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {},
1856        }
1857        assert!(
1858            !dropped.load(std::sync::atomic::Ordering::SeqCst),
1859            "bridge/service destruction must wait for the accepted request"
1860        );
1861
1862        release.notify_one();
1863        assert_eq!(
1864            tokio::time::timeout(std::time::Duration::from_secs(5), response).await???,
1865            hyprstream_rpc::service::dispatch::DISPATCH_DENIED.as_bytes()
1866        );
1867        tokio::time::timeout(std::time::Duration::from_secs(5), &mut second_stop).await??;
1868        assert!(
1869            dropped.load(std::sync::atomic::Ordering::SeqCst),
1870            "second stop must await bridge/service destruction from the original join"
1871        );
1872        assert!(client.call(b"late".to_vec()).await.is_err());
1873        Ok(())
1874    }
1875
1876    fn blocking_panicking_thread_for_stop_test(
1877        armed: std::sync::mpsc::Sender<()>,
1878        shutdown_observed: tokio::sync::oneshot::Sender<()>,
1879        release: Arc<Notify>,
1880    ) -> SpawnedService {
1881        let shutdown = Arc::new(Notify::new());
1882        let worker_shutdown = Arc::clone(&shutdown);
1883        let handle = std::thread::spawn(move || {
1884            let runtime = tokio::runtime::Builder::new_current_thread()
1885                .enable_all()
1886                .build()
1887                .unwrap_or_else(|error| panic!("build stop-error test runtime: {error}"));
1888            runtime.block_on(async move {
1889                let notified = worker_shutdown.notified();
1890                tokio::pin!(notified);
1891                notified.as_mut().enable();
1892                armed
1893                    .send(())
1894                    .unwrap_or_else(|error| panic!("signal stop-error worker readiness: {error}"));
1895                notified.await;
1896                let _ = shutdown_observed.send(());
1897                release.notified().await;
1898                panic!("deliberate direct-stop terminal error")
1899            });
1900        });
1901        SpawnedService::thread(
1902            "direct-stop-terminal-error".to_owned(),
1903            Some(handle),
1904            shutdown,
1905            None,
1906        )
1907    }
1908
1909    fn ready_before_waiter_thread_for_stop_test(
1910        ready: std::sync::mpsc::Sender<()>,
1911        arm_waiter: std::sync::mpsc::Receiver<()>,
1912        shutdown_observed: tokio::sync::oneshot::Sender<()>,
1913    ) -> (SpawnedService, Arc<ThreadStopCompletion>) {
1914        let shutdown = Arc::new(Notify::new());
1915        let worker_shutdown = Arc::clone(&shutdown);
1916        let handle = std::thread::spawn(move || {
1917            ready
1918                .send(())
1919                .unwrap_or_else(|error| panic!("signal ready-before-waiter readiness: {error}"));
1920            arm_waiter
1921                .recv()
1922                .unwrap_or_else(|error| panic!("release ready-before-waiter arm: {error}"));
1923            let runtime = tokio::runtime::Builder::new_current_thread()
1924                .enable_all()
1925                .build()
1926                .unwrap_or_else(|error| panic!("build ready-before-waiter runtime: {error}"));
1927            runtime.block_on(async move {
1928                worker_shutdown.notified().await;
1929                let _ = shutdown_observed.send(());
1930            });
1931        });
1932        let completion = Arc::new(ThreadStopCompletion::new(None, handle));
1933        (
1934            SpawnedService {
1935                id: "ready-before-waiter".to_owned(),
1936                kind: ServiceKind::Thread {
1937                    handle: None,
1938                    shutdown,
1939                    completion: Some(Arc::clone(&completion)),
1940                },
1941                _registration: None,
1942            },
1943            completion,
1944        )
1945    }
1946
1947    #[tokio::test]
1948    async fn threaded_stop_retains_shutdown_for_waiter_armed_after_readiness() -> AnyhowResult<()> {
1949        let (ready_tx, ready_rx) = std::sync::mpsc::channel();
1950        let (arm_waiter_tx, arm_waiter_rx) = std::sync::mpsc::channel();
1951        let (shutdown_observed_tx, shutdown_observed_rx) = tokio::sync::oneshot::channel();
1952        let (mut spawned, completion) =
1953            ready_before_waiter_thread_for_stop_test(ready_tx, arm_waiter_rx, shutdown_observed_tx);
1954
1955        tokio::task::spawn_blocking(move || {
1956            ready_rx
1957                .recv_timeout(std::time::Duration::from_secs(5))
1958                .map_err(|error| anyhow!("thread did not report readiness: {error}"))
1959        })
1960        .await??;
1961
1962        let mut stop = Box::pin(spawned.stop());
1963        tokio::select! {
1964            result = &mut stop => anyhow::bail!(
1965                "stop returned before the thread installed its post-readiness waiter: {result:?}"
1966            ),
1967            _ = async {
1968                while !completion.join_started.load(std::sync::atomic::Ordering::Acquire) {
1969                    tokio::task::yield_now().await;
1970                }
1971            } => {},
1972        }
1973        // `join_started` is set only after stop has requested shutdown. Arm the
1974        // service waiter now, reproducing an immediate stop after readiness.
1975        arm_waiter_tx
1976            .send(())
1977            .map_err(|error| anyhow!("release post-readiness waiter: {error}"))?;
1978        tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_observed_rx)
1979            .await
1980            .map_err(|_| anyhow!("late shutdown waiter did not observe retained stop"))?
1981            .map_err(|error| anyhow!("late shutdown observer dropped: {error}"))?;
1982        tokio::time::timeout(std::time::Duration::from_secs(5), &mut stop)
1983            .await
1984            .map_err(|_| anyhow!("stop did not join the late waiter thread"))??;
1985        Ok(())
1986    }
1987
1988    #[tokio::test]
1989    async fn cancelled_direct_thread_stop_replays_the_same_terminal_error() -> AnyhowResult<()> {
1990        let (armed_tx, armed_rx) = std::sync::mpsc::channel();
1991        let (shutdown_observed_tx, shutdown_observed_rx) = tokio::sync::oneshot::channel();
1992        let release = Arc::new(Notify::new());
1993        let mut spawned = blocking_panicking_thread_for_stop_test(
1994            armed_tx,
1995            shutdown_observed_tx,
1996            Arc::clone(&release),
1997        );
1998        tokio::task::spawn_blocking(move || {
1999            armed_rx
2000                .recv_timeout(std::time::Duration::from_secs(5))
2001                .map_err(|error| anyhow!("stop-error worker did not arm: {error}"))
2002        })
2003        .await??;
2004
2005        let mut first_stop = Box::pin(spawned.stop());
2006        tokio::select! {
2007            result = &mut first_stop => anyhow::bail!(
2008                "first stop returned before the controlled thread terminal: {result:?}"
2009            ),
2010            result = shutdown_observed_rx => result.map_err(|error| anyhow!(
2011                "stop-error worker dropped shutdown observation: {error}"
2012            ))?,
2013        }
2014        drop(first_stop);
2015
2016        let mut second_stop = Box::pin(spawned.stop());
2017        tokio::select! {
2018            result = &mut second_stop => anyhow::bail!(
2019                "second stop returned before the original terminal error: {result:?}"
2020            ),
2021            _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {},
2022        }
2023        release.notify_one();
2024        let second_error = match tokio::time::timeout(
2025            std::time::Duration::from_secs(5),
2026            &mut second_stop,
2027        )
2028        .await?
2029        {
2030            Err(error) => error.to_string(),
2031            Ok(()) => anyhow::bail!("joined panicking worker unexpectedly stopped successfully"),
2032        };
2033        drop(second_stop);
2034        let third_error = match spawned.stop().await {
2035            Err(error) => error.to_string(),
2036            Ok(()) => anyhow::bail!("later stop unexpectedly hid the terminal join error"),
2037        };
2038        assert_eq!(
2039            second_error, third_error,
2040            "every later stop must receive the same terminal error"
2041        );
2042        assert!(
2043            second_error.contains("panicked during shutdown"),
2044            "terminal error must remain truthful: {second_error}"
2045        );
2046        Ok(())
2047    }
2048
2049    #[tokio::test]
2050    async fn cancelled_direct_thread_stop_retains_terminal_after_all_waiters_drop(
2051    ) -> AnyhowResult<()> {
2052        let (armed_tx, armed_rx) = std::sync::mpsc::channel();
2053        let (shutdown_observed_tx, shutdown_observed_rx) = tokio::sync::oneshot::channel();
2054        let release = Arc::new(Notify::new());
2055        let mut spawned = blocking_panicking_thread_for_stop_test(
2056            armed_tx,
2057            shutdown_observed_tx,
2058            Arc::clone(&release),
2059        );
2060        tokio::task::spawn_blocking(move || {
2061            armed_rx
2062                .recv_timeout(std::time::Duration::from_secs(5))
2063                .map_err(|error| anyhow!("no-waiter worker did not arm: {error}"))
2064        })
2065        .await??;
2066
2067        let mut first_stop = Box::pin(spawned.stop());
2068        tokio::select! {
2069            result = &mut first_stop => anyhow::bail!(
2070                "first stop returned before the controlled thread terminal: {result:?}"
2071            ),
2072            result = shutdown_observed_rx => result.map_err(|error| anyhow!(
2073                "no-waiter worker dropped shutdown observation: {error}"
2074            ))?,
2075        }
2076        drop(first_stop);
2077        release.notify_one();
2078
2079        tokio::time::timeout(std::time::Duration::from_secs(5), async {
2080            while spawned.is_running() {
2081                tokio::task::yield_now().await;
2082            }
2083        })
2084        .await
2085        .map_err(|_| anyhow!("joined worker never published a terminal result"))?;
2086        let second_error = match spawned.stop().await {
2087            Err(error) => error.to_string(),
2088            Ok(()) => anyhow::bail!("late stop unexpectedly hid the retained terminal error"),
2089        };
2090        assert!(
2091            second_error.contains("panicked during shutdown"),
2092            "late terminal error must remain truthful: {second_error}"
2093        );
2094        Ok(())
2095    }
2096
2097    #[tokio::test]
2098    async fn both_inproc_owners_drain_entered_request_with_retained_client() -> AnyhowResult<()> {
2099        use hyprstream_rpc::rpc_client::RpcClientImpl;
2100        use hyprstream_rpc::signer::LocalSigner;
2101        use hyprstream_rpc::transport::in_memory::InMemoryTransport;
2102        let (key, store) = spawner_test_hybrid_identity();
2103        for unified in [false, true] {
2104            let endpoint = format!("bridge-drain-owner-{unified}");
2105            let transport = TransportConfig::inproc(&endpoint);
2106            let entered = Arc::new(Notify::new());
2107            let release = Arc::new(Notify::new());
2108            let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
2109            let service = GatedService {
2110                echo: EchoService::new(transport, key.clone()),
2111                entered: entered.clone(),
2112                release: release.clone(),
2113                dropped: dropped.clone(),
2114                panic_on_drop: false,
2115            };
2116            let owner: Box<dyn Spawnable> = if unified {
2117                Box::new(UnifiedServiceConfig::new(service, None))
2118            } else {
2119                Box::new(service)
2120            };
2121            let shutdown = Arc::new(Notify::new());
2122            struct Cleanup(Arc<Notify>);
2123            impl Drop for Cleanup {
2124                fn drop(&mut self) {
2125                    self.0.notify_one();
2126                }
2127            }
2128            let _cleanup = Cleanup(shutdown.clone());
2129            let stopped = shutdown.clone();
2130            let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
2131            let task = tokio::task::spawn_blocking(move || owner.run(stopped, Some(ready_tx)));
2132            tokio::time::timeout(std::time::Duration::from_secs(5), ready_rx).await??;
2133            let processor = hyprstream_rpc::dial::lookup_inproc(&endpoint)
2134                .ok_or_else(|| anyhow::anyhow!("missing ready processor"))?;
2135            let client = Arc::new(
2136                RpcClientImpl::new(
2137                    LocalSigner::new(key.clone()),
2138                    InMemoryTransport::new(processor),
2139                    Some(key.verifying_key()),
2140                )
2141                .with_response_pq_store(store.clone()),
2142            );
2143            let caller = client.clone();
2144            let response = tokio::spawn(async move { caller.call(b"accepted".to_vec()).await });
2145            tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified()).await?;
2146            shutdown.notify_one();
2147            tokio::time::timeout(std::time::Duration::from_secs(5), async {
2148                while hyprstream_rpc::dial::lookup_inproc(&endpoint).is_some() {
2149                    tokio::task::yield_now().await;
2150                }
2151            })
2152            .await?;
2153            assert!(
2154                !task.is_finished(),
2155                "owner returned while accepted request is gated"
2156            );
2157            assert!(!dropped.load(std::sync::atomic::Ordering::SeqCst));
2158            // Endpoint removal follows admission closure, so this retained
2159            // client must be rejected while the accepted call is still gated.
2160            assert!(tokio::time::timeout(
2161                std::time::Duration::from_secs(5),
2162                client.call(b"late".to_vec())
2163            )
2164            .await?
2165            .is_err());
2166            release.notify_one();
2167            assert_eq!(
2168                tokio::time::timeout(std::time::Duration::from_secs(5), response).await???,
2169                hyprstream_rpc::service::dispatch::DISPATCH_DENIED.as_bytes()
2170            );
2171            tokio::time::timeout(std::time::Duration::from_secs(5), task).await???;
2172            assert!(
2173                dropped.load(std::sync::atomic::Ordering::SeqCst),
2174                "retained client pinned service"
2175            );
2176            assert!(client.call(b"late".to_vec()).await.is_err());
2177        }
2178        Ok(())
2179    }
2180
2181    #[tokio::test]
2182    async fn both_inproc_owners_propagate_destructor_panic() -> AnyhowResult<()> {
2183        for unified in [false, true] {
2184            let (key, _) = generate_signing_keypair();
2185            let service = GatedService {
2186                echo: EchoService::new(
2187                    TransportConfig::inproc(format!("bridge-panic-owner-{unified}")),
2188                    key,
2189                ),
2190                entered: Arc::new(Notify::new()),
2191                release: Arc::new(Notify::new()),
2192                dropped: Arc::new(std::sync::atomic::AtomicBool::new(false)),
2193                panic_on_drop: true,
2194            };
2195            let owner: Box<dyn Spawnable> = if unified {
2196                Box::new(UnifiedServiceConfig::new(service, None))
2197            } else {
2198                Box::new(service)
2199            };
2200            let shutdown = Arc::new(Notify::new());
2201            struct Cleanup(Arc<Notify>);
2202            impl Drop for Cleanup {
2203                fn drop(&mut self) {
2204                    self.0.notify_one();
2205                }
2206            }
2207            let _cleanup = Cleanup(shutdown.clone());
2208            let stopped = shutdown.clone();
2209            let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
2210            let task = tokio::task::spawn_blocking(move || owner.run(stopped, Some(ready_tx)));
2211            tokio::time::timeout(std::time::Duration::from_secs(5), ready_rx).await??;
2212            shutdown.notify_one();
2213            let result = tokio::time::timeout(std::time::Duration::from_secs(5), task).await??;
2214            assert!(result.is_err(), "owner hid bridge destructor panic");
2215        }
2216        Ok(())
2217    }
2218
2219    fn loop_config(
2220        iroh_enabled: bool,
2221        iroh_required: bool,
2222        on_iroh_bound: Option<Box<dyn FnOnce(String, [u8; 32]) -> anyhow::Result<()> + Send>>,
2223    ) -> hyprstream_rpc::service::QuicLoopConfig {
2224        hyprstream_rpc::service::QuicLoopConfig {
2225            announcement_cancellation: tokio_util::sync::CancellationToken::new(),
2226            cert_chain: Vec::new(),
2227            key_der: zeroize::Zeroizing::new(Vec::new()),
2228            bind_addr: "127.0.0.1:0"
2229                .parse()
2230                .unwrap_or_else(|error| panic!("test socket address: {error}")),
2231            server_name: "service.test".to_owned(),
2232            protected_resource_json: None,
2233            on_quic_bound: None,
2234            iroh_enabled,
2235            iroh_required,
2236            on_iroh_bound,
2237            moq_relay: None,
2238            moq_relay_server_identity: None,
2239            moq_admission: None,
2240            moq_ingress_authorizer: None,
2241            moq_admission_proof: None,
2242        }
2243    }
2244
2245    #[tokio::test]
2246    async fn test_tokio_spawner() -> hyprstream_rpc::Result<()> {
2247        let (signing_key, _verifying_key) = generate_signing_keypair();
2248        let transport = TransportConfig::inproc("test-spawner-tokio");
2249
2250        // Service is directly Spawnable - no wrapping needed
2251        let service = EchoService::new(transport, signing_key);
2252
2253        let spawner = ServiceSpawner::tokio();
2254        let mut spawned = spawner.spawn(service).await?;
2255
2256        assert!(spawned.is_running());
2257
2258        spawned.stop().await?;
2259        Ok(())
2260    }
2261
2262    #[tokio::test]
2263    async fn test_thread_spawner() -> hyprstream_rpc::Result<()> {
2264        let (signing_key, _verifying_key) = generate_signing_keypair();
2265        let transport = TransportConfig::inproc("test-spawner-thread");
2266
2267        // Service is directly Spawnable - no wrapping needed
2268        let service = EchoService::new(transport, signing_key);
2269
2270        let spawner = ServiceSpawner::threaded();
2271        let mut spawned = spawner.spawn(service).await?;
2272
2273        assert!(spawned.is_running());
2274
2275        spawned.stop().await?;
2276        Ok(())
2277    }
2278
2279    #[test]
2280    fn required_profile_rejects_disabled_iroh_at_spawner_seam() {
2281        let config = loop_config(false, true, None);
2282        let error = match validate_required_iroh_enabled(&config) {
2283            Ok(()) => panic!("required profile accepted a disabled Iroh carrier"),
2284            Err(error) => error,
2285        };
2286        assert!(error.to_string().contains("rejects iroh = false"));
2287    }
2288
2289    /// Exercise actual carrier bind and the publication barrier with a real
2290    /// lifecycle socket. Each outcome runs in its own process (env/singletons).
2291    #[cfg(all(unix, feature = "systemd"))]
2292    #[test]
2293    fn required_profile_systemd_ready_follows_bind_and_publication() -> AnyhowResult<()> {
2294        use std::os::unix::net::UnixDatagram;
2295        use std::time::Duration;
2296        const CHILD: &str = "HYPRSTREAM_REQUIRED_NOTIFY_TEST";
2297        let Ok(outcome) = std::env::var(CHILD) else {
2298            for outcome in ["success", "failure", "cancel"] {
2299                let status = std::process::Command::new(std::env::current_exe()?)
2300                    .args(["--exact", "service::spawner::service::tests::required_profile_systemd_ready_follows_bind_and_publication", "--nocapture"])
2301                    .env(CHILD, outcome).status()?;
2302                anyhow::ensure!(status.success(), "notification case {outcome} failed");
2303            }
2304            return Ok(());
2305        };
2306        let dir = tempfile::tempdir()?;
2307        let socket_path = dir.path().join("notify.sock");
2308        let socket = UnixDatagram::bind(&socket_path)?;
2309        socket.set_read_timeout(Some(Duration::from_millis(100)))?;
2310        // Isolated child; set before creating any runtime or service threads.
2311        std::env::set_var("NOTIFY_SOCKET", &socket_path);
2312        let rpc_path = dir.path().join("echo.sock");
2313        let (signing_key, _) = generate_signing_keypair();
2314        let service = EchoService::new(TransportConfig::ipc(&rpc_path), signing_key);
2315        let mut config = loop_config(true, true, None);
2316        let certified = rcgen::generate_simple_self_signed(vec!["service.test".to_owned()])?;
2317        config.cert_chain = vec![certified.cert.der().to_vec()];
2318        config.key_der = zeroize::Zeroizing::new(certified.key_pair.serialize_der());
2319        let cancellation = config.announcement_cancellation.clone();
2320        let callback_cancellation = cancellation.clone();
2321        let (bound_tx, bound_rx) = std::sync::mpsc::sync_channel(1);
2322        let (publication_tx, publication_rx) = std::sync::mpsc::sync_channel(1);
2323        config.on_iroh_bound = Some(Box::new(move |name, node| {
2324            anyhow::ensure!(name == "echo" && node != [0; 32], "actual bound carrier");
2325            bound_tx.send(())?;
2326            loop {
2327                if callback_cancellation.is_cancelled() {
2328                    anyhow::bail!("publication cancelled");
2329                }
2330                match publication_rx.recv_timeout(Duration::from_millis(10)) {
2331                    Ok(true) => return Ok(()),
2332                    Ok(false) => anyhow::bail!("publication rejected"),
2333                    Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue,
2334                    Err(error) => return Err(error.into()),
2335                }
2336            }
2337        }));
2338        let shutdown = Arc::new(Notify::new());
2339        let shutdown_service = Arc::clone(&shutdown);
2340        let (ready_tx, mut ready_rx) = tokio::sync::oneshot::channel();
2341        let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1);
2342        let worker = std::thread::spawn(move || {
2343            let result = Box::new(UnifiedServiceConfig::new(service, Some(config)))
2344                .run(shutdown_service, Some(ready_tx));
2345            let _ = done_tx.send(result);
2346        });
2347        bound_rx.recv_timeout(Duration::from_secs(20))?;
2348        let mut message = [0; 128];
2349        assert!(
2350            socket.recv(&mut message).is_err(),
2351            "no READY while publication pending"
2352        );
2353        assert!(matches!(
2354            ready_rx.try_recv(),
2355            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2356        ));
2357        assert!(!rpc_path.exists(), "required RPC never binds IPC");
2358        match outcome.as_str() {
2359            "success" => {
2360                publication_tx.send(true)?;
2361                socket.set_read_timeout(Some(Duration::from_secs(5)))?;
2362                let size = socket.recv(&mut message)?;
2363                assert_eq!(&message[..size], b"READY=1");
2364                ready_rx.blocking_recv()?;
2365                shutdown.notify_one();
2366            }
2367            "failure" => publication_tx.send(false)?,
2368            "cancel" => shutdown.notify_one(),
2369            _ => anyhow::bail!("unknown case"),
2370        }
2371        let result = done_rx.recv_timeout(Duration::from_secs(20))?;
2372        worker
2373            .join()
2374            .map_err(|_| anyhow::anyhow!("service thread panicked"))?;
2375        assert_eq!(result.is_ok(), outcome == "success");
2376        assert!(cancellation.is_cancelled());
2377        assert!(!rpc_path.exists());
2378        socket.set_read_timeout(Some(Duration::from_millis(100)))?;
2379        assert!(
2380            socket.recv(&mut message).is_err(),
2381            "failure/cancellation cannot emit READY"
2382        );
2383        Ok(())
2384    }
2385
2386    #[test]
2387    fn required_profile_requires_announcement_while_compatibility_continues() {
2388        let mut required = loop_config(true, true, None);
2389        let error = match announce_iroh_bound(
2390            required.on_iroh_bound.take(),
2391            required.iroh_required,
2392            "echo",
2393            [0x11; 32],
2394        ) {
2395            Ok(()) => panic!("required profile accepted a missing announcement callback"),
2396            Err(error) => error,
2397        };
2398        assert!(error
2399            .to_string()
2400            .contains("no initial Iroh announcement callback"));
2401
2402        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2403        let mut compatibility = loop_config(
2404            true,
2405            false,
2406            Some(Box::new({
2407                let calls = Arc::clone(&calls);
2408                move |_, _| {
2409                    calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2410                    anyhow::bail!("announcement unavailable")
2411                }
2412            })),
2413        );
2414        if let Err(error) = announce_iroh_bound(
2415            compatibility.on_iroh_bound.take(),
2416            compatibility.iroh_required,
2417            "echo",
2418            [0x22; 32],
2419        ) {
2420            panic!("compatibility profile did not continue after announcement failure: {error}");
2421        }
2422        assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
2423    }
2424
2425    #[tokio::test]
2426    async fn required_profile_stop_cancels_pending_first_publication() {
2427        let mut config = loop_config(true, true, None);
2428        let cancellation = config.announcement_cancellation.clone();
2429        let callback_cancellation = cancellation.clone();
2430        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
2431        config.on_iroh_bound = Some(Box::new(move |_, _| {
2432            let _ = started_tx.send(());
2433            let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
2434            while !callback_cancellation.is_cancelled() && std::time::Instant::now() < deadline {
2435                std::thread::sleep(std::time::Duration::from_millis(5));
2436            }
2437            anyhow::bail!("first publication cancelled")
2438        }));
2439        let shutdown = Notify::new();
2440        let (result, ()) = tokio::join!(
2441            announce_iroh_bound_until_shutdown(&mut config, "echo", [0x11; 32], &shutdown),
2442            async {
2443                assert!(started_rx.await.is_ok(), "first callback entered");
2444                shutdown.notify_one();
2445            },
2446        );
2447        assert!(
2448            result.is_err(),
2449            "cancelled first publication cannot permit readiness"
2450        );
2451        assert!(
2452            cancellation.is_cancelled(),
2453            "shutdown must reach the blocked publication"
2454        );
2455    }
2456}