Skip to main content

hyprstream_rpc/transport/
iroh_rpc.rs

1//! Iroh RPC plane — server-side protocol handler for ALPN `hyprstream-rpc/1`.
2//!
3//! Part of Epic #131 Phase 2 (#133). This module ships:
4//!
5//! - [`IrohRpcProtocolHandler`] — iroh [`ProtocolHandler`] that plugs into
6//!   [`crate::transport::iroh_substrate`] under the `hyprstream-rpc/1` ALPN.
7//!   Enforces a server-wide cap on concurrent streams (DoS bound) and
8//!   drains in-flight requests on `ProtocolHandler::shutdown`.
9//! - [`IrohRequestProcessor`] — sealed request processing trait; production
10//!   network sessions admit only the canonical envelope/service bridge.
11//! - [`LocalServiceBridge`] — adapts a [`crate::service::RequestService`]
12//!   (potentially `!Send`) to the Send-bounded [`IrohRequestProcessor`].
13//!
14//! **Trust model**: The protocol handler does not parse `SignedEnvelope` —
15//! it forwards opaque bytes to the [`IrohRequestProcessor`]. Envelope
16//! verification, JWT/DPoP, and `authorize_signer` enforcement happen inside
17//! the processor (which is `LocalServiceBridge` in production, delegating
18//! to [`crate::service::dispatch::process_request`]). The handler does
19//! hold a signing key for trusted-local transport-only processor failures.
20//! Admission failures on the untrusted iroh carrier are reset/dropped without
21//! a signed response, preventing an unauthenticated signing oracle.
22//!
23//! **Wire framing**: each request is the opaque bytes of a Cap'n Proto-encoded
24//! [`crate::envelope::SignedEnvelope`] written to a freshly-opened iroh bidi
25//! stream. The response is symmetric. Both endpoints are our code; iroh's
26//! QUIC TLS authenticates the addressed NodeId only at the carrier layer; the
27//! request envelope independently supplies application identity proof.
28
29use std::future::Future;
30use std::sync::Arc;
31
32use anyhow::{Context, Result};
33use bytes::Bytes;
34use ed25519_dalek::SigningKey;
35use iroh::endpoint::Connection;
36use iroh::protocol::{AcceptError, ProtocolHandler};
37use tokio::sync::Semaphore;
38use tokio_util::sync::CancellationToken;
39
40// Transport-generic core lives in `super::rpc_session`. Re-export the shared
41// surface from here so existing `iroh_rpc::{...}` imports keep working.
42pub use super::rpc_session::{
43    build_error_envelope, from_fn, read_to_cap, serve_rpc_connection, IrohRequestProcessor,
44    DEFAULT_STREAM_LIMIT, MAX_FRAME_BYTES,
45};
46
47// ============================================================================
48// IrohRpcProtocolHandler — iroh ProtocolHandler with concurrency cap + drain
49// ============================================================================
50
51/// Iroh protocol handler that terminates `hyprstream-rpc/1` bidi streams,
52/// dispatches each request through the wrapped [`IrohRequestProcessor`],
53/// caps concurrent streams server-wide (shared semaphore), and drains
54/// in-flight requests
55/// on `ProtocolHandler::shutdown` (called by `Router::shutdown`).
56#[derive(Clone)]
57pub struct IrohRpcProtocolHandler {
58    inner: Arc<HandlerInner>,
59}
60
61#[derive(Clone)]
62struct HandlerInner {
63    processor: Arc<dyn IrohRequestProcessor>,
64    /// Used for signed stub errors only below a trusted-local transport test
65    /// boundary. Production iroh admission failures are silently dropped.
66    /// Authenticated application errors come back from the processor already
67    /// wrapped as signed `ResponseEnvelope` bytes.
68    signing_key: SigningKey,
69    /// Caps concurrent bidi streams in flight. Hold one permit per stream.
70    stream_limit: Arc<Semaphore>,
71    stream_limit_capacity: u32,
72    /// Caps concurrent accepted connections (#165). iroh's Router spawns
73    /// each accepted connection into an unbounded JoinSet — connections beyond
74    /// this cap are rejected (dropped) rather than queued so a peer opening
75    /// many idle connections can't exhaust fd/memory.
76    connection_limit: Arc<Semaphore>,
77    /// Per-stream request-read timeout (#159 slowloris bound).
78    read_timeout: std::time::Duration,
79    /// Accept-boundary carrier classification. Production is always iroh;
80    /// unit tests may substitute inproc to exercise raw transport mechanics
81    /// below the network envelope-policy boundary.
82    carrier: crate::transport::carrier::CarrierContext,
83    /// Level-triggered shutdown signal: once `cancel()` is called, every
84    /// future and current `cancelled().await` resolves immediately. Used
85    /// instead of `tokio::sync::Notify` because `Notify::notify_waiters`
86    /// is edge-triggered and a shutdown signal landing between accept-loop
87    /// iterations would be lost.
88    shutdown: CancellationToken,
89}
90
91impl std::fmt::Debug for IrohRpcProtocolHandler {
92    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
93        f.debug_struct("IrohRpcProtocolHandler")
94            .field("stream_limit", &self.inner.stream_limit_capacity)
95            .finish_non_exhaustive()
96    }
97}
98
99impl IrohRpcProtocolHandler {
100    /// Build a handler with the default server-wide stream limit
101    /// ([`DEFAULT_STREAM_LIMIT`]).
102    pub fn new<P: IrohRequestProcessor>(processor: P, signing_key: SigningKey) -> Self {
103        Self::with_stream_limit(Arc::new(processor), signing_key, DEFAULT_STREAM_LIMIT)
104    }
105
106    /// Build a handler with an explicit server-wide stream limit.
107    pub fn with_stream_limit(
108        processor: Arc<dyn IrohRequestProcessor>,
109        signing_key: SigningKey,
110        stream_limit: usize,
111    ) -> Self {
112        let stream_limit_capacity = u32::try_from(stream_limit).unwrap_or(u32::MAX);
113        Self {
114            inner: Arc::new(HandlerInner {
115                processor,
116                signing_key,
117                stream_limit: Arc::new(Semaphore::new(stream_limit)),
118                stream_limit_capacity,
119                connection_limit: Arc::new(Semaphore::new(
120                    super::rpc_session::DEFAULT_CONNECTION_LIMIT,
121                )),
122                read_timeout: super::rpc_session::REQUEST_READ_TIMEOUT,
123                carrier: crate::transport::carrier::CarrierContext::iroh(),
124                shutdown: CancellationToken::new(),
125            }),
126        }
127    }
128
129    /// Override the per-stream request-read timeout (#159). Primarily for
130    /// tests that need a short slowloris bound; production uses the default
131    /// [`super::rpc_session::REQUEST_READ_TIMEOUT`].
132    pub fn with_read_timeout(mut self, read_timeout: std::time::Duration) -> Self {
133        self.mutate_inner(|i| i.read_timeout = read_timeout);
134        self
135    }
136
137    #[cfg(test)]
138    fn with_test_trusted_carrier(mut self) -> Self {
139        self.mutate_inner(|i| i.carrier = crate::transport::carrier::CarrierContext::inproc());
140        self
141    }
142
143    /// Override the concurrent-connection cap (builder style, mirrors
144    /// [`super::quinn_transport::QuinnRpcServer::with_connection_limit`]).
145    pub fn with_connection_limit(mut self, connection_limit: usize) -> Self {
146        self.mutate_inner(|i| i.connection_limit = Arc::new(Semaphore::new(connection_limit)));
147        self
148    }
149
150    /// Mutate the inner config, cloning `Arc<HandlerInner>` only if it is shared.
151    fn mutate_inner(&mut self, f: impl FnOnce(&mut HandlerInner)) {
152        match Arc::get_mut(&mut self.inner) {
153            Some(inner) => f(inner),
154            None => {
155                let mut cloned = (*self.inner).clone();
156                f(&mut cloned);
157                self.inner = Arc::new(cloned);
158            }
159        }
160    }
161
162    /// Apply all tunables from an [`super::rpc_session::RpcConfig`] in one call (#197).
163    pub fn with_rpc_config(self, cfg: &super::rpc_session::RpcConfig) -> Self {
164        let processor = Arc::clone(&self.inner.processor);
165        let signing_key = self.inner.signing_key.clone();
166        Self::with_stream_limit(processor, signing_key, cfg.stream_limit)
167            .with_read_timeout(cfg.request_read_timeout)
168            .with_connection_limit(cfg.connection_limit)
169    }
170}
171
172impl ProtocolHandler for IrohRpcProtocolHandler {
173    async fn accept(&self, conn: Connection) -> Result<(), AcceptError> {
174        // Connection cap (#165): reject (drop) at cap rather than queue.
175        // iroh's Router spawns each accepted connection into an unbounded
176        // JoinSet — this try_acquire enforces the same bound QuinnRpcServer
177        // and UdsRpcServer apply. The permit is held for the connection's
178        // entire lifetime (dropped when this fn returns).
179        let _conn_permit = match Arc::clone(&self.inner.connection_limit).try_acquire_owned() {
180            Ok(p) => p,
181            Err(_) => {
182                tracing::warn!("iroh-rpc: connection cap reached, rejecting connection");
183                return Ok(());
184            }
185        };
186
187        // Adapt the raw iroh `Connection` to the transport-generic `Session`
188        // abstraction and delegate to the shared accept loop. Drain semantics
189        // (`shutdown` below draining the same `Semaphore`) are unchanged.
190        let session = web_transport_iroh::Session::raw(conn);
191        // INV-2 (#1042): this accept boundary terminates an iroh connection —
192        // an untrusted carrier regardless of direct/relay path or NodeId.
193        serve_rpc_connection(
194            session,
195            Arc::clone(&self.inner.processor),
196            self.inner.signing_key.clone(),
197            Arc::clone(&self.inner.stream_limit),
198            self.inner.read_timeout,
199            self.inner.shutdown.clone(),
200            self.inner.carrier,
201        )
202        .await
203        .map_err(|e| AcceptError::from_err(std::io::Error::other(e.to_string())))
204    }
205
206    async fn shutdown(&self) {
207        // Stop the accept loop from taking new streams. Level-triggered:
208        // even if cancel() lands between iterations, the next time the
209        // loop hits `cancelled().await` it returns immediately.
210        self.inner.shutdown.cancel();
211        let deadline = self
212            .inner
213            .processor
214            .begin_shutdown(tokio::time::Instant::now() + super::rpc_session::DRAIN_TIMEOUT);
215        // Wait for all in-flight streams to release their permits.
216        // `acquire_many` succeeds only once every permit is returned. Bounded
217        // (#159) so a wedged processor/transport can't hang shutdown forever;
218        // on timeout we close() and proceed (remaining tasks die with the conn).
219        let cap = self.inner.stream_limit_capacity;
220        match tokio::time::timeout_at(deadline, self.inner.stream_limit.acquire_many(cap)).await {
221            Ok(Ok(permits)) => {
222                // Keep permits drained so any post-shutdown accept also
223                // sees a closed semaphore.
224                permits.forget();
225                self.inner.stream_limit.close();
226            }
227            Ok(Err(_)) => {
228                // Already closed; nothing to drain.
229            }
230            Err(_) => {
231                tracing::warn!(
232                    timeout = ?super::rpc_session::DRAIN_TIMEOUT,
233                    "iroh-rpc: drain timed out, forcing teardown"
234                );
235                self.inner.stream_limit.close();
236            }
237        }
238    }
239}
240
241/// Client-side helper: open a bidi stream on `hyprstream-rpc/1` against an
242/// already-connected iroh [`Connection`], write the request, read the response.
243///
244/// Primitive used by tests and internally by
245/// [`super::iroh_transport::IrohTransport`] — production callers should
246/// construct an `IrohTransport` + [`crate::rpc_client::RpcClientImpl`] instead.
247pub async fn client_request(conn: &Connection, request: &[u8]) -> Result<Bytes> {
248    let (mut send, mut recv) = conn.open_bi().await.context("open_bi")?;
249    send.write_all(request).await.context("write request")?;
250    send.finish().context("finish send")?;
251    let buf = recv
252        .read_to_end(MAX_FRAME_BYTES)
253        .await
254        .context("read response")?;
255    Ok(Bytes::from(buf))
256}
257
258// ============================================================================
259// LocalServiceBridge — adapt a (possibly `!Send`) RequestService to the Send-bound
260// IrohRequestProcessor trait by running the service on a dedicated LocalSet
261// thread and forwarding requests over an mpsc channel.
262// ============================================================================
263
264/// Per-request payload + response slot exchanged with the bridge thread.
265struct BridgeMessage {
266    request: Bytes,
267    /// Accept-boundary carrier classification (INV-2 #1042), forwarded
268    /// verbatim to the dispatch pipeline's cleartext policy.
269    carrier: crate::transport::carrier::CarrierContext,
270    respond: tokio::sync::oneshot::Sender<Result<Bytes>>,
271}
272
273/// Shared dispatch loop body: relay `BridgeMessage`s from `rx` to `service`,
274/// each in its own `spawn_local` task so a slow handler never head-of-lines
275/// the queue.
276async fn run_bridge_dispatch_loop<S>(
277    service: std::rc::Rc<S>,
278    mut rx: tokio::sync::mpsc::Receiver<BridgeMessage>,
279    nonce_cache: Arc<crate::envelope::InMemoryNonceCache>,
280    shutdown: CancellationToken,
281) -> BridgeShutdownResult
282where
283    S: crate::service::RequestService + 'static,
284{
285    let mut requests = tokio::task::JoinSet::new();
286    let mut result = Ok(());
287    // Closing the receiver rejects new sends but preserves the accepted queue.
288    // The dispatch owner joins every spawned request before returning.
289    loop {
290        let msg = tokio::select! {
291            biased;
292            _ = shutdown.cancelled(), if !rx.is_closed() => {
293                rx.close();
294                continue;
295            }
296            joined = requests.join_next(), if !requests.is_empty() => {
297                if matches!(joined, Some(Err(_))) {
298                    result = Err(BridgeShutdownError::ThreadPanicked);
299                }
300                continue;
301            }
302            msg = rx.recv() => match msg {
303                Some(msg) => msg,
304                None => break,
305            },
306        };
307        let service = std::rc::Rc::clone(&service);
308        let nonce_cache = Arc::clone(&nonce_cache);
309        requests.spawn_local(async move {
310            let signing_key = service.signing_key();
311            let result = crate::service::dispatch::process_request(
312                msg.request.as_ref(),
313                &*service,
314                crate::envelope::EnvelopeVerification::AnySigner,
315                &signing_key,
316                &nonce_cache,
317                msg.carrier,
318            )
319            .await
320            .map(Bytes::from);
321            let _ = msg.respond.send(result);
322        });
323    }
324    while let Some(joined) = requests.join_next().await {
325        if joined.is_err() {
326            result = Err(BridgeShutdownError::ThreadPanicked);
327        }
328    }
329    result
330}
331
332/// Adapt a [`crate::service::RequestService`] to [`IrohRequestProcessor`].
333///
334/// Spins up a dedicated thread running a single-threaded tokio runtime with
335/// a `LocalSet`. The service runs on that thread (compatible with both `Send`
336/// and `!Send` services like inference); requests arriving on the iroh accept
337/// loop are forwarded via an `mpsc` channel and awaited via `oneshot`.
338///
339/// **Lifecycle**: shutdown closes admission, drains queued and running requests
340/// within the same grace period as network RPC, then destroys residual LocalSet
341/// tasks and joins the thread. A retained client cannot pin the service. Final
342/// handle drop provides synchronous cleanup; explicit owners use async shutdown.
343pub struct LocalServiceBridge {
344    tx: tokio::sync::mpsc::Sender<BridgeMessage>,
345    shutdown: CancellationToken,
346    thread: Arc<BridgeThread>,
347    admission: Arc<parking_lot::Mutex<Option<tokio::time::Instant>>>,
348}
349
350/// Stable terminal outcome shared by all explicit shutdown waiters.
351#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
352pub enum BridgeShutdownError {
353    #[error("bridge thread panicked")]
354    ThreadPanicked,
355    #[error("bridge request drain timed out")]
356    DrainTimedOut,
357    #[error("bridge runtime initialization failed: {0}")]
358    Runtime(String),
359    #[error("cannot join bridge from its own thread")]
360    SelfJoin,
361    #[error("bridge join task failed: {0}")]
362    JoinTask(String),
363}
364
365type BridgeShutdownResult = std::result::Result<(), BridgeShutdownError>;
366
367/// Join ownership survives cancellation of an async shutdown waiter.
368struct BridgeThread {
369    id: std::thread::ThreadId,
370    handle: parking_lot::Mutex<Option<std::thread::JoinHandle<BridgeShutdownResult>>>,
371    completed: parking_lot::Mutex<Option<BridgeShutdownResult>>,
372    completion: parking_lot::Condvar,
373}
374
375impl BridgeThread {
376    fn new(handle: std::thread::JoinHandle<BridgeShutdownResult>) -> Self {
377        Self {
378            id: handle.thread().id(),
379            handle: parking_lot::Mutex::new(Some(handle)),
380            completed: parking_lot::Mutex::new(None),
381            completion: parking_lot::Condvar::new(),
382        }
383    }
384
385    fn join(&self) -> BridgeShutdownResult {
386        if self.id == std::thread::current().id() {
387            return Err(BridgeShutdownError::SelfJoin);
388        }
389        let handle = self.handle.lock().take();
390        if let Some(handle) = handle {
391            // No lock needed by any caller or service destructor is held here.
392            let result = handle
393                .join()
394                .unwrap_or(Err(BridgeShutdownError::ThreadPanicked));
395            *self.completed.lock() = Some(result.clone());
396            self.completion.notify_all();
397            result
398        } else {
399            let mut completed = self.completed.lock();
400            loop {
401                if let Some(result) = &*completed {
402                    return result.clone();
403                }
404                self.completion.wait(&mut completed);
405            }
406        }
407    }
408}
409
410async fn finish_bridge_local_set(
411    local: tokio::task::LocalSet,
412    dispatch: tokio::task::JoinHandle<BridgeShutdownResult>,
413    shutdown: CancellationToken,
414    grace: std::time::Duration,
415    admission: Arc<parking_lot::Mutex<Option<tokio::time::Instant>>>,
416) -> BridgeShutdownResult {
417    let result = {
418        // Await the dispatch task (which owns every accepted RPC), not the
419        // whole LocalSet: streaming continuations are residual background work.
420        let run = local.run_until(async move {
421            dispatch
422                .await
423                .unwrap_or(Err(BridgeShutdownError::ThreadPanicked))
424        });
425        tokio::pin!(run);
426        tokio::select! {
427            biased;
428            _ = shutdown.cancelled() => {
429                let deadline = admission.lock().unwrap_or_else(|| tokio::time::Instant::now() + grace);
430                tokio::time::timeout_at(deadline, &mut run).await
431                    .unwrap_or(Err(BridgeShutdownError::DrainTimedOut))
432            }
433            result = &mut run => result,
434        }
435    };
436    // Entered runtime remains alive while residual tasks/services are destroyed.
437    drop(local);
438    result
439}
440
441impl crate::transport::rpc_session::sealed::Sealed for LocalServiceBridge {
442    fn admits_untrusted_carrier(&self) -> bool {
443        true
444    }
445}
446
447impl LocalServiceBridge {
448    /// Spawn a dedicated bridge thread that owns `service` and forwards
449    /// requests through [`crate::service::dispatch::process_request`].
450    ///
451    /// The response signing key is taken from `service.signing_key()` — there
452    /// is no separate parameter to avoid drift between the service's identity
453    /// and the key used to sign responses.
454    ///
455    /// `queue_depth` is the mpsc channel capacity — backpressure point when
456    /// the bridge falls behind. Pass `0` for default (128).
457    pub fn spawn<S>(
458        service: S,
459        nonce_cache: Arc<crate::envelope::InMemoryNonceCache>,
460        queue_depth: usize,
461    ) -> Result<Self>
462    where
463        S: crate::service::RequestService + Send + 'static,
464    {
465        let cap = if queue_depth == 0 { 128 } else { queue_depth };
466        let (tx, rx) = tokio::sync::mpsc::channel::<BridgeMessage>(cap);
467        let service_name = service.name().to_owned();
468
469        let shutdown = CancellationToken::new();
470        let stopped = shutdown.clone();
471        let admission = Arc::new(parking_lot::Mutex::new(None));
472        let thread_admission = Arc::clone(&admission);
473        let thread = std::thread::Builder::new()
474            .name(format!("iroh-rpc-bridge:{service_name}"))
475            .spawn(move || {
476                let rt = match tokio::runtime::Builder::new_current_thread()
477                    .enable_all()
478                    .build()
479                {
480                    Ok(rt) => rt,
481                    Err(e) => {
482                        return Err(BridgeShutdownError::Runtime(e.to_string()));
483                    }
484                };
485                let local = tokio::task::LocalSet::new();
486                let dispatch = local.spawn_local(run_bridge_dispatch_loop(
487                    std::rc::Rc::new(service),
488                    rx,
489                    nonce_cache,
490                    stopped.clone(),
491                ));
492                let result = rt.block_on(finish_bridge_local_set(
493                    local,
494                    dispatch,
495                    stopped,
496                    super::rpc_session::DRAIN_TIMEOUT,
497                    thread_admission,
498                ));
499                drop(rt);
500                result
501            })
502            .map_err(|e| anyhow::anyhow!("spawn iroh-rpc bridge thread: {e}"))?;
503
504        Ok(Self {
505            tx,
506            shutdown,
507            thread: Arc::new(BridgeThread::new(thread)),
508            admission,
509        })
510    }
511
512    /// Like [`LocalServiceBridge::spawn`], but constructs the service ON the
513    /// bridge thread via the async `build` closure. For services whose
514    /// construction is itself `!Send` or async (e.g. GPU init that must happen on
515    /// the serve thread, like `InferenceService`): the built service value never
516    /// has to be `Send`-moved across threads — only the builder's captured inputs
517    /// do (`F: Send`), and the `!Send` result lives only on the bridge thread.
518    ///
519    /// Returns the bridge plus a readiness receiver that resolves once `build`
520    /// completes: `Ok(())` when the service is built and serving, or the build
521    /// error. Callers MUST await it before advertising the service (a build
522    /// failure otherwise surfaces only as "channel closed" on the first request).
523    pub fn spawn_with<F, Fut, S>(
524        thread_name: impl Into<String>,
525        build: F,
526        nonce_cache: Arc<crate::envelope::InMemoryNonceCache>,
527        queue_depth: usize,
528    ) -> Result<(Self, tokio::sync::oneshot::Receiver<Result<()>>)>
529    where
530        F: FnOnce() -> Fut + Send + 'static,
531        Fut: Future<Output = Result<S>>,
532        S: crate::service::RequestService + 'static,
533    {
534        let cap = if queue_depth == 0 { 128 } else { queue_depth };
535        let (tx, rx) = tokio::sync::mpsc::channel::<BridgeMessage>(cap);
536        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::<Result<()>>();
537        let thread_name = thread_name.into();
538
539        let shutdown = CancellationToken::new();
540        let stopped = shutdown.clone();
541        let admission = Arc::new(parking_lot::Mutex::new(None));
542        let thread_admission = Arc::clone(&admission);
543        let thread = std::thread::Builder::new()
544            .name(format!("rpc-bridge:{thread_name}"))
545            .spawn(move || {
546                let rt = match tokio::runtime::Builder::new_current_thread()
547                    .enable_all()
548                    .build()
549                {
550                    Ok(rt) => rt,
551                    Err(e) => {
552                        let _ = ready_tx.send(Err(anyhow::anyhow!("bridge runtime: {e}")));
553                        return Err(BridgeShutdownError::Runtime(e.to_string()));
554                    }
555                };
556                let local = tokio::task::LocalSet::new();
557                let dispatch_shutdown = stopped.clone();
558                let dispatch = local.spawn_local(async move {
559                    // Build on-thread; a failure (e.g. GPU init) is reported via
560                    // the readiness channel and the bridge thread exits.
561                    let built = tokio::select! {
562                        biased;
563                        _ = dispatch_shutdown.cancelled() => return Ok(()),
564                        built = build() => built,
565                    };
566                    let service = match built {
567                        Ok(s) => std::rc::Rc::new(s),
568                        Err(e) => {
569                            let _ = ready_tx.send(Err(e));
570                            return Ok(());
571                        }
572                    };
573                    if ready_tx.send(Ok(())).is_err() {
574                        // Caller gave up waiting; no point serving.
575                        return Ok(());
576                    }
577                    run_bridge_dispatch_loop(service, rx, nonce_cache, dispatch_shutdown).await
578                });
579                let result = rt.block_on(finish_bridge_local_set(
580                    local,
581                    dispatch,
582                    stopped,
583                    super::rpc_session::DRAIN_TIMEOUT,
584                    thread_admission,
585                ));
586                drop(rt);
587                result
588            })
589            .map_err(|e| anyhow::anyhow!("spawn rpc bridge thread: {e}"))?;
590
591        Ok((
592            Self {
593                tx,
594                shutdown,
595                thread: Arc::new(BridgeThread::new(thread)),
596                admission,
597            },
598            ready_rx,
599        ))
600    }
601}
602
603impl LocalServiceBridge {
604    fn close_admission(&self) {
605        self.begin_shutdown(tokio::time::Instant::now() + super::rpc_session::DRAIN_TIMEOUT);
606    }
607
608    /// Begin draining immediately; subsequent callers reuse the first deadline.
609    pub fn begin_shutdown(&self, deadline: tokio::time::Instant) -> tokio::time::Instant {
610        let mut admission = self.admission.lock();
611        let deadline = *admission.get_or_insert(deadline);
612        self.shutdown.cancel();
613        deadline
614    }
615
616    /// Close admission, drain accepted work, and wait for service/runtime destruction.
617    /// Joining runs off the caller's runtime; no thread lock is held while waiting.
618    /// Concurrent close calls share the same terminal result.
619    pub async fn shutdown(&self) -> BridgeShutdownResult {
620        self.close_admission();
621        if self.thread.id == std::thread::current().id() {
622            return Err(BridgeShutdownError::SelfJoin);
623        }
624        let thread = Arc::clone(&self.thread);
625        tokio::task::spawn_blocking(move || thread.join())
626            .await
627            .map_err(|error| BridgeShutdownError::JoinTask(error.to_string()))?
628    }
629}
630
631impl Drop for LocalServiceBridge {
632    fn drop(&mut self) {
633        self.close_admission();
634        // Explicit async shutdown is the service-owner path. Final-handle drop
635        // also owns teardown, including failed or cancelled spawn_with builders.
636        if let Err(error) = self.thread.join() {
637            tracing::error!(%error, "bridge drop teardown failed");
638        }
639    }
640}
641
642impl IrohRequestProcessor for LocalServiceBridge {
643    fn close_admission(&self) {
644        LocalServiceBridge::close_admission(self);
645    }
646
647    fn begin_shutdown(&self, deadline: tokio::time::Instant) -> tokio::time::Instant {
648        LocalServiceBridge::begin_shutdown(self, deadline)
649    }
650
651    fn process(
652        &self,
653        request: Bytes,
654        carrier: crate::transport::carrier::CarrierContext,
655    ) -> std::pin::Pin<Box<dyn Future<Output = Result<Bytes>> + Send + '_>> {
656        let tx = self.tx.clone();
657        Box::pin(async move {
658            let (respond_tx, respond_rx) = tokio::sync::oneshot::channel();
659            let permit = tokio::select! {
660                biased;
661                _ = self.shutdown.cancelled() => anyhow::bail!("iroh-rpc bridge: admission closed"),
662                reserved = tx.reserve() => {
663                    reserved.map_err(|_| anyhow::anyhow!("iroh-rpc bridge: channel closed"))?
664                }
665            };
666            {
667                // Linearize successful enqueue against shutdown, without holding
668                // any lock across queue backpressure, dispatch, or thread join.
669                let _admission = self.admission.lock();
670                anyhow::ensure!(
671                    !self.shutdown.is_cancelled(),
672                    "iroh-rpc bridge: admission closed"
673                );
674                permit.send(BridgeMessage {
675                    request,
676                    carrier,
677                    respond: respond_tx,
678                });
679            }
680            respond_rx
681                .await
682                .map_err(|_| anyhow::anyhow!("iroh-rpc bridge: response dropped"))?
683        })
684    }
685}
686
687#[cfg(test)]
688mod tests {
689    use super::*;
690    use crate::transport::iroh_substrate::{
691        IrohSubstrate, NoopHandler, ALPN_HYPRSTREAM_RPC, ALPN_MOQ_LITE,
692    };
693    use iroh::{EndpointAddr, TransportAddr};
694    use rand::RngCore;
695    use std::time::Duration;
696
697    fn fresh_key() -> [u8; 32] {
698        let mut k = [0u8; 32];
699        rand::thread_rng().fill_bytes(&mut k);
700        k
701    }
702
703    fn fresh_signing_key() -> SigningKey {
704        SigningKey::from_bytes(&fresh_key())
705    }
706
707    fn direct_addr(substrate: &IrohSubstrate) -> EndpointAddr {
708        EndpointAddr::from_parts(
709            substrate.endpoint_id(),
710            substrate
711                .endpoint()
712                .bound_sockets()
713                .into_iter()
714                .map(TransportAddr::Ip),
715        )
716    }
717
718    /// Minimal `RequestService` for `spawn_with` tests.
719    struct BridgeEcho {
720        name: String,
721        transport: crate::transport::TransportConfig,
722        signing_key: SigningKey,
723        drop_probe: Option<BridgeDropProbe>,
724        request_gate: Option<(Arc<tokio::sync::Semaphore>, Arc<tokio::sync::Semaphore>)>,
725    }
726
727    struct BridgeDropProbe {
728        entered: std::sync::mpsc::Sender<()>,
729        release: std::sync::mpsc::Receiver<()>,
730        completed: Arc<std::sync::atomic::AtomicBool>,
731    }
732
733    impl Drop for BridgeDropProbe {
734        fn drop(&mut self) {
735            let _ = self.entered.send(());
736            let _ = self.release.recv_timeout(Duration::from_secs(5));
737            self.completed
738                .store(true, std::sync::atomic::Ordering::SeqCst);
739        }
740    }
741
742    impl BridgeEcho {
743        fn new(signing_key: SigningKey) -> Self {
744            Self {
745                drop_probe: None,
746                request_gate: None,
747                name: "bridge-echo".to_owned(),
748                transport: crate::transport::TransportConfig::inproc("bridge-echo-unused"),
749                signing_key,
750            }
751        }
752    }
753    #[async_trait::async_trait(?Send)]
754    impl crate::service::RequestService for BridgeEcho {
755        async fn handle_request(
756            &self,
757            _ctx: &crate::service::EnvelopeContext,
758            body: &crate::service::DecodedRequestBody,
759        ) -> Result<(Vec<u8>, Option<crate::service::Continuation>)> {
760            let mut out = vec![0xBE];
761            out.extend_from_slice(body.bytes());
762            Ok((out, None))
763        }
764        async fn verify_claims(&self, _ctx: &mut crate::service::EnvelopeContext) -> Result<()> {
765            if let Some((entered, release)) = &self.request_gate {
766                entered.add_permits(1);
767                release.acquire().await?.forget();
768            }
769            anyhow::bail!("gated claims denial")
770        }
771        fn build_error_payload(&self, _request_id: u64, error: &str) -> Vec<u8> {
772            error.as_bytes().to_vec()
773        }
774        fn decode_request_body(
775            &self,
776            signed_body: &[u8],
777        ) -> Result<crate::service::DecodedRequestBody> {
778            Ok(crate::service::DecodedRequestBody::opaque(
779                signed_body.to_vec(),
780            ))
781        }
782        fn name(&self) -> &str {
783            &self.name
784        }
785        fn transport(&self) -> &crate::transport::TransportConfig {
786            &self.transport
787        }
788        fn signing_key(&self) -> SigningKey {
789            self.signing_key.clone()
790        }
791    }
792
793    #[test]
794    fn bridge_drop_waits_for_service_destructor() -> Result<()> {
795        assert_bridge_destructor_completion(false)
796    }
797
798    #[test]
799    fn bridge_spawn_with_drop_waits_for_service_destructor() -> Result<()> {
800        assert_bridge_destructor_completion(true)
801    }
802
803    fn assert_bridge_destructor_completion(build_on_thread: bool) -> Result<()> {
804        let (entered_tx, entered_rx) = std::sync::mpsc::channel();
805        let (release_tx, release_rx) = std::sync::mpsc::channel();
806        let completed = Arc::new(std::sync::atomic::AtomicBool::new(false));
807        let mut service = BridgeEcho::new(fresh_signing_key());
808        service.drop_probe = Some(BridgeDropProbe {
809            entered: entered_tx,
810            release: release_rx,
811            completed: Arc::clone(&completed),
812        });
813        let nonce = Arc::new(crate::envelope::InMemoryNonceCache::new());
814        let bridge = Arc::new(if build_on_thread {
815            let (bridge, ready) = LocalServiceBridge::spawn_with(
816                "destructor-gate",
817                move || async move { Ok(service) },
818                nonce,
819                0,
820            )?;
821            ready.blocking_recv()??;
822            bridge
823        } else {
824            LocalServiceBridge::spawn(service, nonce, 0)?
825        });
826        let retained = Arc::clone(&bridge);
827        drop(retained);
828        assert!(!completed.load(std::sync::atomic::Ordering::SeqCst));
829        let (returned_tx, returned_rx) = std::sync::mpsc::channel();
830        let owner = std::thread::spawn(move || {
831            drop(bridge);
832            let _ = returned_tx.send(completed.load(std::sync::atomic::Ordering::SeqCst));
833        });
834        entered_rx.recv_timeout(Duration::from_secs(5))?;
835        // The destructor is held at a causal gate, not slowed by a sleep.
836        // Teardown must not return while that gate is held.
837        let premature = returned_rx.recv_timeout(Duration::from_millis(100)).ok();
838        release_tx.send(())?;
839        owner
840            .join()
841            .map_err(|_| anyhow::anyhow!("bridge owner panicked"))?;
842        assert!(
843            premature.is_none(),
844            "bridge drop returned before service destruction"
845        );
846        assert!(returned_rx.recv_timeout(Duration::from_secs(5))?);
847        Ok(())
848    }
849
850    #[test]
851    fn bridge_thread_join_never_joins_itself() -> Result<()> {
852        let (owner_tx, owner_rx) = std::sync::mpsc::channel::<Arc<BridgeThread>>();
853        let (returned_tx, returned_rx) = std::sync::mpsc::channel();
854        let worker = std::thread::spawn(move || {
855            if let Ok(owner) = owner_rx.recv() {
856                assert_eq!(owner.join(), Err(BridgeShutdownError::SelfJoin));
857                let _ = returned_tx.send(());
858            }
859            Ok(())
860        });
861        let owner = Arc::new(BridgeThread::new(worker));
862        owner_tx.send(Arc::clone(&owner))?;
863        returned_rx.recv_timeout(Duration::from_secs(5))?;
864        owner.join()?;
865        assert_eq!(*owner.completed.lock(), Some(Ok(())));
866        Ok(())
867    }
868
869    #[tokio::test]
870    async fn bridge_cancelled_shutdown_waiter_keeps_completion_barrier() -> Result<()> {
871        let (entered_tx, entered_rx) = std::sync::mpsc::channel();
872        let (release_tx, release_rx) = std::sync::mpsc::channel();
873        let completed = Arc::new(std::sync::atomic::AtomicBool::new(false));
874        let mut service = BridgeEcho::new(fresh_signing_key());
875        service.drop_probe = Some(BridgeDropProbe {
876            entered: entered_tx,
877            release: release_rx,
878            completed: Arc::clone(&completed),
879        });
880        let bridge = Arc::new(LocalServiceBridge::spawn(
881            service,
882            Arc::new(crate::envelope::InMemoryNonceCache::new()),
883            0,
884        )?);
885        let owner = Arc::clone(&bridge);
886        let first = tokio::spawn(async move { owner.shutdown().await });
887        tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(5)))
888            .await??;
889        first.abort();
890        let _ = first.await;
891        let premature = tokio::time::timeout(Duration::from_millis(100), bridge.shutdown())
892            .await
893            .is_ok();
894        release_tx.send(())?;
895        tokio::time::timeout(Duration::from_secs(5), bridge.shutdown()).await??;
896        assert!(
897            !premature,
898            "cancelled waiter lost the in-progress thread join"
899        );
900        assert!(completed.load(std::sync::atomic::Ordering::SeqCst));
901        Ok(())
902    }
903
904    #[tokio::test]
905    async fn bridge_shutdown_cancels_builder_with_retained_handle() -> Result<()> {
906        struct Dropped(Arc<std::sync::atomic::AtomicBool>);
907        impl Drop for Dropped {
908            fn drop(&mut self) {
909                self.0.store(true, std::sync::atomic::Ordering::SeqCst);
910            }
911        }
912        let completed = Arc::new(std::sync::atomic::AtomicBool::new(false));
913        let guard = Dropped(Arc::clone(&completed));
914        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
915        let (bridge, ready) = LocalServiceBridge::spawn_with::<_, _, BridgeEcho>(
916            "pending-builder",
917            move || async move {
918                let _guard = guard;
919                let _ = entered_tx.send(());
920                std::future::pending().await
921            },
922            Arc::new(crate::envelope::InMemoryNonceCache::new()),
923            0,
924        )?;
925        let bridge = Arc::new(bridge);
926        let retained = Arc::clone(&bridge);
927        tokio::time::timeout(Duration::from_secs(5), entered_rx).await??;
928        tokio::time::timeout(Duration::from_secs(5), bridge.shutdown()).await??;
929        assert!(completed.load(std::sync::atomic::Ordering::SeqCst));
930        assert!(ready.await.is_err());
931        assert!(retained
932            .process(
933                Bytes::new(),
934                crate::transport::carrier::CarrierContext::iroh()
935            )
936            .await
937            .is_err());
938        // Completion is idempotent while another owner still retains the bridge.
939        bridge.shutdown().await?;
940        Ok(())
941    }
942
943    #[tokio::test]
944    async fn bridge_shutdown_drains_accepted_queue_before_residual_cancellation() -> Result<()> {
945        use crate::ToCapnp;
946        let key = fresh_signing_key();
947        let pq = crate::node_identity::derive_mesh_mldsa_key(&key);
948        let mut store = crate::envelope::KeyedPqTrustStore::new();
949        store.bind(
950            key.verifying_key().to_bytes(),
951            &crate::crypto::pq::ml_dsa_sk_to_vk(&pq),
952        );
953        let store = Arc::new(store);
954        crate::envelope::install_verify_config(crate::envelope::EnvelopeVerifyConfig {
955            policy: crate::crypto::CryptoPolicy::Hybrid,
956            pq_store: Some(store.clone()),
957        })?;
958        crate::envelope::install_response_verify_config(crate::envelope::ResponseVerifyConfig {
959            policy: crate::crypto::CryptoPolicy::Hybrid,
960            pq_store: Some(store),
961        })?;
962        let entered = Arc::new(tokio::sync::Semaphore::new(0));
963        let release = Arc::new(tokio::sync::Semaphore::new(0));
964        let mut service = BridgeEcho::new(key.clone());
965        service.request_gate = Some((entered.clone(), release.clone()));
966        let (tx, rx) = tokio::sync::mpsc::channel(2);
967        let mut responses = Vec::new();
968        for payload in [b"one", b"two"] {
969            let envelope = crate::envelope::SignedEnvelope::new_signed_hybrid(
970                crate::envelope::RequestEnvelope::new(payload.to_vec()),
971                &key,
972                &pq,
973            );
974            let mut message = capnp::message::Builder::new_default();
975            envelope.write_to(
976                &mut message.init_root::<crate::common_capnp::signed_envelope::Builder>(),
977            );
978            let mut wire = Vec::new();
979            capnp::serialize::write_message(&mut wire, &message)?;
980            let (respond, response) = tokio::sync::oneshot::channel();
981            tx.try_send(BridgeMessage {
982                request: Bytes::from(wire),
983                carrier: crate::transport::carrier::CarrierContext::inproc(),
984                respond,
985            })
986            .map_err(|_| anyhow::anyhow!("queue admission failed"))?;
987            responses.push(response);
988        }
989        // Both requests are queued BEFORE shutdown, before dispatch is polled.
990        let stopped = CancellationToken::new();
991        stopped.cancel();
992        let local = tokio::task::LocalSet::new();
993        let dispatch = local.spawn_local(run_bridge_dispatch_loop(
994            std::rc::Rc::new(service),
995            rx,
996            Arc::new(crate::envelope::InMemoryNonceCache::new()),
997            stopped.clone(),
998        ));
999        let drain = finish_bridge_local_set(
1000            local,
1001            dispatch,
1002            stopped,
1003            super::super::rpc_session::DRAIN_TIMEOUT,
1004            Arc::new(parking_lot::Mutex::new(None)),
1005        );
1006        let coordinate = async {
1007            tokio::time::timeout(Duration::from_secs(5), entered.acquire_many(2))
1008                .await??
1009                .forget();
1010            assert!(tx.is_closed(), "new admissions remained open during grace");
1011            release.add_permits(2);
1012            for response in responses {
1013                let wire = response.await??;
1014                let (_, payload) =
1015                    crate::envelope::unwrap_response(&wire, Some(&key.verifying_key()))?;
1016                assert_eq!(
1017                    payload,
1018                    crate::service::dispatch::DISPATCH_DENIED.as_bytes()
1019                );
1020            }
1021            anyhow::Ok(())
1022        };
1023        let (drained, coordinated) = tokio::join!(drain, coordinate);
1024        drained?;
1025        coordinated?;
1026        Ok(())
1027    }
1028
1029    #[tokio::test]
1030    async fn bridge_panicked_thread_result_is_shared_by_shutdown_waiters() -> Result<()> {
1031        let worker = std::thread::spawn(|| -> BridgeShutdownResult {
1032            panic!("causal bridge thread panic");
1033        });
1034        let (tx, _rx) = tokio::sync::mpsc::channel(1);
1035        let bridge = Arc::new(LocalServiceBridge {
1036            tx,
1037            shutdown: CancellationToken::new(),
1038            thread: Arc::new(BridgeThread::new(worker)),
1039            admission: Arc::new(parking_lot::Mutex::new(None)),
1040        });
1041        let (first, second) = tokio::join!(bridge.shutdown(), bridge.shutdown());
1042        assert_eq!(first, Err(BridgeShutdownError::ThreadPanicked));
1043        assert_eq!(second, first);
1044        assert_eq!(bridge.shutdown().await, first);
1045        Ok(())
1046    }
1047
1048    #[tokio::test]
1049    async fn bridge_drain_timeout_cancels_accepted_request_and_shares_error() -> Result<()> {
1050        use crate::ToCapnp;
1051        let key = fresh_signing_key();
1052        let pq = crate::node_identity::derive_mesh_mldsa_key(&key);
1053        let mut store = crate::envelope::KeyedPqTrustStore::new();
1054        store.bind(
1055            key.verifying_key().to_bytes(),
1056            &crate::crypto::pq::ml_dsa_sk_to_vk(&pq),
1057        );
1058        crate::envelope::install_verify_config(crate::envelope::EnvelopeVerifyConfig {
1059            policy: crate::crypto::CryptoPolicy::Hybrid,
1060            pq_store: Some(Arc::new(store)),
1061        })?;
1062        let envelope = crate::envelope::SignedEnvelope::new_signed_hybrid(
1063            crate::envelope::RequestEnvelope::new(b"held".to_vec()),
1064            &key,
1065            &pq,
1066        );
1067        let mut message = capnp::message::Builder::new_default();
1068        envelope
1069            .write_to(&mut message.init_root::<crate::common_capnp::signed_envelope::Builder>());
1070        let mut wire = Vec::new();
1071        capnp::serialize::write_message(&mut wire, &message)?;
1072        let entered = Arc::new(tokio::sync::Semaphore::new(0));
1073        let mut service = BridgeEcho::new(key);
1074        service.request_gate = Some((entered.clone(), Arc::new(tokio::sync::Semaphore::new(0))));
1075        let (drop_entered, _) = std::sync::mpsc::channel();
1076        let (release, released) = std::sync::mpsc::channel();
1077        release.send(())?;
1078        let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
1079        service.drop_probe = Some(BridgeDropProbe {
1080            entered: drop_entered,
1081            release: released,
1082            completed: dropped.clone(),
1083        });
1084        let (tx, rx) = tokio::sync::mpsc::channel(1);
1085        let (respond, response) = tokio::sync::oneshot::channel();
1086        tx.try_send(BridgeMessage {
1087            request: Bytes::from(wire),
1088            carrier: crate::transport::carrier::CarrierContext::inproc(),
1089            respond,
1090        })
1091        .map_err(|_| anyhow::anyhow!("queue admission failed"))?;
1092        let stopped = CancellationToken::new();
1093        let thread_stopped = stopped.clone();
1094        let thread = std::thread::spawn(move || {
1095            // Real dispatch/queue/destructor, with only the runtime clock made
1096            // virtual: exercise the unchanged production 40-second bound.
1097            let rt = tokio::runtime::Builder::new_current_thread()
1098                .enable_all()
1099                .start_paused(true)
1100                .build()
1101                .map_err(|e| BridgeShutdownError::Runtime(e.to_string()))?;
1102            let local = tokio::task::LocalSet::new();
1103            let dispatch = local.spawn_local(run_bridge_dispatch_loop(
1104                std::rc::Rc::new(service),
1105                rx,
1106                Arc::new(crate::envelope::InMemoryNonceCache::new()),
1107                thread_stopped.clone(),
1108            ));
1109            rt.block_on(finish_bridge_local_set(
1110                local,
1111                dispatch,
1112                thread_stopped,
1113                super::super::rpc_session::DRAIN_TIMEOUT,
1114                Arc::new(parking_lot::Mutex::new(None)),
1115            ))
1116        });
1117        let bridge = LocalServiceBridge {
1118            tx,
1119            shutdown: stopped,
1120            thread: Arc::new(BridgeThread::new(thread)),
1121            admission: Arc::new(parking_lot::Mutex::new(None)),
1122        };
1123        tokio::time::timeout(Duration::from_secs(5), entered.acquire())
1124            .await??
1125            .forget();
1126        let (first, second) = tokio::join!(bridge.shutdown(), bridge.shutdown());
1127        assert_eq!(first, Err(BridgeShutdownError::DrainTimedOut));
1128        assert_eq!(second, first);
1129        assert_eq!(bridge.shutdown().await, first);
1130        assert!(
1131            response.await.is_err(),
1132            "forced deadline retained the accepted response sender"
1133        );
1134        assert!(dropped.load(std::sync::atomic::Ordering::SeqCst));
1135        Ok(())
1136    }
1137
1138    fn deadline_test_identity() -> Result<SigningKey> {
1139        let key = fresh_signing_key();
1140        let pq = crate::node_identity::derive_mesh_mldsa_key(&key);
1141        let mut store = crate::envelope::KeyedPqTrustStore::new();
1142        store.bind(
1143            key.verifying_key().to_bytes(),
1144            &crate::crypto::pq::ml_dsa_sk_to_vk(&pq),
1145        );
1146        crate::envelope::install_verify_config(crate::envelope::EnvelopeVerifyConfig {
1147            policy: crate::crypto::CryptoPolicy::Hybrid,
1148            pq_store: Some(Arc::new(store)),
1149        })?;
1150        Ok(key)
1151    }
1152
1153    fn deadline_test_bridge(
1154        key: &SigningKey,
1155    ) -> Result<(
1156        Arc<LocalServiceBridge>,
1157        Arc<tokio::sync::Semaphore>,
1158        Arc<tokio::sync::Semaphore>,
1159        Arc<std::sync::atomic::AtomicBool>,
1160    )> {
1161        let entered = Arc::new(tokio::sync::Semaphore::new(0));
1162        let release = Arc::new(tokio::sync::Semaphore::new(0));
1163        let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
1164        let mut service = BridgeEcho::new(key.clone());
1165        service.request_gate = Some((entered.clone(), release.clone()));
1166        let (drop_entered, _) = std::sync::mpsc::channel();
1167        let (drop_release, released) = std::sync::mpsc::channel();
1168        drop_release.send(())?;
1169        service.drop_probe = Some(BridgeDropProbe {
1170            entered: drop_entered,
1171            release: released,
1172            completed: dropped.clone(),
1173        });
1174        let bridge = Arc::new(LocalServiceBridge::spawn(
1175            service,
1176            Arc::new(crate::envelope::InMemoryNonceCache::new()),
1177            0,
1178        )?);
1179        Ok((bridge, entered, release, dropped))
1180    }
1181
1182    async fn exercise_transport_bridge_deadline<T, F, D>(
1183        transport: T,
1184        key: &SigningKey,
1185        bridge: Arc<LocalServiceBridge>,
1186        entered: Arc<tokio::sync::Semaphore>,
1187        release: Arc<tokio::sync::Semaphore>,
1188        dropped: Arc<std::sync::atomic::AtomicBool>,
1189        forced: bool,
1190        drain: D,
1191    ) -> Result<()>
1192    where
1193        T: crate::transport::Transport + 'static,
1194        F: Future<Output = Result<()>> + Send + 'static,
1195        D: FnOnce(tokio::time::Instant) -> F,
1196    {
1197        let mut kem = crate::crypto::hybrid_kem::KeyedKemTrustStore::new();
1198        kem.bind(
1199            key.verifying_key().to_bytes(),
1200            crate::node_identity::derive_mesh_kem_recipient(key)?.public(),
1201        );
1202        let pq = crate::node_identity::derive_mesh_mldsa_key(key);
1203        let mut response_trust = crate::envelope::KeyedPqTrustStore::new();
1204        response_trust.bind(
1205            key.verifying_key().to_bytes(),
1206            &crate::crypto::pq::ml_dsa_sk_to_vk(&pq),
1207        );
1208        let client = Arc::new(
1209            crate::rpc_client::RpcClientImpl::new(
1210                crate::signer::LocalSigner::new(key.clone()),
1211                transport,
1212                Some(key.verifying_key()),
1213            )
1214            .with_request_kem_store(Arc::new(kem))
1215            .with_response_pq_store(Arc::new(response_trust)),
1216        );
1217        // Release on assertion/error too: a regression must fail, not strand
1218        // the real worker behind its fixture gate during test unwinding.
1219        struct ReleaseOnDrop(Arc<tokio::sync::Semaphore>);
1220        impl Drop for ReleaseOnDrop {
1221            fn drop(&mut self) {
1222                self.0.add_permits(1);
1223            }
1224        }
1225        let _cleanup = ReleaseOnDrop(release.clone());
1226        let caller = client.clone();
1227        let response = tokio::spawn(async move {
1228            caller
1229                .call_for_service("bridge-echo", b"held".to_vec())
1230                .await
1231        });
1232        tokio::time::timeout(Duration::from_secs(5), entered.acquire())
1233            .await??
1234            .forget();
1235        let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
1236        assert_eq!(bridge.begin_shutdown(deadline), deadline);
1237        assert_eq!(
1238            bridge.begin_shutdown(deadline + Duration::from_secs(40)),
1239            deadline,
1240            "later carrier shutdown must not extend the first grace"
1241        );
1242        assert!(bridge
1243            .process(
1244                Bytes::new(),
1245                crate::transport::carrier::CarrierContext::inproc()
1246            )
1247            .await
1248            .is_err());
1249        assert!(!dropped.load(std::sync::atomic::Ordering::SeqCst));
1250        let carrier = tokio::spawn(drain(deadline));
1251        if !forced {
1252            release.add_permits(1);
1253            let payload = tokio::time::timeout(Duration::from_secs(2), response).await???;
1254            assert_eq!(
1255                payload,
1256                crate::service::dispatch::DISPATCH_DENIED.as_bytes()
1257            );
1258        } else {
1259            // The accepted RPC is still held: neither a caller cancellation
1260            // nor an already-finished request can make the deadline test pass.
1261            assert!(!response.is_finished());
1262            assert!(
1263                tokio::time::timeout(Duration::from_secs(3), response)
1264                    .await??
1265                    .is_err(),
1266                "held request must be cancelled at deadline"
1267            );
1268        }
1269        tokio::time::timeout(Duration::from_secs(3), carrier).await???;
1270        // Deliberately join AFTER the transport drain too: this must reuse the
1271        // established deadline/result, never grant the bridge another 40s.
1272        let result = tokio::time::timeout(Duration::from_secs(1), bridge.shutdown()).await?;
1273        if forced {
1274            assert_eq!(result, Err(BridgeShutdownError::DrainTimedOut));
1275        } else {
1276            result?;
1277            assert!(
1278                tokio::time::Instant::now() < deadline,
1279                "released request did not drain during grace"
1280            );
1281        }
1282        assert!(
1283            dropped.load(std::sync::atomic::Ordering::SeqCst),
1284            "join returned before destructor despite retained client"
1285        );
1286        assert_eq!(
1287            bridge.begin_shutdown(tokio::time::Instant::now() + Duration::from_secs(40)),
1288            deadline
1289        );
1290        // Keep the actual network client alive through the completed join.
1291        drop(client);
1292        Ok(())
1293    }
1294
1295    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1296    async fn irohrpc_bridge_shared_deadline_replies_or_forces_join() -> Result<()> {
1297        let key = deadline_test_identity()?;
1298        for forced in [false, true] {
1299            let (bridge, entered, release, dropped) = deadline_test_bridge(&key)?;
1300            let server = IrohSubstrate::new_test(
1301                fresh_key(),
1302                NoopHandler::new("moq"),
1303                IrohRpcProtocolHandler::with_stream_limit(
1304                    bridge.clone(),
1305                    key.clone(),
1306                    DEFAULT_STREAM_LIMIT,
1307                ),
1308            )
1309            .await?;
1310            let client = IrohSubstrate::new_test(
1311                fresh_key(),
1312                NoopHandler::new("client moq"),
1313                NoopHandler::new("client rpc"),
1314            )
1315            .await?;
1316            let connection = client
1317                .connect(direct_addr(&server), ALPN_HYPRSTREAM_RPC)
1318                .await?;
1319            exercise_transport_bridge_deadline(
1320                crate::transport::iroh_transport::IrohTransport::new(connection),
1321                &key,
1322                bridge,
1323                entered,
1324                release,
1325                dropped,
1326                forced,
1327                move |_| async move { server.shutdown().await },
1328            )
1329            .await?;
1330            client.shutdown().await?;
1331        }
1332        Ok(())
1333    }
1334
1335    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1336    async fn quinnrpc_bridge_shared_deadline_replies_or_forces_join() -> Result<()> {
1337        use crate::transport::quinn_transport::{connect_pinned, QuinnRpcServer, QuinnTransport};
1338        crate::transport::pq_provider::install_pq_crypto_provider()?;
1339        let key = deadline_test_identity()?;
1340        for forced in [false, true] {
1341            let (bridge, entered, release, dropped) = deadline_test_bridge(&key)?;
1342            let certificate = rcgen::generate_simple_self_signed(vec!["localhost".into()])?;
1343            let cert_der = certificate.cert.der().to_vec();
1344            let private = rustls::pki_types::PrivateKeyDer::Pkcs8(
1345                rustls::pki_types::PrivatePkcs8KeyDer::from(certificate.key_pair.serialize_der()),
1346            );
1347            let server = web_transport_quinn::ServerBuilder::new()
1348                .with_addr("127.0.0.1:0".parse()?)
1349                .with_certificate(
1350                    vec![rustls::pki_types::CertificateDer::from(cert_der.clone())],
1351                    private,
1352                )?;
1353            let address = server.local_addr()?;
1354            let rpc = QuinnRpcServer::with_capacity(
1355                server,
1356                bridge.clone(),
1357                key.clone(),
1358                DEFAULT_STREAM_LIMIT,
1359            );
1360            let limit = rpc.stream_limit();
1361            let capacity = rpc.capacity();
1362            let token = rpc.shutdown_token();
1363            let task = tokio::spawn(rpc.run());
1364            let session = connect_pinned(address, &cert_der).await?;
1365            exercise_transport_bridge_deadline(
1366                QuinnTransport::new(session),
1367                &key,
1368                bridge,
1369                entered,
1370                release,
1371                dropped,
1372                forced,
1373                move |deadline| async move {
1374                    QuinnRpcServer::shutdown_until(&limit, capacity, &token, deadline).await;
1375                    task.await??;
1376                    Ok(())
1377                },
1378            )
1379            .await?;
1380        }
1381        Ok(())
1382    }
1383
1384    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1385    async fn udsrpc_bridge_shared_deadline_replies_or_forces_join() -> Result<()> {
1386        let key = deadline_test_identity()?;
1387        for forced in [false, true] {
1388            let (bridge, entered, release, dropped) = deadline_test_bridge(&key)?;
1389            let directory = std::env::temp_dir()
1390                .join(format!("hyprstream-r3-{}", hex::encode(&fresh_key()[..8])));
1391            std::fs::create_dir(&directory)?;
1392            let path = directory.join("rpc.sock");
1393            let config = crate::transport::TransportConfig::ipc(path.clone());
1394            let shutdown = Arc::new(tokio::sync::Notify::new());
1395            struct StopOnDrop(Arc<tokio::sync::Notify>);
1396            impl Drop for StopOnDrop {
1397                fn drop(&mut self) {
1398                    self.0.notify_one();
1399                }
1400            }
1401            let _stop = StopOnDrop(shutdown.clone());
1402            let (ready, readiness) = tokio::sync::oneshot::channel();
1403            let processor = bridge.clone();
1404            let signing_key = key.clone();
1405            let stopped = shutdown.clone();
1406            let task = tokio::spawn(async move {
1407                crate::service::serve::serve_bridged(
1408                    &config,
1409                    processor,
1410                    signing_key,
1411                    stopped,
1412                    Some(ready),
1413                )
1414                .await
1415            });
1416            tokio::time::timeout(Duration::from_secs(5), readiness).await??;
1417            exercise_transport_bridge_deadline(
1418                crate::transport::lazy_uds::LazyUdsTransport::new(path),
1419                &key,
1420                bridge,
1421                entered,
1422                release,
1423                dropped,
1424                forced,
1425                move |_| async move {
1426                    shutdown.notify_one();
1427                    task.await??;
1428                    Ok(())
1429                },
1430            )
1431            .await?;
1432        }
1433        Ok(())
1434    }
1435
1436    #[tokio::test(start_paused = true)]
1437    async fn transport_shared_deadline_does_not_reset_for_held_response_permits() -> Result<()> {
1438        // Hold real drain permits independently of bridge completion, as a
1439        // stuck response write can do. A later carrier cannot get a new 40s.
1440        let quinn_limit = Arc::new(Semaphore::new(1));
1441        let uds_limit = Arc::new(Semaphore::new(1));
1442        let _quinn_permit = quinn_limit.clone().acquire_owned().await?;
1443        let _uds_permit = uds_limit.clone().acquire_owned().await?;
1444        let deadline = tokio::time::Instant::now() + super::super::rpc_session::DRAIN_TIMEOUT;
1445        tokio::time::advance(Duration::from_secs(30)).await;
1446        let quinn_token = CancellationToken::new();
1447        let uds_token = CancellationToken::new();
1448        let quinn_stopped = quinn_token.clone();
1449        let uds_stopped = uds_token.clone();
1450        let task = tokio::spawn(async move {
1451            tokio::join!(
1452                crate::transport::quinn_transport::QuinnRpcServer::shutdown_until(
1453                    &quinn_limit,
1454                    1,
1455                    &quinn_stopped,
1456                    deadline
1457                ),
1458                crate::transport::uds_server::UdsRpcServer::shutdown_until(
1459                    &uds_limit,
1460                    1,
1461                    &uds_stopped,
1462                    deadline
1463                ),
1464            );
1465            assert!(quinn_limit.is_closed() && uds_limit.is_closed());
1466        });
1467        quinn_token.cancelled().await;
1468        uds_token.cancelled().await;
1469        tokio::time::advance(Duration::from_secs(9)).await;
1470        assert!(
1471            !task.is_finished(),
1472            "drain expired before the shared deadline"
1473        );
1474        tokio::time::advance(Duration::from_secs(1)).await;
1475        tokio::task::yield_now().await;
1476        assert!(
1477            task.is_finished(),
1478            "carrier started a fresh grace instead of using the owner's remaining budget"
1479        );
1480        task.await?;
1481        Ok(())
1482    }
1483
1484    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1485    async fn spawn_with_readiness_ok_on_success_err_on_failure() -> Result<()> {
1486        let nonce = Arc::new(crate::envelope::InMemoryNonceCache::new());
1487
1488        // Builder succeeds → readiness resolves Ok and the bridge serves.
1489        let sk = fresh_signing_key();
1490        let (_bridge, ready) = LocalServiceBridge::spawn_with(
1491            "ok",
1492            move || async move { Ok(BridgeEcho::new(sk)) },
1493            Arc::clone(&nonce),
1494            0,
1495        )?;
1496        ready
1497            .await
1498            .map_err(|_| anyhow::anyhow!("readiness dropped"))??;
1499
1500        // Builder fails (e.g. GPU init error) → the error surfaces on readiness,
1501        // not silently as a later channel-closed.
1502        let (_bridge2, ready2) = LocalServiceBridge::spawn_with::<_, _, BridgeEcho>(
1503            "fail",
1504            move || async move { Err(anyhow::anyhow!("boom")) },
1505            nonce,
1506            0,
1507        )?;
1508        let build_result = ready2
1509            .await
1510            .map_err(|_| anyhow::anyhow!("readiness dropped"))?;
1511        assert!(
1512            build_result.is_err(),
1513            "a build failure must surface on the readiness channel"
1514        );
1515        Ok(())
1516    }
1517
1518    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1519    async fn rpc_request_response_round_trip() -> Result<()> {
1520        let processor = from_fn(|req: Bytes| async move {
1521            let mut out = Vec::with_capacity(1 + req.len());
1522            out.push(0xAB);
1523            out.extend_from_slice(&req);
1524            Ok(Bytes::from(out))
1525        });
1526        let rpc_handler =
1527            IrohRpcProtocolHandler::new(processor, fresh_signing_key()).with_test_trusted_carrier();
1528
1529        let server =
1530            IrohSubstrate::new_test(fresh_key(), NoopHandler::new("moq-not-wired"), rpc_handler)
1531                .await?;
1532        let server_addr = direct_addr(&server);
1533
1534        let client = IrohSubstrate::new_test(
1535            fresh_key(),
1536            NoopHandler::new("client moq"),
1537            NoopHandler::new("client rpc"),
1538        )
1539        .await?;
1540
1541        let conn = client.connect(server_addr, ALPN_HYPRSTREAM_RPC).await?;
1542        let resp = client_request(&conn, b"ping").await?;
1543        assert_eq!(&resp[..], b"\xABping");
1544
1545        let conn2 = client.connect(direct_addr(&server), ALPN_MOQ_LITE).await?;
1546        drop(conn2);
1547
1548        // Release every caller-owned connection before draining the routers.
1549        // Keeping this RPC handle alive races noq's driver teardown and can
1550        // intermittently trip its GSO-batch alignment assertion.
1551        drop(conn);
1552        client.shutdown().await?;
1553        server.shutdown().await?;
1554        Ok(())
1555    }
1556
1557    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1558    async fn rpc_concurrent_requests() -> Result<()> {
1559        let processor = from_fn(|req: Bytes| async move {
1560            let mut out = req.to_vec();
1561            out.push(b'!');
1562            Ok(Bytes::from(out))
1563        });
1564        let server = IrohSubstrate::new_test(
1565            fresh_key(),
1566            NoopHandler::new("moq"),
1567            IrohRpcProtocolHandler::new(processor, fresh_signing_key()).with_test_trusted_carrier(),
1568        )
1569        .await?;
1570        let server_addr = direct_addr(&server);
1571
1572        let client = IrohSubstrate::new_test(
1573            fresh_key(),
1574            NoopHandler::new("c-moq"),
1575            NoopHandler::new("c-rpc"),
1576        )
1577        .await?;
1578        let conn = Arc::new(client.connect(server_addr, ALPN_HYPRSTREAM_RPC).await?);
1579
1580        let mut handles = Vec::new();
1581        for i in 0..8u8 {
1582            let conn = Arc::clone(&conn);
1583            handles.push(tokio::spawn(async move {
1584                let resp = client_request(&conn, &[i]).await?;
1585                assert_eq!(&resp[..], &[i, b'!']);
1586                anyhow::Ok(())
1587            }));
1588        }
1589        for h in handles {
1590            h.await??;
1591        }
1592
1593        client.shutdown().await?;
1594        server.shutdown().await?;
1595        Ok(())
1596    }
1597
1598    /// Processor returns `Err` → client gets a parseable signed
1599    /// `ResponseEnvelope` carrying the error message (request_id = 0), not
1600    /// an opaque EOF / Cap'n Proto parse failure. (Fix for review finding #4.)
1601    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1602    async fn rpc_processor_error_yields_parseable_envelope() -> Result<()> {
1603        let server_signing = fresh_signing_key();
1604        let server_vk = server_signing.verifying_key();
1605
1606        let processor =
1607            from_fn(|_req: Bytes| async move { Err(anyhow::anyhow!("boom from processor")) });
1608        let handler =
1609            IrohRpcProtocolHandler::new(processor, server_signing).with_test_trusted_carrier();
1610
1611        let server = IrohSubstrate::new_test(fresh_key(), NoopHandler::new("moq"), handler).await?;
1612        let server_addr = direct_addr(&server);
1613
1614        let client = IrohSubstrate::new_test(
1615            fresh_key(),
1616            NoopHandler::new("c-moq"),
1617            NoopHandler::new("c-rpc"),
1618        )
1619        .await?;
1620        let conn = client.connect(server_addr, ALPN_HYPRSTREAM_RPC).await?;
1621
1622        // Send any bytes; the processor will Err regardless. The interesting
1623        // assertion is what comes back: a verifiable ResponseEnvelope.
1624        let resp_bytes = client_request(&conn, b"anything").await?;
1625        let (request_id, payload) =
1626            crate::envelope::unwrap_response(&resp_bytes, Some(&server_vk))?;
1627        assert_eq!(request_id, 0, "error envelope uses request_id=0");
1628        let body = std::str::from_utf8(&payload)?;
1629        assert!(body.contains("boom from processor"), "got: {body}");
1630
1631        client.shutdown().await?;
1632        server.shutdown().await?;
1633        Ok(())
1634    }
1635
1636    /// Concurrency cap is enforced: with a stream_limit of 2, a third
1637    /// concurrent slow request must wait for one of the first two to finish.
1638    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1639    async fn rpc_stream_limit_enforced() -> Result<()> {
1640        use std::sync::atomic::{AtomicUsize, Ordering};
1641
1642        let in_flight = Arc::new(AtomicUsize::new(0));
1643        let peak = Arc::new(AtomicUsize::new(0));
1644        let in_flight_c = Arc::clone(&in_flight);
1645        let peak_c = Arc::clone(&peak);
1646
1647        let processor = from_fn(move |_req: Bytes| {
1648            let in_flight = Arc::clone(&in_flight_c);
1649            let peak = Arc::clone(&peak_c);
1650            async move {
1651                let cur = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
1652                // Update peak.
1653                let mut p = peak.load(Ordering::SeqCst);
1654                while cur > p {
1655                    match peak.compare_exchange_weak(p, cur, Ordering::SeqCst, Ordering::SeqCst) {
1656                        Ok(_) => break,
1657                        Err(prev) => p = prev,
1658                    }
1659                }
1660                tokio::time::sleep(Duration::from_millis(200)).await;
1661                in_flight.fetch_sub(1, Ordering::SeqCst);
1662                Ok(Bytes::from_static(b"ok"))
1663            }
1664        });
1665        let handler =
1666            IrohRpcProtocolHandler::with_stream_limit(Arc::new(processor), fresh_signing_key(), 2)
1667                .with_test_trusted_carrier();
1668
1669        let server = IrohSubstrate::new_test(fresh_key(), NoopHandler::new("moq"), handler).await?;
1670        let server_addr = direct_addr(&server);
1671
1672        let client = IrohSubstrate::new_test(
1673            fresh_key(),
1674            NoopHandler::new("c-moq"),
1675            NoopHandler::new("c-rpc"),
1676        )
1677        .await?;
1678        let conn = Arc::new(client.connect(server_addr, ALPN_HYPRSTREAM_RPC).await?);
1679
1680        let mut handles = Vec::new();
1681        for _ in 0..6 {
1682            let conn = Arc::clone(&conn);
1683            handles.push(tokio::spawn(async move {
1684                let _ = client_request(&conn, b"x").await?;
1685                anyhow::Ok(())
1686            }));
1687        }
1688        for h in handles {
1689            h.await??;
1690        }
1691
1692        assert!(
1693            peak.load(Ordering::SeqCst) <= 2,
1694            "stream_limit=2 violated, observed peak={}",
1695            peak.load(Ordering::SeqCst)
1696        );
1697
1698        client.shutdown().await?;
1699        server.shutdown().await?;
1700        Ok(())
1701    }
1702
1703    /// #159: a stalled (slowloris) stream that holds a permit without sending
1704    /// FIN must be abandoned after the read timeout, releasing its permit so a
1705    /// well-behaved request can still be served. With `stream_limit=1`, the
1706    /// stall grabs the only permit; without the read-timeout fix the normal
1707    /// request would block forever and this test would hang.
1708    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1709    async fn rpc_slowloris_stream_is_timed_out_and_permit_released() -> Result<()> {
1710        let processor = from_fn(move |_req: Bytes| async move { Ok(Bytes::from_static(b"ok")) });
1711        let handler =
1712            IrohRpcProtocolHandler::with_stream_limit(Arc::new(processor), fresh_signing_key(), 1)
1713                .with_test_trusted_carrier()
1714                .with_read_timeout(Duration::from_millis(300));
1715
1716        let server = IrohSubstrate::new_test(fresh_key(), NoopHandler::new("moq"), handler).await?;
1717        let server_addr = direct_addr(&server);
1718
1719        let client = IrohSubstrate::new_test(
1720            fresh_key(),
1721            NoopHandler::new("c-moq"),
1722            NoopHandler::new("c-rpc"),
1723        )
1724        .await?;
1725        let conn = client.connect(server_addr, ALPN_HYPRSTREAM_RPC).await?;
1726
1727        // Slowloris: open a bidi stream, dribble one byte, never finish.
1728        let (mut stall_send, _stall_recv) = conn.open_bi().await.context("open_bi stall")?;
1729        stall_send
1730            .write_all(b"x")
1731            .await
1732            .context("write stall byte")?;
1733        // Deliberately DO NOT call finish(); keep the stream (and its server-side
1734        // permit) alive. Hold the handle so it isn't dropped/reset early.
1735
1736        // A normal request must still succeed once the stalled stream's read
1737        // times out (~300ms) and its permit is released. Bound the wait well
1738        // above the read timeout but far below "forever".
1739        let resp = tokio::time::timeout(Duration::from_secs(5), client_request(&conn, b"req"))
1740            .await
1741            .context("normal request hung — permit was not released (slowloris not bounded)")??;
1742        assert_eq!(&resp[..], b"ok");
1743
1744        drop(stall_send);
1745        client.shutdown().await?;
1746        server.shutdown().await?;
1747        Ok(())
1748    }
1749
1750    /// `Router::shutdown` (via `IrohSubstrate::shutdown`) drains in-flight
1751    /// requests before returning — a request in-progress when shutdown
1752    /// starts still gets a clean response. (Fix for review round-1 #1.)
1753    ///
1754    /// Uses an explicit `Notify` to synchronise "processor has entered the
1755    /// long sleep" with "now safe to shut down" — replaces the previous
1756    /// 50ms `tokio::sleep` that was flaky on loaded CI runners.
1757    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1758    async fn rpc_shutdown_drains_in_flight() -> Result<()> {
1759        let entered = Arc::new(tokio::sync::Notify::new());
1760        let entered_c = Arc::clone(&entered);
1761        let processor = from_fn(move |_req: Bytes| {
1762            let entered = Arc::clone(&entered_c);
1763            async move {
1764                entered.notify_one();
1765                tokio::time::sleep(Duration::from_millis(300)).await;
1766                Ok(Bytes::from_static(b"drained-ok"))
1767            }
1768        });
1769        let handler =
1770            IrohRpcProtocolHandler::new(processor, fresh_signing_key()).with_test_trusted_carrier();
1771
1772        let server = IrohSubstrate::new_test(fresh_key(), NoopHandler::new("moq"), handler).await?;
1773        let server_addr = direct_addr(&server);
1774
1775        let client = IrohSubstrate::new_test(
1776            fresh_key(),
1777            NoopHandler::new("c-moq"),
1778            NoopHandler::new("c-rpc"),
1779        )
1780        .await?;
1781        let conn = client.connect(server_addr, ALPN_HYPRSTREAM_RPC).await?;
1782
1783        let req_task = {
1784            let conn = conn.clone();
1785            tokio::spawn(async move { client_request(&conn, b"slow").await })
1786        };
1787
1788        // Synchronise: wait until the processor signals it has entered the
1789        // sleep, *then* shut down. No wall-clock guesses.
1790        tokio::time::timeout(Duration::from_secs(5), entered.notified())
1791            .await
1792            .context("processor was never entered before shutdown")?;
1793        server.shutdown().await?;
1794
1795        let resp = req_task.await??;
1796        assert_eq!(&resp[..], b"drained-ok");
1797
1798        client.shutdown().await?;
1799        Ok(())
1800    }
1801}