Skip to main content

hyprstream_rpc/transport/
lazy_iroh.rs

1//! Lazy-connecting iroh RPC transport (optional NAT-traversing dial).
2//!
3//! Mirrors [`LazyQuinnTransport`](super::lazy_quinn::LazyQuinnTransport): the
4//! [`dial`](crate::dial::dial) factory is synchronous, so this holds the dial
5//! target (the peer's `EndpointId` + addresses) and connects on the **first
6//! `send()`**, caching the session — preserving sync, zero-I/O construction.
7//!
8//! # The shared client endpoint
9//!
10//! Unlike quinn (where `connect_pinned_sha256` builds its own one-shot client),
11//! iroh dials from a long-lived [`iroh::Endpoint`] that holds its transport
12//! secret and bound sockets. There is exactly one client endpoint per
13//! process; it is installed once at startup via
14//! [`install_iroh_client_endpoint`] (the daemon provisions it during bootstrap),
15//! the same install-once pattern as the inproc registry and the envelope verify
16//! config. Until it is installed, the iroh dial arm errors loudly rather than
17//! silently falling back.
18//!
19//! # Target integrity
20//!
21//! iroh binds the connection to the peer's `EndpointId` (its Ed25519 public
22//! key), proving the dial reached the configured carrier address. This is path
23//! integrity only. It does not authenticate a DID, admission subject, assurance
24//! level, response key, or authorization identity (#1031).
25//!
26//! Re-dial/self-heal semantics match `LazyQuinnTransport`: a per-request timeout
27//! keeps the session; a transport-fatal error drops it so the next call
28//! re-dials. Refined liveness-based re-dial + backoff is #156's job.
29
30use std::net::SocketAddr;
31use std::sync::OnceLock;
32use std::time::Duration;
33
34use anyhow::{anyhow, bail, Result};
35use async_trait::async_trait;
36use tokio::sync::Mutex;
37
38use crate::transport::backoff::LazyState;
39use crate::transport::iroh_substrate::{OwnedIrohClientEndpoint, ALPN_HYPRSTREAM_RPC};
40use crate::transport::iroh_transport::{IrohPendingStream, IrohPublishStub, IrohTransport};
41use crate::transport_traits::Transport;
42
43/// Availability failures before a request is dispatched. This deliberately
44/// excludes malformed addresses, missing client setup, peer authentication,
45/// ALPN rejection, and all errors after connection establishment.
46#[derive(Debug, thiserror::Error)]
47pub enum IrohPeerUnavailable {
48    #[error("iroh connect timed out")]
49    ConnectTimeout,
50    #[error("iroh peer is in reconnect backoff")]
51    ReconnectBackoff,
52    #[error("iroh peer address discovery is not available yet")]
53    AddressDiscovery(#[source] iroh::endpoint::ConnectError),
54}
55
56/// Default per-request deadline when the caller passes `None`.
57const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
58
59/// Ceiling on the lazy connect (iroh dial + handshake). The per-request deadline
60/// wraps only the request; without this an unreachable peer would hang the first
61/// `send()` indefinitely.
62const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
63
64/// Process-wide client iroh endpoint, installed once at startup.
65static IROH_CLIENT_ENDPOINT: OnceLock<iroh::Endpoint> = OnceLock::new();
66
67/// Install the process-global client iroh endpoint used to
68/// originate outbound RPC dials. First-write-wins (mirrors
69/// `install_verify_config`); returns `Err(endpoint)` if one is already set.
70///
71/// In production the authenticated OS-owned process bootstrap calls this
72/// FIRST, with a distinct outbound-only carrier endpoint (retained in
73/// `PROCESS_BOOTSTRAP_CARRIER`, transport purpose key
74/// `hyprstream-bootstrap-client-transport-v1`). A service that later binds
75/// its own inbound substrate therefore receives `Err(returned_capability)`;
76/// that is valid and expected — it keeps its substrate as the independent
77/// inbound owner and leaves the global endpoint untouched. The global
78/// endpoint is the outbound dialer only: it is not a DID, subject, tenant,
79/// response-signing key, or authorization authority. The capability can only
80/// be obtained from
81/// [`crate::transport::iroh_substrate::IrohSubstrate::owned_client_endpoint`],
82/// which proves the endpoint was bound with the provider configured for N0
83/// interoperability. Owned Hyprstream ALPNs still require the hybrid group at
84/// the completed-carrier admission hook, while public relay/pkarr HTTPS needs
85/// the provider's X25519 fallback.
86///
87/// Arbitrary already-bound endpoints are rejected by the type system:
88///
89/// ```compile_fail
90/// # let endpoint: iroh::Endpoint = todo!();
91/// hyprstream_rpc::transport::lazy_iroh::install_iroh_client_endpoint(endpoint);
92/// ```
93///
94/// The opaque capability itself also cannot be forged from a raw endpoint:
95///
96/// ```compile_fail
97/// # let endpoint: iroh::Endpoint = todo!();
98/// let _owned =
99///     hyprstream_rpc::transport::iroh_substrate::OwnedIrohClientEndpoint(endpoint);
100/// ```
101pub fn install_iroh_client_endpoint(
102    endpoint: OwnedIrohClientEndpoint,
103) -> Result<(), OwnedIrohClientEndpoint> {
104    endpoint.install_into(&IROH_CLIENT_ENDPOINT)
105}
106
107/// The installed client endpoint, cloned (cheap — iroh `Endpoint` is `Arc`-backed).
108///
109/// Exposed crate-internally so the streaming-plane dialer
110/// ([`crate::dial::dial_stream`]'s iroh arm, #282) can reuse the same install-once
111/// endpoint the RPC plane dials from.
112pub(crate) fn iroh_client_endpoint() -> Option<iroh::Endpoint> {
113    IROH_CLIENT_ENDPOINT.get().cloned()
114}
115
116/// An iroh RPC transport that connects on first use and caches the session.
117pub struct LazyIrohTransport {
118    node_id: [u8; 32],
119    direct_addrs: Vec<SocketAddr>,
120    relay_url: Option<String>,
121    /// Cached session + reconnect backoff (#156).
122    state: Mutex<LazyState<IrohTransport>>,
123    /// Accessed only while holding `state`: a later backoff must not turn a
124    /// prior authentication/configuration rejection into availability.
125    availability_backoff: std::sync::atomic::AtomicBool,
126    /// Test-only dial endpoint override.
127    ///
128    /// Production always dials from the process-global install-once endpoint
129    /// ([`iroh_client_endpoint`]); there is exactly one tokio runtime for the
130    /// whole process, so that endpoint lives for the process lifetime.
131    ///
132    /// Under `cargo test`, however, every `#[tokio::test]` spins up and tears
133    /// down its OWN runtime. iroh's `Endpoint` spawns its magicsock/router
134    /// background tasks on the runtime active at `bind()`, so the install-once
135    /// global endpoint dies with whichever test's runtime installed it. A later
136    /// test that reuses the (now dead) global hits iroh's
137    /// `RemoteStateActorStopped` → "Internal consistency error". Injecting a
138    /// per-test endpoint that lives on the *current* test's runtime makes the
139    /// dialing tests deterministic regardless of order (and lets them avoid
140    /// polluting the global at all).
141    #[cfg(test)]
142    endpoint_override: Option<iroh::Endpoint>,
143}
144
145impl LazyIrohTransport {
146    /// Create a lazy transport for the iroh peer identified by `node_id`. No
147    /// connection is made until the first `send()`.
148    pub fn new(
149        node_id: [u8; 32],
150        direct_addrs: Vec<SocketAddr>,
151        relay_url: Option<String>,
152    ) -> Self {
153        Self {
154            node_id,
155            direct_addrs,
156            relay_url,
157            state: Mutex::new(LazyState::default()),
158            availability_backoff: std::sync::atomic::AtomicBool::new(false),
159            #[cfg(test)]
160            endpoint_override: None,
161        }
162    }
163
164    /// Test-only constructor that dials from a caller-supplied endpoint instead
165    /// of the process-global one. See [`Self::endpoint_override`] for why tests
166    /// need this (per-test tokio runtimes vs. an install-once global endpoint).
167    #[cfg(test)]
168    pub(crate) fn new_with_endpoint(
169        node_id: [u8; 32],
170        direct_addrs: Vec<SocketAddr>,
171        relay_url: Option<String>,
172        endpoint: iroh::Endpoint,
173    ) -> Self {
174        Self {
175            node_id,
176            direct_addrs,
177            relay_url,
178            state: Mutex::new(LazyState::default()),
179            availability_backoff: std::sync::atomic::AtomicBool::new(false),
180            endpoint_override: Some(endpoint),
181        }
182    }
183
184    /// The endpoint to dial from: the injected per-test endpoint when present,
185    /// otherwise the process-global install-once client endpoint. In production
186    /// builds this is exactly [`iroh_client_endpoint`].
187    fn dial_endpoint(&self) -> Option<iroh::Endpoint> {
188        #[cfg(test)]
189        if let Some(ep) = self.endpoint_override.clone() {
190            return Some(ep);
191        }
192        iroh_client_endpoint()
193    }
194
195    /// Build the iroh `EndpointAddr` for the peer from the stored primitives.
196    fn endpoint_addr(&self) -> Result<iroh::EndpointAddr> {
197        use iroh::{EndpointAddr, EndpointId, TransportAddr};
198        let id = EndpointId::from_bytes(&self.node_id)
199            .map_err(|e| anyhow!("invalid iroh node_id: {e}"))?;
200        let mut transport_addrs: Vec<TransportAddr> = self
201            .direct_addrs
202            .iter()
203            .copied()
204            .map(TransportAddr::Ip)
205            .collect();
206        if let Some(url) = &self.relay_url {
207            let relay = url
208                .parse()
209                .map_err(|e| anyhow!("invalid iroh relay_url '{url}': {e}"))?;
210            transport_addrs.push(TransportAddr::Relay(relay));
211        }
212        Ok(EndpointAddr::from_parts(id, transport_addrs))
213    }
214
215    /// Return the cached connected transport, dialing once (single-flight under
216    /// the lock) if not yet connected. Backs off after consecutive failures (#156).
217    async fn connected(&self) -> Result<IrohTransport> {
218        let mut guard = self.state.lock().await;
219        if let Some(transport) = guard.cached.as_ref() {
220            return Ok(transport.clone());
221        }
222        if let Some(remaining) = guard.backoff.cooldown_remaining() {
223            return if self
224                .availability_backoff
225                .load(std::sync::atomic::Ordering::Relaxed)
226            {
227                Err(IrohPeerUnavailable::ReconnectBackoff.into())
228            } else {
229                Err(anyhow!(
230                    "iroh peer is in reconnect backoff — retry in {remaining:.1?}"
231                ))
232            };
233        }
234        let endpoint = self.dial_endpoint().ok_or_else(|| {
235            anyhow!(
236                "no iroh client endpoint installed — call \
237                 install_iroh_client_endpoint() at startup before dialing iroh peers"
238            )
239        })?;
240        let addr = self.endpoint_addr()?;
241        match tokio::time::timeout(CONNECT_TIMEOUT, endpoint.connect(addr, ALPN_HYPRSTREAM_RPC))
242            .await
243        {
244            Err(_) => {
245                guard.backoff.record_failure();
246                self.availability_backoff
247                    .store(true, std::sync::atomic::Ordering::Relaxed);
248                Err(IrohPeerUnavailable::ConnectTimeout.into())
249            }
250            Ok(Err(e)) => {
251                guard.backoff.record_failure();
252                let unavailable = matches!(
253                    &e,
254                    iroh::endpoint::ConnectError::Connect {
255                        source: iroh::endpoint::ConnectWithOptsError::NoAddress { .. },
256                        ..
257                    }
258                );
259                self.availability_backoff
260                    .store(unavailable, std::sync::atomic::Ordering::Relaxed);
261                if unavailable {
262                    Err(IrohPeerUnavailable::AddressDiscovery(e).into())
263                } else {
264                    Err(anyhow::Error::new(e).context("iroh connect"))
265                }
266            }
267            Ok(Ok(conn)) => {
268                guard.backoff.record_success();
269                self.availability_backoff
270                    .store(false, std::sync::atomic::Ordering::Relaxed);
271                let transport = IrohTransport::new(conn);
272                guard.cached = Some(transport.clone());
273                Ok(transport)
274            }
275        }
276    }
277
278    /// Drop the cached session and record a failure so the next `send()`
279    /// re-dials after the backoff cooldown (#156).
280    async fn invalidate(&self) {
281        let mut guard = self.state.lock().await;
282        guard.cached = None;
283        guard.backoff.record_failure();
284        self.availability_backoff
285            .store(false, std::sync::atomic::Ordering::Relaxed);
286    }
287}
288
289#[async_trait]
290impl Transport for LazyIrohTransport {
291    type Sub = IrohPendingStream;
292    type Pub = IrohPublishStub;
293
294    async fn send(&self, payload: Vec<u8>, timeout_ms: Option<i32>) -> Result<Vec<u8>> {
295        let transport = self
296            .connected()
297            .await
298            .map_err(crate::transport_traits::PreDispatchTransportError::new)?;
299
300        // Our deadline is authoritative so a per-request timeout (busy peer —
301        // keep the session) is distinguishable from a transport-fatal error
302        // (re-dial), matching LazyQuinnTransport.
303        let deadline = timeout_ms
304            .map(|ms| Duration::from_millis(ms.max(0) as u64))
305            .unwrap_or(DEFAULT_TIMEOUT);
306        let inner_ceiling_ms = deadline.as_millis().saturating_mul(2).min(i32::MAX as u128) as i32;
307
308        match tokio::time::timeout(deadline, transport.send(payload, Some(inner_ceiling_ms))).await
309        {
310            Err(_elapsed) => Err(anyhow!("iroh RPC timeout after {deadline:?}")),
311            Ok(Ok(resp)) => Ok(resp),
312            Ok(Err(e)) => {
313                self.invalidate().await;
314                Err(e)
315            }
316        }
317    }
318
319    fn forbids_cleartext_envelope(&self) -> bool {
320        true
321    }
322
323    async fn subscribe(&self, _topic: &[u8]) -> Result<Self::Sub> {
324        bail!("lazy iroh RPC transport does not support SUB — streaming is on the moq plane")
325    }
326
327    async fn publish(&self, _topic: &[u8]) -> Result<Self::Pub> {
328        bail!("lazy iroh RPC transport does not support PUB — streaming is on the moq plane")
329    }
330}
331
332#[cfg(test)]
333#[allow(clippy::unwrap_used, clippy::expect_used)]
334mod tests {
335    use super::*;
336    use crate::transport::iroh_substrate::{EchoHandler, IrohSubstrate, NoopHandler};
337
338    fn fresh_key() -> [u8; 32] {
339        use rand::RngCore;
340        let mut k = [0u8; 32];
341        rand::thread_rng().fill_bytes(&mut k);
342        k
343    }
344
345    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
346    async fn lazy_iroh_absent_peer_is_typed_availability_before_dispatch() {
347        let client = IrohSubstrate::new_test(
348            fresh_key(),
349            NoopHandler::new("client moq"),
350            NoopHandler::new("client rpc"),
351        )
352        .await
353        .unwrap();
354        let peer = ed25519_dalek::SigningKey::from_bytes(&fresh_key())
355            .verifying_key()
356            .to_bytes();
357        // Minimal test carrier has neither a relay nor address lookup. No
358        // server exists, so these bytes can never have reached a handler.
359        let transport =
360            LazyIrohTransport::new_with_endpoint(peer, vec![], None, client.endpoint().clone());
361        let error = transport
362            .send(b"not-dispatched".to_vec(), Some(100))
363            .await
364            .unwrap_err();
365        assert!(crate::transport_traits::is_pre_dispatch_transport_error(
366            &error
367        ));
368        assert!(error
369            .chain()
370            .any(|cause| cause.downcast_ref::<IrohPeerUnavailable>().is_some()));
371        assert!(transport.state.lock().await.cached.is_none());
372        // Backoff preserves its explicit availability provenance.
373        let error = transport
374            .send(b"still-not-dispatched".to_vec(), Some(100))
375            .await
376            .unwrap_err();
377        assert!(error.chain().any(|cause| matches!(
378            cause.downcast_ref::<IrohPeerUnavailable>(),
379            Some(IrohPeerUnavailable::ReconnectBackoff)
380        )));
381        drop(transport);
382        client.shutdown().await.unwrap();
383    }
384
385    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
386    async fn lazy_iroh_connects_on_first_send_and_caches() {
387        // Echo server substrate.
388        let server = IrohSubstrate::new_test(fresh_key(), EchoHandler, EchoHandler)
389            .await
390            .unwrap();
391        let server_id = server.endpoint_id();
392        let direct: Vec<SocketAddr> = server.endpoint().bound_sockets().into_iter().collect();
393
394        // Client substrate (originates only). Dial from THIS test's endpoint via
395        // `new_with_endpoint` rather than the process-global install-once one: the
396        // global is bound to whichever test's tokio runtime installed it and dies
397        // with that runtime, which made this test flake with iroh's "Internal
398        // consistency error" (RemoteStateActorStopped) when it ran after another
399        // iroh test. A per-test endpoint on the current runtime is deterministic.
400        let client = IrohSubstrate::new_test(
401            fresh_key(),
402            NoopHandler::new("client moq"),
403            NoopHandler::new("client rpc"),
404        )
405        .await
406        .unwrap();
407
408        let node_id: [u8; 32] = *server_id.as_bytes();
409        let t =
410            LazyIrohTransport::new_with_endpoint(node_id, direct, None, client.endpoint().clone());
411        assert!(
412            t.state.lock().await.cached.is_none(),
413            "no connection before first send"
414        );
415
416        // EchoHandler echoes the bytes written on the bidi stream, which is the
417        // same shape SessionRpcTransport::send uses (write payload → read to
418        // FIN), so the lazy transport round-trips them. (EchoHandler is a
419        // one-shot smoke handler — it closes the connection after a single
420        // stream, so we don't test session *reuse* here; the multiplexed-stream
421        // path is covered by quinn_transport's round-trip test against a real
422        // QuinnRpcServer, which IrohTransport shares via SessionRpcTransport.)
423        let resp = t.send(b"ping".to_vec(), Some(4_000)).await.unwrap();
424        assert_eq!(resp, b"ping");
425        assert!(
426            t.state.lock().await.cached.is_some(),
427            "session cached after first send"
428        );
429
430        // Release the cached connection and endpoint clone before draining both
431        // routers. Abruptly dropping the client substrate while `t` still held a
432        // live connection left noq driver tasks racing runtime teardown, which
433        // intermittently triggered its GSO-batch assertion in the full suite.
434        drop(t);
435        client.shutdown().await.unwrap();
436        server.shutdown().await.unwrap();
437    }
438
439    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
440    async fn wrong_node_id_rejected() {
441        // Server we'll dial the *address* of, but with a different identity.
442        let server = IrohSubstrate::new_test(fresh_key(), EchoHandler, EchoHandler)
443            .await
444            .unwrap();
445        let direct: Vec<SocketAddr> = server.endpoint().bound_sockets().into_iter().collect();
446
447        // Dial from this test's own endpoint (see `lazy_iroh_connects_*` above and
448        // `endpoint_override`'s docs): the install-once global is bound to another
449        // test's runtime and would flake with "Internal consistency error".
450        let client = IrohSubstrate::new_test(
451            fresh_key(),
452            NoopHandler::new("client moq"),
453            NoopHandler::new("client rpc"),
454        )
455        .await
456        .unwrap();
457
458        // A *valid* but wrong EndpointId (a third node's), so this tests handshake
459        // identity rejection — not a malformed key.
460        let other =
461            IrohSubstrate::new_test(fresh_key(), NoopHandler::new("o1"), NoopHandler::new("o2"))
462                .await
463                .unwrap();
464        let wrong_id: [u8; 32] = *other.endpoint_id().as_bytes();
465
466        let t =
467            LazyIrohTransport::new_with_endpoint(wrong_id, direct, None, client.endpoint().clone());
468        let res = tokio::time::timeout(Duration::from_secs(8), t.send(b"x".to_vec(), Some(3_000)))
469            .await
470            .expect("send must complete (with an error) — wrong identity should reject, not hang");
471        let error = res.expect_err("dialing a server's address under a wrong EndpointId must fail");
472        assert!(
473            !error
474                .chain()
475                .any(|cause| cause.downcast_ref::<IrohPeerUnavailable>().is_some()),
476            "a peer-identity handshake rejection must not become retryable availability"
477        );
478        assert!(
479            t.state.lock().await.cached.is_none(),
480            "a failed handshake caches nothing"
481        );
482        let backoff_error = t
483            .send(b"x".to_vec(), Some(3_000))
484            .await
485            .expect_err("rejection backoff");
486        assert!(
487            !backoff_error
488                .chain()
489                .any(|cause| cause.downcast_ref::<IrohPeerUnavailable>().is_some()),
490            "backoff must retain the prior permanent failure classification"
491        );
492
493        // As above, release the transport's endpoint clone before cleanly
494        // draining every substrate owned by this test.
495        drop(t);
496        client.shutdown().await.unwrap();
497        other.shutdown().await.unwrap();
498        server.shutdown().await.unwrap();
499    }
500
501    #[tokio::test]
502    async fn subscribe_publish_bail() {
503        let t = LazyIrohTransport::new([0u8; 32], vec![], None);
504        assert!(t.subscribe(b"topic").await.is_err());
505        assert!(t.publish(b"topic").await.is_err());
506    }
507}