hyprstream_rpc/transport/lazy_iroh.rs
1//! Lazy-connecting iroh RPC transport (optional NAT-traversing dial).
2//!
3//! Mirrors [`LazyQuinnTransport`](super::lazy_quinn::LazyQuinnTransport): the
4//! [`dial`](crate::dial::dial) factory is synchronous, so this holds the dial
5//! target (the peer's `EndpointId` + addresses) and connects on the **first
6//! `send()`**, caching the session — preserving sync, zero-I/O construction.
7//!
8//! # The shared client endpoint
9//!
10//! Unlike quinn (where `connect_pinned_sha256` builds its own one-shot client),
11//! iroh dials from a long-lived [`iroh::Endpoint`] that holds its transport
12//! secret and bound sockets. There is exactly one client endpoint per
13//! process; it is installed once at startup via
14//! [`install_iroh_client_endpoint`] (the daemon provisions it during bootstrap),
15//! the same install-once pattern as the inproc registry and the envelope verify
16//! config. Until it is installed, the iroh dial arm errors loudly rather than
17//! silently falling back.
18//!
19//! # Target integrity
20//!
21//! iroh binds the connection to the peer's `EndpointId` (its Ed25519 public
22//! key), proving the dial reached the configured carrier address. This is path
23//! integrity only. It does not authenticate a DID, admission subject, assurance
24//! level, response key, or authorization identity (#1031).
25//!
26//! Re-dial/self-heal semantics match `LazyQuinnTransport`: a per-request timeout
27//! keeps the session; a transport-fatal error drops it so the next call
28//! re-dials. Refined liveness-based re-dial + backoff is #156's job.
29
30use std::net::SocketAddr;
31use std::sync::OnceLock;
32use std::time::Duration;
33
34use anyhow::{anyhow, bail, Result};
35use async_trait::async_trait;
36use tokio::sync::Mutex;
37
38use crate::transport::backoff::LazyState;
39use crate::transport::iroh_substrate::{OwnedIrohClientEndpoint, ALPN_HYPRSTREAM_RPC};
40use crate::transport::iroh_transport::{IrohPendingStream, IrohPublishStub, IrohTransport};
41use crate::transport_traits::Transport;
42
43/// Availability failures before a request is dispatched. This deliberately
44/// excludes malformed addresses, missing client setup, peer authentication,
45/// ALPN rejection, and all errors after connection establishment.
46#[derive(Debug, thiserror::Error)]
47pub enum IrohPeerUnavailable {
48 #[error("iroh connect timed out")]
49 ConnectTimeout,
50 #[error("iroh peer is in reconnect backoff")]
51 ReconnectBackoff,
52 #[error("iroh peer address discovery is not available yet")]
53 AddressDiscovery(#[source] iroh::endpoint::ConnectError),
54}
55
56/// Default per-request deadline when the caller passes `None`.
57const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
58
59/// Ceiling on the lazy connect (iroh dial + handshake). The per-request deadline
60/// wraps only the request; without this an unreachable peer would hang the first
61/// `send()` indefinitely.
62const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
63
64/// Process-wide client iroh endpoint, installed once at startup.
65static IROH_CLIENT_ENDPOINT: OnceLock<iroh::Endpoint> = OnceLock::new();
66
67/// Install the process-global client iroh endpoint used to
68/// originate outbound RPC dials. First-write-wins (mirrors
69/// `install_verify_config`); returns `Err(endpoint)` if one is already set.
70///
71/// In production the authenticated OS-owned process bootstrap calls this
72/// FIRST, with a distinct outbound-only carrier endpoint (retained in
73/// `PROCESS_BOOTSTRAP_CARRIER`, transport purpose key
74/// `hyprstream-bootstrap-client-transport-v1`). A service that later binds
75/// its own inbound substrate therefore receives `Err(returned_capability)`;
76/// that is valid and expected — it keeps its substrate as the independent
77/// inbound owner and leaves the global endpoint untouched. The global
78/// endpoint is the outbound dialer only: it is not a DID, subject, tenant,
79/// response-signing key, or authorization authority. The capability can only
80/// be obtained from
81/// [`crate::transport::iroh_substrate::IrohSubstrate::owned_client_endpoint`],
82/// which proves the endpoint was bound with the provider configured for N0
83/// interoperability. Owned Hyprstream ALPNs still require the hybrid group at
84/// the completed-carrier admission hook, while public relay/pkarr HTTPS needs
85/// the provider's X25519 fallback.
86///
87/// Arbitrary already-bound endpoints are rejected by the type system:
88///
89/// ```compile_fail
90/// # let endpoint: iroh::Endpoint = todo!();
91/// hyprstream_rpc::transport::lazy_iroh::install_iroh_client_endpoint(endpoint);
92/// ```
93///
94/// The opaque capability itself also cannot be forged from a raw endpoint:
95///
96/// ```compile_fail
97/// # let endpoint: iroh::Endpoint = todo!();
98/// let _owned =
99/// hyprstream_rpc::transport::iroh_substrate::OwnedIrohClientEndpoint(endpoint);
100/// ```
101pub fn install_iroh_client_endpoint(
102 endpoint: OwnedIrohClientEndpoint,
103) -> Result<(), OwnedIrohClientEndpoint> {
104 endpoint.install_into(&IROH_CLIENT_ENDPOINT)
105}
106
107/// The installed client endpoint, cloned (cheap — iroh `Endpoint` is `Arc`-backed).
108///
109/// Exposed crate-internally so the streaming-plane dialer
110/// ([`crate::dial::dial_stream`]'s iroh arm, #282) can reuse the same install-once
111/// endpoint the RPC plane dials from.
112pub(crate) fn iroh_client_endpoint() -> Option<iroh::Endpoint> {
113 IROH_CLIENT_ENDPOINT.get().cloned()
114}
115
116/// An iroh RPC transport that connects on first use and caches the session.
117pub struct LazyIrohTransport {
118 node_id: [u8; 32],
119 direct_addrs: Vec<SocketAddr>,
120 relay_url: Option<String>,
121 /// Cached session + reconnect backoff (#156).
122 state: Mutex<LazyState<IrohTransport>>,
123 /// Accessed only while holding `state`: a later backoff must not turn a
124 /// prior authentication/configuration rejection into availability.
125 availability_backoff: std::sync::atomic::AtomicBool,
126 /// Test-only dial endpoint override.
127 ///
128 /// Production always dials from the process-global install-once endpoint
129 /// ([`iroh_client_endpoint`]); there is exactly one tokio runtime for the
130 /// whole process, so that endpoint lives for the process lifetime.
131 ///
132 /// Under `cargo test`, however, every `#[tokio::test]` spins up and tears
133 /// down its OWN runtime. iroh's `Endpoint` spawns its magicsock/router
134 /// background tasks on the runtime active at `bind()`, so the install-once
135 /// global endpoint dies with whichever test's runtime installed it. A later
136 /// test that reuses the (now dead) global hits iroh's
137 /// `RemoteStateActorStopped` → "Internal consistency error". Injecting a
138 /// per-test endpoint that lives on the *current* test's runtime makes the
139 /// dialing tests deterministic regardless of order (and lets them avoid
140 /// polluting the global at all).
141 #[cfg(test)]
142 endpoint_override: Option<iroh::Endpoint>,
143}
144
145impl LazyIrohTransport {
146 /// Create a lazy transport for the iroh peer identified by `node_id`. No
147 /// connection is made until the first `send()`.
148 pub fn new(
149 node_id: [u8; 32],
150 direct_addrs: Vec<SocketAddr>,
151 relay_url: Option<String>,
152 ) -> Self {
153 Self {
154 node_id,
155 direct_addrs,
156 relay_url,
157 state: Mutex::new(LazyState::default()),
158 availability_backoff: std::sync::atomic::AtomicBool::new(false),
159 #[cfg(test)]
160 endpoint_override: None,
161 }
162 }
163
164 /// Test-only constructor that dials from a caller-supplied endpoint instead
165 /// of the process-global one. See [`Self::endpoint_override`] for why tests
166 /// need this (per-test tokio runtimes vs. an install-once global endpoint).
167 #[cfg(test)]
168 pub(crate) fn new_with_endpoint(
169 node_id: [u8; 32],
170 direct_addrs: Vec<SocketAddr>,
171 relay_url: Option<String>,
172 endpoint: iroh::Endpoint,
173 ) -> Self {
174 Self {
175 node_id,
176 direct_addrs,
177 relay_url,
178 state: Mutex::new(LazyState::default()),
179 availability_backoff: std::sync::atomic::AtomicBool::new(false),
180 endpoint_override: Some(endpoint),
181 }
182 }
183
184 /// The endpoint to dial from: the injected per-test endpoint when present,
185 /// otherwise the process-global install-once client endpoint. In production
186 /// builds this is exactly [`iroh_client_endpoint`].
187 fn dial_endpoint(&self) -> Option<iroh::Endpoint> {
188 #[cfg(test)]
189 if let Some(ep) = self.endpoint_override.clone() {
190 return Some(ep);
191 }
192 iroh_client_endpoint()
193 }
194
195 /// Build the iroh `EndpointAddr` for the peer from the stored primitives.
196 fn endpoint_addr(&self) -> Result<iroh::EndpointAddr> {
197 use iroh::{EndpointAddr, EndpointId, TransportAddr};
198 let id = EndpointId::from_bytes(&self.node_id)
199 .map_err(|e| anyhow!("invalid iroh node_id: {e}"))?;
200 let mut transport_addrs: Vec<TransportAddr> = self
201 .direct_addrs
202 .iter()
203 .copied()
204 .map(TransportAddr::Ip)
205 .collect();
206 if let Some(url) = &self.relay_url {
207 let relay = url
208 .parse()
209 .map_err(|e| anyhow!("invalid iroh relay_url '{url}': {e}"))?;
210 transport_addrs.push(TransportAddr::Relay(relay));
211 }
212 Ok(EndpointAddr::from_parts(id, transport_addrs))
213 }
214
215 /// Return the cached connected transport, dialing once (single-flight under
216 /// the lock) if not yet connected. Backs off after consecutive failures (#156).
217 async fn connected(&self) -> Result<IrohTransport> {
218 let mut guard = self.state.lock().await;
219 if let Some(transport) = guard.cached.as_ref() {
220 return Ok(transport.clone());
221 }
222 if let Some(remaining) = guard.backoff.cooldown_remaining() {
223 return if self
224 .availability_backoff
225 .load(std::sync::atomic::Ordering::Relaxed)
226 {
227 Err(IrohPeerUnavailable::ReconnectBackoff.into())
228 } else {
229 Err(anyhow!(
230 "iroh peer is in reconnect backoff — retry in {remaining:.1?}"
231 ))
232 };
233 }
234 let endpoint = self.dial_endpoint().ok_or_else(|| {
235 anyhow!(
236 "no iroh client endpoint installed — call \
237 install_iroh_client_endpoint() at startup before dialing iroh peers"
238 )
239 })?;
240 let addr = self.endpoint_addr()?;
241 match tokio::time::timeout(CONNECT_TIMEOUT, endpoint.connect(addr, ALPN_HYPRSTREAM_RPC))
242 .await
243 {
244 Err(_) => {
245 guard.backoff.record_failure();
246 self.availability_backoff
247 .store(true, std::sync::atomic::Ordering::Relaxed);
248 Err(IrohPeerUnavailable::ConnectTimeout.into())
249 }
250 Ok(Err(e)) => {
251 guard.backoff.record_failure();
252 let unavailable = matches!(
253 &e,
254 iroh::endpoint::ConnectError::Connect {
255 source: iroh::endpoint::ConnectWithOptsError::NoAddress { .. },
256 ..
257 }
258 );
259 self.availability_backoff
260 .store(unavailable, std::sync::atomic::Ordering::Relaxed);
261 if unavailable {
262 Err(IrohPeerUnavailable::AddressDiscovery(e).into())
263 } else {
264 Err(anyhow::Error::new(e).context("iroh connect"))
265 }
266 }
267 Ok(Ok(conn)) => {
268 guard.backoff.record_success();
269 self.availability_backoff
270 .store(false, std::sync::atomic::Ordering::Relaxed);
271 let transport = IrohTransport::new(conn);
272 guard.cached = Some(transport.clone());
273 Ok(transport)
274 }
275 }
276 }
277
278 /// Drop the cached session and record a failure so the next `send()`
279 /// re-dials after the backoff cooldown (#156).
280 async fn invalidate(&self) {
281 let mut guard = self.state.lock().await;
282 guard.cached = None;
283 guard.backoff.record_failure();
284 self.availability_backoff
285 .store(false, std::sync::atomic::Ordering::Relaxed);
286 }
287}
288
289#[async_trait]
290impl Transport for LazyIrohTransport {
291 type Sub = IrohPendingStream;
292 type Pub = IrohPublishStub;
293
294 async fn send(&self, payload: Vec<u8>, timeout_ms: Option<i32>) -> Result<Vec<u8>> {
295 let transport = self
296 .connected()
297 .await
298 .map_err(crate::transport_traits::PreDispatchTransportError::new)?;
299
300 // Our deadline is authoritative so a per-request timeout (busy peer —
301 // keep the session) is distinguishable from a transport-fatal error
302 // (re-dial), matching LazyQuinnTransport.
303 let deadline = timeout_ms
304 .map(|ms| Duration::from_millis(ms.max(0) as u64))
305 .unwrap_or(DEFAULT_TIMEOUT);
306 let inner_ceiling_ms = deadline.as_millis().saturating_mul(2).min(i32::MAX as u128) as i32;
307
308 match tokio::time::timeout(deadline, transport.send(payload, Some(inner_ceiling_ms))).await
309 {
310 Err(_elapsed) => Err(anyhow!("iroh RPC timeout after {deadline:?}")),
311 Ok(Ok(resp)) => Ok(resp),
312 Ok(Err(e)) => {
313 self.invalidate().await;
314 Err(e)
315 }
316 }
317 }
318
319 fn forbids_cleartext_envelope(&self) -> bool {
320 true
321 }
322
323 async fn subscribe(&self, _topic: &[u8]) -> Result<Self::Sub> {
324 bail!("lazy iroh RPC transport does not support SUB — streaming is on the moq plane")
325 }
326
327 async fn publish(&self, _topic: &[u8]) -> Result<Self::Pub> {
328 bail!("lazy iroh RPC transport does not support PUB — streaming is on the moq plane")
329 }
330}
331
332#[cfg(test)]
333#[allow(clippy::unwrap_used, clippy::expect_used)]
334mod tests {
335 use super::*;
336 use crate::transport::iroh_substrate::{EchoHandler, IrohSubstrate, NoopHandler};
337
338 fn fresh_key() -> [u8; 32] {
339 use rand::RngCore;
340 let mut k = [0u8; 32];
341 rand::thread_rng().fill_bytes(&mut k);
342 k
343 }
344
345 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
346 async fn lazy_iroh_absent_peer_is_typed_availability_before_dispatch() {
347 let client = IrohSubstrate::new_test(
348 fresh_key(),
349 NoopHandler::new("client moq"),
350 NoopHandler::new("client rpc"),
351 )
352 .await
353 .unwrap();
354 let peer = ed25519_dalek::SigningKey::from_bytes(&fresh_key())
355 .verifying_key()
356 .to_bytes();
357 // Minimal test carrier has neither a relay nor address lookup. No
358 // server exists, so these bytes can never have reached a handler.
359 let transport =
360 LazyIrohTransport::new_with_endpoint(peer, vec![], None, client.endpoint().clone());
361 let error = transport
362 .send(b"not-dispatched".to_vec(), Some(100))
363 .await
364 .unwrap_err();
365 assert!(crate::transport_traits::is_pre_dispatch_transport_error(
366 &error
367 ));
368 assert!(error
369 .chain()
370 .any(|cause| cause.downcast_ref::<IrohPeerUnavailable>().is_some()));
371 assert!(transport.state.lock().await.cached.is_none());
372 // Backoff preserves its explicit availability provenance.
373 let error = transport
374 .send(b"still-not-dispatched".to_vec(), Some(100))
375 .await
376 .unwrap_err();
377 assert!(error.chain().any(|cause| matches!(
378 cause.downcast_ref::<IrohPeerUnavailable>(),
379 Some(IrohPeerUnavailable::ReconnectBackoff)
380 )));
381 drop(transport);
382 client.shutdown().await.unwrap();
383 }
384
385 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
386 async fn lazy_iroh_connects_on_first_send_and_caches() {
387 // Echo server substrate.
388 let server = IrohSubstrate::new_test(fresh_key(), EchoHandler, EchoHandler)
389 .await
390 .unwrap();
391 let server_id = server.endpoint_id();
392 let direct: Vec<SocketAddr> = server.endpoint().bound_sockets().into_iter().collect();
393
394 // Client substrate (originates only). Dial from THIS test's endpoint via
395 // `new_with_endpoint` rather than the process-global install-once one: the
396 // global is bound to whichever test's tokio runtime installed it and dies
397 // with that runtime, which made this test flake with iroh's "Internal
398 // consistency error" (RemoteStateActorStopped) when it ran after another
399 // iroh test. A per-test endpoint on the current runtime is deterministic.
400 let client = IrohSubstrate::new_test(
401 fresh_key(),
402 NoopHandler::new("client moq"),
403 NoopHandler::new("client rpc"),
404 )
405 .await
406 .unwrap();
407
408 let node_id: [u8; 32] = *server_id.as_bytes();
409 let t =
410 LazyIrohTransport::new_with_endpoint(node_id, direct, None, client.endpoint().clone());
411 assert!(
412 t.state.lock().await.cached.is_none(),
413 "no connection before first send"
414 );
415
416 // EchoHandler echoes the bytes written on the bidi stream, which is the
417 // same shape SessionRpcTransport::send uses (write payload → read to
418 // FIN), so the lazy transport round-trips them. (EchoHandler is a
419 // one-shot smoke handler — it closes the connection after a single
420 // stream, so we don't test session *reuse* here; the multiplexed-stream
421 // path is covered by quinn_transport's round-trip test against a real
422 // QuinnRpcServer, which IrohTransport shares via SessionRpcTransport.)
423 let resp = t.send(b"ping".to_vec(), Some(4_000)).await.unwrap();
424 assert_eq!(resp, b"ping");
425 assert!(
426 t.state.lock().await.cached.is_some(),
427 "session cached after first send"
428 );
429
430 // Release the cached connection and endpoint clone before draining both
431 // routers. Abruptly dropping the client substrate while `t` still held a
432 // live connection left noq driver tasks racing runtime teardown, which
433 // intermittently triggered its GSO-batch assertion in the full suite.
434 drop(t);
435 client.shutdown().await.unwrap();
436 server.shutdown().await.unwrap();
437 }
438
439 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
440 async fn wrong_node_id_rejected() {
441 // Server we'll dial the *address* of, but with a different identity.
442 let server = IrohSubstrate::new_test(fresh_key(), EchoHandler, EchoHandler)
443 .await
444 .unwrap();
445 let direct: Vec<SocketAddr> = server.endpoint().bound_sockets().into_iter().collect();
446
447 // Dial from this test's own endpoint (see `lazy_iroh_connects_*` above and
448 // `endpoint_override`'s docs): the install-once global is bound to another
449 // test's runtime and would flake with "Internal consistency error".
450 let client = IrohSubstrate::new_test(
451 fresh_key(),
452 NoopHandler::new("client moq"),
453 NoopHandler::new("client rpc"),
454 )
455 .await
456 .unwrap();
457
458 // A *valid* but wrong EndpointId (a third node's), so this tests handshake
459 // identity rejection — not a malformed key.
460 let other =
461 IrohSubstrate::new_test(fresh_key(), NoopHandler::new("o1"), NoopHandler::new("o2"))
462 .await
463 .unwrap();
464 let wrong_id: [u8; 32] = *other.endpoint_id().as_bytes();
465
466 let t =
467 LazyIrohTransport::new_with_endpoint(wrong_id, direct, None, client.endpoint().clone());
468 let res = tokio::time::timeout(Duration::from_secs(8), t.send(b"x".to_vec(), Some(3_000)))
469 .await
470 .expect("send must complete (with an error) — wrong identity should reject, not hang");
471 let error = res.expect_err("dialing a server's address under a wrong EndpointId must fail");
472 assert!(
473 !error
474 .chain()
475 .any(|cause| cause.downcast_ref::<IrohPeerUnavailable>().is_some()),
476 "a peer-identity handshake rejection must not become retryable availability"
477 );
478 assert!(
479 t.state.lock().await.cached.is_none(),
480 "a failed handshake caches nothing"
481 );
482 let backoff_error = t
483 .send(b"x".to_vec(), Some(3_000))
484 .await
485 .expect_err("rejection backoff");
486 assert!(
487 !backoff_error
488 .chain()
489 .any(|cause| cause.downcast_ref::<IrohPeerUnavailable>().is_some()),
490 "backoff must retain the prior permanent failure classification"
491 );
492
493 // As above, release the transport's endpoint clone before cleanly
494 // draining every substrate owned by this test.
495 drop(t);
496 client.shutdown().await.unwrap();
497 other.shutdown().await.unwrap();
498 server.shutdown().await.unwrap();
499 }
500
501 #[tokio::test]
502 async fn subscribe_publish_bail() {
503 let t = LazyIrohTransport::new([0u8; 32], vec![], None);
504 assert!(t.subscribe(b"topic").await.is_err());
505 assert!(t.publish(b"topic").await.is_err());
506 }
507}