hyprstream_rpc/transport/iroh_substrate.rs
1//! Iroh transport substrate — the PRIMARY production transport for Epic #131
2//! (ZMQ → moq-net + iroh), promoted to default in #410.
3//!
4//! Builds a shared `iroh::Endpoint` from an Ed25519 node key and wires an
5//! [`iroh::protocol::Router`] with two ALPN slots:
6//!
7//! - [`ALPN_MOQ_LITE`] (`moql`) — moq-net session handler, wired by the daemon
8//! spawner (#282) to dispatch into the shared global `moq_stream::Origin`
9//! for the streaming plane.
10//! - [`ALPN_HYPRSTREAM_RPC`] (`hyprstream-rpc/1`) — raw Cap'n Proto bidi
11//! handler, wired by the daemon spawner (#282) to terminate
12//! `IrohRpcProtocolHandler` for the RPC plane.
13//!
14//! Phase 1 shipped the substrate, a [`NoopHandler`] placeholder, and a smoke
15//! test verifying that two endpoints can dial each other directly and open a
16//! bidi stream on each ALPN. As of #410, production code DOES consume this
17//! module: the daemon spawner (`hyprstream-service::service::spawner`) binds
18//! one `IrohSubstrate` per QUIC-enabled service as the PRIMARY production
19//! endpoint (on by default; `[quic] iroh = false` opts out to quinn-only). The
20//! real `moql` and `hyprstream-rpc/1` handlers are threaded in by the spawner
21//! (#282). OAuth binds a reach-only endpoint whose inbound ALPNs remain refused
22//! until independently verified application/session proof is wired.
23//!
24//! # D3 — iroh pkarr is LIVENESS-ONLY, never an authority source (#895)
25//!
26//! iroh's `presets::N0` discovery publishes classical **pkarr** records —
27//! signed-but-not-encrypted packets — onto the open BitTorrent mainline DHT to
28//! advertise reach (home relay + direct addresses) for a NodeId. As of #895
29//! (epic #880 Track D / D3) this is the contract for pkarr in hyprstream:
30//!
31//! - **Reach hint, not authority.** A pkarr record is an **unverified dial
32//! candidate**: "NodeId N claims it is reachable at relay R / addr A." It is
33//! signed by N's Ed25519 key, so it is integrity-protected *as a reach
34//! claim* — but that says nothing about *which* federated identity (capsule,
35//! `did:web`, `did:key`) controls N, what its PQ key material is, or whether
36//! it may be admitted. **No identity / trust / admission decision is ever
37//! derived from a pkarr record.** `Connection::remote_id()` authenticates only
38//! the carrier endpoint. Application identity requires independently resolved
39//! current keys plus fresh inside-carrier proof (#1027).
40//! - **Two riders, one DHT, one trust posture.** Both iroh pkarr and the at9p
41//! mainline locator (#890 / C2) ride the *same* mainline DHT. Only at9p is
42//! zero-trust: its records are content-addressed (`did:at9p:<cid512>`) and
43//! made trustworthy by the GATE pipeline (canon → hash → composite-sig), so
44//! the DHT is an **untrusted locator** for at9p, never a trust root. pkarr, by
45//! contrast, is classical-only and trusts the DHT to carry the publisher's
46//! reach claim verbatim — fine for liveness, fatal as an authority source.
47//! - **Not removed.** pkarr stays — it is genuinely useful for NAT traversal /
48//! liveness ("is this peer dialable right now?"). D3 only strips (and forbids
49//! re-deriving) authority from it; it does not disable publication.
50//!
51//! Enforcement seams: the dial path (`crate::dial`) resolves *addresses* from a
52//! NodeId as an address, never trusting pkarr for identity. Admission does not
53//! consume NodeId, `remote_id()`, or pkarr. The wasm pkarr output is typed as an unverified reach hint
54//! ([`crate::iroh_peer::PkarrReachHint`]) so it cannot be conflated with
55//! verified reach.
56//!
57//! The #385 metadata-leak side channel (a passive observer can enumerate which
58//! NodeIds are reachable and correlate lookup traffic) is **unaffected** by this
59//! demotion — pkarr is still public. Its mitigation is oblivious-relay (#361);
60//! until then, operators that consider the side channel actionable should run a
61//! self-hosted `iroh-dns-server` configured through the owned endpoint builder.
62
63use anyhow::Result;
64use iroh::endpoint::{presets, Accepting, AfterHandshakeOutcome, Connection, EndpointHooks};
65use iroh::protocol::{AcceptError, DynProtocolHandler, ProtocolHandler, Router};
66use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey};
67use std::{future::Future, pin::Pin};
68
69/// ALPN for moq-net (moq-lite). Must equal `moq_net::version::ALPN_LITE`.
70pub const ALPN_MOQ_LITE: &[u8] = b"moql";
71
72/// ALPN for the raw Cap'n Proto bidi RPC plane.
73pub const ALPN_HYPRSTREAM_RPC: &[u8] = b"hyprstream-rpc/1";
74
75/// Keep the owned HyprStream ALPNs on a PQ-hybrid carrier while allowing the
76/// endpoint's outer HTTPS clients (N0 relay and pkarr) to interoperate with
77/// classical-only public infrastructure. The provider therefore offers
78/// X25519 as a fallback, but a completed HyprStream carrier handshake that
79/// negotiated it is rejected before the protocol router sees the connection.
80#[derive(Debug, Clone, Copy, Default)]
81struct HybridCarrierHook;
82
83impl EndpointHooks for HybridCarrierHook {
84 async fn after_handshake(&self, conn: &Connection) -> AfterHandshakeOutcome {
85 if !matches!(conn.alpn(), ALPN_MOQ_LITE | ALPN_HYPRSTREAM_RPC) {
86 return AfterHandshakeOutcome::Accept;
87 }
88
89 let Some(data) = conn.handshake_data() else {
90 return AfterHandshakeOutcome::Reject {
91 error_code: 403u32.into(),
92 reason: b"missing iroh handshake data".to_vec(),
93 };
94 };
95 let Ok(data) = data.downcast::<noq::crypto::rustls::HandshakeData>() else {
96 return AfterHandshakeOutcome::Reject {
97 error_code: 403u32.into(),
98 reason: b"unsupported iroh handshake data".to_vec(),
99 };
100 };
101 if data.negotiated_key_exchange_group == Some(rustls::NamedGroup::X25519MLKEM768) {
102 AfterHandshakeOutcome::Accept
103 } else {
104 AfterHandshakeOutcome::Reject {
105 error_code: 403u32.into(),
106 reason: b"hyprstream carrier requires PQ-hybrid TLS".to_vec(),
107 }
108 }
109 }
110}
111
112/// Two-plane iroh substrate: one shared QUIC endpoint, two ALPN-dispatched
113/// protocol handlers.
114///
115/// Drop to abort the accept loop; call [`IrohSubstrate::shutdown`] for a
116/// clean shutdown that drains protocol handlers and closes the endpoint.
117pub struct IrohSubstrate {
118 endpoint: Endpoint,
119 router: Router,
120}
121
122/// Capability proving an outbound iroh endpoint was constructed by
123/// [`IrohSubstrate::new`] with the owned PQ-hybrid carrier policy.
124///
125/// The inner endpoint is intentionally private: an already-bound iroh endpoint
126/// does not expose its effective provider, so arbitrary endpoints cannot be
127/// promoted into the process-global RPC/MoQ dialer.
128#[derive(Clone)]
129pub struct OwnedIrohClientEndpoint(Endpoint);
130
131impl OwnedIrohClientEndpoint {
132 pub(crate) fn install_into(
133 self,
134 slot: &std::sync::OnceLock<Endpoint>,
135 ) -> std::result::Result<(), Self> {
136 slot.set(self.0).map_err(Self)
137 }
138}
139
140impl IrohSubstrate {
141 /// Build the substrate from raw 32-byte Ed25519 secret key material.
142 ///
143 /// Uses iroh's `presets::N0` for discovery (n0 DNS + pkarr) and relay
144 /// fallback. The native carrier remains strict PQ-hybrid; the patched iroh
145 /// builder routes only external CA-validated HTTPS through its compatible
146 /// provider, so public N0 infrastructure can be reached without weakening
147 /// the owned carrier policy.
148 ///
149 /// **pkarr here is liveness-only** — see the module-level "D3" note. The
150 /// published record carries reach hints (relay + direct addrs) for this
151 /// endpoint's NodeId; it derives **zero** identity/trust authority. The
152 /// #385 metadata-leak caveat also applies (mitigated by #361, not by this
153 /// demotion).
154 pub async fn new<M, R>(
155 secret_key_bytes: [u8; 32],
156 moq_handler: M,
157 rpc_handler: R,
158 ) -> Result<Self>
159 where
160 M: Into<Box<dyn DynProtocolHandler>>,
161 R: Into<Box<dyn DynProtocolHandler>>,
162 {
163 // N0 1.0.x configures native resolution through DNS TXT only, while
164 // its publisher writes relay-only reach to the HTTPS pkarr service.
165 // Install the matching HTTPS resolver as well so node-id-only native
166 // announcements can resolve the same reach that N0 publishes.
167 let endpoint = Endpoint::builder(presets::N0)
168 .address_lookup(iroh::address_lookup::PkarrResolver::n0_dns())
169 .secret_key(SecretKey::from_bytes(&secret_key_bytes))
170 .crypto_provider(crate::transport::pq_provider::internal_mesh_crypto_provider())
171 .ca_crypto_provider(crate::transport::pq_provider::pq_crypto_provider())
172 .hooks(HybridCarrierHook)
173 .bind()
174 .await
175 .map_err(|e| anyhow::anyhow!("iroh endpoint bind: {e}"))?;
176
177 Ok(Self::from_owned_endpoint(
178 endpoint,
179 moq_handler,
180 rpc_handler,
181 ))
182 }
183
184 /// Build a hermetic direct-only substrate for unit tests.
185 ///
186 /// Production uses [`presets::N0`] for discovery, pkarr publication, and
187 /// relay fallback. Unit carrier tests construct their target explicitly
188 /// from bound sockets, so those background paths add no coverage and can
189 /// race independent Tokio runtimes during parallel suite teardown. The
190 /// empty preset keeps the same QUIC crypto provider and real UDP carrier
191 /// while disabling discovery and relay workers.
192 #[cfg(test)]
193 pub(crate) async fn new_test<M, R>(
194 secret_key_bytes: [u8; 32],
195 moq_handler: M,
196 rpc_handler: R,
197 ) -> Result<Self>
198 where
199 M: Into<Box<dyn DynProtocolHandler>>,
200 R: Into<Box<dyn DynProtocolHandler>>,
201 {
202 let endpoint = Endpoint::builder(presets::Empty)
203 .secret_key(SecretKey::from_bytes(&secret_key_bytes))
204 .crypto_provider(crate::transport::pq_provider::internal_mesh_crypto_provider())
205 .bind()
206 .await
207 .map_err(|e| anyhow::anyhow!("test iroh endpoint bind: {e}"))?;
208
209 Ok(Self::from_owned_endpoint(
210 endpoint,
211 moq_handler,
212 rpc_handler,
213 ))
214 }
215
216 /// Attach the owned ALPNs to an endpoint constructed immediately above.
217 /// Kept private because `Endpoint` does not reveal its effective provider.
218 ///
219 /// The wrapper completes the full TLS handshake before delegating to a
220 /// protocol handler. Iroh otherwise permits a handler's `on_accepting`
221 /// hook to consume 0-RTT data before `EndpointHooks` runs; owned ALPNs must
222 /// not expose that early-data path while carrier admission is enforced
223 /// after the handshake.
224 fn from_owned_endpoint<M, R>(endpoint: Endpoint, moq_handler: M, rpc_handler: R) -> Self
225 where
226 M: Into<Box<dyn DynProtocolHandler>>,
227 R: Into<Box<dyn DynProtocolHandler>>,
228 {
229 let router = Router::builder(endpoint.clone())
230 .accept(
231 ALPN_MOQ_LITE,
232 Box::new(FullHandshakeHandler::new(moq_handler.into()))
233 as Box<dyn DynProtocolHandler>,
234 )
235 .accept(
236 ALPN_HYPRSTREAM_RPC,
237 Box::new(FullHandshakeHandler::new(rpc_handler.into()))
238 as Box<dyn DynProtocolHandler>,
239 )
240 .spawn();
241 Self { endpoint, router }
242 }
243
244 /// Return this endpoint's carrier address for reach advertisement and
245 /// diagnostics. It is not a DID/JWKS key or application identity proof.
246 pub fn endpoint_id(&self) -> EndpointId {
247 self.endpoint.id()
248 }
249
250 /// Borrow the underlying endpoint, e.g. to issue outbound connects on
251 /// an ALPN not served by this substrate's router.
252 pub fn endpoint(&self) -> &Endpoint {
253 &self.endpoint
254 }
255
256 /// Clone the endpoint as an owned-provider capability for installation as
257 /// the shared outbound RPC and MoQ dial endpoint.
258 pub fn owned_client_endpoint(&self) -> OwnedIrohClientEndpoint {
259 OwnedIrohClientEndpoint(self.endpoint.clone())
260 }
261
262 /// Borrow the router (e.g. to inspect or to share lifetime).
263 pub fn router(&self) -> &Router {
264 &self.router
265 }
266
267 /// Dial a peer for the given ALPN.
268 pub async fn connect(&self, addr: EndpointAddr, alpn: &[u8]) -> Result<Connection> {
269 self.endpoint
270 .connect(addr, alpn)
271 .await
272 .map_err(|e| anyhow::anyhow!("iroh connect: {e}"))
273 }
274
275 /// Drain handlers and close the endpoint.
276 pub async fn shutdown(self) -> Result<()> {
277 self.router
278 .shutdown()
279 .await
280 .map_err(|e| anyhow::anyhow!("router shutdown: {e}"))?;
281 Ok(())
282 }
283}
284
285/// Restrict owned ALPN handlers to the full-handshake `accept(Connection)`
286/// path. This prevents a custom handler from consuming 0-RTT before the
287/// post-handshake carrier policy has run.
288#[derive(Debug)]
289struct FullHandshakeHandler {
290 inner: Box<dyn DynProtocolHandler>,
291}
292
293impl FullHandshakeHandler {
294 fn new(inner: Box<dyn DynProtocolHandler>) -> Self {
295 Self { inner }
296 }
297}
298
299impl DynProtocolHandler for FullHandshakeHandler {
300 fn on_accepting(
301 &self,
302 accepting: Accepting,
303 ) -> Pin<Box<dyn Future<Output = Result<Connection, AcceptError>> + Send + '_>> {
304 Box::pin(async move { accepting.await.map_err(AcceptError::from_err) })
305 }
306
307 fn accept(
308 &self,
309 connection: Connection,
310 ) -> Pin<Box<dyn Future<Output = Result<(), AcceptError>> + Send + '_>> {
311 self.inner.accept(connection)
312 }
313
314 fn shutdown(&self) -> Pin<Box<dyn Future<Output = ()> + Send + '_>> {
315 self.inner.shutdown()
316 }
317}
318
319/// Placeholder protocol handler that accepts an incoming connection and
320/// immediately returns. Used until Phase 2 / Phase 3 wire the real handlers.
321#[derive(Debug, Clone, Default)]
322pub struct NoopHandler {
323 reason: &'static str,
324}
325
326impl NoopHandler {
327 pub fn new(reason: &'static str) -> Self {
328 Self { reason }
329 }
330}
331
332impl ProtocolHandler for NoopHandler {
333 async fn accept(&self, _conn: Connection) -> Result<(), AcceptError> {
334 tracing::trace!(reason = %self.reason, "NoopHandler accepted (no-op)");
335 Ok(())
336 }
337}
338
339/// Fail-closed protocol handler for an ALPN that is intentionally disabled.
340///
341/// Unlike [`NoopHandler`], this explicitly closes the carrier connection. It is
342/// used where an endpoint may exist for outbound reach but no trustworthy
343/// application/session proof is available for inbound requests.
344#[derive(Debug, Clone)]
345pub struct RefuseHandler {
346 reason: &'static str,
347}
348
349impl RefuseHandler {
350 pub fn new(reason: &'static str) -> Self {
351 Self { reason }
352 }
353}
354
355impl ProtocolHandler for RefuseHandler {
356 async fn accept(&self, conn: Connection) -> Result<(), AcceptError> {
357 tracing::warn!(reason = %self.reason, "refusing disabled iroh ALPN");
358 conn.close(0u32.into(), self.reason.as_bytes());
359 Ok(())
360 }
361}
362
363/// Test-only handler that accepts one bidi stream, reads one length-prefixed
364/// message, echoes it back, then closes. Used by the Phase 1 smoke test to
365/// verify that both ALPN slots route correctly.
366#[derive(Debug, Clone, Default)]
367pub struct EchoHandler;
368
369impl ProtocolHandler for EchoHandler {
370 async fn accept(&self, conn: Connection) -> Result<(), AcceptError> {
371 let (mut send, mut recv) = conn
372 .accept_bi()
373 .await
374 .map_err(|e| AcceptError::from_err(e))?;
375 let buf = recv
376 .read_to_end(1024)
377 .await
378 .map_err(|e| AcceptError::from_err(e))?;
379 send.write_all(&buf)
380 .await
381 .map_err(|e| AcceptError::from_err(e))?;
382 send.finish().map_err(|e| AcceptError::from_err(e))?;
383 // Cleanly wait for the peer to acknowledge by reading the final FIN.
384 let _ = send.stopped().await;
385 Ok(())
386 }
387}
388
389#[cfg(test)]
390#[allow(clippy::expect_used)]
391mod tests {
392 use super::*;
393 use iroh::TransportAddr;
394 use noq::crypto::rustls::HandshakeData;
395 use rand::RngCore;
396 use std::sync::{
397 atomic::{AtomicBool, Ordering},
398 Arc,
399 };
400
401 fn fresh_key() -> [u8; 32] {
402 let mut k = [0u8; 32];
403 rand::thread_rng().fill_bytes(&mut k);
404 k
405 }
406
407 #[derive(Debug, Clone)]
408 struct EarlyDataProbe(Arc<AtomicBool>);
409
410 impl ProtocolHandler for EarlyDataProbe {
411 async fn on_accepting(&self, _accepting: Accepting) -> Result<Connection, AcceptError> {
412 self.0.store(true, Ordering::SeqCst);
413 Err(AcceptError::from_err(std::io::Error::other(
414 "early-data hook must not be delegated",
415 )))
416 }
417
418 async fn accept(&self, _connection: Connection) -> Result<(), AcceptError> {
419 Ok(())
420 }
421 }
422
423 /// Build an `EndpointAddr` for a server directly from its bound sockets +
424 /// endpoint id. Skips the n0 relay/pkarr resolution path, which keeps the
425 /// smoke test hermetic — no DNS lookup, no network egress.
426 fn direct_addr(substrate: &IrohSubstrate) -> EndpointAddr {
427 EndpointAddr::from_parts(
428 substrate.endpoint_id(),
429 substrate
430 .endpoint()
431 .bound_sockets()
432 .into_iter()
433 .map(TransportAddr::Ip),
434 )
435 }
436
437 fn assert_hybrid_handshake(conn: &Connection, alpn: &[u8]) {
438 let data = conn
439 .handshake_data()
440 .expect("completed iroh connection has handshake data")
441 .downcast::<HandshakeData>()
442 .expect("iroh uses noq rustls handshake data");
443 assert_eq!(data.protocol.as_deref(), Some(alpn));
444 assert_eq!(
445 data.negotiated_key_exchange_group,
446 Some(rustls::NamedGroup::X25519MLKEM768),
447 "owned iroh mesh must negotiate X25519MLKEM768"
448 );
449 }
450
451 /// Smoke test: build two substrates, dial direct (no DNS/pkarr), echo a
452 /// message on each ALPN. Verifies the router dispatches by ALPN and that
453 /// both planes terminate distinct handlers.
454 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
455 async fn smoke_two_endpoints_two_alpns() -> Result<()> {
456 // Server with EchoHandler on both ALPNs.
457 let server = IrohSubstrate::new_test(fresh_key(), EchoHandler, EchoHandler).await?;
458 let server_id = server.endpoint_id();
459 let server_addr = direct_addr(&server);
460
461 // Client with no-op handlers (it only originates).
462 let client = IrohSubstrate::new_test(
463 fresh_key(),
464 NoopHandler::new("client moq"),
465 NoopHandler::new("client rpc"),
466 )
467 .await?;
468
469 // RPC plane round-trip.
470 {
471 let conn = client
472 .connect(server_addr.clone(), ALPN_HYPRSTREAM_RPC)
473 .await?;
474 assert_hybrid_handshake(&conn, ALPN_HYPRSTREAM_RPC);
475 let (mut send, mut recv) = conn.open_bi().await?;
476 send.write_all(b"hello rpc").await?;
477 send.finish()?;
478 let got = recv.read_to_end(64).await?;
479 assert_eq!(&got, b"hello rpc");
480 }
481
482 // Streaming plane round-trip (echo here; real moq-net wiring lands in Phase 3).
483 {
484 let conn = client.connect(server_addr.clone(), ALPN_MOQ_LITE).await?;
485 assert_hybrid_handshake(&conn, ALPN_MOQ_LITE);
486 let (mut send, mut recv) = conn.open_bi().await?;
487 send.write_all(b"hello moq").await?;
488 send.finish()?;
489 let got = recv.read_to_end(64).await?;
490 assert_eq!(&got, b"hello moq");
491 }
492
493 // Sanity: server id is stable.
494 assert_eq!(server.endpoint_id(), server_id);
495
496 client.shutdown().await?;
497 server.shutdown().await?;
498 Ok(())
499 }
500
501 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
502 async fn owned_handlers_complete_handshake_before_delegation() -> Result<()> {
503 let called = Arc::new(AtomicBool::new(false));
504 let server = IrohSubstrate::new_test(
505 fresh_key(),
506 EarlyDataProbe(called.clone()),
507 NoopHandler::new("rpc"),
508 )
509 .await?;
510 let client = IrohSubstrate::new_test(
511 fresh_key(),
512 NoopHandler::new("moq"),
513 NoopHandler::new("rpc"),
514 )
515 .await?;
516
517 let conn = client.connect(direct_addr(&server), ALPN_MOQ_LITE).await?;
518 assert_hybrid_handshake(&conn, ALPN_MOQ_LITE);
519 assert!(!called.load(Ordering::SeqCst));
520
521 client.shutdown().await?;
522 server.shutdown().await?;
523 Ok(())
524 }
525
526 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
527 async fn internal_iroh_mesh_rejects_classical_only_peer() -> Result<()> {
528 let server = IrohSubstrate::new(fresh_key(), EchoHandler, EchoHandler).await?;
529 let server_addr = direct_addr(&server);
530 let classical_client = Endpoint::builder(presets::N0)
531 .secret_key(SecretKey::from_bytes(&fresh_key()))
532 .crypto_provider(Arc::new(rustls::crypto::ring::default_provider()))
533 .bind()
534 .await
535 .map_err(|e| anyhow::anyhow!("classical mutation endpoint bind: {e}"))?;
536
537 for alpn in [ALPN_HYPRSTREAM_RPC, ALPN_MOQ_LITE] {
538 let result = tokio::time::timeout(
539 std::time::Duration::from_secs(5),
540 classical_client.connect(server_addr.clone(), alpn),
541 )
542 .await
543 .expect("classical-only iroh mutation handshake must terminate");
544 match result {
545 Err(_) => {}
546 Ok(conn) => {
547 tokio::time::timeout(std::time::Duration::from_secs(5), conn.closed())
548 .await
549 .expect("classical-only carrier must be closed by the hybrid hook");
550 }
551 }
552 }
553
554 classical_client.close().await;
555 server.shutdown().await?;
556 Ok(())
557 }
558}