Skip to main content

hyprstream_rpc/transport/
iroh_substrate.rs

1//! Iroh transport substrate — the PRIMARY production transport for Epic #131
2//! (ZMQ → moq-net + iroh), promoted to default in #410.
3//!
4//! Builds a shared `iroh::Endpoint` from an Ed25519 node key and wires an
5//! [`iroh::protocol::Router`] with two ALPN slots:
6//!
7//! - [`ALPN_MOQ_LITE`] (`moql`) — moq-net session handler, wired by the daemon
8//!   spawner (#282) to dispatch into the shared global `moq_stream::Origin`
9//!   for the streaming plane.
10//! - [`ALPN_HYPRSTREAM_RPC`] (`hyprstream-rpc/1`) — raw Cap'n Proto bidi
11//!   handler, wired by the daemon spawner (#282) to terminate
12//!   `IrohRpcProtocolHandler` for the RPC plane.
13//!
14//! Phase 1 shipped the substrate, a [`NoopHandler`] placeholder, and a smoke
15//! test verifying that two endpoints can dial each other directly and open a
16//! bidi stream on each ALPN. As of #410, production code DOES consume this
17//! module: the daemon spawner (`hyprstream-service::service::spawner`) binds
18//! one `IrohSubstrate` per QUIC-enabled service as the PRIMARY production
19//! endpoint (on by default; `[quic] iroh = false` opts out to quinn-only). The
20//! real `moql` and `hyprstream-rpc/1` handlers are threaded in by the spawner
21//! (#282). OAuth binds a reach-only endpoint whose inbound ALPNs remain refused
22//! until independently verified application/session proof is wired.
23//!
24//! # D3 — iroh pkarr is LIVENESS-ONLY, never an authority source (#895)
25//!
26//! iroh's `presets::N0` discovery publishes classical **pkarr** records —
27//! signed-but-not-encrypted packets — onto the open BitTorrent mainline DHT to
28//! advertise reach (home relay + direct addresses) for a NodeId. As of #895
29//! (epic #880 Track D / D3) this is the contract for pkarr in hyprstream:
30//!
31//! - **Reach hint, not authority.** A pkarr record is an **unverified dial
32//!   candidate**: "NodeId N claims it is reachable at relay R / addr A." It is
33//!   signed by N's Ed25519 key, so it is integrity-protected *as a reach
34//!   claim* — but that says nothing about *which* federated identity (capsule,
35//!   `did:web`, `did:key`) controls N, what its PQ key material is, or whether
36//!   it may be admitted. **No identity / trust / admission decision is ever
37//!   derived from a pkarr record.** `Connection::remote_id()` authenticates only
38//!   the carrier endpoint. Application identity requires independently resolved
39//!   current keys plus fresh inside-carrier proof (#1027).
40//! - **Two riders, one DHT, one trust posture.** Both iroh pkarr and the at9p
41//!   mainline locator (#890 / C2) ride the *same* mainline DHT. Only at9p is
42//!   zero-trust: its records are content-addressed (`did:at9p:<cid512>`) and
43//!   made trustworthy by the GATE pipeline (canon → hash → composite-sig), so
44//!   the DHT is an **untrusted locator** for at9p, never a trust root. pkarr, by
45//!   contrast, is classical-only and trusts the DHT to carry the publisher's
46//!   reach claim verbatim — fine for liveness, fatal as an authority source.
47//! - **Not removed.** pkarr stays — it is genuinely useful for NAT traversal /
48//!   liveness ("is this peer dialable right now?"). D3 only strips (and forbids
49//!   re-deriving) authority from it; it does not disable publication.
50//!
51//! Enforcement seams: the dial path (`crate::dial`) resolves *addresses* from a
52//! NodeId as an address, never trusting pkarr for identity. Admission does not
53//! consume NodeId, `remote_id()`, or pkarr. The wasm pkarr output is typed as an unverified reach hint
54//! ([`crate::iroh_peer::PkarrReachHint`]) so it cannot be conflated with
55//! verified reach.
56//!
57//! The #385 metadata-leak side channel (a passive observer can enumerate which
58//! NodeIds are reachable and correlate lookup traffic) is **unaffected** by this
59//! demotion — pkarr is still public. Its mitigation is oblivious-relay (#361);
60//! until then, operators that consider the side channel actionable should run a
61//! self-hosted `iroh-dns-server` configured through the owned endpoint builder.
62
63use anyhow::Result;
64use iroh::endpoint::{presets, Accepting, AfterHandshakeOutcome, Connection, EndpointHooks};
65use iroh::protocol::{AcceptError, DynProtocolHandler, ProtocolHandler, Router};
66use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey};
67use std::{future::Future, pin::Pin};
68
69/// ALPN for moq-net (moq-lite). Must equal `moq_net::version::ALPN_LITE`.
70pub const ALPN_MOQ_LITE: &[u8] = b"moql";
71
72/// ALPN for the raw Cap'n Proto bidi RPC plane.
73pub const ALPN_HYPRSTREAM_RPC: &[u8] = b"hyprstream-rpc/1";
74
75/// Keep the owned HyprStream ALPNs on a PQ-hybrid carrier while allowing the
76/// endpoint's outer HTTPS clients (N0 relay and pkarr) to interoperate with
77/// classical-only public infrastructure. The provider therefore offers
78/// X25519 as a fallback, but a completed HyprStream carrier handshake that
79/// negotiated it is rejected before the protocol router sees the connection.
80#[derive(Debug, Clone, Copy, Default)]
81struct HybridCarrierHook;
82
83impl EndpointHooks for HybridCarrierHook {
84    async fn after_handshake(&self, conn: &Connection) -> AfterHandshakeOutcome {
85        if !matches!(conn.alpn(), ALPN_MOQ_LITE | ALPN_HYPRSTREAM_RPC) {
86            return AfterHandshakeOutcome::Accept;
87        }
88
89        let Some(data) = conn.handshake_data() else {
90            return AfterHandshakeOutcome::Reject {
91                error_code: 403u32.into(),
92                reason: b"missing iroh handshake data".to_vec(),
93            };
94        };
95        let Ok(data) = data.downcast::<noq::crypto::rustls::HandshakeData>() else {
96            return AfterHandshakeOutcome::Reject {
97                error_code: 403u32.into(),
98                reason: b"unsupported iroh handshake data".to_vec(),
99            };
100        };
101        if data.negotiated_key_exchange_group == Some(rustls::NamedGroup::X25519MLKEM768) {
102            AfterHandshakeOutcome::Accept
103        } else {
104            AfterHandshakeOutcome::Reject {
105                error_code: 403u32.into(),
106                reason: b"hyprstream carrier requires PQ-hybrid TLS".to_vec(),
107            }
108        }
109    }
110}
111
112/// Two-plane iroh substrate: one shared QUIC endpoint, two ALPN-dispatched
113/// protocol handlers.
114///
115/// Drop to abort the accept loop; call [`IrohSubstrate::shutdown`] for a
116/// clean shutdown that drains protocol handlers and closes the endpoint.
117pub struct IrohSubstrate {
118    endpoint: Endpoint,
119    router: Router,
120}
121
122/// Capability proving an outbound iroh endpoint was constructed by
123/// [`IrohSubstrate::new`] with the owned PQ-hybrid carrier policy.
124///
125/// The inner endpoint is intentionally private: an already-bound iroh endpoint
126/// does not expose its effective provider, so arbitrary endpoints cannot be
127/// promoted into the process-global RPC/MoQ dialer.
128#[derive(Clone)]
129pub struct OwnedIrohClientEndpoint(Endpoint);
130
131impl OwnedIrohClientEndpoint {
132    pub(crate) fn install_into(
133        self,
134        slot: &std::sync::OnceLock<Endpoint>,
135    ) -> std::result::Result<(), Self> {
136        slot.set(self.0).map_err(Self)
137    }
138}
139
140impl IrohSubstrate {
141    /// Build the substrate from raw 32-byte Ed25519 secret key material.
142    ///
143    /// Uses iroh's `presets::N0` for discovery (n0 DNS + pkarr) and relay
144    /// fallback. The native carrier remains strict PQ-hybrid; the patched iroh
145    /// builder routes only external CA-validated HTTPS through its compatible
146    /// provider, so public N0 infrastructure can be reached without weakening
147    /// the owned carrier policy.
148    ///
149    /// **pkarr here is liveness-only** — see the module-level "D3" note. The
150    /// published record carries reach hints (relay + direct addrs) for this
151    /// endpoint's NodeId; it derives **zero** identity/trust authority. The
152    /// #385 metadata-leak caveat also applies (mitigated by #361, not by this
153    /// demotion).
154    pub async fn new<M, R>(
155        secret_key_bytes: [u8; 32],
156        moq_handler: M,
157        rpc_handler: R,
158    ) -> Result<Self>
159    where
160        M: Into<Box<dyn DynProtocolHandler>>,
161        R: Into<Box<dyn DynProtocolHandler>>,
162    {
163        // N0 1.0.x configures native resolution through DNS TXT only, while
164        // its publisher writes relay-only reach to the HTTPS pkarr service.
165        // Install the matching HTTPS resolver as well so node-id-only native
166        // announcements can resolve the same reach that N0 publishes.
167        let endpoint = Endpoint::builder(presets::N0)
168            .address_lookup(iroh::address_lookup::PkarrResolver::n0_dns())
169            .secret_key(SecretKey::from_bytes(&secret_key_bytes))
170            .crypto_provider(crate::transport::pq_provider::internal_mesh_crypto_provider())
171            .ca_crypto_provider(crate::transport::pq_provider::pq_crypto_provider())
172            .hooks(HybridCarrierHook)
173            .bind()
174            .await
175            .map_err(|e| anyhow::anyhow!("iroh endpoint bind: {e}"))?;
176
177        Ok(Self::from_owned_endpoint(
178            endpoint,
179            moq_handler,
180            rpc_handler,
181        ))
182    }
183
184    /// Build a hermetic direct-only substrate for unit tests.
185    ///
186    /// Production uses [`presets::N0`] for discovery, pkarr publication, and
187    /// relay fallback. Unit carrier tests construct their target explicitly
188    /// from bound sockets, so those background paths add no coverage and can
189    /// race independent Tokio runtimes during parallel suite teardown. The
190    /// empty preset keeps the same QUIC crypto provider and real UDP carrier
191    /// while disabling discovery and relay workers.
192    #[cfg(test)]
193    pub(crate) async fn new_test<M, R>(
194        secret_key_bytes: [u8; 32],
195        moq_handler: M,
196        rpc_handler: R,
197    ) -> Result<Self>
198    where
199        M: Into<Box<dyn DynProtocolHandler>>,
200        R: Into<Box<dyn DynProtocolHandler>>,
201    {
202        let endpoint = Endpoint::builder(presets::Empty)
203            .secret_key(SecretKey::from_bytes(&secret_key_bytes))
204            .crypto_provider(crate::transport::pq_provider::internal_mesh_crypto_provider())
205            .bind()
206            .await
207            .map_err(|e| anyhow::anyhow!("test iroh endpoint bind: {e}"))?;
208
209        Ok(Self::from_owned_endpoint(
210            endpoint,
211            moq_handler,
212            rpc_handler,
213        ))
214    }
215
216    /// Attach the owned ALPNs to an endpoint constructed immediately above.
217    /// Kept private because `Endpoint` does not reveal its effective provider.
218    ///
219    /// The wrapper completes the full TLS handshake before delegating to a
220    /// protocol handler. Iroh otherwise permits a handler's `on_accepting`
221    /// hook to consume 0-RTT data before `EndpointHooks` runs; owned ALPNs must
222    /// not expose that early-data path while carrier admission is enforced
223    /// after the handshake.
224    fn from_owned_endpoint<M, R>(endpoint: Endpoint, moq_handler: M, rpc_handler: R) -> Self
225    where
226        M: Into<Box<dyn DynProtocolHandler>>,
227        R: Into<Box<dyn DynProtocolHandler>>,
228    {
229        let router = Router::builder(endpoint.clone())
230            .accept(
231                ALPN_MOQ_LITE,
232                Box::new(FullHandshakeHandler::new(moq_handler.into()))
233                    as Box<dyn DynProtocolHandler>,
234            )
235            .accept(
236                ALPN_HYPRSTREAM_RPC,
237                Box::new(FullHandshakeHandler::new(rpc_handler.into()))
238                    as Box<dyn DynProtocolHandler>,
239            )
240            .spawn();
241        Self { endpoint, router }
242    }
243
244    /// Return this endpoint's carrier address for reach advertisement and
245    /// diagnostics. It is not a DID/JWKS key or application identity proof.
246    pub fn endpoint_id(&self) -> EndpointId {
247        self.endpoint.id()
248    }
249
250    /// Borrow the underlying endpoint, e.g. to issue outbound connects on
251    /// an ALPN not served by this substrate's router.
252    pub fn endpoint(&self) -> &Endpoint {
253        &self.endpoint
254    }
255
256    /// Clone the endpoint as an owned-provider capability for installation as
257    /// the shared outbound RPC and MoQ dial endpoint.
258    pub fn owned_client_endpoint(&self) -> OwnedIrohClientEndpoint {
259        OwnedIrohClientEndpoint(self.endpoint.clone())
260    }
261
262    /// Borrow the router (e.g. to inspect or to share lifetime).
263    pub fn router(&self) -> &Router {
264        &self.router
265    }
266
267    /// Dial a peer for the given ALPN.
268    pub async fn connect(&self, addr: EndpointAddr, alpn: &[u8]) -> Result<Connection> {
269        self.endpoint
270            .connect(addr, alpn)
271            .await
272            .map_err(|e| anyhow::anyhow!("iroh connect: {e}"))
273    }
274
275    /// Drain handlers and close the endpoint.
276    pub async fn shutdown(self) -> Result<()> {
277        self.router
278            .shutdown()
279            .await
280            .map_err(|e| anyhow::anyhow!("router shutdown: {e}"))?;
281        Ok(())
282    }
283}
284
285/// Restrict owned ALPN handlers to the full-handshake `accept(Connection)`
286/// path. This prevents a custom handler from consuming 0-RTT before the
287/// post-handshake carrier policy has run.
288#[derive(Debug)]
289struct FullHandshakeHandler {
290    inner: Box<dyn DynProtocolHandler>,
291}
292
293impl FullHandshakeHandler {
294    fn new(inner: Box<dyn DynProtocolHandler>) -> Self {
295        Self { inner }
296    }
297}
298
299impl DynProtocolHandler for FullHandshakeHandler {
300    fn on_accepting(
301        &self,
302        accepting: Accepting,
303    ) -> Pin<Box<dyn Future<Output = Result<Connection, AcceptError>> + Send + '_>> {
304        Box::pin(async move { accepting.await.map_err(AcceptError::from_err) })
305    }
306
307    fn accept(
308        &self,
309        connection: Connection,
310    ) -> Pin<Box<dyn Future<Output = Result<(), AcceptError>> + Send + '_>> {
311        self.inner.accept(connection)
312    }
313
314    fn shutdown(&self) -> Pin<Box<dyn Future<Output = ()> + Send + '_>> {
315        self.inner.shutdown()
316    }
317}
318
319/// Placeholder protocol handler that accepts an incoming connection and
320/// immediately returns. Used until Phase 2 / Phase 3 wire the real handlers.
321#[derive(Debug, Clone, Default)]
322pub struct NoopHandler {
323    reason: &'static str,
324}
325
326impl NoopHandler {
327    pub fn new(reason: &'static str) -> Self {
328        Self { reason }
329    }
330}
331
332impl ProtocolHandler for NoopHandler {
333    async fn accept(&self, _conn: Connection) -> Result<(), AcceptError> {
334        tracing::trace!(reason = %self.reason, "NoopHandler accepted (no-op)");
335        Ok(())
336    }
337}
338
339/// Fail-closed protocol handler for an ALPN that is intentionally disabled.
340///
341/// Unlike [`NoopHandler`], this explicitly closes the carrier connection. It is
342/// used where an endpoint may exist for outbound reach but no trustworthy
343/// application/session proof is available for inbound requests.
344#[derive(Debug, Clone)]
345pub struct RefuseHandler {
346    reason: &'static str,
347}
348
349impl RefuseHandler {
350    pub fn new(reason: &'static str) -> Self {
351        Self { reason }
352    }
353}
354
355impl ProtocolHandler for RefuseHandler {
356    async fn accept(&self, conn: Connection) -> Result<(), AcceptError> {
357        tracing::warn!(reason = %self.reason, "refusing disabled iroh ALPN");
358        conn.close(0u32.into(), self.reason.as_bytes());
359        Ok(())
360    }
361}
362
363/// Test-only handler that accepts one bidi stream, reads one length-prefixed
364/// message, echoes it back, then closes. Used by the Phase 1 smoke test to
365/// verify that both ALPN slots route correctly.
366#[derive(Debug, Clone, Default)]
367pub struct EchoHandler;
368
369impl ProtocolHandler for EchoHandler {
370    async fn accept(&self, conn: Connection) -> Result<(), AcceptError> {
371        let (mut send, mut recv) = conn
372            .accept_bi()
373            .await
374            .map_err(|e| AcceptError::from_err(e))?;
375        let buf = recv
376            .read_to_end(1024)
377            .await
378            .map_err(|e| AcceptError::from_err(e))?;
379        send.write_all(&buf)
380            .await
381            .map_err(|e| AcceptError::from_err(e))?;
382        send.finish().map_err(|e| AcceptError::from_err(e))?;
383        // Cleanly wait for the peer to acknowledge by reading the final FIN.
384        let _ = send.stopped().await;
385        Ok(())
386    }
387}
388
389#[cfg(test)]
390#[allow(clippy::expect_used)]
391mod tests {
392    use super::*;
393    use iroh::TransportAddr;
394    use noq::crypto::rustls::HandshakeData;
395    use rand::RngCore;
396    use std::sync::{
397        atomic::{AtomicBool, Ordering},
398        Arc,
399    };
400
401    fn fresh_key() -> [u8; 32] {
402        let mut k = [0u8; 32];
403        rand::thread_rng().fill_bytes(&mut k);
404        k
405    }
406
407    #[derive(Debug, Clone)]
408    struct EarlyDataProbe(Arc<AtomicBool>);
409
410    impl ProtocolHandler for EarlyDataProbe {
411        async fn on_accepting(&self, _accepting: Accepting) -> Result<Connection, AcceptError> {
412            self.0.store(true, Ordering::SeqCst);
413            Err(AcceptError::from_err(std::io::Error::other(
414                "early-data hook must not be delegated",
415            )))
416        }
417
418        async fn accept(&self, _connection: Connection) -> Result<(), AcceptError> {
419            Ok(())
420        }
421    }
422
423    /// Build an `EndpointAddr` for a server directly from its bound sockets +
424    /// endpoint id. Skips the n0 relay/pkarr resolution path, which keeps the
425    /// smoke test hermetic — no DNS lookup, no network egress.
426    fn direct_addr(substrate: &IrohSubstrate) -> EndpointAddr {
427        EndpointAddr::from_parts(
428            substrate.endpoint_id(),
429            substrate
430                .endpoint()
431                .bound_sockets()
432                .into_iter()
433                .map(TransportAddr::Ip),
434        )
435    }
436
437    fn assert_hybrid_handshake(conn: &Connection, alpn: &[u8]) {
438        let data = conn
439            .handshake_data()
440            .expect("completed iroh connection has handshake data")
441            .downcast::<HandshakeData>()
442            .expect("iroh uses noq rustls handshake data");
443        assert_eq!(data.protocol.as_deref(), Some(alpn));
444        assert_eq!(
445            data.negotiated_key_exchange_group,
446            Some(rustls::NamedGroup::X25519MLKEM768),
447            "owned iroh mesh must negotiate X25519MLKEM768"
448        );
449    }
450
451    /// Smoke test: build two substrates, dial direct (no DNS/pkarr), echo a
452    /// message on each ALPN. Verifies the router dispatches by ALPN and that
453    /// both planes terminate distinct handlers.
454    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
455    async fn smoke_two_endpoints_two_alpns() -> Result<()> {
456        // Server with EchoHandler on both ALPNs.
457        let server = IrohSubstrate::new_test(fresh_key(), EchoHandler, EchoHandler).await?;
458        let server_id = server.endpoint_id();
459        let server_addr = direct_addr(&server);
460
461        // Client with no-op handlers (it only originates).
462        let client = IrohSubstrate::new_test(
463            fresh_key(),
464            NoopHandler::new("client moq"),
465            NoopHandler::new("client rpc"),
466        )
467        .await?;
468
469        // RPC plane round-trip.
470        {
471            let conn = client
472                .connect(server_addr.clone(), ALPN_HYPRSTREAM_RPC)
473                .await?;
474            assert_hybrid_handshake(&conn, ALPN_HYPRSTREAM_RPC);
475            let (mut send, mut recv) = conn.open_bi().await?;
476            send.write_all(b"hello rpc").await?;
477            send.finish()?;
478            let got = recv.read_to_end(64).await?;
479            assert_eq!(&got, b"hello rpc");
480        }
481
482        // Streaming plane round-trip (echo here; real moq-net wiring lands in Phase 3).
483        {
484            let conn = client.connect(server_addr.clone(), ALPN_MOQ_LITE).await?;
485            assert_hybrid_handshake(&conn, ALPN_MOQ_LITE);
486            let (mut send, mut recv) = conn.open_bi().await?;
487            send.write_all(b"hello moq").await?;
488            send.finish()?;
489            let got = recv.read_to_end(64).await?;
490            assert_eq!(&got, b"hello moq");
491        }
492
493        // Sanity: server id is stable.
494        assert_eq!(server.endpoint_id(), server_id);
495
496        client.shutdown().await?;
497        server.shutdown().await?;
498        Ok(())
499    }
500
501    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
502    async fn owned_handlers_complete_handshake_before_delegation() -> Result<()> {
503        let called = Arc::new(AtomicBool::new(false));
504        let server = IrohSubstrate::new_test(
505            fresh_key(),
506            EarlyDataProbe(called.clone()),
507            NoopHandler::new("rpc"),
508        )
509        .await?;
510        let client = IrohSubstrate::new_test(
511            fresh_key(),
512            NoopHandler::new("moq"),
513            NoopHandler::new("rpc"),
514        )
515        .await?;
516
517        let conn = client.connect(direct_addr(&server), ALPN_MOQ_LITE).await?;
518        assert_hybrid_handshake(&conn, ALPN_MOQ_LITE);
519        assert!(!called.load(Ordering::SeqCst));
520
521        client.shutdown().await?;
522        server.shutdown().await?;
523        Ok(())
524    }
525
526    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
527    async fn internal_iroh_mesh_rejects_classical_only_peer() -> Result<()> {
528        let server = IrohSubstrate::new(fresh_key(), EchoHandler, EchoHandler).await?;
529        let server_addr = direct_addr(&server);
530        let classical_client = Endpoint::builder(presets::N0)
531            .secret_key(SecretKey::from_bytes(&fresh_key()))
532            .crypto_provider(Arc::new(rustls::crypto::ring::default_provider()))
533            .bind()
534            .await
535            .map_err(|e| anyhow::anyhow!("classical mutation endpoint bind: {e}"))?;
536
537        for alpn in [ALPN_HYPRSTREAM_RPC, ALPN_MOQ_LITE] {
538            let result = tokio::time::timeout(
539                std::time::Duration::from_secs(5),
540                classical_client.connect(server_addr.clone(), alpn),
541            )
542            .await
543            .expect("classical-only iroh mutation handshake must terminate");
544            match result {
545                Err(_) => {}
546                Ok(conn) => {
547                    tokio::time::timeout(std::time::Duration::from_secs(5), conn.closed())
548                        .await
549                        .expect("classical-only carrier must be closed by the hybrid hook");
550                }
551            }
552        }
553
554        classical_client.close().await;
555        server.shutdown().await?;
556        Ok(())
557    }
558}