Skip to main content

hyprstream_core/services/
event_network.rs

1//! Native Event transport bootstrap. Identity admission and MAC are independent.
2use anyhow::{Context, Result};
3use hyprstream_discovery::admission_roster::{self, DeploymentRosterSource};
4use hyprstream_rpc::moq_authz::PeerIdentity;
5use hyprstream_rpc::transport::iroh_moq::SharedIngressAuthorizer;
6use hyprstream_rpc::transport::moql_admission::MoqlAdmissionProof;
7use std::collections::BTreeSet;
8use std::sync::Arc;
9
10/// The fixed Event tree belongs only to the provisioned local tenant. The
11/// process's own proof must be a current accepted deployment service identity
12/// — resolved from the live checkpoint store (#1652), never from a DID-valued
13/// config binding that could go stale at identity churn.
14pub fn require_event_tenant(proof: &MoqlAdmissionProof) -> Result<()> {
15    let roster = hyprstream_discovery::production_deployment_roster()?;
16    admission_roster::require_admitted_service_binding(&roster, proof)?;
17    super::native_event_policy::require_enrolled_event_identity(proof)
18}
19
20/// Initialize both CLI and service Event clients according to the deployment
21/// profile. Required mode never opens a UDS, including when credentials are absent.
22pub fn ensure_event_origin_for_profile() -> Result<()> {
23    let config = crate::config::HyprConfig::load()?;
24    if !config.quic.iroh_required() && !hyprstream_discovery::native_network_required() {
25        hyprstream_rpc::moq_event::ensure_event_client_origin(hyprstream_rpc::paths::event_socket());
26        return Ok(());
27    }
28    let proof = hyprstream_rpc::moq_stream::global_moq_admission_proof()
29        .context("native Event client requires an installed checkpointed admission proof")?
30        .clone();
31    require_event_tenant(&proof)?;
32    hyprstream_rpc::events::install_network_event_identity(&proof)?;
33    if let Some(origin) = hyprstream_rpc::moq_event::install_event_network_client_origin() {
34        tokio::spawn(async move {
35            let mut failures = 0u32;
36            loop {
37                let started = std::time::Instant::now();
38                let result: Result<()> = async {
39                    let (transport, identity) =
40                        hyprstream_discovery::production_moq_event_target().await?;
41                    let mut proof = proof.clone();
42                    proof.expected_server = identity;
43                    hyprstream_rpc::moq_event::run_event_network_link(&origin, &transport, &proof)
44                        .await
45                }
46                .await;
47                if let Err(error) = result {
48                    tracing::warn!(%error, "native Event link unavailable");
49                }
50                // No authorization-failure notification exists at this layer.
51                // Slow bounded retries permit recovery without an idle deny
52                // storm. Reset only after a genuinely sustained connection.
53                if started.elapsed() >= std::time::Duration::from_secs(60) {
54                    failures = 0;
55                }
56                failures = failures.saturating_add(1);
57                tokio::time::sleep(event_retry_delay(failures)).await;
58            }
59        });
60    }
61    Ok(())
62}
63
64fn event_retry_delay(failures: u32) -> std::time::Duration {
65    let seconds = 2u64.saturating_pow(failures.min(8)).min(300);
66    // Positive jitter is bounded and never weakens the minimum retry delay.
67    std::time::Duration::from_millis(seconds * 1000 + u64::from(rand::random::<u16>() % 1000))
68}
69
70/// The Event ingress grant: the already-admitted peer's verified DID resolves
71/// (through the same live accepted-state roster admission used) to a
72/// deployment service NAME, which must be a member of the operator's
73/// name-keyed `quic.event_publishers` set (#1652). Names survive identity
74/// churn, so the grant never goes stale when a service's DID changes; an
75/// empty set denies every peer — read-only, never all-publishers.
76pub fn event_ingress_authorizer(
77    roster: Arc<dyn DeploymentRosterSource>,
78    publishers: Arc<BTreeSet<String>>,
79) -> SharedIngressAuthorizer {
80    Arc::new(move |peer: &PeerIdentity, tenant: &str| {
81        tenant == admission_roster::LOCAL_TENANT
82            && peer
83                .subject
84                .as_deref()
85                .and_then(|did| {
86                    let snapshot = roster.roster();
87                    admission_roster::roster_service_name(
88                        &snapshot,
89                        did,
90                        hyprstream_rpc::envelope::current_timestamp(),
91                    )
92                })
93                .is_some_and(|name| publishers.contains(name.as_str()))
94    })
95}
96
97/// Build the Event-owned admission authenticator and the independent explicit
98/// ingress grant. An absent MAC policy still denies every Event source.
99pub fn event_handler(
100    ctx: &hyprstream_service::ServiceContext,
101    origin: &hyprstream_rpc::moq_event::MoqEventOrigin,
102    node_id: [u8; 32],
103) -> Result<hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler> {
104    use hyprstream_rpc::transport::iroh_moq::{
105        IrohMoqProtocolHandler, MoqAuthzConfig, OriginShared,
106    };
107    use hyprstream_rpc::transport::moql_admission::{
108        MoqlAdmissionAuthenticator, MoqlServerIdentityProof,
109    };
110    let config = crate::config::HyprConfig::load()?;
111    let proof = ctx
112        .moql_admission_proof("event")?
113        .context("native Event server proof missing")?;
114    // Self-binding: the checkpoint store must resolve THIS process's own DID
115    // to THIS service, and every configured publisher name must be carried by
116    // a live deployment identity. Refusal fails the spawn (fail-closed),
117    // replacing the former empty-tenant-map refusal.
118    let roster = hyprstream_discovery::production_deployment_roster()?;
119    admission_roster::require_service_self_binding(&roster, "event", &proof)?;
120    super::native_event_policy::require_enrolled_event_identity(&proof)?;
121    admission_roster::require_publisher_roster(&roster, &config.quic.event_publishers)?;
122    hyprstream_rpc::events::install_network_event_identity(&proof)?;
123    // Authentication stays on the per-DID authority; only grant resolution
124    // runs over the roster (uniqueness needs the whole store).
125    let admission = MoqlAdmissionAuthenticator::new(
126        hyprstream_discovery::production_moql_accepted_state_authority()?,
127        admission_roster::derived_tenant_resolver(Arc::clone(&roster)),
128    );
129    admission.install_server_identity(
130        MoqlServerIdentityProof::from_local_admission_proof(&proof)?,
131        node_id,
132    )?;
133    let authz = hyprstream_rpc::events::installed_event_authz()
134        .context("Event MAC reference monitor missing")?;
135    Ok(IrohMoqProtocolHandler::with_origin(OriginShared::from_pair(
136        origin.producer(),
137        origin.consumer(),
138    ))
139    .with_event_authz(authz)
140    .with_authz(
141        MoqAuthzConfig::default()
142            .with_admission(Arc::new(admission))
143            .with_ingress_authorizer(event_ingress_authorizer(
144                roster,
145                Arc::new(config.quic.event_publishers),
146            )),
147    ))
148}
149
150/// Owns the Event carrier and announcement cancellation for the service lifetime.
151pub struct EventNetworkService {
152    secret: [u8; 32],
153    node_id: [u8; 32],
154    handler: hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler,
155    announce: hyprstream_service::service::factory::NativeIrohAnnouncementCallback,
156}
157
158impl EventNetworkService {
159    pub fn new(
160        ctx: &hyprstream_service::ServiceContext,
161        origin: &hyprstream_rpc::moq_event::MoqEventOrigin,
162    ) -> Result<Self> {
163        let key = hyprstream_rpc::node_identity::derive_purpose_key(
164            &ctx.service_signing_key("event"),
165            "hyprstream-iroh-transport-v1",
166        );
167        let node_id = key.verifying_key().to_bytes();
168        Ok(Self {
169            secret: key.to_bytes(),
170            node_id,
171            handler: event_handler(ctx, origin, node_id)?,
172            announce: ctx.native_iroh_announcement_callback("event")?,
173        })
174    }
175
176    async fn run_until_shutdown(
177        self,
178        shutdown: Arc<tokio::sync::Notify>,
179        ready: impl FnOnce(),
180    ) -> hyprstream_rpc::error::Result<()> {
181        use hyprstream_rpc::error::RpcError;
182        // Keep the same registered waiter through bind, publication and readiness.
183        // Replacing it after READY can lose notify_waiters() from an owner that
184        // stops the service as soon as readiness is observed.
185        let stopped = shutdown.notified();
186        tokio::pin!(stopped);
187        stopped.as_mut().enable();
188        let substrate = hyprstream_rpc::transport::iroh_substrate::IrohSubstrate::new(
189            self.secret,
190            self.handler,
191            hyprstream_rpc::transport::iroh_substrate::RefuseHandler::new(
192                "Event provides only authenticated moql",
193            ),
194        )
195        .await
196        .map_err(|e| RpcError::SpawnFailed(format!("native Event bind: {e}")))?;
197        let cancellation = tokio_util::sync::CancellationToken::new();
198        let _cancel_on_exit = cancellation.clone().drop_guard();
199        let announce_cancel = cancellation.clone();
200        let announce = self.announce;
201        let node_id = self.node_id;
202        let initial = tokio::select! {
203            biased;
204            _ = &mut stopped => Err(RpcError::SpawnFailed("Event stopped before announcement".into())),
205            result = tokio::task::spawn_blocking(move || announce(announce_cancel, node_id)) => {
206                result.map_err(|e| RpcError::SpawnFailed(format!("Event announcement task: {e}")))
207                    .and_then(|r| r.map_err(|e| RpcError::SpawnFailed(format!("Event initial announcement: {e}"))))
208            }
209        };
210        if let Err(error) = initial {
211            cancellation.cancel();
212            let _ = substrate.shutdown().await;
213            return Err(error);
214        }
215        ready();
216        tokio::select! {
217            _ = &mut stopped => {},
218            _ = cancellation.cancelled() => {},
219            _ = async {
220                while !substrate.router().is_shutdown() && !substrate.endpoint().is_closed() {
221                    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
222                }
223            } => {},
224        }
225        cancellation.cancel();
226        substrate
227            .shutdown()
228            .await
229            .map_err(|e| RpcError::SpawnFailed(format!("Event shutdown: {e}")))
230    }
231}
232
233impl hyprstream_rpc::service::Spawnable for EventNetworkService {
234    fn name(&self) -> &str {
235        "event"
236    }
237    fn registrations(
238        &self,
239    ) -> Vec<(
240        hyprstream_rpc::registry::SocketKind,
241        hyprstream_rpc::transport::TransportConfig,
242    )> {
243        vec![]
244    }
245    fn run(
246        self: Box<Self>,
247        shutdown: Arc<tokio::sync::Notify>,
248        on_ready: Option<tokio::sync::oneshot::Sender<()>>,
249    ) -> hyprstream_rpc::error::Result<()> {
250        use hyprstream_rpc::error::RpcError;
251        let runtime = tokio::runtime::Builder::new_current_thread()
252            .enable_all()
253            .build()
254            .map_err(|e| RpcError::SpawnFailed(e.to_string()))?;
255        runtime.block_on((*self).run_until_shutdown(shutdown, move || {
256            if let Some(ready) = on_ready {
257                let _ = ready.send(());
258            }
259            let _ = hyprstream_rpc::notify::ready();
260        }))
261    }
262}
263
264/// Probe the native Event capability: authenticate to the freshly resolved
265/// carrier and receive an actual frame from the authorized system source.
266/// A carrier handshake alone, or a local retained Event, cannot satisfy it.
267pub async fn probe_event_network(timeout: std::time::Duration) -> Result<()> {
268    tokio::time::timeout(timeout, async {
269        let config = crate::config::HyprConfig::load()?;
270        anyhow::ensure!(config.quic.iroh_required() || hyprstream_discovery::native_network_required(), "Event network probe requires network-iroh-required");
271        let mut proof = hyprstream_rpc::moq_stream::global_moq_admission_proof()
272            .context("Event probe requires a checkpointed client proof")?.clone();
273        require_event_tenant(&proof)?;
274        let (transport, identity) = hyprstream_discovery::production_moq_event_target().await?;
275        proof.expected_server = identity;
276        let stream = hyprstream_rpc::dial::dial_stream_authenticated(&transport, &proof).await?;
277        let origin = hyprstream_rpc::moq_event::MoqEventOrigin::new();
278        // Subscriber-only: a health probe never announces or publishes.
279        let client = moq_net::Client::new().with_consume(origin.producer());
280        let session = stream.connect_moq(&client).await?;
281        let consumer = origin.consumer();
282        let evidence = async {
283            let broadcast = consumer.announced_broadcast("local/events/system").await.context("system Event source unavailable")?;
284            let mut track = broadcast.subscribe_track(&moq_net::Track::new(hyprstream_rpc::moq_event::EVENT_TRACK))?;
285            let mut group = track.next_group().await?.context("system Event track has no group")?;
286            group.read_frame().await?.context("system Event track has no frame")?;
287            Ok(())
288        };
289        tokio::select! {
290            result = evidence => result,
291            _ = session.closed() => anyhow::bail!("Event probe session closed before authorized track delivery"),
292        }
293    }).await.context("native Event capability probe timed out")?
294}
295
296#[cfg(test)]
297mod tests {
298    use super::*;
299
300    #[test]
301    fn native_event_retries_have_positive_jitter_and_a_finite_cap() {
302        for failures in [1, 2, 3, 8, 9, 100, u32::MAX] {
303            let delay = event_retry_delay(failures);
304            let base = 2u64.saturating_pow(failures.min(8)).min(300);
305            assert!(delay >= std::time::Duration::from_secs(base));
306            assert!(delay < std::time::Duration::from_secs(base + 1));
307            assert!(delay < std::time::Duration::from_secs(257));
308        }
309    }
310
311    fn test_event_service(
312        announce: hyprstream_service::service::factory::NativeIrohAnnouncementCallback,
313    ) -> EventNetworkService {
314        use hyprstream_rpc::transport::iroh_moq::{IrohMoqProtocolHandler, OriginShared};
315        let key = ed25519_dalek::SigningKey::from_bytes(&rand::random());
316        let origin = hyprstream_rpc::moq_event::MoqEventOrigin::new();
317        EventNetworkService {
318            secret: key.to_bytes(),
319            node_id: key.verifying_key().to_bytes(),
320            handler: IrohMoqProtocolHandler::with_origin(OriginShared::from_pair(
321                origin.producer(),
322                origin.consumer(),
323            )),
324            announce,
325        }
326    }
327
328    #[tokio::test]
329    async fn event_shutdown_at_readiness_is_retained() -> Result<()> {
330        let shutdown = Arc::new(tokio::sync::Notify::new());
331        let cancellation = tokio_util::sync::CancellationToken::new();
332        let announced = Arc::new(parking_lot::Mutex::new(cancellation.clone()));
333        let observed = Arc::clone(&announced);
334        let service = test_event_service(Arc::new(move |token, _| {
335            *observed.lock() = token;
336            Ok(())
337        }));
338        let ready = std::cell::Cell::new(false);
339        tokio::time::timeout(
340            std::time::Duration::from_secs(10),
341            service.run_until_shutdown(Arc::clone(&shutdown), || {
342                ready.set(true);
343                // Deterministically stop inside readiness, before the final
344                // serving select is polled. A newly created waiter loses this.
345                shutdown.notify_waiters();
346            }),
347        )
348        .await
349        .context("Event lost shutdown at readiness")??;
350        anyhow::ensure!(ready.get(), "Event never reached readiness");
351        anyhow::ensure!(
352            announced.lock().is_cancelled(),
353            "announcement was not cancelled"
354        );
355        Ok(())
356    }
357
358    #[tokio::test]
359    async fn event_shutdown_during_announcement_prevents_readiness() -> Result<()> {
360        let shutdown = Arc::new(tokio::sync::Notify::new());
361        let stopping = Arc::clone(&shutdown);
362        let service = test_event_service(Arc::new(move |_, _| {
363            stopping.notify_waiters();
364            Ok(())
365        }));
366        let ready = std::cell::Cell::new(false);
367        let result = tokio::time::timeout(
368            std::time::Duration::from_secs(10),
369            service.run_until_shutdown(shutdown, || ready.set(true)),
370        )
371        .await
372        .context("Event did not stop during announcement")?;
373        anyhow::ensure!(result.is_err(), "startup shutdown must fail readiness");
374        anyhow::ensure!(!ready.get(), "Event became ready after startup shutdown");
375        Ok(())
376    }
377
378    /// A required CLI/service initializer must reject absent proof before it
379    /// installs an origin or spawns a compatibility socket task. Isolate process
380    /// globals so this tests the real initializer without fixture order effects.
381    #[test]
382    fn required_event_client_missing_proof_has_no_local_fallback() -> Result<()> {
383        const CHILD: &str = "HYPRSTREAM_EVENT_REQUIRED_TEST_CHILD";
384        if std::env::var_os(CHILD).is_none() {
385            let status = std::process::Command::new(std::env::current_exe()?)
386                .args(["--exact", "services::event_network::tests::required_event_client_missing_proof_has_no_local_fallback", "--nocapture"])
387                .env(CHILD, "1")
388                .env("HYPRSTREAM__QUIC__NATIVE_NETWORK_PROFILE", "network-iroh-required")
389                .status()?;
390            anyhow::ensure!(
391                status.success(),
392                "required Event initializer isolation failed"
393            );
394            return Ok(());
395        }
396        anyhow::ensure!(hyprstream_rpc::moq_event::global_moq_event_origin().is_none());
397        let error = ensure_event_origin_for_profile()
398            .err()
399            .context("missing proof did not reject required Event initialization")?;
400        anyhow::ensure!(
401            error.to_string().contains("checkpointed admission proof"),
402            "unexpected error: {error}"
403        );
404        anyhow::ensure!(
405            hyprstream_rpc::moq_event::global_moq_event_origin().is_none(),
406            "required failure installed a local origin"
407        );
408        Ok(())
409    }
410
411    fn state_with_services(
412        service_ids: &[&str],
413    ) -> hyprstream_rpc::transport::moql_admission::AcceptedIdentityState {
414        use hyprstream_rpc::transport::moql_admission::{
415            AcceptedIdentityState, AcceptedSubjectKey,
416        };
417        AcceptedIdentityState {
418            epoch: 1,
419            head_digest: [1; 64],
420            subject_keys: vec![AcceptedSubjectKey {
421                ed25519: [2; 32],
422                ml_dsa_65: vec![3; 1952],
423            }],
424            service_ids: service_ids.iter().map(ToString::to_string).collect(),
425            // Deployment invariants: successor epoch + bounded expiry — the
426            // derivation rejects genesis-only/unbounded states.
427            expires_at_unix_ms: Some(hyprstream_rpc::envelope::current_timestamp() + 3_600_000),
428        }
429    }
430
431    fn peer_of(did: &str) -> hyprstream_rpc::moq_authz::PeerIdentity {
432        hyprstream_rpc::moq_authz::PeerIdentity::authenticated(did.to_owned())
433    }
434
435    /// #1652 churn test (Event ingress grant): when a service's DID changes —
436    /// same service name re-bound to a new DID, as at an R1 store re-init —
437    /// the name-keyed grant keeps admitting that service in the SAME process,
438    /// no restart, no config change. Under the former DID-keyed set the new
439    /// DID was silently unlisted (read-only) until config was re-projected
440    /// and every container restarted. A foreign record squatting the service
441    /// name beside the genuine identity denies BOTH (fail-closed), instead of
442    /// self-selecting into the grant.
443    #[test]
444    fn event_ingress_follows_service_names_across_did_churn() {
445        use hyprstream_discovery::admission_roster::DeploymentRosterSource;
446        use std::collections::BTreeMap;
447        let original = "did:at9p:original";
448        let successor = "did:at9p:successor";
449        let foreign = "did:at9p:foreign-record";
450        let squatter = "did:at9p:squatter";
451        let states = Arc::new(parking_lot::RwLock::new(BTreeMap::from([
452            (original.to_owned(), state_with_services(&["#event"])),
453            (foreign.to_owned(), state_with_services(&["#ns"])),
454        ])));
455        let roster: Arc<dyn DeploymentRosterSource> = {
456            let states = Arc::clone(&states);
457            Arc::new(move || states.read().clone().into_iter().collect::<Vec<_>>())
458        };
459        let publishers: Arc<std::collections::BTreeSet<String>> =
460            Arc::new(["event".to_owned()].into());
461        let authorizer = event_ingress_authorizer(roster, publishers);
462
463        assert!(authorizer.authorize_ingress(&peer_of(original), "local"));
464        // Fail-closed matrix: foreign store record, unknown DID, wrong tenant.
465        assert!(!authorizer.authorize_ingress(&peer_of(foreign), "local"));
466        assert!(!authorizer.authorize_ingress(&peer_of("did:at9p:unknown"), "local"));
467        assert!(!authorizer.authorize_ingress(&peer_of(original), "other"));
468
469        // Identity churn: the same service name now lives at a new DID.
470        states.write().remove(original);
471        states
472            .write()
473            .insert(successor.to_owned(), state_with_services(&["#event"]));
474        assert!(
475            !authorizer.authorize_ingress(&peer_of(original), "local"),
476            "superseded DID must lose the grant"
477        );
478        assert!(
479            authorizer.authorize_ingress(&peer_of(successor), "local"),
480            "successor DID keeps ingress through the name-keyed grant, no restart"
481        );
482
483        // Name squatting: a foreign record claiming "#event" beside the
484        // genuine identity makes the name ambiguous — deny both, escalate to
485        // neither (the store cannot distinguish provenance; uniqueness is the
486        // discriminator).
487        states
488            .write()
489            .insert(squatter.to_owned(), state_with_services(&["#event"]));
490        assert!(!authorizer.authorize_ingress(&peer_of(successor), "local"));
491        assert!(!authorizer.authorize_ingress(&peer_of(squatter), "local"));
492
493        // Empty set = read-only for everyone, including deployment identities.
494        states.write().remove(squatter);
495        let empty: Arc<std::collections::BTreeSet<String>> = Arc::default();
496        let deny_all = event_ingress_authorizer(
497            {
498                let states = Arc::clone(&states);
499                Arc::new(move || states.read().clone().into_iter().collect::<Vec<_>>())
500            },
501            empty,
502        );
503        assert!(!deny_all.authorize_ingress(&peer_of(successor), "local"));
504    }
505}