hyprstream_core/services/
event_network.rs1use anyhow::{Context, Result};
3use hyprstream_discovery::admission_roster::{self, DeploymentRosterSource};
4use hyprstream_rpc::moq_authz::PeerIdentity;
5use hyprstream_rpc::transport::iroh_moq::SharedIngressAuthorizer;
6use hyprstream_rpc::transport::moql_admission::MoqlAdmissionProof;
7use std::collections::BTreeSet;
8use std::sync::Arc;
9
10pub fn require_event_tenant(proof: &MoqlAdmissionProof) -> Result<()> {
15 let roster = hyprstream_discovery::production_deployment_roster()?;
16 admission_roster::require_admitted_service_binding(&roster, proof)?;
17 super::native_event_policy::require_enrolled_event_identity(proof)
18}
19
20pub fn ensure_event_origin_for_profile() -> Result<()> {
23 let config = crate::config::HyprConfig::load()?;
24 if !config.quic.iroh_required() && !hyprstream_discovery::native_network_required() {
25 hyprstream_rpc::moq_event::ensure_event_client_origin(hyprstream_rpc::paths::event_socket());
26 return Ok(());
27 }
28 let proof = hyprstream_rpc::moq_stream::global_moq_admission_proof()
29 .context("native Event client requires an installed checkpointed admission proof")?
30 .clone();
31 require_event_tenant(&proof)?;
32 hyprstream_rpc::events::install_network_event_identity(&proof)?;
33 if let Some(origin) = hyprstream_rpc::moq_event::install_event_network_client_origin() {
34 tokio::spawn(async move {
35 let mut failures = 0u32;
36 loop {
37 let started = std::time::Instant::now();
38 let result: Result<()> = async {
39 let (transport, identity) =
40 hyprstream_discovery::production_moq_event_target().await?;
41 let mut proof = proof.clone();
42 proof.expected_server = identity;
43 hyprstream_rpc::moq_event::run_event_network_link(&origin, &transport, &proof)
44 .await
45 }
46 .await;
47 if let Err(error) = result {
48 tracing::warn!(%error, "native Event link unavailable");
49 }
50 if started.elapsed() >= std::time::Duration::from_secs(60) {
54 failures = 0;
55 }
56 failures = failures.saturating_add(1);
57 tokio::time::sleep(event_retry_delay(failures)).await;
58 }
59 });
60 }
61 Ok(())
62}
63
64fn event_retry_delay(failures: u32) -> std::time::Duration {
65 let seconds = 2u64.saturating_pow(failures.min(8)).min(300);
66 std::time::Duration::from_millis(seconds * 1000 + u64::from(rand::random::<u16>() % 1000))
68}
69
70pub fn event_ingress_authorizer(
77 roster: Arc<dyn DeploymentRosterSource>,
78 publishers: Arc<BTreeSet<String>>,
79) -> SharedIngressAuthorizer {
80 Arc::new(move |peer: &PeerIdentity, tenant: &str| {
81 tenant == admission_roster::LOCAL_TENANT
82 && peer
83 .subject
84 .as_deref()
85 .and_then(|did| {
86 let snapshot = roster.roster();
87 admission_roster::roster_service_name(
88 &snapshot,
89 did,
90 hyprstream_rpc::envelope::current_timestamp(),
91 )
92 })
93 .is_some_and(|name| publishers.contains(name.as_str()))
94 })
95}
96
97pub fn event_handler(
100 ctx: &hyprstream_service::ServiceContext,
101 origin: &hyprstream_rpc::moq_event::MoqEventOrigin,
102 node_id: [u8; 32],
103) -> Result<hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler> {
104 use hyprstream_rpc::transport::iroh_moq::{
105 IrohMoqProtocolHandler, MoqAuthzConfig, OriginShared,
106 };
107 use hyprstream_rpc::transport::moql_admission::{
108 MoqlAdmissionAuthenticator, MoqlServerIdentityProof,
109 };
110 let config = crate::config::HyprConfig::load()?;
111 let proof = ctx
112 .moql_admission_proof("event")?
113 .context("native Event server proof missing")?;
114 let roster = hyprstream_discovery::production_deployment_roster()?;
119 admission_roster::require_service_self_binding(&roster, "event", &proof)?;
120 super::native_event_policy::require_enrolled_event_identity(&proof)?;
121 admission_roster::require_publisher_roster(&roster, &config.quic.event_publishers)?;
122 hyprstream_rpc::events::install_network_event_identity(&proof)?;
123 let admission = MoqlAdmissionAuthenticator::new(
126 hyprstream_discovery::production_moql_accepted_state_authority()?,
127 admission_roster::derived_tenant_resolver(Arc::clone(&roster)),
128 );
129 admission.install_server_identity(
130 MoqlServerIdentityProof::from_local_admission_proof(&proof)?,
131 node_id,
132 )?;
133 let authz = hyprstream_rpc::events::installed_event_authz()
134 .context("Event MAC reference monitor missing")?;
135 Ok(IrohMoqProtocolHandler::with_origin(OriginShared::from_pair(
136 origin.producer(),
137 origin.consumer(),
138 ))
139 .with_event_authz(authz)
140 .with_authz(
141 MoqAuthzConfig::default()
142 .with_admission(Arc::new(admission))
143 .with_ingress_authorizer(event_ingress_authorizer(
144 roster,
145 Arc::new(config.quic.event_publishers),
146 )),
147 ))
148}
149
150pub struct EventNetworkService {
152 secret: [u8; 32],
153 node_id: [u8; 32],
154 handler: hyprstream_rpc::transport::iroh_moq::IrohMoqProtocolHandler,
155 announce: hyprstream_service::service::factory::NativeIrohAnnouncementCallback,
156}
157
158impl EventNetworkService {
159 pub fn new(
160 ctx: &hyprstream_service::ServiceContext,
161 origin: &hyprstream_rpc::moq_event::MoqEventOrigin,
162 ) -> Result<Self> {
163 let key = hyprstream_rpc::node_identity::derive_purpose_key(
164 &ctx.service_signing_key("event"),
165 "hyprstream-iroh-transport-v1",
166 );
167 let node_id = key.verifying_key().to_bytes();
168 Ok(Self {
169 secret: key.to_bytes(),
170 node_id,
171 handler: event_handler(ctx, origin, node_id)?,
172 announce: ctx.native_iroh_announcement_callback("event")?,
173 })
174 }
175
176 async fn run_until_shutdown(
177 self,
178 shutdown: Arc<tokio::sync::Notify>,
179 ready: impl FnOnce(),
180 ) -> hyprstream_rpc::error::Result<()> {
181 use hyprstream_rpc::error::RpcError;
182 let stopped = shutdown.notified();
186 tokio::pin!(stopped);
187 stopped.as_mut().enable();
188 let substrate = hyprstream_rpc::transport::iroh_substrate::IrohSubstrate::new(
189 self.secret,
190 self.handler,
191 hyprstream_rpc::transport::iroh_substrate::RefuseHandler::new(
192 "Event provides only authenticated moql",
193 ),
194 )
195 .await
196 .map_err(|e| RpcError::SpawnFailed(format!("native Event bind: {e}")))?;
197 let cancellation = tokio_util::sync::CancellationToken::new();
198 let _cancel_on_exit = cancellation.clone().drop_guard();
199 let announce_cancel = cancellation.clone();
200 let announce = self.announce;
201 let node_id = self.node_id;
202 let initial = tokio::select! {
203 biased;
204 _ = &mut stopped => Err(RpcError::SpawnFailed("Event stopped before announcement".into())),
205 result = tokio::task::spawn_blocking(move || announce(announce_cancel, node_id)) => {
206 result.map_err(|e| RpcError::SpawnFailed(format!("Event announcement task: {e}")))
207 .and_then(|r| r.map_err(|e| RpcError::SpawnFailed(format!("Event initial announcement: {e}"))))
208 }
209 };
210 if let Err(error) = initial {
211 cancellation.cancel();
212 let _ = substrate.shutdown().await;
213 return Err(error);
214 }
215 ready();
216 tokio::select! {
217 _ = &mut stopped => {},
218 _ = cancellation.cancelled() => {},
219 _ = async {
220 while !substrate.router().is_shutdown() && !substrate.endpoint().is_closed() {
221 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
222 }
223 } => {},
224 }
225 cancellation.cancel();
226 substrate
227 .shutdown()
228 .await
229 .map_err(|e| RpcError::SpawnFailed(format!("Event shutdown: {e}")))
230 }
231}
232
233impl hyprstream_rpc::service::Spawnable for EventNetworkService {
234 fn name(&self) -> &str {
235 "event"
236 }
237 fn registrations(
238 &self,
239 ) -> Vec<(
240 hyprstream_rpc::registry::SocketKind,
241 hyprstream_rpc::transport::TransportConfig,
242 )> {
243 vec![]
244 }
245 fn run(
246 self: Box<Self>,
247 shutdown: Arc<tokio::sync::Notify>,
248 on_ready: Option<tokio::sync::oneshot::Sender<()>>,
249 ) -> hyprstream_rpc::error::Result<()> {
250 use hyprstream_rpc::error::RpcError;
251 let runtime = tokio::runtime::Builder::new_current_thread()
252 .enable_all()
253 .build()
254 .map_err(|e| RpcError::SpawnFailed(e.to_string()))?;
255 runtime.block_on((*self).run_until_shutdown(shutdown, move || {
256 if let Some(ready) = on_ready {
257 let _ = ready.send(());
258 }
259 let _ = hyprstream_rpc::notify::ready();
260 }))
261 }
262}
263
264pub async fn probe_event_network(timeout: std::time::Duration) -> Result<()> {
268 tokio::time::timeout(timeout, async {
269 let config = crate::config::HyprConfig::load()?;
270 anyhow::ensure!(config.quic.iroh_required() || hyprstream_discovery::native_network_required(), "Event network probe requires network-iroh-required");
271 let mut proof = hyprstream_rpc::moq_stream::global_moq_admission_proof()
272 .context("Event probe requires a checkpointed client proof")?.clone();
273 require_event_tenant(&proof)?;
274 let (transport, identity) = hyprstream_discovery::production_moq_event_target().await?;
275 proof.expected_server = identity;
276 let stream = hyprstream_rpc::dial::dial_stream_authenticated(&transport, &proof).await?;
277 let origin = hyprstream_rpc::moq_event::MoqEventOrigin::new();
278 let client = moq_net::Client::new().with_consume(origin.producer());
280 let session = stream.connect_moq(&client).await?;
281 let consumer = origin.consumer();
282 let evidence = async {
283 let broadcast = consumer.announced_broadcast("local/events/system").await.context("system Event source unavailable")?;
284 let mut track = broadcast.subscribe_track(&moq_net::Track::new(hyprstream_rpc::moq_event::EVENT_TRACK))?;
285 let mut group = track.next_group().await?.context("system Event track has no group")?;
286 group.read_frame().await?.context("system Event track has no frame")?;
287 Ok(())
288 };
289 tokio::select! {
290 result = evidence => result,
291 _ = session.closed() => anyhow::bail!("Event probe session closed before authorized track delivery"),
292 }
293 }).await.context("native Event capability probe timed out")?
294}
295
296#[cfg(test)]
297mod tests {
298 use super::*;
299
300 #[test]
301 fn native_event_retries_have_positive_jitter_and_a_finite_cap() {
302 for failures in [1, 2, 3, 8, 9, 100, u32::MAX] {
303 let delay = event_retry_delay(failures);
304 let base = 2u64.saturating_pow(failures.min(8)).min(300);
305 assert!(delay >= std::time::Duration::from_secs(base));
306 assert!(delay < std::time::Duration::from_secs(base + 1));
307 assert!(delay < std::time::Duration::from_secs(257));
308 }
309 }
310
311 fn test_event_service(
312 announce: hyprstream_service::service::factory::NativeIrohAnnouncementCallback,
313 ) -> EventNetworkService {
314 use hyprstream_rpc::transport::iroh_moq::{IrohMoqProtocolHandler, OriginShared};
315 let key = ed25519_dalek::SigningKey::from_bytes(&rand::random());
316 let origin = hyprstream_rpc::moq_event::MoqEventOrigin::new();
317 EventNetworkService {
318 secret: key.to_bytes(),
319 node_id: key.verifying_key().to_bytes(),
320 handler: IrohMoqProtocolHandler::with_origin(OriginShared::from_pair(
321 origin.producer(),
322 origin.consumer(),
323 )),
324 announce,
325 }
326 }
327
328 #[tokio::test]
329 async fn event_shutdown_at_readiness_is_retained() -> Result<()> {
330 let shutdown = Arc::new(tokio::sync::Notify::new());
331 let cancellation = tokio_util::sync::CancellationToken::new();
332 let announced = Arc::new(parking_lot::Mutex::new(cancellation.clone()));
333 let observed = Arc::clone(&announced);
334 let service = test_event_service(Arc::new(move |token, _| {
335 *observed.lock() = token;
336 Ok(())
337 }));
338 let ready = std::cell::Cell::new(false);
339 tokio::time::timeout(
340 std::time::Duration::from_secs(10),
341 service.run_until_shutdown(Arc::clone(&shutdown), || {
342 ready.set(true);
343 shutdown.notify_waiters();
346 }),
347 )
348 .await
349 .context("Event lost shutdown at readiness")??;
350 anyhow::ensure!(ready.get(), "Event never reached readiness");
351 anyhow::ensure!(
352 announced.lock().is_cancelled(),
353 "announcement was not cancelled"
354 );
355 Ok(())
356 }
357
358 #[tokio::test]
359 async fn event_shutdown_during_announcement_prevents_readiness() -> Result<()> {
360 let shutdown = Arc::new(tokio::sync::Notify::new());
361 let stopping = Arc::clone(&shutdown);
362 let service = test_event_service(Arc::new(move |_, _| {
363 stopping.notify_waiters();
364 Ok(())
365 }));
366 let ready = std::cell::Cell::new(false);
367 let result = tokio::time::timeout(
368 std::time::Duration::from_secs(10),
369 service.run_until_shutdown(shutdown, || ready.set(true)),
370 )
371 .await
372 .context("Event did not stop during announcement")?;
373 anyhow::ensure!(result.is_err(), "startup shutdown must fail readiness");
374 anyhow::ensure!(!ready.get(), "Event became ready after startup shutdown");
375 Ok(())
376 }
377
378 #[test]
382 fn required_event_client_missing_proof_has_no_local_fallback() -> Result<()> {
383 const CHILD: &str = "HYPRSTREAM_EVENT_REQUIRED_TEST_CHILD";
384 if std::env::var_os(CHILD).is_none() {
385 let status = std::process::Command::new(std::env::current_exe()?)
386 .args(["--exact", "services::event_network::tests::required_event_client_missing_proof_has_no_local_fallback", "--nocapture"])
387 .env(CHILD, "1")
388 .env("HYPRSTREAM__QUIC__NATIVE_NETWORK_PROFILE", "network-iroh-required")
389 .status()?;
390 anyhow::ensure!(
391 status.success(),
392 "required Event initializer isolation failed"
393 );
394 return Ok(());
395 }
396 anyhow::ensure!(hyprstream_rpc::moq_event::global_moq_event_origin().is_none());
397 let error = ensure_event_origin_for_profile()
398 .err()
399 .context("missing proof did not reject required Event initialization")?;
400 anyhow::ensure!(
401 error.to_string().contains("checkpointed admission proof"),
402 "unexpected error: {error}"
403 );
404 anyhow::ensure!(
405 hyprstream_rpc::moq_event::global_moq_event_origin().is_none(),
406 "required failure installed a local origin"
407 );
408 Ok(())
409 }
410
411 fn state_with_services(
412 service_ids: &[&str],
413 ) -> hyprstream_rpc::transport::moql_admission::AcceptedIdentityState {
414 use hyprstream_rpc::transport::moql_admission::{
415 AcceptedIdentityState, AcceptedSubjectKey,
416 };
417 AcceptedIdentityState {
418 epoch: 1,
419 head_digest: [1; 64],
420 subject_keys: vec![AcceptedSubjectKey {
421 ed25519: [2; 32],
422 ml_dsa_65: vec![3; 1952],
423 }],
424 service_ids: service_ids.iter().map(ToString::to_string).collect(),
425 expires_at_unix_ms: Some(hyprstream_rpc::envelope::current_timestamp() + 3_600_000),
428 }
429 }
430
431 fn peer_of(did: &str) -> hyprstream_rpc::moq_authz::PeerIdentity {
432 hyprstream_rpc::moq_authz::PeerIdentity::authenticated(did.to_owned())
433 }
434
435 #[test]
444 fn event_ingress_follows_service_names_across_did_churn() {
445 use hyprstream_discovery::admission_roster::DeploymentRosterSource;
446 use std::collections::BTreeMap;
447 let original = "did:at9p:original";
448 let successor = "did:at9p:successor";
449 let foreign = "did:at9p:foreign-record";
450 let squatter = "did:at9p:squatter";
451 let states = Arc::new(parking_lot::RwLock::new(BTreeMap::from([
452 (original.to_owned(), state_with_services(&["#event"])),
453 (foreign.to_owned(), state_with_services(&["#ns"])),
454 ])));
455 let roster: Arc<dyn DeploymentRosterSource> = {
456 let states = Arc::clone(&states);
457 Arc::new(move || states.read().clone().into_iter().collect::<Vec<_>>())
458 };
459 let publishers: Arc<std::collections::BTreeSet<String>> =
460 Arc::new(["event".to_owned()].into());
461 let authorizer = event_ingress_authorizer(roster, publishers);
462
463 assert!(authorizer.authorize_ingress(&peer_of(original), "local"));
464 assert!(!authorizer.authorize_ingress(&peer_of(foreign), "local"));
466 assert!(!authorizer.authorize_ingress(&peer_of("did:at9p:unknown"), "local"));
467 assert!(!authorizer.authorize_ingress(&peer_of(original), "other"));
468
469 states.write().remove(original);
471 states
472 .write()
473 .insert(successor.to_owned(), state_with_services(&["#event"]));
474 assert!(
475 !authorizer.authorize_ingress(&peer_of(original), "local"),
476 "superseded DID must lose the grant"
477 );
478 assert!(
479 authorizer.authorize_ingress(&peer_of(successor), "local"),
480 "successor DID keeps ingress through the name-keyed grant, no restart"
481 );
482
483 states
488 .write()
489 .insert(squatter.to_owned(), state_with_services(&["#event"]));
490 assert!(!authorizer.authorize_ingress(&peer_of(successor), "local"));
491 assert!(!authorizer.authorize_ingress(&peer_of(squatter), "local"));
492
493 states.write().remove(squatter);
495 let empty: Arc<std::collections::BTreeSet<String>> = Arc::default();
496 let deny_all = event_ingress_authorizer(
497 {
498 let states = Arc::clone(&states);
499 Arc::new(move || states.read().clone().into_iter().collect::<Vec<_>>())
500 },
501 empty,
502 );
503 assert!(!deny_all.authorize_ingress(&peer_of(successor), "local"));
504 }
505}