hyprstream_rpc/moq_event.rs
1//! moq-lite event bus plane — replaces the ZMQ XPUB/XSUB ProxyService (#167).
2//!
3//! # Architecture
4//!
5//! The event bus is a process-global [`MoqEventOrigin`] rooted at `local/events`.
6//! Each service registers its own broadcast under `local/events/{source}` with a
7//! single `events` track. Each published frame carries the full dot-separated topic
8//! (`{source}.{entity}.{event}`) and the raw payload, packed as:
9//!
10//! ```text
11//! [4 bytes topic_len BE][topic UTF-8][payload bytes]
12//! ```
13//!
14//! Subscribers connect to the origin, discover source broadcasts via
15//! `OriginConsumer::announced()`, subscribe to their `events` track, and do
16//! in-memory topic-prefix filtering (matching ZMQ's prefix-filter semantics).
17//!
18//! This is the "Live" preset for the event bus: fan-out, unbounded, at-most-once.
19//! No chained HMAC or policy axes — events are best-effort lifecycle signals, not
20//! auditable streams. [`crate::events::EventPublisher`] / [`crate::events::EventSubscriber`]
21//! (the canonical broadcast type — see `crate::events`) layer group-key encryption
22//! on top of this transport, selectable via [`crate::crypto::event_crypto::EventPrivacy`].
23//!
24//! ## Two origins: `moq_event` vs `moq_stream`
25//!
26//! Broadcast publish/subscribe (fan-out, N subscribers per source/topic) is
27//! consolidated exclusively on THIS origin (`moq_event` / `MoqEventOrigin`).
28//! The sibling `moq_stream` origin (`crate::moq_stream`) is reserved for
29//! point-to-point token streaming (one producer, one consumer per stream,
30//! DH-derived `StreamKeys`) — it is not a second broadcast plane and new
31//! fan-out use cases should not be added to it. Anything that needs "N
32//! interested parties learn about an event" belongs on `moq_event`; anything
33//! that is a private request/response or token stream between two parties
34//! belongs on `moq_stream`.
35//!
36//! ## QoS (#606)
37//!
38//! `MoqEventSubscriber` reuses the EXISTING `StreamOpt` contract from
39//! `streaming.capnp` / [`crate::stream_info`] (#213/#273) rather than a parallel
40//! event-specific QoS type — see [`crate::stream_info::EventLive`] (default) and
41//! [`crate::stream_info::EventReliable`] (at-least-once + retained, for events
42//! that must not be silently dropped, e.g. `model.loaded`). Only `delivery` and
43//! `overflow_policy` are enforced client-side today (`ordering`/`completion` are
44//! reserved for a future chained-integrity layer, same as the flat at-most-once
45//! default already implies `Ordering::Ordered` is not currently checked).
46//!
47//! # #393 — per-OID broadcast paths
48//!
49//! The original design (above) is a flat single-track fan-out: every event for
50//! every model OID lands on the same `events` track, and subscribers filter
51//! post-read. That is the firehose problem — O(whole-network) reads. The fix
52//! mirrors the streaming plane (`moq_stream`: `{tenant}/{service}/{topic}/{instance}`):
53//! each model OID gets its OWN broadcast path, `local/events/publications/{oid_hash}`,
54//! so moq's `scope(&[Path])` makes wire-level selectivity automatic (one QUIC
55//! uni-stream per group per subscribed track). A node tracking N of M OIDs reads
56//! N tracks, not M.
57//!
58//! The flat `local/events/{source}` track is RETAINED as a transition fallback:
59//! publishers that mirror to both paths keep legacy subscribers working while
60//! new subscribers read the selective per-OID track. No capnp change — the event
61//! payload format is unchanged; only the track naming/routing changes.
62//!
63//! ## Late-join retention (decision A: firehose-backfill)
64//!
65//! moq's per-track cache evicts groups older than `MAX_GROUP_AGE` (5s) — too
66//! short for a scheduler reconstructing publication history. On first
67//! subscription to an OID's track, [`BackfillMode`] checks whether history is
68//! available via the optional atproto firehose (`sh.tangled.git.refUpdate`) /
69//! registry; if so it replays that history before switching to live MoQ. If the
70//! firehose is unavailable the subscriber starts live-only (graceful
71//! degradation) — the firehose is the cold-start path, MoQ is the live path.
72
73use std::collections::{HashMap, HashSet, VecDeque};
74use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
75use std::sync::{Arc, OnceLock};
76use std::time::Duration;
77
78use anyhow::{anyhow, Result};
79use bytes::Bytes;
80use moq_net::{
81 BroadcastProducer, Group, Origin, OriginConsumer, OriginProducer, Path, Track, TrackProducer,
82};
83use parking_lot::Mutex;
84use sha2::{Digest, Sha256};
85use tokio::sync::mpsc;
86
87use crate::stream_info::{OverflowPolicy, StreamOpt, StreamOptPreset};
88
89// ============================================================================
90// Process-global moq event bus origin — set once at startup by the factory.
91// ============================================================================
92
93static GLOBAL_MOQ_EVENT_ORIGIN: OnceLock<MoqEventOrigin> = OnceLock::new();
94
95/// Register the process-global moq event bus origin.
96///
97/// Must be called once at startup (from the event-service factory) before any
98/// `EventPublisher::new()` or `EventSubscriber::new()` call.
99/// Returns `true` on the first call, `false` if already initialized (idempotent).
100pub fn init_global_moq_event_origin(origin: MoqEventOrigin) -> bool {
101 GLOBAL_MOQ_EVENT_ORIGIN.set(origin).is_ok()
102}
103
104/// Borrow the process-global moq event bus origin, if initialized.
105///
106/// Returns `None` when the event bus has not yet been wired (unit tests, or
107/// before `create_event_service` factory has run).
108pub fn global_moq_event_origin() -> Option<&'static MoqEventOrigin> {
109 GLOBAL_MOQ_EVENT_ORIGIN.get()
110}
111
112/// Track name for the event fanout track within each source broadcast.
113pub const EVENT_TRACK: &str = "events";
114
115/// Broadcast path prefix (the `local/events` root under which per-source
116/// broadcasts are registered).
117pub const EVENT_PREFIX: &str = "local/events";
118
119/// Broadcast path segment under which per-OID (#393) publication broadcasts
120/// live: `local/events/publications/{oid_hash}`. Mirrors the streaming plane's
121/// `{tenant}/{service}/{topic}/{instance}` shape so `scope(&[Path])` gives
122/// wire-level selectivity — a node tracking N of M OIDs reads N tracks, not M.
123pub const PUBLICATIONS_SEGMENT: &str = "publications";
124
125/// Full prefix for per-OID publication broadcasts.
126pub const PUBLICATIONS_PREFIX: &str = "local/events/publications";
127
128/// Number of hex characters of the SHA-256 OID hash used in the broadcast path.
129/// 16 hex chars (64 bits) keeps paths short while making collisions negligible
130/// across any realistic OID namespace (birthday bound ≈ 2^32 models).
131pub const OID_HASH_LEN: usize = 16;
132
133/// Stable, filesystem/moq-safe identifier for a model OID.
134///
135/// The OID is hashed (SHA-256, truncated to [`OID_HASH_LEN`] hex chars) so the
136/// broadcast path is unguessable from the OID alone and contains no characters
137/// that are illegal in a moq path segment. Two distinct OIDs map to distinct
138/// paths with overwhelming probability.
139pub fn oid_hash(oid: &str) -> String {
140 let digest = Sha256::digest(oid.as_bytes());
141 hex::encode(&digest[..OID_HASH_LEN / 2])
142}
143
144/// Build the per-OID publication broadcast path: `local/events/publications/{oid_hash}`.
145pub fn publication_broadcast_path(oid: &str) -> String {
146 format!("{}/{}", PUBLICATIONS_PREFIX, oid_hash(oid))
147}
148
149// ============================================================================
150// MoqEventOrigin
151// ============================================================================
152
153/// Shared moq origin for the event bus plane.
154///
155/// Replaces the ZMQ XPUB/XSUB `ProxyService`. Publishers register broadcasts
156/// under `local/events/{source}`; subscribers watch the origin consumer for
157/// announcements and subscribe to the `events` track of each source broadcast.
158#[derive(Clone)]
159pub struct MoqEventOrigin {
160 inner: Arc<EventOriginInner>,
161}
162
163struct EventOriginInner {
164 /// Root-scoped producer: publishes under `local/events/{source}`.
165 producer: OriginProducer,
166 /// Consumer over the same origin tree (for subscribers).
167 consumer: OriginConsumer,
168 /// Keep `BroadcastProducer`s alive so their broadcasts stay announced.
169 /// Keyed by source name so re-registration replaces the stale producer.
170 broadcasts: Mutex<HashMap<String, BroadcastProducer>>,
171}
172
173impl MoqEventOrigin {
174 /// Create a standalone event origin (used at startup / in unit tests).
175 pub fn new() -> Self {
176 let producer = Origin::random().produce();
177 let consumer = producer.consume();
178 Self {
179 inner: Arc::new(EventOriginInner {
180 producer,
181 consumer,
182 broadcasts: Mutex::new(HashMap::new()),
183 }),
184 }
185 }
186
187 /// Create from an existing producer/consumer pair (e.g. shared with the
188 /// iroh moq substrate so external relay subscribers see the same tree).
189 pub fn from_pair(producer: OriginProducer, consumer: OriginConsumer) -> Self {
190 Self {
191 inner: Arc::new(EventOriginInner {
192 producer,
193 consumer,
194 broadcasts: Mutex::new(HashMap::new()),
195 }),
196 }
197 }
198
199 /// Create an event publisher for `source` (e.g. `"worker"`, `"system"`).
200 ///
201 /// Registers the broadcast `local/events/{source}` and opens its `events` track.
202 pub fn publisher(&self, source: &str) -> Result<MoqEventPublisher> {
203 let path = format!("{}/{}", EVENT_PREFIX, source);
204 let mut broadcast = self
205 .inner
206 .producer
207 .create_broadcast(path.as_str())
208 .ok_or_else(|| anyhow!("create_broadcast denied for {path}"))?;
209
210 let track = broadcast.create_track(Track::new(EVENT_TRACK))?;
211
212 // Retain the broadcast producer so it stays announced for the publisher's lifetime.
213 // HashMap keyed by source: re-registration replaces the stale producer atomically.
214 self.inner
215 .broadcasts
216 .lock()
217 .insert(source.to_owned(), broadcast);
218
219 Ok(MoqEventPublisher {
220 track,
221 source: source.to_owned(),
222 next_group: 0,
223 oid_track: None,
224 })
225 }
226
227 /// Create an event publisher for `source` (e.g. `"worker"`, `"system"`) that
228 /// ALSO mirrors every event to the per-OID (#393) publication track for `oid`.
229 ///
230 /// This is the transition path: the publisher writes each event to BOTH the
231 /// legacy flat `local/events/{source}` broadcast (so existing subscribers
232 /// keep working) AND the selective `local/events/publications/{oid_hash}`
233 /// broadcast (so #393 subscribers get wire-level per-OID selectivity). Once
234 /// all subscribers have migrated to per-OID tracks the flat mirror can be
235 /// dropped (see [`MoqEventOrigin::publisher_oid_only`]).
236 ///
237 /// `oid` is the model OID whose publications this publisher emits; it is
238 /// hashed via [`oid_hash`] for the broadcast path.
239 pub fn publisher_with_oid(&self, source: &str, oid: &str) -> Result<MoqEventPublisher> {
240 let flat = self.publisher(source)?;
241 let oid_track = self.oid_track(oid)?;
242 Ok(MoqEventPublisher {
243 track: flat.track,
244 source: flat.source,
245 next_group: flat.next_group,
246 oid_track: Some(oid_track),
247 })
248 }
249
250 /// Create an event publisher that writes ONLY to the per-OID (#393)
251 /// publication track for `oid` — no flat-track mirror. Use once every
252 /// subscriber of this source has migrated to per-OID subscription.
253 ///
254 /// `source` is retained for topic-prefix semantics and `publish()` topic
255 /// construction, but nothing is announced under `local/events/{source}`.
256 pub fn publisher_oid_only(&self, source: &str, oid: &str) -> Result<MoqEventPublisher> {
257 let oid_track = self.oid_track(oid)?;
258 Ok(MoqEventPublisher {
259 track: oid_track,
260 source: source.to_owned(),
261 next_group: 0,
262 oid_track: None,
263 })
264 }
265
266 /// Open (creating if needed) the per-OID publication track for `oid` and
267 /// retain its broadcast producer. Returns the track producer.
268 fn oid_track(&self, oid: &str) -> Result<TrackProducer> {
269 let path = publication_broadcast_path(oid);
270 let mut broadcast = self
271 .inner
272 .producer
273 .create_broadcast(path.as_str())
274 .ok_or_else(|| anyhow!("create_broadcast denied for {path}"))?;
275 let track = broadcast.create_track(Track::new(EVENT_TRACK))?;
276 // Keyed by the full path so distinct OIDs accumulate distinct producers
277 // and re-registration of the same OID replaces the stale producer.
278 self.inner.broadcasts.lock().insert(path, broadcast);
279 Ok(track)
280 }
281
282 /// Clone the origin consumer (for subscriber background tasks).
283 pub fn consumer(&self) -> OriginConsumer {
284 self.inner.consumer.clone()
285 }
286
287 /// Clone the origin producer.
288 ///
289 /// Needed to wire this origin into a `moq_net::Server`/`Client` with
290 /// `with_origin` / `with_consume` so externally-published broadcasts ingest
291 /// into the same tree subscribers read from.
292 pub fn producer(&self) -> OriginProducer {
293 self.inner.producer.clone()
294 }
295}
296
297impl Default for MoqEventOrigin {
298 fn default() -> Self {
299 Self::new()
300 }
301}
302
303// ============================================================================
304// Cross-process event plane over UDS (#275)
305//
306// In the same-process (`InprocManager`) deployment every service shares one
307// process-global `MoqEventOrigin`, so an in-process publisher and subscriber
308// see the same tree. In the systemd / `--ipc` deployment each service runs in
309// its OWN process; only the `event` service's process initializes the global
310// origin via the factory. Other processes (notably `worker`) had no origin at
311// all — `EventPublisher::new` failed with "moq event bus not initialized".
312//
313// The fix mirrors the streaming plane (`moq_stream::serve_moq_uds_background`):
314//
315// * The `event` service SERVES its origin over a well-known UDS path
316// ([`serve_event_moq_uds_background`]) using `moq_net::Server::with_origin`
317// so the session is BIDIRECTIONAL — it both serves announced broadcasts to
318// subscribers AND ingests broadcasts a connected client publishes.
319//
320// * Every NON-event process creates its own local `MoqEventOrigin`, registers
321// it as the process global, then spawns a background task
322// ([`connect_event_moq_uds_background`]) that connects a `moq_net::Client`
323// to the event service's UDS with `with_origin(local_producer)`. That makes
324// the session bidirectional too: locally-published event broadcasts are
325// announced UP to the event service (which fans them out to all
326// subscribers), and broadcasts from other processes flow DOWN into the
327// local origin so local subscribers see them.
328//
329// The UDS path is the stable, cross-process [`crate::paths::event_socket`] —
330// not a per-PID path — so any process can find it without an RPC round-trip.
331// ============================================================================
332
333/// Reconnect backoff for the client-side event plane link.
334const EVENT_RECONNECT_DELAY: Duration = Duration::from_millis(500);
335
336/// Serve the event-bus origin over a UDS socket so cross-process publishers and
337/// subscribers reach the same tree (#275).
338///
339/// Mirrors [`crate::moq_stream::serve_moq_uds_background`], but the moq server is
340/// built with `with_origin` (publish + consume) rather than `with_publish`
341/// alone, because the event plane is bidirectional: a connected client both
342/// announces its own event broadcasts (ingested into this origin) and subscribes
343/// to broadcasts from every other process.
344///
345/// Called once by the event-service factory. Idempotent: a second call is a
346/// no-op (the first bind wins). Connection errors are logged, not fatal.
347pub fn serve_event_moq_uds_background(origin: MoqEventOrigin, path: std::path::PathBuf) {
348 use crate::transport::uds_session::{accept_uds, PLANE_MOQ};
349 use moq_net::Server as MoqServer;
350
351 // Ensure the parent directory exists (runtime_dir may not be created yet).
352 if let Some(parent) = path.parent() {
353 let _ = std::fs::create_dir_all(parent);
354 }
355 // Remove a stale socket from a previous run (best-effort).
356 let _ = std::fs::remove_file(&path);
357
358 let listener = match std::os::unix::net::UnixListener::bind(&path) {
359 Ok(l) => l,
360 Err(e) => {
361 tracing::error!(path = %path.display(), "event moq UDS bind failed: {e}");
362 return;
363 }
364 };
365
366 // 0o600: owner read/write only; SO_PEERCRED enforces uid match (uds_server.rs).
367 #[cfg(unix)]
368 {
369 use std::os::unix::fs::PermissionsExt;
370 let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600));
371 }
372
373 // tokio requires the listener be non-blocking before adoption.
374 if let Err(e) = listener.set_nonblocking(true) {
375 tracing::error!("event moq UDS set_nonblocking failed: {e}");
376 return;
377 }
378
379 let listener = match tokio::net::UnixListener::from_std(listener) {
380 Ok(l) => l,
381 Err(e) => {
382 tracing::error!("event moq UDS listener conversion failed: {e}");
383 return;
384 }
385 };
386
387 tracing::info!(path = %path.display(), "event moq UDS listener ready");
388
389 tokio::spawn(async move {
390 loop {
391 let stream = match listener.accept().await {
392 Ok((s, _)) => s,
393 Err(e) => {
394 tracing::warn!("event moq UDS accept error: {e}");
395 continue;
396 }
397 };
398 // Each session gets its own producer/consumer view of the SAME origin
399 // tree, so a client's published broadcasts are ingested and re-served
400 // to every other connected session.
401 let producer = origin.producer();
402 tokio::spawn(async move {
403 let (plane, session) = match accept_uds(stream).await {
404 Ok(pair) => pair,
405 Err(e) => {
406 tracing::debug!("event moq UDS handshake error: {e}");
407 return;
408 }
409 };
410 if plane != PLANE_MOQ {
411 tracing::debug!("event moq UDS: unexpected plane 0x{plane:02x} — dropping");
412 return;
413 }
414 // `accept` returns once the handshake completes; the protocol then
415 // runs in the background. The returned `Session` MUST be held —
416 // dropping it closes the connection. Hold it until it closes so
417 // the bidirectional bridge stays live (publish + consume).
418 match MoqServer::new().with_origin(producer).accept(session).await {
419 Ok(moq_session) => {
420 let reason = moq_session.closed().await;
421 tracing::debug!("event moq UDS session closed: {reason:?}");
422 }
423 Err(e) => tracing::debug!("event moq UDS accept error: {e}"),
424 }
425 });
426 }
427 });
428}
429
430/// Ensure this process can publish/subscribe events, wiring a CLIENT-mode
431/// event origin connected to the event service's UDS plane if needed (#275).
432///
433/// Idempotent and safe to call from any process at startup:
434///
435/// * If the process-global origin is ALREADY initialized (this process hosts
436/// the `event` service, which calls [`init_global_moq_event_origin`] +
437/// [`serve_event_moq_uds_background`] in its factory), this is a no-op —
438/// the in-process origin is authoritative.
439///
440/// * Otherwise it creates a fresh local origin, registers it as the global,
441/// and spawns the background link to the event service's UDS
442/// ([`connect_event_moq_uds_background`]). After this returns,
443/// `EventPublisher::new` / `EventSubscriber` succeed in this process and
444/// events flow cross-process to the shared bus.
445///
446/// Must be called from within a tokio runtime (it spawns a background task).
447/// `path` is the event service's UDS socket (use [`crate::paths::event_socket`]).
448pub fn ensure_event_client_origin(path: std::path::PathBuf) {
449 if global_moq_event_origin().is_some() {
450 // The event service is co-located in this process; nothing to dial.
451 return;
452 }
453 let origin = MoqEventOrigin::new();
454 // Register first so any `EventPublisher::new` after this point resolves the
455 // global immediately, even before the background link is established (moq
456 // buffers/announces the broadcast and it flows up once the link connects).
457 if !init_global_moq_event_origin(origin.clone()) {
458 // Lost a race with the event factory or a concurrent caller; the global
459 // that won is authoritative, so drop ours.
460 return;
461 }
462 connect_event_moq_uds_background(origin, path);
463}
464
465/// Install a client origin without opening any local socket.
466pub fn install_event_network_client_origin() -> Option<MoqEventOrigin> {
467 if global_moq_event_origin().is_some() {
468 return None;
469 }
470 let origin = MoqEventOrigin::new();
471 init_global_moq_event_origin(origin.clone()).then_some(origin)
472}
473
474/// One authenticated network link. Target identity must accompany a fresh
475/// checkpoint-resolved Iroh transport; callers re-resolve before reconnecting.
476pub async fn run_event_network_link(
477 origin: &MoqEventOrigin,
478 transport: &crate::transport::TransportConfig,
479 proof: &crate::transport::moql_admission::MoqlAdmissionProof,
480) -> Result<()> {
481 anyhow::ensure!(
482 matches!(
483 transport.endpoint,
484 crate::transport::EndpointType::Iroh { .. }
485 ),
486 "Event network link requires Iroh"
487 );
488 let stream = crate::dial::dial_stream_authenticated(transport, proof).await?;
489 let client = moq_net::Client::new().with_origin(origin.producer());
490 let session = stream.connect_moq(&client).await?;
491 let _ = session.closed().await;
492 anyhow::bail!("authenticated Event link closed")
493}
494
495/// Connect a local event origin to the event service's UDS plane (#275).
496///
497/// Spawns a background task that connects a `moq_net::Client` (built with
498/// `with_origin(local_producer)`) to `path`, keeping the link alive and
499/// reconnecting on failure. The session is bidirectional: locally-published
500/// event broadcasts are announced UP to the event service, and broadcasts from
501/// other processes flow DOWN into `origin` so local subscribers see them.
502///
503/// Called once, in non-event processes, AFTER registering `origin` as the
504/// process global. The task runs for the process lifetime.
505pub fn connect_event_moq_uds_background(origin: MoqEventOrigin, path: std::path::PathBuf) {
506 tokio::spawn(async move {
507 loop {
508 if let Err(e) = run_event_client_link(&origin, &path).await {
509 tracing::debug!(
510 path = %path.display(),
511 "event moq client link ended: {e}; reconnecting in {:?}",
512 EVENT_RECONNECT_DELAY
513 );
514 }
515 tokio::time::sleep(EVENT_RECONNECT_DELAY).await;
516 }
517 });
518}
519
520/// One connect-and-pump cycle of the client-side event link. Returns when the
521/// session closes (so the caller can reconnect).
522async fn run_event_client_link(origin: &MoqEventOrigin, path: &std::path::Path) -> Result<()> {
523 use crate::transport::uds_session::{connect_uds, PLANE_MOQ};
524 use moq_net::Client as MoqClient;
525
526 let session = connect_uds(path, PLANE_MOQ)
527 .await
528 .map_err(|e| anyhow!("event moq UDS connect {}: {e}", path.display()))?;
529
530 // `with_origin` makes this a bidirectional link: publish local broadcasts UP
531 // and consume remote broadcasts DOWN into the SAME origin tree subscribers read.
532 let moq_client = MoqClient::new().with_origin(origin.producer());
533 let moq_session = moq_client
534 .connect(session)
535 .await
536 .map_err(|e| anyhow!("event moq handshake: {e}"))?;
537
538 tracing::info!(path = %path.display(), "event moq client link established");
539
540 // Hold the session open until it closes; broadcasts flow in both directions
541 // for the duration via the origin producer/consumer wired above.
542 let reason = moq_session.closed().await;
543 Err(anyhow!("event moq session closed: {reason:?}"))
544}
545
546// ============================================================================
547// MoqEventPublisher
548// ============================================================================
549
550/// In-process moq event publisher.
551///
552/// Publishes topic+payload pairs to the `local/events/{source}` broadcast.
553/// Each `publish_raw` call writes one moq Group (one Frame) on the `events` track.
554///
555/// When constructed via [`MoqEventOrigin::publisher_with_oid`], each event is
556/// ALSO mirrored to the per-OID (#393) publication track so selective
557/// subscribers receive it without the firehose. When constructed via
558/// [`MoqEventOrigin::publisher_oid_only`], `track` IS the OID track and
559/// `oid_track` is `None`.
560pub struct MoqEventPublisher {
561 track: TrackProducer,
562 source: String,
563 next_group: u64,
564 /// Per-OID mirror track. `Some` when the publisher writes to BOTH the flat
565 /// source track and the OID track (transition path); `None` when `track` is
566 /// already the OID track (post-migration) or when no OID was supplied.
567 oid_track: Option<TrackProducer>,
568}
569
570impl MoqEventPublisher {
571 /// Publish a raw topic + payload.
572 ///
573 /// Frame format: `[4 bytes topic_len BE][topic bytes][payload bytes]`.
574 ///
575 /// If this publisher was created with an OID mirror, the frame is written to
576 /// both the flat source track and the per-OID publication track (one Group
577 /// on each, sharing the same group id so consumers on either track see a
578 /// consistent sequence).
579 pub fn publish_raw(&mut self, topic: &str, payload: &[u8]) -> Result<()> {
580 let topic_bytes = topic.as_bytes();
581 let topic_len = topic_bytes.len() as u32;
582
583 let mut frame = Vec::with_capacity(4 + topic_bytes.len() + payload.len());
584 frame.extend_from_slice(&topic_len.to_be_bytes());
585 frame.extend_from_slice(topic_bytes);
586 frame.extend_from_slice(payload);
587
588 let group_id = self.next_group;
589 self.next_group += 1;
590
591 // Write to the primary track.
592 write_group(&mut self.track, group_id, &frame)?;
593 // Mirror to the per-OID track if one is attached (#393 transition).
594 if let Some(oid_track) = &mut self.oid_track {
595 write_group(oid_track, group_id, &frame)?;
596 }
597 Ok(())
598 }
599
600 /// Publish `{source}.{entity}.{event}` with payload.
601 pub fn publish(&mut self, entity: &str, event: &str, payload: &[u8]) -> Result<()> {
602 if entity.contains('.') {
603 return Err(anyhow!("Entity name cannot contain '.': {}", entity));
604 }
605 if event.contains('.') {
606 return Err(anyhow!("Event name cannot contain '.': {}", event));
607 }
608 let topic = format!("{}.{}.{}", self.source, entity, event);
609 self.publish_raw(&topic, payload)
610 }
611
612 /// Source name for this publisher.
613 pub fn source(&self) -> &str {
614 &self.source
615 }
616
617 /// True if this publisher mirrors events to a per-OID (#393) publication track.
618 pub fn has_oid_mirror(&self) -> bool {
619 self.oid_track.is_some()
620 }
621}
622
623/// Write one Group (one Frame) carrying `frame` bytes to `track` at `group_id`.
624fn write_group(track: &mut TrackProducer, group_id: u64, frame: &[u8]) -> Result<()> {
625 let mut group = track.create_group(Group::from(group_id))?;
626 group.write_frame(Bytes::copy_from_slice(frame))?;
627 group.finish()?;
628 Ok(())
629}
630
631// ============================================================================
632// #393 — Firehose backfill for late-join (decision A)
633//
634// moq's per-track cache evicts groups older than MAX_GROUP_AGE (5s). A scheduler
635// reconstructing publication history for an OID needs more than 5s of backlog.
636// On first subscription to an OID's track we consult an optional BackfillSource
637// (atproto firehose `sh.tangled.git.refUpdate` / registry); if it can serve
638// history for that OID we replay it before switching to live MoQ. If no source
639// is available the subscriber starts live-only (graceful degradation): the
640// firehose is the cold-start path, MoQ is the live path.
641// ============================================================================
642
643/// A source of historical publication events for firehose-backfill late-join (#393).
644///
645/// Implementations typically wrap the atproto firehose (`sh.tangled.git.refUpdate`)
646/// or a registry snapshot. All methods are fallible and best-effort: returning an
647/// error or empty iterator from any of them causes [`BackfillMode`] to gracefully
648/// degrade to live-only MoQ (no backfill), never to fail the subscription.
649///
650/// This is a trait object interface so the transport-agnostic `hyprstream-rpc`
651/// crate can express the backfill contract without depending on the atproto /
652/// firehose wiring (which lives in the `hyprstream` app crate).
653pub trait BackfillSource: Send + Sync {
654 /// True if this source can serve history for `oid` right now.
655 ///
656 /// Returning `false` (e.g. firehose offline, registry unreachable) makes the
657 /// subscriber skip backfill and start live-only.
658 fn has_history(&self, oid: &str) -> bool;
659
660 /// Replay historical publication events for `oid` to `tx`, oldest-first.
661 ///
662 /// Each item is the same `(topic, payload)` pair a live MoQ subscriber would
663 /// yield. Implementations should bound the replay (event count / wall time)
664 /// and return once the backlog is drained; the caller then switches to live.
665 /// Errors are logged and treated as "no history available".
666 fn replay<'a>(
667 &'a self,
668 oid: &'a str,
669 tx: &'a mpsc::Sender<(String, Vec<u8>)>,
670 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send + 'a>>;
671}
672
673/// Late-join retention mode for a per-OID (#393) subscriber.
674///
675/// Decision A: on first subscription to an OID's track, optionally replay
676/// history from a [`BackfillSource`] (atproto firehose / registry), then switch
677/// to live MoQ. The firehose is the cold-start path; MoQ is the live path.
678#[derive(Default)]
679pub enum BackfillMode {
680 /// No backfill: subscribe live-only. The default; matches pre-#393 behaviour
681 /// and is the graceful-degradation fallback when no firehose is available.
682 #[default]
683 LiveOnly,
684 /// Attempt backfill from the supplied source for `oid` before going live.
685 /// If the source reports no history (`has_history` false) or `replay` errors,
686 /// the subscriber silently falls back to [`BackfillMode::LiveOnly`].
687 FirehoseBackfill {
688 oid: String,
689 source: Arc<dyn BackfillSource>,
690 },
691}
692
693// ============================================================================
694// Event delivery channel — QoS-aware overflow policy (#606)
695// ============================================================================
696//
697// Replaces the previous unconditional `mpsc::channel(256)` + blocking
698// `.send().await`. That gave every subscriber a single fixed-size lossless
699// channel: a slow consumer that stopped calling `recv()` would back up its
700// OWN per-broadcast read tasks indefinitely (no resubscribe, no eviction).
701//
702// `StreamOpt::overflow_policy` (reused, not reinvented — see module docs)
703// now selects the behaviour per subscriber:
704// - `Block` — lossless backpressure, same semantics as the old
705// `mpsc` channel (correct for the `EventReliable` /
706// at-least-once preset, which must not silently drop).
707// - `DropOldest` — ring-buffer: a full queue evicts its oldest entry
708// rather than stalling the producer-side read tasks
709// (the new default via `EventLive`, #606 finding: a
710// shared/slow consumer must not wedge delivery).
711//
712// This is per-subscriber state (each `MoqEventSubscriber` owns one channel;
713// its own per-broadcast read tasks share the sender half via `clone()`), so
714// a slow subscriber only ever affects its own delivery — not other
715// subscribers reading the same broadcast.
716
717type EventItem = (String, Vec<u8>);
718
719struct EventChannelState {
720 queue: VecDeque<EventItem>,
721 capacity: usize,
722 overflow: OverflowPolicy,
723 closed: bool,
724}
725
726struct EventChannelInner {
727 state: Mutex<EventChannelState>,
728 item_ready: tokio::sync::Notify,
729 space_ready: tokio::sync::Notify,
730 /// Live `EventChannelSender` clones. The last drop closes the channel
731 /// (see `EventChannelSender`'s `Clone`/`Drop` impls) so a parked
732 /// `recv()` wakes with `None` instead of waiting forever.
733 sender_count: AtomicUsize,
734}
735
736/// Sender half of a subscriber's QoS-aware delivery channel. Cloneable —
737/// each per-broadcast read task gets its own clone. Closes the channel when
738/// the last clone is dropped (manual `Clone`/`Drop`, refcounted via
739/// `EventChannelInner::sender_count` — `#[derive(Clone)]` would NOT track
740/// this, which is why this impl is hand-written rather than derived).
741struct EventChannelSender {
742 inner: Arc<EventChannelInner>,
743 dropped_count: Arc<AtomicU64>,
744}
745
746/// Receiver half, owned by the [`MoqEventSubscriber`].
747struct EventChannelReceiver {
748 inner: Arc<EventChannelInner>,
749}
750
751/// Overflow capacity for a channel built from `overflow_policy`: the
752/// `DropOldest` high-water-mark, or a fixed depth for lossless `Block`
753/// channels (matches the historic `mpsc::channel(256)` default).
754fn event_channel_capacity(overflow: &OverflowPolicy) -> usize {
755 match overflow {
756 OverflowPolicy::Block => 256,
757 OverflowPolicy::DropOldest { high_water_mark } => (*high_water_mark).max(1) as usize,
758 }
759}
760
761fn event_channel(
762 overflow: OverflowPolicy,
763 dropped_count: Arc<AtomicU64>,
764) -> (EventChannelSender, EventChannelReceiver) {
765 let capacity = event_channel_capacity(&overflow);
766 let inner = Arc::new(EventChannelInner {
767 state: Mutex::new(EventChannelState {
768 queue: VecDeque::with_capacity(capacity.min(1024)),
769 capacity,
770 overflow,
771 closed: false,
772 }),
773 item_ready: tokio::sync::Notify::new(),
774 space_ready: tokio::sync::Notify::new(),
775 sender_count: AtomicUsize::new(1),
776 });
777 (
778 EventChannelSender {
779 inner: inner.clone(),
780 dropped_count,
781 },
782 EventChannelReceiver { inner },
783 )
784}
785
786impl EventChannelSender {
787 /// Send an item honouring the configured overflow policy. Returns `false`
788 /// once the receiver has been dropped — the caller should stop reading.
789 async fn send(&self, item: EventItem) -> bool {
790 loop {
791 {
792 let mut state = self.inner.state.lock();
793 if state.closed {
794 return false;
795 }
796 if state.queue.len() < state.capacity {
797 state.queue.push_back(item);
798 drop(state);
799 self.inner.item_ready.notify_one();
800 return true;
801 }
802 // At capacity.
803 match state.overflow {
804 OverflowPolicy::Block => {
805 // Fall through: wait for the receiver to free space.
806 }
807 OverflowPolicy::DropOldest { .. } => {
808 state.queue.pop_front();
809 state.queue.push_back(item);
810 drop(state);
811 self.dropped_count.fetch_add(1, Ordering::Relaxed);
812 self.inner.item_ready.notify_one();
813 return true;
814 }
815 }
816 }
817 self.inner.space_ready.notified().await;
818 }
819 }
820}
821
822impl Clone for EventChannelSender {
823 fn clone(&self) -> Self {
824 self.inner.sender_count.fetch_add(1, Ordering::AcqRel);
825 Self {
826 inner: self.inner.clone(),
827 dropped_count: self.dropped_count.clone(),
828 }
829 }
830}
831
832impl Drop for EventChannelSender {
833 fn drop(&mut self) {
834 // Last sender clone gone: close the channel and wake a parked
835 // `recv()` so it observes `closed` instead of waiting forever.
836 // In-flight queued items are still delivered first — `recv()` only
837 // returns `None` once the queue has also drained (see `recv()`).
838 if self.inner.sender_count.fetch_sub(1, Ordering::AcqRel) == 1 {
839 let mut state = self.inner.state.lock();
840 state.closed = true;
841 drop(state);
842 self.inner.item_ready.notify_waiters();
843 }
844 }
845}
846
847impl EventChannelReceiver {
848 async fn recv(&mut self) -> Option<EventItem> {
849 loop {
850 {
851 let mut state = self.inner.state.lock();
852 if let Some(item) = state.queue.pop_front() {
853 drop(state);
854 self.inner.space_ready.notify_one();
855 return Some(item);
856 }
857 if state.closed {
858 return None;
859 }
860 }
861 self.inner.item_ready.notified().await;
862 }
863 }
864
865 fn try_recv(&mut self) -> Result<Option<EventItem>, ()> {
866 let mut state = self.inner.state.lock();
867 if let Some(item) = state.queue.pop_front() {
868 drop(state);
869 self.inner.space_ready.notify_one();
870 return Ok(Some(item));
871 }
872 if state.closed {
873 return Err(());
874 }
875 Ok(None)
876 }
877}
878
879// ============================================================================
880// MoqEventSubscriber
881// ============================================================================
882
883/// In-process moq event subscriber.
884///
885/// Subscribes to one or more topic-prefix patterns. Patterns use the same
886/// dot-separated prefix semantics as ZMQ (`"worker."` matches all worker events,
887/// `""` matches everything). Backed by a background Tokio task that watches the
888/// origin consumer for new source broadcasts and reads their `events` tracks.
889///
890/// # #393 — per-OID subscription
891///
892/// [`Self::subscribe_oid`] scopes the subscriber to a single model OID's
893/// publication track (`local/events/publications/{oid_hash}`), so only that
894/// OID's events cross the wire — not the whole firehose. This is mutually
895/// exclusive with the flat-pattern [`Self::subscribe`] API: a subscriber is
896/// either pattern-based (legacy firehose + in-memory filter) or OID-scoped.
897pub struct MoqEventSubscriber {
898 /// Patterns added via `subscribe()`. Finalized before `recv()` is called.
899 /// Empty when this subscriber is OID-scoped (no flat-track filtering).
900 patterns: Vec<String>,
901 /// The OID whose per-OID track this subscriber reads (`None` ⇒ legacy
902 /// pattern-based subscription over the flat `local/events/{source}` tracks).
903 oid_subscription: Option<String>,
904 /// Late-join retention mode (decision A). Defaults to [`BackfillMode::LiveOnly`].
905 backfill: BackfillMode,
906 /// Delivery QoS (#606). Reuses [`StreamOpt`] — defaults to
907 /// [`crate::stream_info::EventLive`] (at-most-once, drop-oldest), matching
908 /// the pre-#606 behaviour but with a bounded, never-wedging queue instead
909 /// of an unconditionally-blocking one.
910 qos: StreamOpt,
911 /// Skip live groups with `sequence <= resume_from` (offset-resume, #606).
912 /// `None` (the default) skips nothing — sequence numbers start at 0, so a
913 /// bare `u64` threshold could not distinguish "unset" from "resume after
914 /// the very first group". Only meaningful per-broadcast; see
915 /// [`Self::with_resume_from`] docs for the multi-source-pattern caveat.
916 resume_from: Option<u64>,
917 /// Highest live-group sequence delivered so far, shared with the
918 /// background read task(s). Best-effort hint for callers implementing
919 /// resume — see [`Self::last_sequence`].
920 last_sequence: Arc<AtomicU64>,
921 /// Count of items evicted under `OverflowPolicy::DropOldest` (#606).
922 dropped_count: Arc<AtomicU64>,
923 /// Count of idle-timeout-and-resume cycles on the live read loop (#606) —
924 /// a quiet (low event rate) but still-live broadcast no longer causes the
925 /// subscriber to silently exit; this counts how often that recovery fired.
926 idle_resumes: Arc<AtomicU64>,
927 /// Explicit origin consumer for tests (bypasses the process-global
928 /// `OnceLock`, which is single-writer and so cannot be reset between tests
929 /// in one binary). `None` in production ⇒ resolve the global at `recv()`.
930 #[cfg(test)]
931 test_consumer: Option<OriginConsumer>,
932 /// Receiving end of the background task's QoS-aware channel (#606).
933 /// `None` until first `recv()`.
934 rx: Option<EventChannelReceiver>,
935 /// Background task handle (kept alive for the subscriber's lifetime).
936 _task: Option<tokio::task::JoinHandle<()>>,
937}
938
939impl MoqEventSubscriber {
940 /// Create a new subscriber. No background task is started yet.
941 pub fn new() -> Self {
942 Self {
943 patterns: Vec::new(),
944 oid_subscription: None,
945 backfill: BackfillMode::LiveOnly,
946 qos: crate::stream_info::EventLive::stream_opt(),
947 resume_from: None,
948 last_sequence: Arc::new(AtomicU64::new(0)),
949 dropped_count: Arc::new(AtomicU64::new(0)),
950 idle_resumes: Arc::new(AtomicU64::new(0)),
951 #[cfg(test)]
952 test_consumer: None,
953 rx: None,
954 _task: None,
955 }
956 }
957
958 /// Create a subscriber bound to an explicit origin consumer (tests only).
959 ///
960 /// Production code resolves the process-global origin at `recv()` time (see
961 /// [`ensure_started`]); that global is a `OnceLock` and cannot be reset
962 /// between tests in a single binary, so tests that need an isolated origin
963 /// pass its consumer here instead.
964 #[cfg(test)]
965 pub(crate) fn new_for_test(consumer: OriginConsumer) -> Self {
966 let mut s = Self::new();
967 s.test_consumer = Some(consumer);
968 s
969 }
970
971 /// Add a topic-prefix filter (prefix match, ZMQ semantics).
972 ///
973 /// - `"worker."` — all events from source `worker`
974 /// - `"worker.sandbox123.started"` — exact topic
975 /// - `""` — all events (subscribe-all)
976 ///
977 /// Must be called before the first `recv()`. Mutually exclusive with
978 /// [`Self::subscribe_oid`] (legacy flat-track subscription).
979 pub fn subscribe(&mut self, pattern: &str) -> Result<()> {
980 if self.rx.is_some() {
981 return Err(anyhow!("subscribe() must be called before recv()"));
982 }
983 if self.oid_subscription.is_some() {
984 return Err(anyhow!(
985 "subscribe() is mutually exclusive with subscribe_oid()"
986 ));
987 }
988 self.patterns.push(pattern.to_owned());
989 Ok(())
990 }
991
992 /// Subscribe to all events (`pattern = ""`).
993 pub fn subscribe_all(&mut self) -> Result<()> {
994 self.subscribe("")
995 }
996
997 /// Unsubscribe from a pattern. Must be called before `recv()`.
998 pub fn unsubscribe(&mut self, pattern: &str) -> Result<()> {
999 if self.rx.is_some() {
1000 return Err(anyhow!("unsubscribe() must be called before recv()"));
1001 }
1002 self.patterns.retain(|p| p != pattern);
1003 Ok(())
1004 }
1005
1006 /// Subscribe to a single model OID's per-OID publication track (#393).
1007 ///
1008 /// The subscriber scopes its origin consumer to
1009 /// `local/events/publications/{oid_hash}` and reads only that track, so only
1010 /// events published for `oid` cross the wire (one QUIC uni-stream per group
1011 /// on the wire). This replaces the flat firehose + in-memory filter.
1012 ///
1013 /// Mutually exclusive with [`Self::subscribe`] / [`Self::subscribe_all`].
1014 /// Must be called before the first `recv()`.
1015 pub fn subscribe_oid(&mut self, oid: &str) -> Result<()> {
1016 if self.rx.is_some() {
1017 return Err(anyhow!("subscribe_oid() must be called before recv()"));
1018 }
1019 if !self.patterns.is_empty() {
1020 return Err(anyhow!(
1021 "subscribe_oid() is mutually exclusive with subscribe()"
1022 ));
1023 }
1024 self.oid_subscription = Some(oid.to_owned());
1025 Ok(())
1026 }
1027
1028 /// Set the late-join retention mode (decision A: firehose-backfill).
1029 ///
1030 /// Only meaningful for OID-scoped subscribers (see [`Self::subscribe_oid`]):
1031 /// on first `recv()`, a [`BackfillMode::FirehoseBackfill`] subscriber asks
1032 /// its [`BackfillSource`] for `oid` history and replays it before going live.
1033 /// If the source is unavailable the subscriber silently degrades to
1034 /// live-only (never fails). Must be called before the first `recv()`.
1035 pub fn with_backfill(&mut self, mode: BackfillMode) -> Result<()> {
1036 if self.rx.is_some() {
1037 return Err(anyhow!("with_backfill() must be called before recv()"));
1038 }
1039 self.backfill = mode;
1040 Ok(())
1041 }
1042
1043 /// Select the delivery QoS (#606). Reuses [`StreamOpt`] — pass
1044 /// [`crate::stream_info::EventReliable::stream_opt()`] for at-least-once
1045 /// delivery (events that must not be silently dropped, e.g.
1046 /// `model.loaded`), or any other `StreamOpt` combination. Defaults to
1047 /// [`crate::stream_info::EventLive`] if never called. Must be called
1048 /// before the first `recv()`.
1049 pub fn with_qos(&mut self, qos: StreamOpt) -> Result<()> {
1050 if self.rx.is_some() {
1051 return Err(anyhow!("with_qos() must be called before recv()"));
1052 }
1053 self.qos = qos;
1054 Ok(())
1055 }
1056
1057 /// Skip live groups with `sequence <= resume_from` (offset-resume, #606).
1058 /// Combine with [`Self::last_sequence`] to persist a cursor across
1059 /// reconnects for `EventReliable`-style at-least-once delivery.
1060 ///
1061 /// Caveat: a `MoqEventSubscriber` using the legacy flat-pattern
1062 /// [`Self::subscribe`] API may read from MULTIPLE independent source
1063 /// broadcasts (each with its own group-sequence counter); `resume_from`
1064 /// is applied per-broadcast, so a single threshold is only unambiguous
1065 /// for [`Self::subscribe_oid`] (one broadcast). Multi-source resume
1066 /// requires a per-source cursor map, which is out of scope here.
1067 /// Must be called before the first `recv()`.
1068 pub fn with_resume_from(&mut self, sequence: u64) -> Result<()> {
1069 if self.rx.is_some() {
1070 return Err(anyhow!("with_resume_from() must be called before recv()"));
1071 }
1072 self.resume_from = Some(sequence);
1073 Ok(())
1074 }
1075
1076 /// Highest live-group sequence delivered so far (best-effort hint for
1077 /// resume; see [`Self::with_resume_from`]). `0` before the first delivery.
1078 pub fn last_sequence(&self) -> u64 {
1079 self.last_sequence.load(Ordering::Relaxed)
1080 }
1081
1082 /// Count of items evicted under `OverflowPolicy::DropOldest` (#606).
1083 pub fn dropped_count(&self) -> u64 {
1084 self.dropped_count.load(Ordering::Relaxed)
1085 }
1086
1087 /// Count of idle-timeout-and-resume cycles on the live read loop (#606).
1088 pub fn idle_resumes(&self) -> u64 {
1089 self.idle_resumes.load(Ordering::Relaxed)
1090 }
1091
1092 /// Lazily start the background task and return a mutable reference to the channel receiver.
1093 fn ensure_started(&mut self) -> Result<&mut EventChannelReceiver> {
1094 if self.rx.is_none() {
1095 #[cfg(test)]
1096 let consumer = if let Some(c) = self.test_consumer.clone() {
1097 c
1098 } else {
1099 global_moq_event_origin()
1100 .ok_or_else(|| anyhow!("moq event bus not initialized; call init_global_moq_event_origin first"))?
1101 .consumer()
1102 };
1103 #[cfg(not(test))]
1104 let consumer = global_moq_event_origin()
1105 .ok_or_else(|| {
1106 anyhow!(
1107 "moq event bus not initialized; call init_global_moq_event_origin first"
1108 )
1109 })?
1110 .consumer();
1111
1112 let (tx, rx) =
1113 event_channel(self.qos.overflow_policy.clone(), self.dropped_count.clone());
1114 let resume_from = self.resume_from;
1115 let last_sequence = self.last_sequence.clone();
1116 let idle_resumes = self.idle_resumes.clone();
1117
1118 let task = if let Some(oid) = self.oid_subscription.clone() {
1119 // #393 per-OID subscription: scope to the OID's track path and
1120 // run the (optional) backfill before going live.
1121 let backfill = std::mem::replace(&mut self.backfill, BackfillMode::LiveOnly);
1122 tokio::spawn(run_oid_subscriber_task(
1123 consumer,
1124 oid,
1125 backfill,
1126 tx,
1127 resume_from,
1128 last_sequence,
1129 idle_resumes,
1130 ))
1131 } else {
1132 // Legacy flat-track pattern subscription.
1133 let patterns = self.patterns.clone();
1134 tokio::spawn(run_subscriber_task(
1135 consumer,
1136 patterns,
1137 tx,
1138 resume_from,
1139 last_sequence,
1140 idle_resumes,
1141 ))
1142 };
1143 self.rx = Some(rx);
1144 self._task = Some(task);
1145 }
1146 self.rx
1147 .as_mut()
1148 .ok_or_else(|| anyhow!("subscriber channel not initialized"))
1149 }
1150
1151 /// Receive the next event. Blocks until an event arrives or the bus is closed.
1152 pub async fn recv(&mut self) -> Result<(String, Vec<u8>)> {
1153 let rx = self.ensure_started()?;
1154 rx.recv()
1155 .await
1156 .ok_or_else(|| anyhow!("event subscriber closed"))
1157 }
1158
1159 /// Receive with timeout.
1160 pub async fn recv_timeout(&mut self, timeout: Duration) -> Result<Option<(String, Vec<u8>)>> {
1161 match tokio::time::timeout(timeout, self.recv()).await {
1162 Ok(result) => result.map(Some),
1163 Err(_) => Ok(None),
1164 }
1165 }
1166
1167 /// Try to receive without blocking.
1168 pub fn try_recv(&mut self) -> Result<Option<(String, Vec<u8>)>> {
1169 let rx = self.ensure_started()?;
1170 match rx.try_recv() {
1171 Ok(msg) => Ok(msg),
1172 Err(()) => Err(anyhow!("event subscriber closed")),
1173 }
1174 }
1175}
1176
1177impl Default for MoqEventSubscriber {
1178 fn default() -> Self {
1179 Self::new()
1180 }
1181}
1182
1183// ============================================================================
1184// Background subscriber task
1185// ============================================================================
1186
1187/// Background task: watches the origin consumer for announced source broadcasts,
1188/// subscribes to each `events` track, and relays matching topic+payload pairs to `tx`.
1189async fn run_subscriber_task(
1190 mut consumer: OriginConsumer,
1191 patterns: Vec<String>,
1192 tx: EventChannelSender,
1193 resume_from: Option<u64>,
1194 last_sequence: Arc<AtomicU64>,
1195 idle_resumes: Arc<AtomicU64>,
1196) {
1197 use tokio::task::JoinSet;
1198
1199 let patterns = Arc::new(patterns);
1200
1201 // Scope the consumer to the `local/events` subtree, then further narrow by
1202 // source names extracted from the patterns.
1203 match scope_consumer_for_patterns(&consumer, &patterns) {
1204 Some(c) => consumer = c,
1205 None => {
1206 // Origin has no `local/events` tree yet — using the root consumer would
1207 // expose all RPC traffic to the event channel. Abort instead.
1208 tracing::warn!("event subscriber: origin has no event scope; subscriber task exiting");
1209 return;
1210 }
1211 }
1212
1213 // Track per-broadcast read tasks so they are cancelled when this loop exits.
1214 let mut tasks: JoinSet<()> = JoinSet::new();
1215
1216 loop {
1217 let (path, broadcast) = match consumer.announced().await {
1218 None => break, // origin closed
1219 Some((_, None)) => continue, // unannounce
1220 Some((path, Some(b))) => (path, b),
1221 };
1222
1223 let tx2 = tx.clone();
1224 let patterns2 = patterns.clone();
1225 let path_str = path.as_str().to_owned();
1226 let last_sequence2 = last_sequence.clone();
1227 let idle_resumes2 = idle_resumes.clone();
1228
1229 tasks.spawn(async move {
1230 read_event_broadcast(
1231 broadcast,
1232 path_str,
1233 patterns2,
1234 tx2,
1235 resume_from,
1236 last_sequence2,
1237 idle_resumes2,
1238 )
1239 .await;
1240 });
1241 }
1242 // Dropping `tasks` here cancels all per-broadcast read tasks.
1243}
1244
1245/// Scope the origin consumer to `local/events/{sources}` based on patterns.
1246///
1247/// Returns `None` only when `local/events` itself has no tree yet (the event
1248/// origin has never announced anything) — the caller treats that as "nothing
1249/// to subscribe to" and aborts. For a subscribe-all pattern (`""`), or when no
1250/// source names can be extracted from `patterns`, the broader `local/events`
1251/// consumer is returned.
1252///
1253/// A source-specific second-level scope (e.g. narrowing to `local/events/worker`)
1254/// can ALSO return `None` even though `local/events` exists, simply because
1255/// that particular source hasn't announced its broadcast yet — a subscriber
1256/// commonly starts before its publisher does. That is not "no event scope";
1257/// it just means the narrower subtree doesn't exist *yet*. Treating it the
1258/// same as the top-level `None` case used to make the subscriber task abort
1259/// permanently (see `run_subscriber_task`), so a subscriber that started
1260/// before its publisher would never receive anything even after the publisher
1261/// came up. Instead, fall back to the broader `local/events`-scoped consumer:
1262/// per-frame pattern filtering (`topic_matches_patterns` in
1263/// `read_event_broadcast`) still narrows correctly once the source announces.
1264fn scope_consumer_for_patterns(
1265 consumer: &OriginConsumer,
1266 patterns: &[String],
1267) -> Option<OriginConsumer> {
1268 // Scope to local/events first
1269 let events_consumer = consumer.scope(&[Path::new(EVENT_PREFIX)])?;
1270
1271 // If subscribe-all pattern present, return events-scoped consumer
1272 if patterns.iter().any(String::is_empty) {
1273 return Some(events_consumer);
1274 }
1275
1276 // Extract unique source names from patterns (e.g., "worker." → "worker")
1277 let sources: Vec<String> = patterns
1278 .iter()
1279 .filter_map(|p| {
1280 let src = p.split('.').next()?;
1281 if src.is_empty() {
1282 None
1283 } else {
1284 Some(src.to_owned())
1285 }
1286 })
1287 .collect::<HashSet<String>>()
1288 .into_iter()
1289 .collect();
1290
1291 if sources.is_empty() {
1292 return Some(events_consumer);
1293 }
1294
1295 let path_refs: Vec<Path<'_>> = sources.iter().map(|s| Path::new(s.as_str())).collect();
1296 // Narrowing can legitimately fail if none of `sources` have announced yet;
1297 // fall back to the events-scoped consumer rather than propagating `None`
1298 // (which the caller treats as a fatal "no event scope" abort).
1299 Some(events_consumer.scope(&path_refs).unwrap_or(events_consumer))
1300}
1301
1302/// Read all groups from a single source broadcast's `events` track and relay
1303/// matching events to `tx`.
1304async fn read_event_broadcast(
1305 broadcast: moq_net::BroadcastConsumer,
1306 path: String,
1307 patterns: Arc<Vec<String>>,
1308 tx: EventChannelSender,
1309 resume_from: Option<u64>,
1310 last_sequence: Arc<AtomicU64>,
1311 idle_resumes: Arc<AtomicU64>,
1312) {
1313 let mut track = match broadcast.subscribe_track(&Track::new(EVENT_TRACK)) {
1314 Ok(t) => t,
1315 Err(_) => return,
1316 };
1317
1318 loop {
1319 let mut group =
1320 match tokio::time::timeout(crate::moq_stream::GROUP_IDLE_TIMEOUT, track.next_group())
1321 .await
1322 {
1323 Ok(Ok(Some(g))) => g,
1324 Ok(Ok(None)) | Ok(Err(_)) => break,
1325 Err(_elapsed) => {
1326 // #606: a quiet (low event rate) broadcast is NOT the same as a
1327 // gone one. Previously this `break`-ed, permanently dropping
1328 // the subscriber after one idle period — silently losing every
1329 // later event. Keep waiting; only `next_group()` itself
1330 // returning `None`/`Err` (above) means the broadcast actually
1331 // ended. `next_group()` is safe to re-poll after its previous
1332 // future was dropped by the timeout (it does not consume
1333 // internal cursor state on cancellation).
1334 tracing::debug!(
1335 "event subscriber idle — source broadcast still live, resuming wait"
1336 );
1337 idle_resumes.fetch_add(1, Ordering::Relaxed);
1338 continue;
1339 }
1340 };
1341
1342 // Offset-resume (#606): skip groups already delivered in a prior
1343 // session. Only unambiguous for a single-broadcast subscription; see
1344 // `MoqEventSubscriber::with_resume_from` docs for the multi-source
1345 // caveat that also applies here.
1346 if let Some(threshold) = resume_from {
1347 if group.sequence <= threshold {
1348 continue;
1349 }
1350 }
1351
1352 let frame = match group.read_frame().await {
1353 Ok(Some(f)) => f,
1354 Ok(None) => break, // group ended without a frame
1355 Err(e) => {
1356 tracing::warn!(error = %e, "event broadcast: frame read error");
1357 break;
1358 }
1359 };
1360
1361 // Decode: [4 bytes topic_len BE][topic][payload]
1362 if frame.len() < 4 {
1363 continue;
1364 }
1365 let topic_len = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize;
1366 if frame.len() < 4 + topic_len {
1367 continue;
1368 }
1369 let topic = match std::str::from_utf8(&frame[4..4 + topic_len]) {
1370 Ok(t) => t.to_owned(),
1371 Err(_) => continue,
1372 };
1373 let payload = frame[4 + topic_len..].to_vec();
1374
1375 // Apply pattern filter (prefix match, ZMQ semantics)
1376 if topic_matches_patterns(&topic, &patterns) {
1377 last_sequence.fetch_max(group.sequence, Ordering::Relaxed);
1378 if !tx.send((topic, payload)).await {
1379 break; // receiver dropped
1380 }
1381 }
1382 }
1383
1384 let _ = path; // kept for debugging context; unused in the hot path
1385}
1386
1387/// #393 per-OID subscriber task.
1388///
1389/// Scopes the origin consumer to `local/events/publications/{oid_hash}` so only
1390/// that OID's events cross the wire. If `backfill` carries a [`BackfillSource`]
1391/// that reports history for `oid`, that history is replayed to `tx` first
1392/// (cold-start), then the task subscribes to the live track. Any backfill error
1393/// or "no history" result degrades silently to live-only — the subscription
1394/// never fails just because the firehose is down.
1395async fn run_oid_subscriber_task(
1396 mut consumer: OriginConsumer,
1397 oid: String,
1398 backfill: BackfillMode,
1399 tx: EventChannelSender,
1400 resume_from: Option<u64>,
1401 last_sequence: Arc<AtomicU64>,
1402 idle_resumes: Arc<AtomicU64>,
1403) {
1404 // 1) Optional cold-start backfill (decision A). Run BEFORE scoping the live
1405 // consumer so the scheduler sees history before new events regardless of
1406 // when (or whether) the live track is announced.
1407 //
1408 // `BackfillSource::replay` is a PUBLIC trait implemented outside this
1409 // crate (#393 doc) and is typed against `mpsc::Sender` — #606 does not
1410 // change that public contract. Instead, bridge: replay into a small
1411 // plain `mpsc` channel and forward each item into the QoS-aware `tx` so
1412 // backfilled items get the same overflow-policy treatment as live ones.
1413 if let BackfillMode::FirehoseBackfill {
1414 oid: bf_oid,
1415 source,
1416 } = &backfill
1417 {
1418 if bf_oid == &oid && source.has_history(&oid) {
1419 let (bridge_tx, mut bridge_rx) = mpsc::channel::<(String, Vec<u8>)>(64);
1420 let fwd_tx = tx.clone();
1421 let forward = tokio::spawn(async move {
1422 while let Some(item) = bridge_rx.recv().await {
1423 if !fwd_tx.send(item).await {
1424 break;
1425 }
1426 }
1427 });
1428 let replay_result = source.replay(&oid, &bridge_tx).await;
1429 drop(bridge_tx); // close so the forwarder's recv() loop ends
1430 let _ = forward.await;
1431 match replay_result {
1432 Ok(()) => tracing::debug!(oid = %oid, "event backfill replay complete"),
1433 Err(e) => {
1434 tracing::warn!(oid = %oid, error = %e, "event backfill failed; continuing live-only");
1435 }
1436 }
1437 } else {
1438 tracing::debug!(oid = %oid, "no backfill history available; starting live-only");
1439 }
1440 }
1441
1442 // 2) Scope the consumer to the publications subtree. We scope to the PREFIX
1443 // only (not the OID hash) so the announced() stream keeps flowing even
1444 // before this OID's broadcast exists — late-join then picks up the OID's
1445 // broadcast by matching its hash in the announcement path. This avoids the
1446 // race where a subscriber starts before the publisher announces.
1447 let want_hash = oid_hash(&oid);
1448 let scoped = match consumer.scope(&[Path::new(PUBLICATIONS_PREFIX)]) {
1449 Some(c) => c,
1450 None => {
1451 tracing::debug!(oid = %oid, "publications subtree not announced; subscriber idle");
1452 return;
1453 }
1454 };
1455 consumer = scoped;
1456
1457 // 3) Wait for the OID's broadcast to be announced and read its `events` track.
1458 // We filter announcements by the OID hash so a per-OID subscriber sees
1459 // exactly one broadcast (the OID's) — no in-memory pattern filtering on
1460 // event topics is needed; the scope + hash match already narrowed the wire.
1461 loop {
1462 let (path, broadcast) = match consumer.announced().await {
1463 None => break, // origin closed
1464 Some((_, None)) => continue, // unannounce — re-await
1465 Some((path, Some(b))) => (path, b),
1466 };
1467 // Only read announcements whose final path segment is this OID's hash.
1468 // Other OIDs' broadcasts are ignored — they stay on their own tracks.
1469 // Match the full final segment (not a bare suffix) so two hashes that
1470 // happen to share a tail cannot cross-subscribe.
1471 let last_segment = path.as_str().rsplit('/').next().unwrap_or("");
1472 if last_segment != want_hash.as_str() {
1473 continue;
1474 }
1475 read_oid_event_broadcast(
1476 broadcast,
1477 tx.clone(),
1478 resume_from,
1479 last_sequence.clone(),
1480 idle_resumes.clone(),
1481 )
1482 .await;
1483 }
1484}
1485
1486/// Read all groups from a per-OID publication broadcast's `events` track and
1487/// relay every frame to `tx`. Unlike [`read_event_broadcast`] there is NO
1488/// pattern filtering — the consumer scope already limited the wire to one OID.
1489async fn read_oid_event_broadcast(
1490 broadcast: moq_net::BroadcastConsumer,
1491 tx: EventChannelSender,
1492 resume_from: Option<u64>,
1493 last_sequence: Arc<AtomicU64>,
1494 idle_resumes: Arc<AtomicU64>,
1495) {
1496 let mut track = match broadcast.subscribe_track(&Track::new(EVENT_TRACK)) {
1497 Ok(t) => t,
1498 Err(_) => return,
1499 };
1500
1501 loop {
1502 let mut group =
1503 match tokio::time::timeout(crate::moq_stream::GROUP_IDLE_TIMEOUT, track.next_group())
1504 .await
1505 {
1506 Ok(Ok(Some(g))) => g,
1507 Ok(Ok(None)) | Ok(Err(_)) => break,
1508 Err(_elapsed) => {
1509 // #606: see read_event_broadcast — quiet != gone, keep waiting.
1510 tracing::debug!(
1511 "per-OID event subscriber idle — broadcast still live, resuming wait"
1512 );
1513 idle_resumes.fetch_add(1, Ordering::Relaxed);
1514 continue;
1515 }
1516 };
1517
1518 // Offset-resume (#606): unambiguous here — exactly one broadcast.
1519 if let Some(threshold) = resume_from {
1520 if group.sequence <= threshold {
1521 continue;
1522 }
1523 }
1524
1525 let frame = match group.read_frame().await {
1526 Ok(Some(f)) => f,
1527 Ok(None) => break,
1528 Err(e) => {
1529 tracing::warn!(error = %e, "per-OID event broadcast: frame read error");
1530 break;
1531 }
1532 };
1533
1534 // Decode: [4 bytes topic_len BE][topic][payload]
1535 if frame.len() < 4 {
1536 continue;
1537 }
1538 let topic_len = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize;
1539 if frame.len() < 4 + topic_len {
1540 continue;
1541 }
1542 let topic = match std::str::from_utf8(&frame[4..4 + topic_len]) {
1543 Ok(t) => t.to_owned(),
1544 Err(_) => continue,
1545 };
1546 let payload = frame[4 + topic_len..].to_vec();
1547
1548 last_sequence.fetch_max(group.sequence, Ordering::Relaxed);
1549 if !tx.send((topic, payload)).await {
1550 break; // receiver dropped
1551 }
1552 }
1553}
1554
1555/// True if `topic` matches any pattern in `patterns` (prefix match).
1556fn topic_matches_patterns(topic: &str, patterns: &[String]) -> bool {
1557 for pat in patterns {
1558 if pat.is_empty() {
1559 return true; // subscribe-all
1560 }
1561 if pat.ends_with('.') {
1562 if topic.starts_with(pat.as_str()) {
1563 return true;
1564 }
1565 } else if topic == pat.as_str() {
1566 return true;
1567 }
1568 }
1569 false
1570}
1571
1572#[cfg(test)]
1573#[allow(clippy::expect_used, clippy::unwrap_used)]
1574mod tests {
1575 use super::*;
1576
1577 #[test]
1578 fn test_topic_matches_patterns() {
1579 let patterns = vec!["worker.".to_owned()];
1580 assert!(topic_matches_patterns(
1581 "worker.sandbox123.started",
1582 &patterns
1583 ));
1584 assert!(!topic_matches_patterns("system.event.ready", &patterns));
1585 assert!(!topic_matches_patterns("worker", &patterns)); // no trailing dot match
1586
1587 let patterns_all = vec!["".to_owned()];
1588 assert!(topic_matches_patterns("any.thing", &patterns_all));
1589
1590 let patterns_exact = vec!["worker.abc.started".to_owned()];
1591 assert!(topic_matches_patterns(
1592 "worker.abc.started",
1593 &patterns_exact
1594 ));
1595 assert!(!topic_matches_patterns(
1596 "worker.abc.stopped",
1597 &patterns_exact
1598 ));
1599 }
1600
1601 // ── #606: EventChannel unit tests (no MoQ transport — deterministic) ──────
1602
1603 #[tokio::test]
1604 async fn event_channel_drop_oldest_evicts_and_counts() {
1605 let dropped = Arc::new(AtomicU64::new(0));
1606 let (tx, mut rx) = event_channel(
1607 OverflowPolicy::DropOldest { high_water_mark: 2 },
1608 dropped.clone(),
1609 );
1610
1611 // Fill to capacity, then overflow by one: item "a" must be evicted.
1612 assert!(tx.send(("t".into(), b"a".to_vec())).await);
1613 assert!(tx.send(("t".into(), b"b".to_vec())).await);
1614 assert!(tx.send(("t".into(), b"c".to_vec())).await);
1615
1616 assert_eq!(dropped.load(Ordering::Relaxed), 1, "exactly one eviction");
1617
1618 let (_, first) = rx.recv().await.expect("first item");
1619 let (_, second) = rx.recv().await.expect("second item");
1620 assert_eq!(first, b"b", "oldest (\"a\") was evicted, \"b\" survives");
1621 assert_eq!(second, b"c");
1622 }
1623
1624 #[tokio::test]
1625 async fn event_channel_block_policy_waits_for_space_not_drop() {
1626 let dropped = Arc::new(AtomicU64::new(0));
1627 let (tx, mut rx) = event_channel(OverflowPolicy::Block, dropped.clone());
1628
1629 // Capacity for OverflowPolicy::Block is the fixed 256 default — send a
1630 // third send concurrently while only 2 are buffered; it must complete
1631 // once we drain one, never by silently dropping (dropped_count stays 0).
1632 assert!(tx.send(("t".into(), b"a".to_vec())).await);
1633 assert!(tx.send(("t".into(), b"b".to_vec())).await);
1634
1635 let (item, _) = rx.recv().await.expect("drain one");
1636 assert_eq!(item, "t");
1637 assert_eq!(
1638 dropped.load(Ordering::Relaxed),
1639 0,
1640 "Block policy never evicts"
1641 );
1642 }
1643
1644 #[tokio::test]
1645 async fn event_channel_closes_when_all_senders_dropped() {
1646 let dropped = Arc::new(AtomicU64::new(0));
1647 let (tx, mut rx) = event_channel(OverflowPolicy::Block, dropped);
1648 let tx2 = tx.clone();
1649 drop(tx);
1650 drop(tx2);
1651 assert_eq!(rx.recv().await, None, "recv must wake with None, not hang");
1652 }
1653
1654 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1655 async fn moq_event_pub_sub_roundtrip() -> Result<()> {
1656 let origin = MoqEventOrigin::new();
1657
1658 // Isolate this test from the process-global OnceLock. A subscriber
1659 // created via `MoqEventSubscriber::new()` reads the global origin; if
1660 // another test in the same binary set that global first, this test's
1661 // publisher and subscriber would resolve against DIFFERENT origins and
1662 // `recv()` would race/fail ("event subscriber closed"). `new_for_test`
1663 // binds the subscriber to THIS origin's consumer, making the pub/sub
1664 // pairing deterministic regardless of test order.
1665 let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
1666
1667 // Source-prefix subscription (not subscribe-all): exercises the
1668 // two-level `scope()` path in `scope_consumer_for_patterns` (scope to
1669 // `local/events`, then narrow to `worker`), now that its fallback to
1670 // the broader events-scoped consumer is fixed.
1671 sub.subscribe("worker.")?;
1672
1673 // Publish BEFORE recv so the broadcast (`local/events/worker`) is already
1674 // announced when the subscriber's background task starts — no timing race.
1675 let mut pub_ = origin.publisher("worker")?;
1676 pub_.publish("sandbox123", "started", b"payload")?;
1677
1678 let result = tokio::time::timeout(std::time::Duration::from_secs(5), sub.recv()).await;
1679
1680 match result {
1681 Ok(Ok((topic, payload))) => {
1682 assert_eq!(topic, "worker.sandbox123.started");
1683 assert_eq!(payload, b"payload");
1684 }
1685 Ok(Err(e)) => panic!("recv error: {e}"),
1686 Err(_) => panic!("recv timeout"),
1687 }
1688
1689 Ok(())
1690 }
1691
1692 /// Regression test for the `scope_consumer_for_patterns` fallback fix: a
1693 /// subscriber that source-prefix-subscribes (`"worker."`) BEFORE its
1694 /// publisher has announced anything must still receive events once the
1695 /// publisher starts — not abort permanently. Before the fix, the
1696 /// second-level `scope()` narrowing to `worker` found no node yet (nothing
1697 /// announced under `local/events/worker`) and returned `None`, which
1698 /// `scope_consumer_for_patterns` propagated as "no event scope", causing
1699 /// `run_subscriber_task` to exit immediately and silently drop the
1700 /// subscriber for good.
1701 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1702 async fn event_subscriber_can_start_before_publisher_announces() -> Result<()> {
1703 let origin = MoqEventOrigin::new();
1704 let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
1705
1706 // Subscribe to the source prefix BEFORE any publisher for "worker"
1707 // exists — `local/events/worker` has not been announced yet.
1708 sub.subscribe("worker.")?;
1709
1710 // Now create the publisher and publish; this is the first time
1711 // `local/events/worker` is announced.
1712 let mut pub_ = origin.publisher("worker")?;
1713 pub_.publish("sandbox123", "started", b"payload")?;
1714
1715 let result = tokio::time::timeout(std::time::Duration::from_secs(5), sub.recv()).await;
1716
1717 match result {
1718 Ok(Ok((topic, payload))) => {
1719 assert_eq!(topic, "worker.sandbox123.started");
1720 assert_eq!(payload, b"payload");
1721 }
1722 Ok(Err(e)) => panic!("recv error: {e}"),
1723 Err(_) => panic!("recv timeout — subscriber likely aborted on the pre-fix scope-race"),
1724 }
1725
1726 Ok(())
1727 }
1728
1729 /// #275 cross-process bug: a non-event process publishes events through a
1730 /// CLIENT-mode origin linked over UDS to a separate SERVER-mode origin
1731 /// (the event service). A subscriber on the SERVER origin must see the
1732 /// event — proving the bus is no longer a process-local island.
1733 ///
1734 /// This mirrors the systemd / --ipc deployment: the "event service" origin
1735 /// is served via `serve_event_moq_uds_background`; the "worker" origin is a
1736 /// distinct origin (as in a separate process) wired via
1737 /// `connect_event_moq_uds_background`. The publisher resolves against the
1738 /// client origin (no in-process event factory ran for it), and the event
1739 /// reaches a subscriber reading the server origin's consumer.
1740 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1741 async fn cross_process_event_over_uds_client_link() -> Result<()> {
1742 use std::time::Duration;
1743
1744 // Unique, stable socket path for this test (cross-"process" rendezvous).
1745 let sock =
1746 std::env::temp_dir().join(format!("hyprstream-event-test-{}.sock", std::process::id()));
1747
1748 // ── "event service" process: serve a SERVER-mode origin over UDS ──────
1749 let server_origin = MoqEventOrigin::new();
1750 serve_event_moq_uds_background(server_origin.clone(), sock.clone());
1751 // Let the listener bind.
1752 tokio::time::sleep(Duration::from_millis(50)).await;
1753
1754 // ── "worker" process: a DISTINCT origin linked to the event UDS ───────
1755 let client_origin = MoqEventOrigin::new();
1756 connect_event_moq_uds_background(client_origin.clone(), sock.clone());
1757
1758 // A publisher built on the client origin (the failing path before #275).
1759 let mut publisher = client_origin.publisher("worker")?;
1760
1761 // Drive the link + announcement by publishing repeatedly; read the
1762 // subscriber on the SERVER origin's consumer (a "different process").
1763 // Read the server consumer directly (the same data a server-side
1764 // EventSubscriber would receive) to avoid depending on the global.
1765 let consumer = server_origin.consumer();
1766 let scoped = consumer
1767 .scope(&[Path::new(EVENT_PREFIX)])
1768 .ok_or_else(|| anyhow!("server origin has no event scope yet"))?;
1769
1770 let recv = tokio::spawn(async move {
1771 let mut scoped = scoped;
1772 loop {
1773 let (_path, bc) = match scoped.announced().await {
1774 None => return None,
1775 Some((_, None)) => continue,
1776 Some((path, Some(b))) => (path, b),
1777 };
1778 let mut track = match bc.subscribe_track(&Track::new(EVENT_TRACK)) {
1779 Ok(t) => t,
1780 Err(_) => continue,
1781 };
1782 let mut group = match track.next_group().await {
1783 Ok(Some(g)) => g,
1784 _ => continue,
1785 };
1786 if let Ok(Some(frame)) = group.read_frame().await {
1787 return Some(frame.to_vec());
1788 }
1789 }
1790 });
1791
1792 // Re-publish periodically (the moq broadcast is retained by the origin,
1793 // but re-asserting it ensures the announcement is live once the
1794 // client→server link establishes). Concurrently await the receiver.
1795 let publish_loop = async {
1796 for _ in 0..50 {
1797 let _ = publisher.publish("sandbox123", "started", b"payload");
1798 tokio::time::sleep(Duration::from_millis(100)).await;
1799 }
1800 // Keep the publisher (and its broadcast) alive until the receiver
1801 // finishes by parking here.
1802 std::future::pending::<()>().await;
1803 };
1804
1805 let got: Option<Vec<u8>> = tokio::select! {
1806 r = recv => r.ok().flatten(),
1807 _ = publish_loop => None,
1808 _ = tokio::time::sleep(Duration::from_secs(10)) => None,
1809 };
1810
1811 let _ = std::fs::remove_file(&sock);
1812 let frame = got.expect("event did not cross the UDS link to the server origin");
1813
1814 // Decode [4 bytes topic_len BE][topic][payload].
1815 assert!(frame.len() >= 4, "frame too short");
1816 let topic_len = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize;
1817 let topic = std::str::from_utf8(&frame[4..4 + topic_len]).unwrap();
1818 let payload = &frame[4 + topic_len..];
1819 assert_eq!(topic, "worker.sandbox123.started");
1820 assert_eq!(payload, b"payload");
1821
1822 Ok(())
1823 }
1824
1825 // ========================================================================
1826 // #393 — per-OID broadcast paths + firehose-backfill late-join
1827 // ========================================================================
1828
1829 #[test]
1830 fn oid_hash_is_stable_and_distinct() {
1831 let a = oid_hash("at://did:web:node.example.com/models/qwen3-4b/v1");
1832 let b = oid_hash("at://did:web:node.example.com/models/qwen3-4b/v1");
1833 assert_eq!(a, b, "same OID must hash to the same path");
1834 assert_eq!(
1835 a.len(),
1836 OID_HASH_LEN,
1837 "hash is truncated to OID_HASH_LEN hex chars"
1838 );
1839
1840 let c = oid_hash("at://did:web:node.example.com/models/llama3-8b/v1");
1841 assert_ne!(a, c, "distinct OIDs must hash to distinct paths");
1842
1843 // The hash must contain only filesystem/moq-safe characters (hex).
1844 assert!(
1845 a.chars().all(|ch| ch.is_ascii_hexdigit()),
1846 "non-hex char in {a}"
1847 );
1848 }
1849
1850 #[test]
1851 fn publication_broadcast_path_shape() {
1852 // Mirrors the streaming plane's path-selective shape.
1853 let path = publication_broadcast_path("my-oid");
1854 assert_eq!(
1855 path,
1856 format!("{}/{}", PUBLICATIONS_PREFIX, oid_hash("my-oid")),
1857 "path must be {PUBLICATIONS_PREFIX}/{{oid_hash}}"
1858 );
1859 assert!(path.starts_with(PUBLICATIONS_PREFIX));
1860 }
1861
1862 /// A per-OID subscriber receives ONLY that OID's events — not events for
1863 /// other OIDs and not the flat firehose. This is the core #393 fix: a node
1864 /// tracking N of M OIDs reads N tracks, not M.
1865 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1866 async fn per_oid_subscription_receives_only_that_oid() -> Result<()> {
1867 let origin = MoqEventOrigin::new();
1868
1869 // Two publishers for two distinct OIDs, each mirroring to its own track.
1870 let mut pub_a = origin.publisher_with_oid("registry", "oid-alpha")?;
1871 let mut pub_b = origin.publisher_with_oid("registry", "oid-beta")?;
1872
1873 // Subscriber scoped to oid-alpha only (uses an explicit origin consumer
1874 // so it does not fight other tests over the process-global OnceLock).
1875 let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
1876 sub.subscribe_oid("oid-alpha")?;
1877
1878 // Publish one event to each OID's track.
1879 pub_a.publish("qwen3-4b", "published", b"alpha-event")?;
1880 pub_b.publish("llama3-8b", "published", b"beta-event")?;
1881
1882 // Let the announcement propagate.
1883 tokio::time::sleep(Duration::from_millis(30)).await;
1884
1885 // The subscriber should receive oid-alpha's event...
1886 let got = tokio::time::timeout(Duration::from_secs(5), sub.recv()).await;
1887 let (topic, payload) = match got {
1888 Ok(Ok(tp)) => tp,
1889 Ok(Err(e)) => panic!("recv error: {e}"),
1890 Err(_) => panic!("recv timeout — oid-alpha event never arrived"),
1891 };
1892 assert_eq!(topic, "registry.qwen3-4b.published");
1893 assert_eq!(payload, b"alpha-event");
1894
1895 // ...and NOT oid-beta's. Poll briefly: any further recv should time out
1896 // (the subscriber is scoped to oid-alpha, so beta's event never reaches it).
1897 let next = sub.recv_timeout(Duration::from_millis(300)).await?;
1898 assert!(
1899 next.is_none(),
1900 "per-OID subscriber received an event for a DIFFERENT OID: {next:?}"
1901 );
1902
1903 // Hold the publishers alive so their broadcasts stay announced for the
1904 // lifetime of the subscriber task.
1905 drop(pub_a);
1906 drop(pub_b);
1907 Ok(())
1908 }
1909
1910 /// The legacy flat `events` track still receives events from a publisher
1911 /// created with `publisher_with_oid` (back-compat fallback during the #393
1912 /// transition). A `publisher_with_oid` mirrors to BOTH the flat
1913 /// `local/events/registry` track and the per-OID publication track, so a
1914 /// legacy subscriber reading the flat track sees the same event as a #393
1915 /// per-OID subscriber.
1916 ///
1917 /// This reads the flat broadcast directly (the way `EventSubscriber` does in
1918 /// production via `OriginConsumer::announced()` + `subscribe_track`), rather
1919 /// than going through the `MoqEventSubscriber` background task, to isolate
1920 /// the back-compat guarantee (#393 must not break the flat track) from a
1921 /// pre-existing two-level-scope race in `scope_consumer_for_patterns`.
1922 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1923 async fn flat_track_back_compat_with_oid_mirror() -> Result<()> {
1924 use moq_net::Track as MoqTrack;
1925
1926 let origin = MoqEventOrigin::new();
1927
1928 // Publisher mirrors to both flat `local/events/registry` and per-OID track.
1929 let mut pub_ = origin.publisher_with_oid("registry", "oid-x")?;
1930 assert!(
1931 pub_.has_oid_mirror(),
1932 "publisher_with_oid must mirror to an OID track"
1933 );
1934 pub_.publish("qwen3-4b", "published", b"mirrored")?;
1935
1936 // Read the flat `local/events/registry` broadcast directly.
1937 let consumer = origin.consumer();
1938 let scoped = consumer
1939 .scope(&[Path::new(EVENT_PREFIX)])
1940 .ok_or_else(|| anyhow!("no events scope"))?;
1941
1942 let recv = tokio::spawn(async move {
1943 let mut scoped = scoped;
1944 loop {
1945 let (_path, bc) = match scoped.announced().await {
1946 None => return None,
1947 Some((_, None)) => continue,
1948 Some((p, Some(b))) => (p, b),
1949 };
1950 let mut track = match bc.subscribe_track(&MoqTrack::new(EVENT_TRACK)) {
1951 Ok(t) => t,
1952 Err(_) => continue,
1953 };
1954 let mut group = match track.next_group().await {
1955 Ok(Some(g)) => g,
1956 _ => continue,
1957 };
1958 if let Ok(Some(frame)) = group.read_frame().await {
1959 return Some(frame.to_vec());
1960 }
1961 }
1962 });
1963
1964 // Re-publish to cover the announcement-propagation race.
1965 let publish_loop = async {
1966 for _ in 0..50 {
1967 let _ = pub_.publish("qwen3-4b", "published", b"mirrored");
1968 tokio::time::sleep(Duration::from_millis(100)).await;
1969 }
1970 std::future::pending::<()>().await;
1971 };
1972
1973 let got: Option<Vec<u8>> = tokio::select! {
1974 r = recv => r.ok().flatten(),
1975 _ = publish_loop => None,
1976 _ = tokio::time::sleep(Duration::from_secs(10)) => None,
1977 };
1978
1979 let frame = got.expect("flat-track subscriber never received the mirrored event");
1980 assert!(frame.len() >= 4, "frame too short");
1981 let topic_len = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize;
1982 let topic = std::str::from_utf8(&frame[4..4 + topic_len]).unwrap();
1983 let payload = &frame[4 + topic_len..];
1984 assert_eq!(topic, "registry.qwen3-4b.published");
1985 assert_eq!(payload, b"mirrored");
1986
1987 Ok(())
1988 }
1989
1990 /// subscribe() and subscribe_oid() are mutually exclusive — a subscriber is
1991 /// either flat-pattern-based or OID-scoped, never both.
1992 #[test]
1993 fn subscribe_and_subscribe_oid_are_mutually_exclusive() -> Result<()> {
1994 let mut s = MoqEventSubscriber::new();
1995 s.subscribe("worker.")?;
1996 assert!(
1997 s.subscribe_oid("oid").is_err(),
1998 "subscribe_oid after subscribe must fail"
1999 );
2000
2001 let mut s = MoqEventSubscriber::new();
2002 s.subscribe_oid("oid")?;
2003 assert!(
2004 s.subscribe("worker.").is_err(),
2005 "subscribe after subscribe_oid must fail"
2006 );
2007
2008 Ok(())
2009 }
2010
2011 /// A no-op BackfillSource used to test firehose-backfill graceful degradation.
2012 struct NullBackfillSource {
2013 /// What `has_history` returns.
2014 has_history: bool,
2015 /// History to replay (if any).
2016 history: Vec<(String, Vec<u8>)>,
2017 }
2018
2019 impl BackfillSource for NullBackfillSource {
2020 fn has_history(&self, _oid: &str) -> bool {
2021 self.has_history
2022 }
2023 fn replay<'a>(
2024 &'a self,
2025 _oid: &'a str,
2026 tx: &'a mpsc::Sender<(String, Vec<u8>)>,
2027 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send + 'a>> {
2028 let history = self.history.clone();
2029 Box::pin(async move {
2030 for (topic, payload) in history {
2031 if tx.send((topic, payload)).await.is_err() {
2032 break;
2033 }
2034 }
2035 Ok(())
2036 })
2037 }
2038 }
2039
2040 /// When the firehose has history, backfill replays it BEFORE live events
2041 /// arrive (cold-start path). The live MoQ events then follow.
2042 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2043 async fn backfill_replays_history_before_live() -> Result<()> {
2044 let origin = MoqEventOrigin::new();
2045
2046 let source = Arc::new(NullBackfillSource {
2047 has_history: true,
2048 history: vec![
2049 (
2050 "registry.qwen3-4b.published".to_owned(),
2051 b"backfill-1".to_vec(),
2052 ),
2053 (
2054 "registry.qwen3-4b.published".to_owned(),
2055 b"backfill-2".to_vec(),
2056 ),
2057 ],
2058 });
2059
2060 let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
2061 sub.subscribe_oid("oid-backfill")?;
2062 sub.with_backfill(BackfillMode::FirehoseBackfill {
2063 oid: "oid-backfill".to_owned(),
2064 source,
2065 })?;
2066
2067 // Drive the backfill + a live publish concurrently.
2068 let mut pub_ = origin.publisher_oid_only("registry", "oid-backfill")?;
2069 pub_.publish("qwen3-4b", "published", b"live")?;
2070
2071 // First two events should be the backfilled history (cold-start), then live.
2072 let e1 = tokio::time::timeout(Duration::from_secs(5), sub.recv())
2073 .await
2074 .expect("backfill[0] timed out")?;
2075 assert_eq!(e1.1, b"backfill-1");
2076
2077 let e2 = tokio::time::timeout(Duration::from_secs(5), sub.recv())
2078 .await
2079 .expect("backfill[1] timed out")?;
2080 assert_eq!(e2.1, b"backfill-2");
2081
2082 // Then the live event.
2083 let e3 = tokio::time::timeout(Duration::from_secs(5), sub.recv())
2084 .await
2085 .expect("live event timed out")?;
2086 assert_eq!(e3.1, b"live");
2087
2088 drop(pub_);
2089 Ok(())
2090 }
2091
2092 /// When the firehose is UNAVAILABLE (has_history false), backfill mode
2093 /// gracefully degrades to live-only — the subscription does not fail and
2094 /// live events still arrive (the firehose is the cold-start path; MoQ is
2095 /// the live path).
2096 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2097 async fn backfill_gracefully_degrades_when_firehose_unavailable() -> Result<()> {
2098 let origin = MoqEventOrigin::new();
2099
2100 let source = Arc::new(NullBackfillSource {
2101 has_history: false, // firehose offline / no history
2102 history: vec![("never".to_owned(), b"never".to_vec())],
2103 });
2104
2105 let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
2106 sub.subscribe_oid("oid-degrade")?;
2107 sub.with_backfill(BackfillMode::FirehoseBackfill {
2108 oid: "oid-degrade".to_owned(),
2109 source,
2110 })?;
2111
2112 let mut pub_ = origin.publisher_oid_only("registry", "oid-degrade")?;
2113 pub_.publish("qwen3-4b", "published", b"live")?;
2114
2115 // Despite requesting backfill, with the firehose down we still get the
2116 // live event (and never the spurious "never" history item).
2117 let got = tokio::time::timeout(Duration::from_secs(5), sub.recv())
2118 .await
2119 .expect("live event timed out (degradation failed)")?;
2120 assert_eq!(
2121 got.1, b"live",
2122 "degraded subscriber must receive live events"
2123 );
2124 assert_ne!(
2125 got.1, b"never",
2126 "firehose history must NOT leak when unavailable"
2127 );
2128
2129 drop(pub_);
2130 Ok(())
2131 }
2132}