Skip to main content

hyprstream_rpc/
moq_event.rs

1//! moq-lite event bus plane — replaces the ZMQ XPUB/XSUB ProxyService (#167).
2//!
3//! # Architecture
4//!
5//! The event bus is a process-global [`MoqEventOrigin`] rooted at `local/events`.
6//! Each service registers its own broadcast under `local/events/{source}` with a
7//! single `events` track. Each published frame carries the full dot-separated topic
8//! (`{source}.{entity}.{event}`) and the raw payload, packed as:
9//!
10//! ```text
11//!   [4 bytes topic_len BE][topic UTF-8][payload bytes]
12//! ```
13//!
14//! Subscribers connect to the origin, discover source broadcasts via
15//! `OriginConsumer::announced()`, subscribe to their `events` track, and do
16//! in-memory topic-prefix filtering (matching ZMQ's prefix-filter semantics).
17//!
18//! This is the "Live" preset for the event bus: fan-out, unbounded, at-most-once.
19//! No chained HMAC or policy axes — events are best-effort lifecycle signals, not
20//! auditable streams. [`crate::events::EventPublisher`] / [`crate::events::EventSubscriber`]
21//! (the canonical broadcast type — see `crate::events`) layer group-key encryption
22//! on top of this transport, selectable via [`crate::crypto::event_crypto::EventPrivacy`].
23//!
24//! ## Two origins: `moq_event` vs `moq_stream`
25//!
26//! Broadcast publish/subscribe (fan-out, N subscribers per source/topic) is
27//! consolidated exclusively on THIS origin (`moq_event` / `MoqEventOrigin`).
28//! The sibling `moq_stream` origin (`crate::moq_stream`) is reserved for
29//! point-to-point token streaming (one producer, one consumer per stream,
30//! DH-derived `StreamKeys`) — it is not a second broadcast plane and new
31//! fan-out use cases should not be added to it. Anything that needs "N
32//! interested parties learn about an event" belongs on `moq_event`; anything
33//! that is a private request/response or token stream between two parties
34//! belongs on `moq_stream`.
35//!
36//! ## QoS (#606)
37//!
38//! `MoqEventSubscriber` reuses the EXISTING `StreamOpt` contract from
39//! `streaming.capnp` / [`crate::stream_info`] (#213/#273) rather than a parallel
40//! event-specific QoS type — see [`crate::stream_info::EventLive`] (default) and
41//! [`crate::stream_info::EventReliable`] (at-least-once + retained, for events
42//! that must not be silently dropped, e.g. `model.loaded`). Only `delivery` and
43//! `overflow_policy` are enforced client-side today (`ordering`/`completion` are
44//! reserved for a future chained-integrity layer, same as the flat at-most-once
45//! default already implies `Ordering::Ordered` is not currently checked).
46//!
47//! # #393 — per-OID broadcast paths
48//!
49//! The original design (above) is a flat single-track fan-out: every event for
50//! every model OID lands on the same `events` track, and subscribers filter
51//! post-read. That is the firehose problem — O(whole-network) reads. The fix
52//! mirrors the streaming plane (`moq_stream`: `{tenant}/{service}/{topic}/{instance}`):
53//! each model OID gets its OWN broadcast path, `local/events/publications/{oid_hash}`,
54//! so moq's `scope(&[Path])` makes wire-level selectivity automatic (one QUIC
55//! uni-stream per group per subscribed track). A node tracking N of M OIDs reads
56//! N tracks, not M.
57//!
58//! The flat `local/events/{source}` track is RETAINED as a transition fallback:
59//! publishers that mirror to both paths keep legacy subscribers working while
60//! new subscribers read the selective per-OID track. No capnp change — the event
61//! payload format is unchanged; only the track naming/routing changes.
62//!
63//! ## Late-join retention (decision A: firehose-backfill)
64//!
65//! moq's per-track cache evicts groups older than `MAX_GROUP_AGE` (5s) — too
66//! short for a scheduler reconstructing publication history. On first
67//! subscription to an OID's track, [`BackfillMode`] checks whether history is
68//! available via the optional atproto firehose (`sh.tangled.git.refUpdate`) /
69//! registry; if so it replays that history before switching to live MoQ. If the
70//! firehose is unavailable the subscriber starts live-only (graceful
71//! degradation) — the firehose is the cold-start path, MoQ is the live path.
72
73use std::collections::{HashMap, HashSet, VecDeque};
74use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
75use std::sync::{Arc, OnceLock};
76use std::time::Duration;
77
78use anyhow::{anyhow, Result};
79use bytes::Bytes;
80use moq_net::{
81    BroadcastProducer, Group, Origin, OriginConsumer, OriginProducer, Path, Track, TrackProducer,
82};
83use parking_lot::Mutex;
84use sha2::{Digest, Sha256};
85use tokio::sync::mpsc;
86
87use crate::stream_info::{OverflowPolicy, StreamOpt, StreamOptPreset};
88
89// ============================================================================
90// Process-global moq event bus origin — set once at startup by the factory.
91// ============================================================================
92
93static GLOBAL_MOQ_EVENT_ORIGIN: OnceLock<MoqEventOrigin> = OnceLock::new();
94
95/// Register the process-global moq event bus origin.
96///
97/// Must be called once at startup (from the event-service factory) before any
98/// `EventPublisher::new()` or `EventSubscriber::new()` call.
99/// Returns `true` on the first call, `false` if already initialized (idempotent).
100pub fn init_global_moq_event_origin(origin: MoqEventOrigin) -> bool {
101    GLOBAL_MOQ_EVENT_ORIGIN.set(origin).is_ok()
102}
103
104/// Borrow the process-global moq event bus origin, if initialized.
105///
106/// Returns `None` when the event bus has not yet been wired (unit tests, or
107/// before `create_event_service` factory has run).
108pub fn global_moq_event_origin() -> Option<&'static MoqEventOrigin> {
109    GLOBAL_MOQ_EVENT_ORIGIN.get()
110}
111
112/// Track name for the event fanout track within each source broadcast.
113pub const EVENT_TRACK: &str = "events";
114
115/// Broadcast path prefix (the `local/events` root under which per-source
116/// broadcasts are registered).
117pub const EVENT_PREFIX: &str = "local/events";
118
119/// Broadcast path segment under which per-OID (#393) publication broadcasts
120/// live: `local/events/publications/{oid_hash}`. Mirrors the streaming plane's
121/// `{tenant}/{service}/{topic}/{instance}` shape so `scope(&[Path])` gives
122/// wire-level selectivity — a node tracking N of M OIDs reads N tracks, not M.
123pub const PUBLICATIONS_SEGMENT: &str = "publications";
124
125/// Full prefix for per-OID publication broadcasts.
126pub const PUBLICATIONS_PREFIX: &str = "local/events/publications";
127
128/// Number of hex characters of the SHA-256 OID hash used in the broadcast path.
129/// 16 hex chars (64 bits) keeps paths short while making collisions negligible
130/// across any realistic OID namespace (birthday bound ≈ 2^32 models).
131pub const OID_HASH_LEN: usize = 16;
132
133/// Stable, filesystem/moq-safe identifier for a model OID.
134///
135/// The OID is hashed (SHA-256, truncated to [`OID_HASH_LEN`] hex chars) so the
136/// broadcast path is unguessable from the OID alone and contains no characters
137/// that are illegal in a moq path segment. Two distinct OIDs map to distinct
138/// paths with overwhelming probability.
139pub fn oid_hash(oid: &str) -> String {
140    let digest = Sha256::digest(oid.as_bytes());
141    hex::encode(&digest[..OID_HASH_LEN / 2])
142}
143
144/// Build the per-OID publication broadcast path: `local/events/publications/{oid_hash}`.
145pub fn publication_broadcast_path(oid: &str) -> String {
146    format!("{}/{}", PUBLICATIONS_PREFIX, oid_hash(oid))
147}
148
149// ============================================================================
150// MoqEventOrigin
151// ============================================================================
152
153/// Shared moq origin for the event bus plane.
154///
155/// Replaces the ZMQ XPUB/XSUB `ProxyService`. Publishers register broadcasts
156/// under `local/events/{source}`; subscribers watch the origin consumer for
157/// announcements and subscribe to the `events` track of each source broadcast.
158#[derive(Clone)]
159pub struct MoqEventOrigin {
160    inner: Arc<EventOriginInner>,
161}
162
163struct EventOriginInner {
164    /// Root-scoped producer: publishes under `local/events/{source}`.
165    producer: OriginProducer,
166    /// Consumer over the same origin tree (for subscribers).
167    consumer: OriginConsumer,
168    /// Keep `BroadcastProducer`s alive so their broadcasts stay announced.
169    /// Keyed by source name so re-registration replaces the stale producer.
170    broadcasts: Mutex<HashMap<String, BroadcastProducer>>,
171}
172
173impl MoqEventOrigin {
174    /// Create a standalone event origin (used at startup / in unit tests).
175    pub fn new() -> Self {
176        let producer = Origin::random().produce();
177        let consumer = producer.consume();
178        Self {
179            inner: Arc::new(EventOriginInner {
180                producer,
181                consumer,
182                broadcasts: Mutex::new(HashMap::new()),
183            }),
184        }
185    }
186
187    /// Create from an existing producer/consumer pair (e.g. shared with the
188    /// iroh moq substrate so external relay subscribers see the same tree).
189    pub fn from_pair(producer: OriginProducer, consumer: OriginConsumer) -> Self {
190        Self {
191            inner: Arc::new(EventOriginInner {
192                producer,
193                consumer,
194                broadcasts: Mutex::new(HashMap::new()),
195            }),
196        }
197    }
198
199    /// Create an event publisher for `source` (e.g. `"worker"`, `"system"`).
200    ///
201    /// Registers the broadcast `local/events/{source}` and opens its `events` track.
202    pub fn publisher(&self, source: &str) -> Result<MoqEventPublisher> {
203        let path = format!("{}/{}", EVENT_PREFIX, source);
204        let mut broadcast = self
205            .inner
206            .producer
207            .create_broadcast(path.as_str())
208            .ok_or_else(|| anyhow!("create_broadcast denied for {path}"))?;
209
210        let track = broadcast.create_track(Track::new(EVENT_TRACK))?;
211
212        // Retain the broadcast producer so it stays announced for the publisher's lifetime.
213        // HashMap keyed by source: re-registration replaces the stale producer atomically.
214        self.inner
215            .broadcasts
216            .lock()
217            .insert(source.to_owned(), broadcast);
218
219        Ok(MoqEventPublisher {
220            track,
221            source: source.to_owned(),
222            next_group: 0,
223            oid_track: None,
224        })
225    }
226
227    /// Create an event publisher for `source` (e.g. `"worker"`, `"system"`) that
228    /// ALSO mirrors every event to the per-OID (#393) publication track for `oid`.
229    ///
230    /// This is the transition path: the publisher writes each event to BOTH the
231    /// legacy flat `local/events/{source}` broadcast (so existing subscribers
232    /// keep working) AND the selective `local/events/publications/{oid_hash}`
233    /// broadcast (so #393 subscribers get wire-level per-OID selectivity). Once
234    /// all subscribers have migrated to per-OID tracks the flat mirror can be
235    /// dropped (see [`MoqEventOrigin::publisher_oid_only`]).
236    ///
237    /// `oid` is the model OID whose publications this publisher emits; it is
238    /// hashed via [`oid_hash`] for the broadcast path.
239    pub fn publisher_with_oid(&self, source: &str, oid: &str) -> Result<MoqEventPublisher> {
240        let flat = self.publisher(source)?;
241        let oid_track = self.oid_track(oid)?;
242        Ok(MoqEventPublisher {
243            track: flat.track,
244            source: flat.source,
245            next_group: flat.next_group,
246            oid_track: Some(oid_track),
247        })
248    }
249
250    /// Create an event publisher that writes ONLY to the per-OID (#393)
251    /// publication track for `oid` — no flat-track mirror. Use once every
252    /// subscriber of this source has migrated to per-OID subscription.
253    ///
254    /// `source` is retained for topic-prefix semantics and `publish()` topic
255    /// construction, but nothing is announced under `local/events/{source}`.
256    pub fn publisher_oid_only(&self, source: &str, oid: &str) -> Result<MoqEventPublisher> {
257        let oid_track = self.oid_track(oid)?;
258        Ok(MoqEventPublisher {
259            track: oid_track,
260            source: source.to_owned(),
261            next_group: 0,
262            oid_track: None,
263        })
264    }
265
266    /// Open (creating if needed) the per-OID publication track for `oid` and
267    /// retain its broadcast producer. Returns the track producer.
268    fn oid_track(&self, oid: &str) -> Result<TrackProducer> {
269        let path = publication_broadcast_path(oid);
270        let mut broadcast = self
271            .inner
272            .producer
273            .create_broadcast(path.as_str())
274            .ok_or_else(|| anyhow!("create_broadcast denied for {path}"))?;
275        let track = broadcast.create_track(Track::new(EVENT_TRACK))?;
276        // Keyed by the full path so distinct OIDs accumulate distinct producers
277        // and re-registration of the same OID replaces the stale producer.
278        self.inner.broadcasts.lock().insert(path, broadcast);
279        Ok(track)
280    }
281
282    /// Clone the origin consumer (for subscriber background tasks).
283    pub fn consumer(&self) -> OriginConsumer {
284        self.inner.consumer.clone()
285    }
286
287    /// Clone the origin producer.
288    ///
289    /// Needed to wire this origin into a `moq_net::Server`/`Client` with
290    /// `with_origin` / `with_consume` so externally-published broadcasts ingest
291    /// into the same tree subscribers read from.
292    pub fn producer(&self) -> OriginProducer {
293        self.inner.producer.clone()
294    }
295}
296
297impl Default for MoqEventOrigin {
298    fn default() -> Self {
299        Self::new()
300    }
301}
302
303// ============================================================================
304// Cross-process event plane over UDS (#275)
305//
306// In the same-process (`InprocManager`) deployment every service shares one
307// process-global `MoqEventOrigin`, so an in-process publisher and subscriber
308// see the same tree. In the systemd / `--ipc` deployment each service runs in
309// its OWN process; only the `event` service's process initializes the global
310// origin via the factory. Other processes (notably `worker`) had no origin at
311// all — `EventPublisher::new` failed with "moq event bus not initialized".
312//
313// The fix mirrors the streaming plane (`moq_stream::serve_moq_uds_background`):
314//
315//   * The `event` service SERVES its origin over a well-known UDS path
316//     ([`serve_event_moq_uds_background`]) using `moq_net::Server::with_origin`
317//     so the session is BIDIRECTIONAL — it both serves announced broadcasts to
318//     subscribers AND ingests broadcasts a connected client publishes.
319//
320//   * Every NON-event process creates its own local `MoqEventOrigin`, registers
321//     it as the process global, then spawns a background task
322//     ([`connect_event_moq_uds_background`]) that connects a `moq_net::Client`
323//     to the event service's UDS with `with_origin(local_producer)`. That makes
324//     the session bidirectional too: locally-published event broadcasts are
325//     announced UP to the event service (which fans them out to all
326//     subscribers), and broadcasts from other processes flow DOWN into the
327//     local origin so local subscribers see them.
328//
329// The UDS path is the stable, cross-process [`crate::paths::event_socket`] —
330// not a per-PID path — so any process can find it without an RPC round-trip.
331// ============================================================================
332
333/// Reconnect backoff for the client-side event plane link.
334const EVENT_RECONNECT_DELAY: Duration = Duration::from_millis(500);
335
336/// Serve the event-bus origin over a UDS socket so cross-process publishers and
337/// subscribers reach the same tree (#275).
338///
339/// Mirrors [`crate::moq_stream::serve_moq_uds_background`], but the moq server is
340/// built with `with_origin` (publish + consume) rather than `with_publish`
341/// alone, because the event plane is bidirectional: a connected client both
342/// announces its own event broadcasts (ingested into this origin) and subscribes
343/// to broadcasts from every other process.
344///
345/// Called once by the event-service factory. Idempotent: a second call is a
346/// no-op (the first bind wins). Connection errors are logged, not fatal.
347pub fn serve_event_moq_uds_background(origin: MoqEventOrigin, path: std::path::PathBuf) {
348    use crate::transport::uds_session::{accept_uds, PLANE_MOQ};
349    use moq_net::Server as MoqServer;
350
351    // Ensure the parent directory exists (runtime_dir may not be created yet).
352    if let Some(parent) = path.parent() {
353        let _ = std::fs::create_dir_all(parent);
354    }
355    // Remove a stale socket from a previous run (best-effort).
356    let _ = std::fs::remove_file(&path);
357
358    let listener = match std::os::unix::net::UnixListener::bind(&path) {
359        Ok(l) => l,
360        Err(e) => {
361            tracing::error!(path = %path.display(), "event moq UDS bind failed: {e}");
362            return;
363        }
364    };
365
366    // 0o600: owner read/write only; SO_PEERCRED enforces uid match (uds_server.rs).
367    #[cfg(unix)]
368    {
369        use std::os::unix::fs::PermissionsExt;
370        let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600));
371    }
372
373    // tokio requires the listener be non-blocking before adoption.
374    if let Err(e) = listener.set_nonblocking(true) {
375        tracing::error!("event moq UDS set_nonblocking failed: {e}");
376        return;
377    }
378
379    let listener = match tokio::net::UnixListener::from_std(listener) {
380        Ok(l) => l,
381        Err(e) => {
382            tracing::error!("event moq UDS listener conversion failed: {e}");
383            return;
384        }
385    };
386
387    tracing::info!(path = %path.display(), "event moq UDS listener ready");
388
389    tokio::spawn(async move {
390        loop {
391            let stream = match listener.accept().await {
392                Ok((s, _)) => s,
393                Err(e) => {
394                    tracing::warn!("event moq UDS accept error: {e}");
395                    continue;
396                }
397            };
398            // Each session gets its own producer/consumer view of the SAME origin
399            // tree, so a client's published broadcasts are ingested and re-served
400            // to every other connected session.
401            let producer = origin.producer();
402            tokio::spawn(async move {
403                let (plane, session) = match accept_uds(stream).await {
404                    Ok(pair) => pair,
405                    Err(e) => {
406                        tracing::debug!("event moq UDS handshake error: {e}");
407                        return;
408                    }
409                };
410                if plane != PLANE_MOQ {
411                    tracing::debug!("event moq UDS: unexpected plane 0x{plane:02x} — dropping");
412                    return;
413                }
414                // `accept` returns once the handshake completes; the protocol then
415                // runs in the background. The returned `Session` MUST be held —
416                // dropping it closes the connection. Hold it until it closes so
417                // the bidirectional bridge stays live (publish + consume).
418                match MoqServer::new().with_origin(producer).accept(session).await {
419                    Ok(moq_session) => {
420                        let reason = moq_session.closed().await;
421                        tracing::debug!("event moq UDS session closed: {reason:?}");
422                    }
423                    Err(e) => tracing::debug!("event moq UDS accept error: {e}"),
424                }
425            });
426        }
427    });
428}
429
430/// Ensure this process can publish/subscribe events, wiring a CLIENT-mode
431/// event origin connected to the event service's UDS plane if needed (#275).
432///
433/// Idempotent and safe to call from any process at startup:
434///
435///   * If the process-global origin is ALREADY initialized (this process hosts
436///     the `event` service, which calls [`init_global_moq_event_origin`] +
437///     [`serve_event_moq_uds_background`] in its factory), this is a no-op —
438///     the in-process origin is authoritative.
439///
440///   * Otherwise it creates a fresh local origin, registers it as the global,
441///     and spawns the background link to the event service's UDS
442///     ([`connect_event_moq_uds_background`]). After this returns,
443///     `EventPublisher::new` / `EventSubscriber` succeed in this process and
444///     events flow cross-process to the shared bus.
445///
446/// Must be called from within a tokio runtime (it spawns a background task).
447/// `path` is the event service's UDS socket (use [`crate::paths::event_socket`]).
448pub fn ensure_event_client_origin(path: std::path::PathBuf) {
449    if global_moq_event_origin().is_some() {
450        // The event service is co-located in this process; nothing to dial.
451        return;
452    }
453    let origin = MoqEventOrigin::new();
454    // Register first so any `EventPublisher::new` after this point resolves the
455    // global immediately, even before the background link is established (moq
456    // buffers/announces the broadcast and it flows up once the link connects).
457    if !init_global_moq_event_origin(origin.clone()) {
458        // Lost a race with the event factory or a concurrent caller; the global
459        // that won is authoritative, so drop ours.
460        return;
461    }
462    connect_event_moq_uds_background(origin, path);
463}
464
465/// Install a client origin without opening any local socket.
466pub fn install_event_network_client_origin() -> Option<MoqEventOrigin> {
467    if global_moq_event_origin().is_some() {
468        return None;
469    }
470    let origin = MoqEventOrigin::new();
471    init_global_moq_event_origin(origin.clone()).then_some(origin)
472}
473
474/// One authenticated network link. Target identity must accompany a fresh
475/// checkpoint-resolved Iroh transport; callers re-resolve before reconnecting.
476pub async fn run_event_network_link(
477    origin: &MoqEventOrigin,
478    transport: &crate::transport::TransportConfig,
479    proof: &crate::transport::moql_admission::MoqlAdmissionProof,
480) -> Result<()> {
481    anyhow::ensure!(
482        matches!(
483            transport.endpoint,
484            crate::transport::EndpointType::Iroh { .. }
485        ),
486        "Event network link requires Iroh"
487    );
488    let stream = crate::dial::dial_stream_authenticated(transport, proof).await?;
489    let client = moq_net::Client::new().with_origin(origin.producer());
490    let session = stream.connect_moq(&client).await?;
491    let _ = session.closed().await;
492    anyhow::bail!("authenticated Event link closed")
493}
494
495/// Connect a local event origin to the event service's UDS plane (#275).
496///
497/// Spawns a background task that connects a `moq_net::Client` (built with
498/// `with_origin(local_producer)`) to `path`, keeping the link alive and
499/// reconnecting on failure. The session is bidirectional: locally-published
500/// event broadcasts are announced UP to the event service, and broadcasts from
501/// other processes flow DOWN into `origin` so local subscribers see them.
502///
503/// Called once, in non-event processes, AFTER registering `origin` as the
504/// process global. The task runs for the process lifetime.
505pub fn connect_event_moq_uds_background(origin: MoqEventOrigin, path: std::path::PathBuf) {
506    tokio::spawn(async move {
507        loop {
508            if let Err(e) = run_event_client_link(&origin, &path).await {
509                tracing::debug!(
510                    path = %path.display(),
511                    "event moq client link ended: {e}; reconnecting in {:?}",
512                    EVENT_RECONNECT_DELAY
513                );
514            }
515            tokio::time::sleep(EVENT_RECONNECT_DELAY).await;
516        }
517    });
518}
519
520/// One connect-and-pump cycle of the client-side event link. Returns when the
521/// session closes (so the caller can reconnect).
522async fn run_event_client_link(origin: &MoqEventOrigin, path: &std::path::Path) -> Result<()> {
523    use crate::transport::uds_session::{connect_uds, PLANE_MOQ};
524    use moq_net::Client as MoqClient;
525
526    let session = connect_uds(path, PLANE_MOQ)
527        .await
528        .map_err(|e| anyhow!("event moq UDS connect {}: {e}", path.display()))?;
529
530    // `with_origin` makes this a bidirectional link: publish local broadcasts UP
531    // and consume remote broadcasts DOWN into the SAME origin tree subscribers read.
532    let moq_client = MoqClient::new().with_origin(origin.producer());
533    let moq_session = moq_client
534        .connect(session)
535        .await
536        .map_err(|e| anyhow!("event moq handshake: {e}"))?;
537
538    tracing::info!(path = %path.display(), "event moq client link established");
539
540    // Hold the session open until it closes; broadcasts flow in both directions
541    // for the duration via the origin producer/consumer wired above.
542    let reason = moq_session.closed().await;
543    Err(anyhow!("event moq session closed: {reason:?}"))
544}
545
546// ============================================================================
547// MoqEventPublisher
548// ============================================================================
549
550/// In-process moq event publisher.
551///
552/// Publishes topic+payload pairs to the `local/events/{source}` broadcast.
553/// Each `publish_raw` call writes one moq Group (one Frame) on the `events` track.
554///
555/// When constructed via [`MoqEventOrigin::publisher_with_oid`], each event is
556/// ALSO mirrored to the per-OID (#393) publication track so selective
557/// subscribers receive it without the firehose. When constructed via
558/// [`MoqEventOrigin::publisher_oid_only`], `track` IS the OID track and
559/// `oid_track` is `None`.
560pub struct MoqEventPublisher {
561    track: TrackProducer,
562    source: String,
563    next_group: u64,
564    /// Per-OID mirror track. `Some` when the publisher writes to BOTH the flat
565    /// source track and the OID track (transition path); `None` when `track` is
566    /// already the OID track (post-migration) or when no OID was supplied.
567    oid_track: Option<TrackProducer>,
568}
569
570impl MoqEventPublisher {
571    /// Publish a raw topic + payload.
572    ///
573    /// Frame format: `[4 bytes topic_len BE][topic bytes][payload bytes]`.
574    ///
575    /// If this publisher was created with an OID mirror, the frame is written to
576    /// both the flat source track and the per-OID publication track (one Group
577    /// on each, sharing the same group id so consumers on either track see a
578    /// consistent sequence).
579    pub fn publish_raw(&mut self, topic: &str, payload: &[u8]) -> Result<()> {
580        let topic_bytes = topic.as_bytes();
581        let topic_len = topic_bytes.len() as u32;
582
583        let mut frame = Vec::with_capacity(4 + topic_bytes.len() + payload.len());
584        frame.extend_from_slice(&topic_len.to_be_bytes());
585        frame.extend_from_slice(topic_bytes);
586        frame.extend_from_slice(payload);
587
588        let group_id = self.next_group;
589        self.next_group += 1;
590
591        // Write to the primary track.
592        write_group(&mut self.track, group_id, &frame)?;
593        // Mirror to the per-OID track if one is attached (#393 transition).
594        if let Some(oid_track) = &mut self.oid_track {
595            write_group(oid_track, group_id, &frame)?;
596        }
597        Ok(())
598    }
599
600    /// Publish `{source}.{entity}.{event}` with payload.
601    pub fn publish(&mut self, entity: &str, event: &str, payload: &[u8]) -> Result<()> {
602        if entity.contains('.') {
603            return Err(anyhow!("Entity name cannot contain '.': {}", entity));
604        }
605        if event.contains('.') {
606            return Err(anyhow!("Event name cannot contain '.': {}", event));
607        }
608        let topic = format!("{}.{}.{}", self.source, entity, event);
609        self.publish_raw(&topic, payload)
610    }
611
612    /// Source name for this publisher.
613    pub fn source(&self) -> &str {
614        &self.source
615    }
616
617    /// True if this publisher mirrors events to a per-OID (#393) publication track.
618    pub fn has_oid_mirror(&self) -> bool {
619        self.oid_track.is_some()
620    }
621}
622
623/// Write one Group (one Frame) carrying `frame` bytes to `track` at `group_id`.
624fn write_group(track: &mut TrackProducer, group_id: u64, frame: &[u8]) -> Result<()> {
625    let mut group = track.create_group(Group::from(group_id))?;
626    group.write_frame(Bytes::copy_from_slice(frame))?;
627    group.finish()?;
628    Ok(())
629}
630
631// ============================================================================
632// #393 — Firehose backfill for late-join (decision A)
633//
634// moq's per-track cache evicts groups older than MAX_GROUP_AGE (5s). A scheduler
635// reconstructing publication history for an OID needs more than 5s of backlog.
636// On first subscription to an OID's track we consult an optional BackfillSource
637// (atproto firehose `sh.tangled.git.refUpdate` / registry); if it can serve
638// history for that OID we replay it before switching to live MoQ. If no source
639// is available the subscriber starts live-only (graceful degradation): the
640// firehose is the cold-start path, MoQ is the live path.
641// ============================================================================
642
643/// A source of historical publication events for firehose-backfill late-join (#393).
644///
645/// Implementations typically wrap the atproto firehose (`sh.tangled.git.refUpdate`)
646/// or a registry snapshot. All methods are fallible and best-effort: returning an
647/// error or empty iterator from any of them causes [`BackfillMode`] to gracefully
648/// degrade to live-only MoQ (no backfill), never to fail the subscription.
649///
650/// This is a trait object interface so the transport-agnostic `hyprstream-rpc`
651/// crate can express the backfill contract without depending on the atproto /
652/// firehose wiring (which lives in the `hyprstream` app crate).
653pub trait BackfillSource: Send + Sync {
654    /// True if this source can serve history for `oid` right now.
655    ///
656    /// Returning `false` (e.g. firehose offline, registry unreachable) makes the
657    /// subscriber skip backfill and start live-only.
658    fn has_history(&self, oid: &str) -> bool;
659
660    /// Replay historical publication events for `oid` to `tx`, oldest-first.
661    ///
662    /// Each item is the same `(topic, payload)` pair a live MoQ subscriber would
663    /// yield. Implementations should bound the replay (event count / wall time)
664    /// and return once the backlog is drained; the caller then switches to live.
665    /// Errors are logged and treated as "no history available".
666    fn replay<'a>(
667        &'a self,
668        oid: &'a str,
669        tx: &'a mpsc::Sender<(String, Vec<u8>)>,
670    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send + 'a>>;
671}
672
673/// Late-join retention mode for a per-OID (#393) subscriber.
674///
675/// Decision A: on first subscription to an OID's track, optionally replay
676/// history from a [`BackfillSource`] (atproto firehose / registry), then switch
677/// to live MoQ. The firehose is the cold-start path; MoQ is the live path.
678#[derive(Default)]
679pub enum BackfillMode {
680    /// No backfill: subscribe live-only. The default; matches pre-#393 behaviour
681    /// and is the graceful-degradation fallback when no firehose is available.
682    #[default]
683    LiveOnly,
684    /// Attempt backfill from the supplied source for `oid` before going live.
685    /// If the source reports no history (`has_history` false) or `replay` errors,
686    /// the subscriber silently falls back to [`BackfillMode::LiveOnly`].
687    FirehoseBackfill {
688        oid: String,
689        source: Arc<dyn BackfillSource>,
690    },
691}
692
693// ============================================================================
694// Event delivery channel — QoS-aware overflow policy (#606)
695// ============================================================================
696//
697// Replaces the previous unconditional `mpsc::channel(256)` + blocking
698// `.send().await`. That gave every subscriber a single fixed-size lossless
699// channel: a slow consumer that stopped calling `recv()` would back up its
700// OWN per-broadcast read tasks indefinitely (no resubscribe, no eviction).
701//
702// `StreamOpt::overflow_policy` (reused, not reinvented — see module docs)
703// now selects the behaviour per subscriber:
704//   - `Block`      — lossless backpressure, same semantics as the old
705//                    `mpsc` channel (correct for the `EventReliable` /
706//                    at-least-once preset, which must not silently drop).
707//   - `DropOldest` — ring-buffer: a full queue evicts its oldest entry
708//                    rather than stalling the producer-side read tasks
709//                    (the new default via `EventLive`, #606 finding: a
710//                    shared/slow consumer must not wedge delivery).
711//
712// This is per-subscriber state (each `MoqEventSubscriber` owns one channel;
713// its own per-broadcast read tasks share the sender half via `clone()`), so
714// a slow subscriber only ever affects its own delivery — not other
715// subscribers reading the same broadcast.
716
717type EventItem = (String, Vec<u8>);
718
719struct EventChannelState {
720    queue: VecDeque<EventItem>,
721    capacity: usize,
722    overflow: OverflowPolicy,
723    closed: bool,
724}
725
726struct EventChannelInner {
727    state: Mutex<EventChannelState>,
728    item_ready: tokio::sync::Notify,
729    space_ready: tokio::sync::Notify,
730    /// Live `EventChannelSender` clones. The last drop closes the channel
731    /// (see `EventChannelSender`'s `Clone`/`Drop` impls) so a parked
732    /// `recv()` wakes with `None` instead of waiting forever.
733    sender_count: AtomicUsize,
734}
735
736/// Sender half of a subscriber's QoS-aware delivery channel. Cloneable —
737/// each per-broadcast read task gets its own clone. Closes the channel when
738/// the last clone is dropped (manual `Clone`/`Drop`, refcounted via
739/// `EventChannelInner::sender_count` — `#[derive(Clone)]` would NOT track
740/// this, which is why this impl is hand-written rather than derived).
741struct EventChannelSender {
742    inner: Arc<EventChannelInner>,
743    dropped_count: Arc<AtomicU64>,
744}
745
746/// Receiver half, owned by the [`MoqEventSubscriber`].
747struct EventChannelReceiver {
748    inner: Arc<EventChannelInner>,
749}
750
751/// Overflow capacity for a channel built from `overflow_policy`: the
752/// `DropOldest` high-water-mark, or a fixed depth for lossless `Block`
753/// channels (matches the historic `mpsc::channel(256)` default).
754fn event_channel_capacity(overflow: &OverflowPolicy) -> usize {
755    match overflow {
756        OverflowPolicy::Block => 256,
757        OverflowPolicy::DropOldest { high_water_mark } => (*high_water_mark).max(1) as usize,
758    }
759}
760
761fn event_channel(
762    overflow: OverflowPolicy,
763    dropped_count: Arc<AtomicU64>,
764) -> (EventChannelSender, EventChannelReceiver) {
765    let capacity = event_channel_capacity(&overflow);
766    let inner = Arc::new(EventChannelInner {
767        state: Mutex::new(EventChannelState {
768            queue: VecDeque::with_capacity(capacity.min(1024)),
769            capacity,
770            overflow,
771            closed: false,
772        }),
773        item_ready: tokio::sync::Notify::new(),
774        space_ready: tokio::sync::Notify::new(),
775        sender_count: AtomicUsize::new(1),
776    });
777    (
778        EventChannelSender {
779            inner: inner.clone(),
780            dropped_count,
781        },
782        EventChannelReceiver { inner },
783    )
784}
785
786impl EventChannelSender {
787    /// Send an item honouring the configured overflow policy. Returns `false`
788    /// once the receiver has been dropped — the caller should stop reading.
789    async fn send(&self, item: EventItem) -> bool {
790        loop {
791            {
792                let mut state = self.inner.state.lock();
793                if state.closed {
794                    return false;
795                }
796                if state.queue.len() < state.capacity {
797                    state.queue.push_back(item);
798                    drop(state);
799                    self.inner.item_ready.notify_one();
800                    return true;
801                }
802                // At capacity.
803                match state.overflow {
804                    OverflowPolicy::Block => {
805                        // Fall through: wait for the receiver to free space.
806                    }
807                    OverflowPolicy::DropOldest { .. } => {
808                        state.queue.pop_front();
809                        state.queue.push_back(item);
810                        drop(state);
811                        self.dropped_count.fetch_add(1, Ordering::Relaxed);
812                        self.inner.item_ready.notify_one();
813                        return true;
814                    }
815                }
816            }
817            self.inner.space_ready.notified().await;
818        }
819    }
820}
821
822impl Clone for EventChannelSender {
823    fn clone(&self) -> Self {
824        self.inner.sender_count.fetch_add(1, Ordering::AcqRel);
825        Self {
826            inner: self.inner.clone(),
827            dropped_count: self.dropped_count.clone(),
828        }
829    }
830}
831
832impl Drop for EventChannelSender {
833    fn drop(&mut self) {
834        // Last sender clone gone: close the channel and wake a parked
835        // `recv()` so it observes `closed` instead of waiting forever.
836        // In-flight queued items are still delivered first — `recv()` only
837        // returns `None` once the queue has also drained (see `recv()`).
838        if self.inner.sender_count.fetch_sub(1, Ordering::AcqRel) == 1 {
839            let mut state = self.inner.state.lock();
840            state.closed = true;
841            drop(state);
842            self.inner.item_ready.notify_waiters();
843        }
844    }
845}
846
847impl EventChannelReceiver {
848    async fn recv(&mut self) -> Option<EventItem> {
849        loop {
850            {
851                let mut state = self.inner.state.lock();
852                if let Some(item) = state.queue.pop_front() {
853                    drop(state);
854                    self.inner.space_ready.notify_one();
855                    return Some(item);
856                }
857                if state.closed {
858                    return None;
859                }
860            }
861            self.inner.item_ready.notified().await;
862        }
863    }
864
865    fn try_recv(&mut self) -> Result<Option<EventItem>, ()> {
866        let mut state = self.inner.state.lock();
867        if let Some(item) = state.queue.pop_front() {
868            drop(state);
869            self.inner.space_ready.notify_one();
870            return Ok(Some(item));
871        }
872        if state.closed {
873            return Err(());
874        }
875        Ok(None)
876    }
877}
878
879// ============================================================================
880// MoqEventSubscriber
881// ============================================================================
882
883/// In-process moq event subscriber.
884///
885/// Subscribes to one or more topic-prefix patterns. Patterns use the same
886/// dot-separated prefix semantics as ZMQ (`"worker."` matches all worker events,
887/// `""` matches everything). Backed by a background Tokio task that watches the
888/// origin consumer for new source broadcasts and reads their `events` tracks.
889///
890/// # #393 — per-OID subscription
891///
892/// [`Self::subscribe_oid`] scopes the subscriber to a single model OID's
893/// publication track (`local/events/publications/{oid_hash}`), so only that
894/// OID's events cross the wire — not the whole firehose. This is mutually
895/// exclusive with the flat-pattern [`Self::subscribe`] API: a subscriber is
896/// either pattern-based (legacy firehose + in-memory filter) or OID-scoped.
897pub struct MoqEventSubscriber {
898    /// Patterns added via `subscribe()`. Finalized before `recv()` is called.
899    /// Empty when this subscriber is OID-scoped (no flat-track filtering).
900    patterns: Vec<String>,
901    /// The OID whose per-OID track this subscriber reads (`None` ⇒ legacy
902    /// pattern-based subscription over the flat `local/events/{source}` tracks).
903    oid_subscription: Option<String>,
904    /// Late-join retention mode (decision A). Defaults to [`BackfillMode::LiveOnly`].
905    backfill: BackfillMode,
906    /// Delivery QoS (#606). Reuses [`StreamOpt`] — defaults to
907    /// [`crate::stream_info::EventLive`] (at-most-once, drop-oldest), matching
908    /// the pre-#606 behaviour but with a bounded, never-wedging queue instead
909    /// of an unconditionally-blocking one.
910    qos: StreamOpt,
911    /// Skip live groups with `sequence <= resume_from` (offset-resume, #606).
912    /// `None` (the default) skips nothing — sequence numbers start at 0, so a
913    /// bare `u64` threshold could not distinguish "unset" from "resume after
914    /// the very first group". Only meaningful per-broadcast; see
915    /// [`Self::with_resume_from`] docs for the multi-source-pattern caveat.
916    resume_from: Option<u64>,
917    /// Highest live-group sequence delivered so far, shared with the
918    /// background read task(s). Best-effort hint for callers implementing
919    /// resume — see [`Self::last_sequence`].
920    last_sequence: Arc<AtomicU64>,
921    /// Count of items evicted under `OverflowPolicy::DropOldest` (#606).
922    dropped_count: Arc<AtomicU64>,
923    /// Count of idle-timeout-and-resume cycles on the live read loop (#606) —
924    /// a quiet (low event rate) but still-live broadcast no longer causes the
925    /// subscriber to silently exit; this counts how often that recovery fired.
926    idle_resumes: Arc<AtomicU64>,
927    /// Explicit origin consumer for tests (bypasses the process-global
928    /// `OnceLock`, which is single-writer and so cannot be reset between tests
929    /// in one binary). `None` in production ⇒ resolve the global at `recv()`.
930    #[cfg(test)]
931    test_consumer: Option<OriginConsumer>,
932    /// Receiving end of the background task's QoS-aware channel (#606).
933    /// `None` until first `recv()`.
934    rx: Option<EventChannelReceiver>,
935    /// Background task handle (kept alive for the subscriber's lifetime).
936    _task: Option<tokio::task::JoinHandle<()>>,
937}
938
939impl MoqEventSubscriber {
940    /// Create a new subscriber. No background task is started yet.
941    pub fn new() -> Self {
942        Self {
943            patterns: Vec::new(),
944            oid_subscription: None,
945            backfill: BackfillMode::LiveOnly,
946            qos: crate::stream_info::EventLive::stream_opt(),
947            resume_from: None,
948            last_sequence: Arc::new(AtomicU64::new(0)),
949            dropped_count: Arc::new(AtomicU64::new(0)),
950            idle_resumes: Arc::new(AtomicU64::new(0)),
951            #[cfg(test)]
952            test_consumer: None,
953            rx: None,
954            _task: None,
955        }
956    }
957
958    /// Create a subscriber bound to an explicit origin consumer (tests only).
959    ///
960    /// Production code resolves the process-global origin at `recv()` time (see
961    /// [`ensure_started`]); that global is a `OnceLock` and cannot be reset
962    /// between tests in a single binary, so tests that need an isolated origin
963    /// pass its consumer here instead.
964    #[cfg(test)]
965    pub(crate) fn new_for_test(consumer: OriginConsumer) -> Self {
966        let mut s = Self::new();
967        s.test_consumer = Some(consumer);
968        s
969    }
970
971    /// Add a topic-prefix filter (prefix match, ZMQ semantics).
972    ///
973    /// - `"worker."` — all events from source `worker`
974    /// - `"worker.sandbox123.started"` — exact topic
975    /// - `""` — all events (subscribe-all)
976    ///
977    /// Must be called before the first `recv()`. Mutually exclusive with
978    /// [`Self::subscribe_oid`] (legacy flat-track subscription).
979    pub fn subscribe(&mut self, pattern: &str) -> Result<()> {
980        if self.rx.is_some() {
981            return Err(anyhow!("subscribe() must be called before recv()"));
982        }
983        if self.oid_subscription.is_some() {
984            return Err(anyhow!(
985                "subscribe() is mutually exclusive with subscribe_oid()"
986            ));
987        }
988        self.patterns.push(pattern.to_owned());
989        Ok(())
990    }
991
992    /// Subscribe to all events (`pattern = ""`).
993    pub fn subscribe_all(&mut self) -> Result<()> {
994        self.subscribe("")
995    }
996
997    /// Unsubscribe from a pattern. Must be called before `recv()`.
998    pub fn unsubscribe(&mut self, pattern: &str) -> Result<()> {
999        if self.rx.is_some() {
1000            return Err(anyhow!("unsubscribe() must be called before recv()"));
1001        }
1002        self.patterns.retain(|p| p != pattern);
1003        Ok(())
1004    }
1005
1006    /// Subscribe to a single model OID's per-OID publication track (#393).
1007    ///
1008    /// The subscriber scopes its origin consumer to
1009    /// `local/events/publications/{oid_hash}` and reads only that track, so only
1010    /// events published for `oid` cross the wire (one QUIC uni-stream per group
1011    /// on the wire). This replaces the flat firehose + in-memory filter.
1012    ///
1013    /// Mutually exclusive with [`Self::subscribe`] / [`Self::subscribe_all`].
1014    /// Must be called before the first `recv()`.
1015    pub fn subscribe_oid(&mut self, oid: &str) -> Result<()> {
1016        if self.rx.is_some() {
1017            return Err(anyhow!("subscribe_oid() must be called before recv()"));
1018        }
1019        if !self.patterns.is_empty() {
1020            return Err(anyhow!(
1021                "subscribe_oid() is mutually exclusive with subscribe()"
1022            ));
1023        }
1024        self.oid_subscription = Some(oid.to_owned());
1025        Ok(())
1026    }
1027
1028    /// Set the late-join retention mode (decision A: firehose-backfill).
1029    ///
1030    /// Only meaningful for OID-scoped subscribers (see [`Self::subscribe_oid`]):
1031    /// on first `recv()`, a [`BackfillMode::FirehoseBackfill`] subscriber asks
1032    /// its [`BackfillSource`] for `oid` history and replays it before going live.
1033    /// If the source is unavailable the subscriber silently degrades to
1034    /// live-only (never fails). Must be called before the first `recv()`.
1035    pub fn with_backfill(&mut self, mode: BackfillMode) -> Result<()> {
1036        if self.rx.is_some() {
1037            return Err(anyhow!("with_backfill() must be called before recv()"));
1038        }
1039        self.backfill = mode;
1040        Ok(())
1041    }
1042
1043    /// Select the delivery QoS (#606). Reuses [`StreamOpt`] — pass
1044    /// [`crate::stream_info::EventReliable::stream_opt()`] for at-least-once
1045    /// delivery (events that must not be silently dropped, e.g.
1046    /// `model.loaded`), or any other `StreamOpt` combination. Defaults to
1047    /// [`crate::stream_info::EventLive`] if never called. Must be called
1048    /// before the first `recv()`.
1049    pub fn with_qos(&mut self, qos: StreamOpt) -> Result<()> {
1050        if self.rx.is_some() {
1051            return Err(anyhow!("with_qos() must be called before recv()"));
1052        }
1053        self.qos = qos;
1054        Ok(())
1055    }
1056
1057    /// Skip live groups with `sequence <= resume_from` (offset-resume, #606).
1058    /// Combine with [`Self::last_sequence`] to persist a cursor across
1059    /// reconnects for `EventReliable`-style at-least-once delivery.
1060    ///
1061    /// Caveat: a `MoqEventSubscriber` using the legacy flat-pattern
1062    /// [`Self::subscribe`] API may read from MULTIPLE independent source
1063    /// broadcasts (each with its own group-sequence counter); `resume_from`
1064    /// is applied per-broadcast, so a single threshold is only unambiguous
1065    /// for [`Self::subscribe_oid`] (one broadcast). Multi-source resume
1066    /// requires a per-source cursor map, which is out of scope here.
1067    /// Must be called before the first `recv()`.
1068    pub fn with_resume_from(&mut self, sequence: u64) -> Result<()> {
1069        if self.rx.is_some() {
1070            return Err(anyhow!("with_resume_from() must be called before recv()"));
1071        }
1072        self.resume_from = Some(sequence);
1073        Ok(())
1074    }
1075
1076    /// Highest live-group sequence delivered so far (best-effort hint for
1077    /// resume; see [`Self::with_resume_from`]). `0` before the first delivery.
1078    pub fn last_sequence(&self) -> u64 {
1079        self.last_sequence.load(Ordering::Relaxed)
1080    }
1081
1082    /// Count of items evicted under `OverflowPolicy::DropOldest` (#606).
1083    pub fn dropped_count(&self) -> u64 {
1084        self.dropped_count.load(Ordering::Relaxed)
1085    }
1086
1087    /// Count of idle-timeout-and-resume cycles on the live read loop (#606).
1088    pub fn idle_resumes(&self) -> u64 {
1089        self.idle_resumes.load(Ordering::Relaxed)
1090    }
1091
1092    /// Lazily start the background task and return a mutable reference to the channel receiver.
1093    fn ensure_started(&mut self) -> Result<&mut EventChannelReceiver> {
1094        if self.rx.is_none() {
1095            #[cfg(test)]
1096            let consumer = if let Some(c) = self.test_consumer.clone() {
1097                c
1098            } else {
1099                global_moq_event_origin()
1100                    .ok_or_else(|| anyhow!("moq event bus not initialized; call init_global_moq_event_origin first"))?
1101                    .consumer()
1102            };
1103            #[cfg(not(test))]
1104            let consumer = global_moq_event_origin()
1105                .ok_or_else(|| {
1106                    anyhow!(
1107                        "moq event bus not initialized; call init_global_moq_event_origin first"
1108                    )
1109                })?
1110                .consumer();
1111
1112            let (tx, rx) =
1113                event_channel(self.qos.overflow_policy.clone(), self.dropped_count.clone());
1114            let resume_from = self.resume_from;
1115            let last_sequence = self.last_sequence.clone();
1116            let idle_resumes = self.idle_resumes.clone();
1117
1118            let task = if let Some(oid) = self.oid_subscription.clone() {
1119                // #393 per-OID subscription: scope to the OID's track path and
1120                // run the (optional) backfill before going live.
1121                let backfill = std::mem::replace(&mut self.backfill, BackfillMode::LiveOnly);
1122                tokio::spawn(run_oid_subscriber_task(
1123                    consumer,
1124                    oid,
1125                    backfill,
1126                    tx,
1127                    resume_from,
1128                    last_sequence,
1129                    idle_resumes,
1130                ))
1131            } else {
1132                // Legacy flat-track pattern subscription.
1133                let patterns = self.patterns.clone();
1134                tokio::spawn(run_subscriber_task(
1135                    consumer,
1136                    patterns,
1137                    tx,
1138                    resume_from,
1139                    last_sequence,
1140                    idle_resumes,
1141                ))
1142            };
1143            self.rx = Some(rx);
1144            self._task = Some(task);
1145        }
1146        self.rx
1147            .as_mut()
1148            .ok_or_else(|| anyhow!("subscriber channel not initialized"))
1149    }
1150
1151    /// Receive the next event. Blocks until an event arrives or the bus is closed.
1152    pub async fn recv(&mut self) -> Result<(String, Vec<u8>)> {
1153        let rx = self.ensure_started()?;
1154        rx.recv()
1155            .await
1156            .ok_or_else(|| anyhow!("event subscriber closed"))
1157    }
1158
1159    /// Receive with timeout.
1160    pub async fn recv_timeout(&mut self, timeout: Duration) -> Result<Option<(String, Vec<u8>)>> {
1161        match tokio::time::timeout(timeout, self.recv()).await {
1162            Ok(result) => result.map(Some),
1163            Err(_) => Ok(None),
1164        }
1165    }
1166
1167    /// Try to receive without blocking.
1168    pub fn try_recv(&mut self) -> Result<Option<(String, Vec<u8>)>> {
1169        let rx = self.ensure_started()?;
1170        match rx.try_recv() {
1171            Ok(msg) => Ok(msg),
1172            Err(()) => Err(anyhow!("event subscriber closed")),
1173        }
1174    }
1175}
1176
1177impl Default for MoqEventSubscriber {
1178    fn default() -> Self {
1179        Self::new()
1180    }
1181}
1182
1183// ============================================================================
1184// Background subscriber task
1185// ============================================================================
1186
1187/// Background task: watches the origin consumer for announced source broadcasts,
1188/// subscribes to each `events` track, and relays matching topic+payload pairs to `tx`.
1189async fn run_subscriber_task(
1190    mut consumer: OriginConsumer,
1191    patterns: Vec<String>,
1192    tx: EventChannelSender,
1193    resume_from: Option<u64>,
1194    last_sequence: Arc<AtomicU64>,
1195    idle_resumes: Arc<AtomicU64>,
1196) {
1197    use tokio::task::JoinSet;
1198
1199    let patterns = Arc::new(patterns);
1200
1201    // Scope the consumer to the `local/events` subtree, then further narrow by
1202    // source names extracted from the patterns.
1203    match scope_consumer_for_patterns(&consumer, &patterns) {
1204        Some(c) => consumer = c,
1205        None => {
1206            // Origin has no `local/events` tree yet — using the root consumer would
1207            // expose all RPC traffic to the event channel. Abort instead.
1208            tracing::warn!("event subscriber: origin has no event scope; subscriber task exiting");
1209            return;
1210        }
1211    }
1212
1213    // Track per-broadcast read tasks so they are cancelled when this loop exits.
1214    let mut tasks: JoinSet<()> = JoinSet::new();
1215
1216    loop {
1217        let (path, broadcast) = match consumer.announced().await {
1218            None => break,               // origin closed
1219            Some((_, None)) => continue, // unannounce
1220            Some((path, Some(b))) => (path, b),
1221        };
1222
1223        let tx2 = tx.clone();
1224        let patterns2 = patterns.clone();
1225        let path_str = path.as_str().to_owned();
1226        let last_sequence2 = last_sequence.clone();
1227        let idle_resumes2 = idle_resumes.clone();
1228
1229        tasks.spawn(async move {
1230            read_event_broadcast(
1231                broadcast,
1232                path_str,
1233                patterns2,
1234                tx2,
1235                resume_from,
1236                last_sequence2,
1237                idle_resumes2,
1238            )
1239            .await;
1240        });
1241    }
1242    // Dropping `tasks` here cancels all per-broadcast read tasks.
1243}
1244
1245/// Scope the origin consumer to `local/events/{sources}` based on patterns.
1246///
1247/// Returns `None` only when `local/events` itself has no tree yet (the event
1248/// origin has never announced anything) — the caller treats that as "nothing
1249/// to subscribe to" and aborts. For a subscribe-all pattern (`""`), or when no
1250/// source names can be extracted from `patterns`, the broader `local/events`
1251/// consumer is returned.
1252///
1253/// A source-specific second-level scope (e.g. narrowing to `local/events/worker`)
1254/// can ALSO return `None` even though `local/events` exists, simply because
1255/// that particular source hasn't announced its broadcast yet — a subscriber
1256/// commonly starts before its publisher does. That is not "no event scope";
1257/// it just means the narrower subtree doesn't exist *yet*. Treating it the
1258/// same as the top-level `None` case used to make the subscriber task abort
1259/// permanently (see `run_subscriber_task`), so a subscriber that started
1260/// before its publisher would never receive anything even after the publisher
1261/// came up. Instead, fall back to the broader `local/events`-scoped consumer:
1262/// per-frame pattern filtering (`topic_matches_patterns` in
1263/// `read_event_broadcast`) still narrows correctly once the source announces.
1264fn scope_consumer_for_patterns(
1265    consumer: &OriginConsumer,
1266    patterns: &[String],
1267) -> Option<OriginConsumer> {
1268    // Scope to local/events first
1269    let events_consumer = consumer.scope(&[Path::new(EVENT_PREFIX)])?;
1270
1271    // If subscribe-all pattern present, return events-scoped consumer
1272    if patterns.iter().any(String::is_empty) {
1273        return Some(events_consumer);
1274    }
1275
1276    // Extract unique source names from patterns (e.g., "worker." → "worker")
1277    let sources: Vec<String> = patterns
1278        .iter()
1279        .filter_map(|p| {
1280            let src = p.split('.').next()?;
1281            if src.is_empty() {
1282                None
1283            } else {
1284                Some(src.to_owned())
1285            }
1286        })
1287        .collect::<HashSet<String>>()
1288        .into_iter()
1289        .collect();
1290
1291    if sources.is_empty() {
1292        return Some(events_consumer);
1293    }
1294
1295    let path_refs: Vec<Path<'_>> = sources.iter().map(|s| Path::new(s.as_str())).collect();
1296    // Narrowing can legitimately fail if none of `sources` have announced yet;
1297    // fall back to the events-scoped consumer rather than propagating `None`
1298    // (which the caller treats as a fatal "no event scope" abort).
1299    Some(events_consumer.scope(&path_refs).unwrap_or(events_consumer))
1300}
1301
1302/// Read all groups from a single source broadcast's `events` track and relay
1303/// matching events to `tx`.
1304async fn read_event_broadcast(
1305    broadcast: moq_net::BroadcastConsumer,
1306    path: String,
1307    patterns: Arc<Vec<String>>,
1308    tx: EventChannelSender,
1309    resume_from: Option<u64>,
1310    last_sequence: Arc<AtomicU64>,
1311    idle_resumes: Arc<AtomicU64>,
1312) {
1313    let mut track = match broadcast.subscribe_track(&Track::new(EVENT_TRACK)) {
1314        Ok(t) => t,
1315        Err(_) => return,
1316    };
1317
1318    loop {
1319        let mut group =
1320            match tokio::time::timeout(crate::moq_stream::GROUP_IDLE_TIMEOUT, track.next_group())
1321                .await
1322            {
1323                Ok(Ok(Some(g))) => g,
1324                Ok(Ok(None)) | Ok(Err(_)) => break,
1325                Err(_elapsed) => {
1326                    // #606: a quiet (low event rate) broadcast is NOT the same as a
1327                    // gone one. Previously this `break`-ed, permanently dropping
1328                    // the subscriber after one idle period — silently losing every
1329                    // later event. Keep waiting; only `next_group()` itself
1330                    // returning `None`/`Err` (above) means the broadcast actually
1331                    // ended. `next_group()` is safe to re-poll after its previous
1332                    // future was dropped by the timeout (it does not consume
1333                    // internal cursor state on cancellation).
1334                    tracing::debug!(
1335                        "event subscriber idle — source broadcast still live, resuming wait"
1336                    );
1337                    idle_resumes.fetch_add(1, Ordering::Relaxed);
1338                    continue;
1339                }
1340            };
1341
1342        // Offset-resume (#606): skip groups already delivered in a prior
1343        // session. Only unambiguous for a single-broadcast subscription; see
1344        // `MoqEventSubscriber::with_resume_from` docs for the multi-source
1345        // caveat that also applies here.
1346        if let Some(threshold) = resume_from {
1347            if group.sequence <= threshold {
1348                continue;
1349            }
1350        }
1351
1352        let frame = match group.read_frame().await {
1353            Ok(Some(f)) => f,
1354            Ok(None) => break, // group ended without a frame
1355            Err(e) => {
1356                tracing::warn!(error = %e, "event broadcast: frame read error");
1357                break;
1358            }
1359        };
1360
1361        // Decode: [4 bytes topic_len BE][topic][payload]
1362        if frame.len() < 4 {
1363            continue;
1364        }
1365        let topic_len = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize;
1366        if frame.len() < 4 + topic_len {
1367            continue;
1368        }
1369        let topic = match std::str::from_utf8(&frame[4..4 + topic_len]) {
1370            Ok(t) => t.to_owned(),
1371            Err(_) => continue,
1372        };
1373        let payload = frame[4 + topic_len..].to_vec();
1374
1375        // Apply pattern filter (prefix match, ZMQ semantics)
1376        if topic_matches_patterns(&topic, &patterns) {
1377            last_sequence.fetch_max(group.sequence, Ordering::Relaxed);
1378            if !tx.send((topic, payload)).await {
1379                break; // receiver dropped
1380            }
1381        }
1382    }
1383
1384    let _ = path; // kept for debugging context; unused in the hot path
1385}
1386
1387/// #393 per-OID subscriber task.
1388///
1389/// Scopes the origin consumer to `local/events/publications/{oid_hash}` so only
1390/// that OID's events cross the wire. If `backfill` carries a [`BackfillSource`]
1391/// that reports history for `oid`, that history is replayed to `tx` first
1392/// (cold-start), then the task subscribes to the live track. Any backfill error
1393/// or "no history" result degrades silently to live-only — the subscription
1394/// never fails just because the firehose is down.
1395async fn run_oid_subscriber_task(
1396    mut consumer: OriginConsumer,
1397    oid: String,
1398    backfill: BackfillMode,
1399    tx: EventChannelSender,
1400    resume_from: Option<u64>,
1401    last_sequence: Arc<AtomicU64>,
1402    idle_resumes: Arc<AtomicU64>,
1403) {
1404    // 1) Optional cold-start backfill (decision A). Run BEFORE scoping the live
1405    //    consumer so the scheduler sees history before new events regardless of
1406    //    when (or whether) the live track is announced.
1407    //
1408    //    `BackfillSource::replay` is a PUBLIC trait implemented outside this
1409    //    crate (#393 doc) and is typed against `mpsc::Sender` — #606 does not
1410    //    change that public contract. Instead, bridge: replay into a small
1411    //    plain `mpsc` channel and forward each item into the QoS-aware `tx` so
1412    //    backfilled items get the same overflow-policy treatment as live ones.
1413    if let BackfillMode::FirehoseBackfill {
1414        oid: bf_oid,
1415        source,
1416    } = &backfill
1417    {
1418        if bf_oid == &oid && source.has_history(&oid) {
1419            let (bridge_tx, mut bridge_rx) = mpsc::channel::<(String, Vec<u8>)>(64);
1420            let fwd_tx = tx.clone();
1421            let forward = tokio::spawn(async move {
1422                while let Some(item) = bridge_rx.recv().await {
1423                    if !fwd_tx.send(item).await {
1424                        break;
1425                    }
1426                }
1427            });
1428            let replay_result = source.replay(&oid, &bridge_tx).await;
1429            drop(bridge_tx); // close so the forwarder's recv() loop ends
1430            let _ = forward.await;
1431            match replay_result {
1432                Ok(()) => tracing::debug!(oid = %oid, "event backfill replay complete"),
1433                Err(e) => {
1434                    tracing::warn!(oid = %oid, error = %e, "event backfill failed; continuing live-only");
1435                }
1436            }
1437        } else {
1438            tracing::debug!(oid = %oid, "no backfill history available; starting live-only");
1439        }
1440    }
1441
1442    // 2) Scope the consumer to the publications subtree. We scope to the PREFIX
1443    //    only (not the OID hash) so the announced() stream keeps flowing even
1444    //    before this OID's broadcast exists — late-join then picks up the OID's
1445    //    broadcast by matching its hash in the announcement path. This avoids the
1446    //    race where a subscriber starts before the publisher announces.
1447    let want_hash = oid_hash(&oid);
1448    let scoped = match consumer.scope(&[Path::new(PUBLICATIONS_PREFIX)]) {
1449        Some(c) => c,
1450        None => {
1451            tracing::debug!(oid = %oid, "publications subtree not announced; subscriber idle");
1452            return;
1453        }
1454    };
1455    consumer = scoped;
1456
1457    // 3) Wait for the OID's broadcast to be announced and read its `events` track.
1458    //    We filter announcements by the OID hash so a per-OID subscriber sees
1459    //    exactly one broadcast (the OID's) — no in-memory pattern filtering on
1460    //    event topics is needed; the scope + hash match already narrowed the wire.
1461    loop {
1462        let (path, broadcast) = match consumer.announced().await {
1463            None => break,               // origin closed
1464            Some((_, None)) => continue, // unannounce — re-await
1465            Some((path, Some(b))) => (path, b),
1466        };
1467        // Only read announcements whose final path segment is this OID's hash.
1468        // Other OIDs' broadcasts are ignored — they stay on their own tracks.
1469        // Match the full final segment (not a bare suffix) so two hashes that
1470        // happen to share a tail cannot cross-subscribe.
1471        let last_segment = path.as_str().rsplit('/').next().unwrap_or("");
1472        if last_segment != want_hash.as_str() {
1473            continue;
1474        }
1475        read_oid_event_broadcast(
1476            broadcast,
1477            tx.clone(),
1478            resume_from,
1479            last_sequence.clone(),
1480            idle_resumes.clone(),
1481        )
1482        .await;
1483    }
1484}
1485
1486/// Read all groups from a per-OID publication broadcast's `events` track and
1487/// relay every frame to `tx`. Unlike [`read_event_broadcast`] there is NO
1488/// pattern filtering — the consumer scope already limited the wire to one OID.
1489async fn read_oid_event_broadcast(
1490    broadcast: moq_net::BroadcastConsumer,
1491    tx: EventChannelSender,
1492    resume_from: Option<u64>,
1493    last_sequence: Arc<AtomicU64>,
1494    idle_resumes: Arc<AtomicU64>,
1495) {
1496    let mut track = match broadcast.subscribe_track(&Track::new(EVENT_TRACK)) {
1497        Ok(t) => t,
1498        Err(_) => return,
1499    };
1500
1501    loop {
1502        let mut group =
1503            match tokio::time::timeout(crate::moq_stream::GROUP_IDLE_TIMEOUT, track.next_group())
1504                .await
1505            {
1506                Ok(Ok(Some(g))) => g,
1507                Ok(Ok(None)) | Ok(Err(_)) => break,
1508                Err(_elapsed) => {
1509                    // #606: see read_event_broadcast — quiet != gone, keep waiting.
1510                    tracing::debug!(
1511                        "per-OID event subscriber idle — broadcast still live, resuming wait"
1512                    );
1513                    idle_resumes.fetch_add(1, Ordering::Relaxed);
1514                    continue;
1515                }
1516            };
1517
1518        // Offset-resume (#606): unambiguous here — exactly one broadcast.
1519        if let Some(threshold) = resume_from {
1520            if group.sequence <= threshold {
1521                continue;
1522            }
1523        }
1524
1525        let frame = match group.read_frame().await {
1526            Ok(Some(f)) => f,
1527            Ok(None) => break,
1528            Err(e) => {
1529                tracing::warn!(error = %e, "per-OID event broadcast: frame read error");
1530                break;
1531            }
1532        };
1533
1534        // Decode: [4 bytes topic_len BE][topic][payload]
1535        if frame.len() < 4 {
1536            continue;
1537        }
1538        let topic_len = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize;
1539        if frame.len() < 4 + topic_len {
1540            continue;
1541        }
1542        let topic = match std::str::from_utf8(&frame[4..4 + topic_len]) {
1543            Ok(t) => t.to_owned(),
1544            Err(_) => continue,
1545        };
1546        let payload = frame[4 + topic_len..].to_vec();
1547
1548        last_sequence.fetch_max(group.sequence, Ordering::Relaxed);
1549        if !tx.send((topic, payload)).await {
1550            break; // receiver dropped
1551        }
1552    }
1553}
1554
1555/// True if `topic` matches any pattern in `patterns` (prefix match).
1556fn topic_matches_patterns(topic: &str, patterns: &[String]) -> bool {
1557    for pat in patterns {
1558        if pat.is_empty() {
1559            return true; // subscribe-all
1560        }
1561        if pat.ends_with('.') {
1562            if topic.starts_with(pat.as_str()) {
1563                return true;
1564            }
1565        } else if topic == pat.as_str() {
1566            return true;
1567        }
1568    }
1569    false
1570}
1571
1572#[cfg(test)]
1573#[allow(clippy::expect_used, clippy::unwrap_used)]
1574mod tests {
1575    use super::*;
1576
1577    #[test]
1578    fn test_topic_matches_patterns() {
1579        let patterns = vec!["worker.".to_owned()];
1580        assert!(topic_matches_patterns(
1581            "worker.sandbox123.started",
1582            &patterns
1583        ));
1584        assert!(!topic_matches_patterns("system.event.ready", &patterns));
1585        assert!(!topic_matches_patterns("worker", &patterns)); // no trailing dot match
1586
1587        let patterns_all = vec!["".to_owned()];
1588        assert!(topic_matches_patterns("any.thing", &patterns_all));
1589
1590        let patterns_exact = vec!["worker.abc.started".to_owned()];
1591        assert!(topic_matches_patterns(
1592            "worker.abc.started",
1593            &patterns_exact
1594        ));
1595        assert!(!topic_matches_patterns(
1596            "worker.abc.stopped",
1597            &patterns_exact
1598        ));
1599    }
1600
1601    // ── #606: EventChannel unit tests (no MoQ transport — deterministic) ──────
1602
1603    #[tokio::test]
1604    async fn event_channel_drop_oldest_evicts_and_counts() {
1605        let dropped = Arc::new(AtomicU64::new(0));
1606        let (tx, mut rx) = event_channel(
1607            OverflowPolicy::DropOldest { high_water_mark: 2 },
1608            dropped.clone(),
1609        );
1610
1611        // Fill to capacity, then overflow by one: item "a" must be evicted.
1612        assert!(tx.send(("t".into(), b"a".to_vec())).await);
1613        assert!(tx.send(("t".into(), b"b".to_vec())).await);
1614        assert!(tx.send(("t".into(), b"c".to_vec())).await);
1615
1616        assert_eq!(dropped.load(Ordering::Relaxed), 1, "exactly one eviction");
1617
1618        let (_, first) = rx.recv().await.expect("first item");
1619        let (_, second) = rx.recv().await.expect("second item");
1620        assert_eq!(first, b"b", "oldest (\"a\") was evicted, \"b\" survives");
1621        assert_eq!(second, b"c");
1622    }
1623
1624    #[tokio::test]
1625    async fn event_channel_block_policy_waits_for_space_not_drop() {
1626        let dropped = Arc::new(AtomicU64::new(0));
1627        let (tx, mut rx) = event_channel(OverflowPolicy::Block, dropped.clone());
1628
1629        // Capacity for OverflowPolicy::Block is the fixed 256 default — send a
1630        // third send concurrently while only 2 are buffered; it must complete
1631        // once we drain one, never by silently dropping (dropped_count stays 0).
1632        assert!(tx.send(("t".into(), b"a".to_vec())).await);
1633        assert!(tx.send(("t".into(), b"b".to_vec())).await);
1634
1635        let (item, _) = rx.recv().await.expect("drain one");
1636        assert_eq!(item, "t");
1637        assert_eq!(
1638            dropped.load(Ordering::Relaxed),
1639            0,
1640            "Block policy never evicts"
1641        );
1642    }
1643
1644    #[tokio::test]
1645    async fn event_channel_closes_when_all_senders_dropped() {
1646        let dropped = Arc::new(AtomicU64::new(0));
1647        let (tx, mut rx) = event_channel(OverflowPolicy::Block, dropped);
1648        let tx2 = tx.clone();
1649        drop(tx);
1650        drop(tx2);
1651        assert_eq!(rx.recv().await, None, "recv must wake with None, not hang");
1652    }
1653
1654    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1655    async fn moq_event_pub_sub_roundtrip() -> Result<()> {
1656        let origin = MoqEventOrigin::new();
1657
1658        // Isolate this test from the process-global OnceLock. A subscriber
1659        // created via `MoqEventSubscriber::new()` reads the global origin; if
1660        // another test in the same binary set that global first, this test's
1661        // publisher and subscriber would resolve against DIFFERENT origins and
1662        // `recv()` would race/fail ("event subscriber closed"). `new_for_test`
1663        // binds the subscriber to THIS origin's consumer, making the pub/sub
1664        // pairing deterministic regardless of test order.
1665        let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
1666
1667        // Source-prefix subscription (not subscribe-all): exercises the
1668        // two-level `scope()` path in `scope_consumer_for_patterns` (scope to
1669        // `local/events`, then narrow to `worker`), now that its fallback to
1670        // the broader events-scoped consumer is fixed.
1671        sub.subscribe("worker.")?;
1672
1673        // Publish BEFORE recv so the broadcast (`local/events/worker`) is already
1674        // announced when the subscriber's background task starts — no timing race.
1675        let mut pub_ = origin.publisher("worker")?;
1676        pub_.publish("sandbox123", "started", b"payload")?;
1677
1678        let result = tokio::time::timeout(std::time::Duration::from_secs(5), sub.recv()).await;
1679
1680        match result {
1681            Ok(Ok((topic, payload))) => {
1682                assert_eq!(topic, "worker.sandbox123.started");
1683                assert_eq!(payload, b"payload");
1684            }
1685            Ok(Err(e)) => panic!("recv error: {e}"),
1686            Err(_) => panic!("recv timeout"),
1687        }
1688
1689        Ok(())
1690    }
1691
1692    /// Regression test for the `scope_consumer_for_patterns` fallback fix: a
1693    /// subscriber that source-prefix-subscribes (`"worker."`) BEFORE its
1694    /// publisher has announced anything must still receive events once the
1695    /// publisher starts — not abort permanently. Before the fix, the
1696    /// second-level `scope()` narrowing to `worker` found no node yet (nothing
1697    /// announced under `local/events/worker`) and returned `None`, which
1698    /// `scope_consumer_for_patterns` propagated as "no event scope", causing
1699    /// `run_subscriber_task` to exit immediately and silently drop the
1700    /// subscriber for good.
1701    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1702    async fn event_subscriber_can_start_before_publisher_announces() -> Result<()> {
1703        let origin = MoqEventOrigin::new();
1704        let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
1705
1706        // Subscribe to the source prefix BEFORE any publisher for "worker"
1707        // exists — `local/events/worker` has not been announced yet.
1708        sub.subscribe("worker.")?;
1709
1710        // Now create the publisher and publish; this is the first time
1711        // `local/events/worker` is announced.
1712        let mut pub_ = origin.publisher("worker")?;
1713        pub_.publish("sandbox123", "started", b"payload")?;
1714
1715        let result = tokio::time::timeout(std::time::Duration::from_secs(5), sub.recv()).await;
1716
1717        match result {
1718            Ok(Ok((topic, payload))) => {
1719                assert_eq!(topic, "worker.sandbox123.started");
1720                assert_eq!(payload, b"payload");
1721            }
1722            Ok(Err(e)) => panic!("recv error: {e}"),
1723            Err(_) => panic!("recv timeout — subscriber likely aborted on the pre-fix scope-race"),
1724        }
1725
1726        Ok(())
1727    }
1728
1729    /// #275 cross-process bug: a non-event process publishes events through a
1730    /// CLIENT-mode origin linked over UDS to a separate SERVER-mode origin
1731    /// (the event service). A subscriber on the SERVER origin must see the
1732    /// event — proving the bus is no longer a process-local island.
1733    ///
1734    /// This mirrors the systemd / --ipc deployment: the "event service" origin
1735    /// is served via `serve_event_moq_uds_background`; the "worker" origin is a
1736    /// distinct origin (as in a separate process) wired via
1737    /// `connect_event_moq_uds_background`. The publisher resolves against the
1738    /// client origin (no in-process event factory ran for it), and the event
1739    /// reaches a subscriber reading the server origin's consumer.
1740    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1741    async fn cross_process_event_over_uds_client_link() -> Result<()> {
1742        use std::time::Duration;
1743
1744        // Unique, stable socket path for this test (cross-"process" rendezvous).
1745        let sock =
1746            std::env::temp_dir().join(format!("hyprstream-event-test-{}.sock", std::process::id()));
1747
1748        // ── "event service" process: serve a SERVER-mode origin over UDS ──────
1749        let server_origin = MoqEventOrigin::new();
1750        serve_event_moq_uds_background(server_origin.clone(), sock.clone());
1751        // Let the listener bind.
1752        tokio::time::sleep(Duration::from_millis(50)).await;
1753
1754        // ── "worker" process: a DISTINCT origin linked to the event UDS ───────
1755        let client_origin = MoqEventOrigin::new();
1756        connect_event_moq_uds_background(client_origin.clone(), sock.clone());
1757
1758        // A publisher built on the client origin (the failing path before #275).
1759        let mut publisher = client_origin.publisher("worker")?;
1760
1761        // Drive the link + announcement by publishing repeatedly; read the
1762        // subscriber on the SERVER origin's consumer (a "different process").
1763        // Read the server consumer directly (the same data a server-side
1764        // EventSubscriber would receive) to avoid depending on the global.
1765        let consumer = server_origin.consumer();
1766        let scoped = consumer
1767            .scope(&[Path::new(EVENT_PREFIX)])
1768            .ok_or_else(|| anyhow!("server origin has no event scope yet"))?;
1769
1770        let recv = tokio::spawn(async move {
1771            let mut scoped = scoped;
1772            loop {
1773                let (_path, bc) = match scoped.announced().await {
1774                    None => return None,
1775                    Some((_, None)) => continue,
1776                    Some((path, Some(b))) => (path, b),
1777                };
1778                let mut track = match bc.subscribe_track(&Track::new(EVENT_TRACK)) {
1779                    Ok(t) => t,
1780                    Err(_) => continue,
1781                };
1782                let mut group = match track.next_group().await {
1783                    Ok(Some(g)) => g,
1784                    _ => continue,
1785                };
1786                if let Ok(Some(frame)) = group.read_frame().await {
1787                    return Some(frame.to_vec());
1788                }
1789            }
1790        });
1791
1792        // Re-publish periodically (the moq broadcast is retained by the origin,
1793        // but re-asserting it ensures the announcement is live once the
1794        // client→server link establishes). Concurrently await the receiver.
1795        let publish_loop = async {
1796            for _ in 0..50 {
1797                let _ = publisher.publish("sandbox123", "started", b"payload");
1798                tokio::time::sleep(Duration::from_millis(100)).await;
1799            }
1800            // Keep the publisher (and its broadcast) alive until the receiver
1801            // finishes by parking here.
1802            std::future::pending::<()>().await;
1803        };
1804
1805        let got: Option<Vec<u8>> = tokio::select! {
1806            r = recv => r.ok().flatten(),
1807            _ = publish_loop => None,
1808            _ = tokio::time::sleep(Duration::from_secs(10)) => None,
1809        };
1810
1811        let _ = std::fs::remove_file(&sock);
1812        let frame = got.expect("event did not cross the UDS link to the server origin");
1813
1814        // Decode [4 bytes topic_len BE][topic][payload].
1815        assert!(frame.len() >= 4, "frame too short");
1816        let topic_len = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize;
1817        let topic = std::str::from_utf8(&frame[4..4 + topic_len]).unwrap();
1818        let payload = &frame[4 + topic_len..];
1819        assert_eq!(topic, "worker.sandbox123.started");
1820        assert_eq!(payload, b"payload");
1821
1822        Ok(())
1823    }
1824
1825    // ========================================================================
1826    // #393 — per-OID broadcast paths + firehose-backfill late-join
1827    // ========================================================================
1828
1829    #[test]
1830    fn oid_hash_is_stable_and_distinct() {
1831        let a = oid_hash("at://did:web:node.example.com/models/qwen3-4b/v1");
1832        let b = oid_hash("at://did:web:node.example.com/models/qwen3-4b/v1");
1833        assert_eq!(a, b, "same OID must hash to the same path");
1834        assert_eq!(
1835            a.len(),
1836            OID_HASH_LEN,
1837            "hash is truncated to OID_HASH_LEN hex chars"
1838        );
1839
1840        let c = oid_hash("at://did:web:node.example.com/models/llama3-8b/v1");
1841        assert_ne!(a, c, "distinct OIDs must hash to distinct paths");
1842
1843        // The hash must contain only filesystem/moq-safe characters (hex).
1844        assert!(
1845            a.chars().all(|ch| ch.is_ascii_hexdigit()),
1846            "non-hex char in {a}"
1847        );
1848    }
1849
1850    #[test]
1851    fn publication_broadcast_path_shape() {
1852        // Mirrors the streaming plane's path-selective shape.
1853        let path = publication_broadcast_path("my-oid");
1854        assert_eq!(
1855            path,
1856            format!("{}/{}", PUBLICATIONS_PREFIX, oid_hash("my-oid")),
1857            "path must be {PUBLICATIONS_PREFIX}/{{oid_hash}}"
1858        );
1859        assert!(path.starts_with(PUBLICATIONS_PREFIX));
1860    }
1861
1862    /// A per-OID subscriber receives ONLY that OID's events — not events for
1863    /// other OIDs and not the flat firehose. This is the core #393 fix: a node
1864    /// tracking N of M OIDs reads N tracks, not M.
1865    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1866    async fn per_oid_subscription_receives_only_that_oid() -> Result<()> {
1867        let origin = MoqEventOrigin::new();
1868
1869        // Two publishers for two distinct OIDs, each mirroring to its own track.
1870        let mut pub_a = origin.publisher_with_oid("registry", "oid-alpha")?;
1871        let mut pub_b = origin.publisher_with_oid("registry", "oid-beta")?;
1872
1873        // Subscriber scoped to oid-alpha only (uses an explicit origin consumer
1874        // so it does not fight other tests over the process-global OnceLock).
1875        let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
1876        sub.subscribe_oid("oid-alpha")?;
1877
1878        // Publish one event to each OID's track.
1879        pub_a.publish("qwen3-4b", "published", b"alpha-event")?;
1880        pub_b.publish("llama3-8b", "published", b"beta-event")?;
1881
1882        // Let the announcement propagate.
1883        tokio::time::sleep(Duration::from_millis(30)).await;
1884
1885        // The subscriber should receive oid-alpha's event...
1886        let got = tokio::time::timeout(Duration::from_secs(5), sub.recv()).await;
1887        let (topic, payload) = match got {
1888            Ok(Ok(tp)) => tp,
1889            Ok(Err(e)) => panic!("recv error: {e}"),
1890            Err(_) => panic!("recv timeout — oid-alpha event never arrived"),
1891        };
1892        assert_eq!(topic, "registry.qwen3-4b.published");
1893        assert_eq!(payload, b"alpha-event");
1894
1895        // ...and NOT oid-beta's. Poll briefly: any further recv should time out
1896        // (the subscriber is scoped to oid-alpha, so beta's event never reaches it).
1897        let next = sub.recv_timeout(Duration::from_millis(300)).await?;
1898        assert!(
1899            next.is_none(),
1900            "per-OID subscriber received an event for a DIFFERENT OID: {next:?}"
1901        );
1902
1903        // Hold the publishers alive so their broadcasts stay announced for the
1904        // lifetime of the subscriber task.
1905        drop(pub_a);
1906        drop(pub_b);
1907        Ok(())
1908    }
1909
1910    /// The legacy flat `events` track still receives events from a publisher
1911    /// created with `publisher_with_oid` (back-compat fallback during the #393
1912    /// transition). A `publisher_with_oid` mirrors to BOTH the flat
1913    /// `local/events/registry` track and the per-OID publication track, so a
1914    /// legacy subscriber reading the flat track sees the same event as a #393
1915    /// per-OID subscriber.
1916    ///
1917    /// This reads the flat broadcast directly (the way `EventSubscriber` does in
1918    /// production via `OriginConsumer::announced()` + `subscribe_track`), rather
1919    /// than going through the `MoqEventSubscriber` background task, to isolate
1920    /// the back-compat guarantee (#393 must not break the flat track) from a
1921    /// pre-existing two-level-scope race in `scope_consumer_for_patterns`.
1922    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1923    async fn flat_track_back_compat_with_oid_mirror() -> Result<()> {
1924        use moq_net::Track as MoqTrack;
1925
1926        let origin = MoqEventOrigin::new();
1927
1928        // Publisher mirrors to both flat `local/events/registry` and per-OID track.
1929        let mut pub_ = origin.publisher_with_oid("registry", "oid-x")?;
1930        assert!(
1931            pub_.has_oid_mirror(),
1932            "publisher_with_oid must mirror to an OID track"
1933        );
1934        pub_.publish("qwen3-4b", "published", b"mirrored")?;
1935
1936        // Read the flat `local/events/registry` broadcast directly.
1937        let consumer = origin.consumer();
1938        let scoped = consumer
1939            .scope(&[Path::new(EVENT_PREFIX)])
1940            .ok_or_else(|| anyhow!("no events scope"))?;
1941
1942        let recv = tokio::spawn(async move {
1943            let mut scoped = scoped;
1944            loop {
1945                let (_path, bc) = match scoped.announced().await {
1946                    None => return None,
1947                    Some((_, None)) => continue,
1948                    Some((p, Some(b))) => (p, b),
1949                };
1950                let mut track = match bc.subscribe_track(&MoqTrack::new(EVENT_TRACK)) {
1951                    Ok(t) => t,
1952                    Err(_) => continue,
1953                };
1954                let mut group = match track.next_group().await {
1955                    Ok(Some(g)) => g,
1956                    _ => continue,
1957                };
1958                if let Ok(Some(frame)) = group.read_frame().await {
1959                    return Some(frame.to_vec());
1960                }
1961            }
1962        });
1963
1964        // Re-publish to cover the announcement-propagation race.
1965        let publish_loop = async {
1966            for _ in 0..50 {
1967                let _ = pub_.publish("qwen3-4b", "published", b"mirrored");
1968                tokio::time::sleep(Duration::from_millis(100)).await;
1969            }
1970            std::future::pending::<()>().await;
1971        };
1972
1973        let got: Option<Vec<u8>> = tokio::select! {
1974            r = recv => r.ok().flatten(),
1975            _ = publish_loop => None,
1976            _ = tokio::time::sleep(Duration::from_secs(10)) => None,
1977        };
1978
1979        let frame = got.expect("flat-track subscriber never received the mirrored event");
1980        assert!(frame.len() >= 4, "frame too short");
1981        let topic_len = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize;
1982        let topic = std::str::from_utf8(&frame[4..4 + topic_len]).unwrap();
1983        let payload = &frame[4 + topic_len..];
1984        assert_eq!(topic, "registry.qwen3-4b.published");
1985        assert_eq!(payload, b"mirrored");
1986
1987        Ok(())
1988    }
1989
1990    /// subscribe() and subscribe_oid() are mutually exclusive — a subscriber is
1991    /// either flat-pattern-based or OID-scoped, never both.
1992    #[test]
1993    fn subscribe_and_subscribe_oid_are_mutually_exclusive() -> Result<()> {
1994        let mut s = MoqEventSubscriber::new();
1995        s.subscribe("worker.")?;
1996        assert!(
1997            s.subscribe_oid("oid").is_err(),
1998            "subscribe_oid after subscribe must fail"
1999        );
2000
2001        let mut s = MoqEventSubscriber::new();
2002        s.subscribe_oid("oid")?;
2003        assert!(
2004            s.subscribe("worker.").is_err(),
2005            "subscribe after subscribe_oid must fail"
2006        );
2007
2008        Ok(())
2009    }
2010
2011    /// A no-op BackfillSource used to test firehose-backfill graceful degradation.
2012    struct NullBackfillSource {
2013        /// What `has_history` returns.
2014        has_history: bool,
2015        /// History to replay (if any).
2016        history: Vec<(String, Vec<u8>)>,
2017    }
2018
2019    impl BackfillSource for NullBackfillSource {
2020        fn has_history(&self, _oid: &str) -> bool {
2021            self.has_history
2022        }
2023        fn replay<'a>(
2024            &'a self,
2025            _oid: &'a str,
2026            tx: &'a mpsc::Sender<(String, Vec<u8>)>,
2027        ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send + 'a>> {
2028            let history = self.history.clone();
2029            Box::pin(async move {
2030                for (topic, payload) in history {
2031                    if tx.send((topic, payload)).await.is_err() {
2032                        break;
2033                    }
2034                }
2035                Ok(())
2036            })
2037        }
2038    }
2039
2040    /// When the firehose has history, backfill replays it BEFORE live events
2041    /// arrive (cold-start path). The live MoQ events then follow.
2042    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2043    async fn backfill_replays_history_before_live() -> Result<()> {
2044        let origin = MoqEventOrigin::new();
2045
2046        let source = Arc::new(NullBackfillSource {
2047            has_history: true,
2048            history: vec![
2049                (
2050                    "registry.qwen3-4b.published".to_owned(),
2051                    b"backfill-1".to_vec(),
2052                ),
2053                (
2054                    "registry.qwen3-4b.published".to_owned(),
2055                    b"backfill-2".to_vec(),
2056                ),
2057            ],
2058        });
2059
2060        let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
2061        sub.subscribe_oid("oid-backfill")?;
2062        sub.with_backfill(BackfillMode::FirehoseBackfill {
2063            oid: "oid-backfill".to_owned(),
2064            source,
2065        })?;
2066
2067        // Drive the backfill + a live publish concurrently.
2068        let mut pub_ = origin.publisher_oid_only("registry", "oid-backfill")?;
2069        pub_.publish("qwen3-4b", "published", b"live")?;
2070
2071        // First two events should be the backfilled history (cold-start), then live.
2072        let e1 = tokio::time::timeout(Duration::from_secs(5), sub.recv())
2073            .await
2074            .expect("backfill[0] timed out")?;
2075        assert_eq!(e1.1, b"backfill-1");
2076
2077        let e2 = tokio::time::timeout(Duration::from_secs(5), sub.recv())
2078            .await
2079            .expect("backfill[1] timed out")?;
2080        assert_eq!(e2.1, b"backfill-2");
2081
2082        // Then the live event.
2083        let e3 = tokio::time::timeout(Duration::from_secs(5), sub.recv())
2084            .await
2085            .expect("live event timed out")?;
2086        assert_eq!(e3.1, b"live");
2087
2088        drop(pub_);
2089        Ok(())
2090    }
2091
2092    /// When the firehose is UNAVAILABLE (has_history false), backfill mode
2093    /// gracefully degrades to live-only — the subscription does not fail and
2094    /// live events still arrive (the firehose is the cold-start path; MoQ is
2095    /// the live path).
2096    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2097    async fn backfill_gracefully_degrades_when_firehose_unavailable() -> Result<()> {
2098        let origin = MoqEventOrigin::new();
2099
2100        let source = Arc::new(NullBackfillSource {
2101            has_history: false, // firehose offline / no history
2102            history: vec![("never".to_owned(), b"never".to_vec())],
2103        });
2104
2105        let mut sub = MoqEventSubscriber::new_for_test(origin.consumer());
2106        sub.subscribe_oid("oid-degrade")?;
2107        sub.with_backfill(BackfillMode::FirehoseBackfill {
2108            oid: "oid-degrade".to_owned(),
2109            source,
2110        })?;
2111
2112        let mut pub_ = origin.publisher_oid_only("registry", "oid-degrade")?;
2113        pub_.publish("qwen3-4b", "published", b"live")?;
2114
2115        // Despite requesting backfill, with the firehose down we still get the
2116        // live event (and never the spurious "never" history item).
2117        let got = tokio::time::timeout(Duration::from_secs(5), sub.recv())
2118            .await
2119            .expect("live event timed out (degradation failed)")?;
2120        assert_eq!(
2121            got.1, b"live",
2122            "degraded subscriber must receive live events"
2123        );
2124        assert_ne!(
2125            got.1, b"never",
2126            "firehose history must NOT leak when unavailable"
2127        );
2128
2129        drop(pub_);
2130        Ok(())
2131    }
2132}