hyprstream_rpc/latch.rs
1//! Latched completion / retained terminal state (EV7, epic #600).
2//!
3//! First-class fan-out completion: a resource's terminal value (task exit,
4//! model-load result, fd close/EOF, …) is retained host-side in a
5//! [`TerminalStore`] (the authoritative value, plaintext-at-owner — NOT the
6//! content-blind relay, NOT [`crate::stream_info::StreamOpt::Retention`]), and
7//! served to late watchers immediately while early watchers block on the
8//! EventService edge.
9//!
10//! [`read_then_subscribe`] is the one helper that subsumes `load --wait` and
11//! the P9 `/task/<id>/exit` pattern under one shape: serve the retained value
12//! if present, else subscribe to the live edge and block until a terminal
13//! event arrives.
14//!
15//! # Await contract
16//!
17//! [`read_then_subscribe`] is a real awaitable `Future` — it `.await`s
18//! [`crate::events::EventSubscriber::recv`], which registers a waker with the
19//! runtime. It is NOT correct-only-under-poll-spin. A consumer that drives VFS
20//! futures via a noop-waker spin loop (e.g. the TUI `chat_app.rs` `poll_local`
21//! driver, being replaced with a real async driver in the workers/P9-task lane,
22//! epic #608) would busy-spin on a blocking read here — consumers MUST drive
23//! this with a real async runtime.
24//!
25//! # Lane split (epic #600 vs epic #608)
26//!
27//! This module is the **generic EventService-side primitive**. The concrete
28//! per-resource Mount-served files (`/task/<id>/{exit,status,fd}`) and the
29//! per-type terminal payloads (`Exited(code)|Killed(sig)|Failed(reason)`,
30//! `Loaded(handle)|LoadFailed`, fd close/EOF) are owned by the workers/P9-task
31//! lane (epic #608): they instantiate [`TerminalState`]-style payloads, own a
32//! [`TerminalStore`], and call [`read_then_subscribe`]. This crate provides
33//! only the generic envelope + store + helper + the await/liveness contracts.
34
35use std::collections::HashMap;
36use std::hash::Hash;
37use std::sync::Arc;
38
39use anyhow::{anyhow, Result};
40use parking_lot::RwLock;
41
42use crate::events::EventSubscriber;
43
44/// A resource identifier for a latched terminal value (e.g. a task id, a
45/// model load-attempt id). Cheap to clone/hash/compare.
46pub trait ResourceKey: Eq + Hash + Clone + Send + Sync + std::fmt::Debug {}
47impl<T: Eq + Hash + Clone + Send + Sync + std::fmt::Debug> ResourceKey for T {}
48
49/// Marker for a per-type terminal payload. Implementations live in the consumer
50/// (epic #608): task-exit, model-load, fd-close, … They carry their own value
51/// type; this trait only pins the "I am a terminal payload" contract so the
52/// store/helper stay generic.
53pub trait TerminalState: Clone + Send + Sync {}
54impl<T: Clone + Send + Sync> TerminalState for T {}
55
56/// A latched terminal value — monotonic write-once. The owner of a resource
57/// latches exactly one of these when the resource terminates.
58///
59/// Generic over the per-type payload `V`. A successful latch IS the terminal
60/// edge; a second latch attempt on an already-terminal resource is rejected
61/// ([`TerminalStore::latch`] returns `false`), so the value can never be
62/// overwritten — matching the Plan9-style "terminal is monotonic write-once"
63/// contract.
64#[derive(Clone, Debug)]
65pub struct Terminal<V: TerminalState> {
66 /// The per-type terminal payload.
67 pub value: V,
68 /// Identity that latched this terminal (the resource owner / publisher, or
69 /// the reaper for an abnormal-death fallback). Auditable.
70 pub latched_by: String,
71}
72
73/// Monotonic-write-once retained-terminal-state store — the authoritative,
74/// host-side, plaintext-at-owner value for each resource.
75///
76/// This is the "file holds the truth" half of Plan9-style completion: a late
77/// watcher (one that subscribes AFTER termination) is served from here
78/// immediately, without replaying the event stream. Distinct from
79/// [`crate::stream_info::StreamOpt::Retention`] (a best-effort content-blind
80/// transport replay buffer) and from the relay (which never holds plaintext).
81#[derive(Default)]
82pub struct TerminalStore<K: ResourceKey, V: TerminalState> {
83 entries: Arc<RwLock<HashMap<K, Terminal<V>>>>,
84}
85
86impl<K: ResourceKey, V: TerminalState> TerminalStore<K, V> {
87 pub fn new() -> Self {
88 Self {
89 entries: Arc::new(RwLock::new(HashMap::new())),
90 }
91 }
92
93 /// Latch a terminal value for `key`. Monotonic write-once: returns `true`
94 /// if this call latched (the key was not already terminal), `false` if the
95 /// key was already terminal (the existing value is NEVER overwritten).
96 pub fn latch(&self, key: K, terminal: Terminal<V>) -> bool {
97 let mut entries = self.entries.write();
98 if entries.contains_key(&key) {
99 return false;
100 }
101 entries.insert(key, terminal);
102 true
103 }
104
105 /// Return the retained terminal for `key`, if any. Serves a late watcher
106 /// (subject to [`TerminalAuthz`] at the call site — see
107 /// [`read_then_subscribe`]).
108 pub fn get(&self, key: &K) -> Option<Terminal<V>> {
109 self.entries.read().get(key).cloned()
110 }
111
112 /// Whether `key` is already terminal.
113 pub fn is_latched(&self, key: &K) -> bool {
114 self.entries.read().contains_key(key)
115 }
116
117 /// Latch a fallback terminal for every `key` that is not already terminal.
118 /// Called by the resource owner's death-detection (reaper / liveness
119 /// timeout) so a fan-out [`read_then_subscribe`] waiter on a resource whose
120 /// owner died never blocks forever — the EV6-liveness obligation. Returns
121 /// the keys that were reaped (newly latched with `fallback`).
122 ///
123 /// Already-latched keys are left untouched — a real terminal always beats a
124 /// synthetic `Failed`/`EOF`. The caller (not this store) owns owner→key
125 /// bookkeeping and death-detection; this just applies the fallback to the
126 /// supplied keys.
127 pub fn reap_unlatched<'k, I, F>(&self, keys: I, fallback: F) -> Vec<K>
128 where
129 I: IntoIterator<Item = &'k K>,
130 K: 'k,
131 F: Fn(&K) -> Terminal<V>,
132 {
133 let mut reaped = Vec::new();
134 let mut entries = self.entries.write();
135 for key in keys {
136 if entries.contains_key(key) {
137 continue;
138 }
139 let terminal = fallback(key);
140 entries.insert(key.clone(), terminal);
141 reaped.push(key.clone());
142 }
143 reaped
144 }
145}
146
147impl<K: ResourceKey, V: TerminalState> Clone for TerminalStore<K, V> {
148 /// Cheap clone — the store is `Arc`-shared internally (same authoritative
149 /// state visible to all clones), so a watcher and the owner can hold their
150 /// own handles to the same store.
151 fn clone(&self) -> Self {
152 Self {
153 entries: Arc::clone(&self.entries),
154 }
155 }
156}
157
158/// Authorization hook for a late read of a retained terminal value. The
159/// late-watcher file read must enforce the same MAC as the subscribe; until
160/// EV4 (#604) wires real MAC this is **fail-closed** — [`DenyAllTerminalAuthz`]
161/// denies every late read, so a late watcher cannot observe a retained value
162/// yet. An EARLY watcher (one that subscribes before the edge) is unaffected:
163/// it receives the live edge through the normal subscribe path (whose authz is
164/// separate).
165//
166// TODO(EV4 #604): replace DenyAllTerminalAuthz with a MAC-backed impl that
167// enforces subject.ctx ⊒ object.label on the retained-value read.
168pub trait TerminalAuthz<K: ResourceKey> {
169 /// Authorize `watcher` reading the retained terminal for `key`.
170 fn authorize_read(&self, key: &K, watcher: &str) -> bool;
171}
172
173/// Fail-closed default [`TerminalAuthz`] until EV4 (#604). Denies all late
174/// reads — a watcher must subscribe before the edge to receive the terminal
175/// value live. Correct ZSP default; not a security gap (no plaintext is
176/// released to a late watcher the MAC hasn't authorized).
177pub struct DenyAllTerminalAuthz;
178impl<K: ResourceKey> TerminalAuthz<K> for DenyAllTerminalAuthz {
179 fn authorize_read(&self, _key: &K, _watcher: &str) -> bool {
180 false
181 }
182}
183
184/// Permissive [`TerminalAuthz`] for a Public (plaintext) operational plane.
185///
186/// Counterpart to [`DenyAllTerminalAuthz`] for resources whose terminal value
187/// rides a plaintext EventService source — e.g. the model-lifecycle plane
188/// (`model.lifecycle.{loaded,failed}`), which EV5/#605 deliberately kept
189/// plaintext (the encrypted-default flip is deferred to #555). On such a plane
190/// a late read of the retained terminal releases no plaintext the live edge
191/// would not, so permitting the [`read_then_subscribe`] fast path is consistent
192/// with the plane's current posture.
193///
194/// This is NOT the right authz for a labelled/encrypted plane — there the late
195/// read must enforce MAC (`subject.ctx ⊒ object.label`); that impl lands under
196/// EV4 (#604). Use [`DenyAllTerminalAuthz`] (the default) until the plane has
197/// labels and a real MAC-backed impl exists.
198pub struct AllowAllTerminalAuthz;
199impl<K: ResourceKey> TerminalAuthz<K> for AllowAllTerminalAuthz {
200 fn authorize_read(&self, _key: &K, _watcher: &str) -> bool {
201 true
202 }
203}
204
205/// Latched-completion read: serve the retained terminal for `key` if already
206/// latched (and `authz` permits the late read), else subscribe to the live
207/// EventService edge via `subscriber` and block until an event that `classify`
208/// identifies as terminal, then latch + return it.
209///
210/// Subsumes `load --wait` and P9 `/task/<id>/exit` under one pattern. The
211/// caller must have already configured `subscriber` with the relevant
212/// subscription pattern(s) before the first `.await`.
213///
214/// **Await contract:** this is a real `Future` (it `.await`s
215/// [`EventSubscriber::recv`]). Consumers MUST drive it with a real async
216/// runtime — a noop-waker poll-spin driver busy-loops (see module docs).
217///
218/// No lock is held across `.await`: the fast-path store read drops its guard
219/// before the `subscriber.recv().await` loop.
220pub async fn read_then_subscribe<K, V, F>(
221 store: &TerminalStore<K, V>,
222 subscriber: &mut EventSubscriber,
223 authz: &impl TerminalAuthz<K>,
224 watcher: &str,
225 key: K,
226 classify: F,
227) -> Result<Terminal<V>>
228where
229 K: ResourceKey,
230 V: TerminalState,
231 F: Fn(&str, &[u8]) -> Option<V>,
232{
233 // Fast path — late watcher: the owner already latched the terminal. Serve
234 // from the retain-store (subject to MAC authz on the read).
235 if let Some(existing) = store.get(&key) {
236 if authz.authorize_read(&key, watcher) {
237 return Ok(existing);
238 }
239 return Err(anyhow!(
240 "late read of terminal {key:?} denied (EV4 authz stub deny-all)"
241 ));
242 }
243
244 // Slow path — early watcher: subscribe to the live edge until a terminal
245 // event arrives. The store guard from `get` is already dropped; nothing is
246 // held across the `.await`.
247 loop {
248 let (topic, payload) = subscriber.recv().await?;
249 if let Some(value) = classify(&topic, &payload) {
250 let terminal = Terminal {
251 value,
252 latched_by: watcher.to_owned(),
253 };
254 // Best-effort latch: the owner (or a racing watcher) may have
255 // latched first — first latch wins, the value is the same. This
256 // fan-out latch is what lets a *subsequent* late watcher hit the
257 // fast path instead of re-subscribing.
258 let _ = store.latch(key.clone(), terminal.clone());
259 return Ok(terminal);
260 }
261 }
262}
263
264#[cfg(test)]
265#[allow(clippy::unwrap_used)]
266mod tests {
267 use super::*;
268
269 // Trivial example terminal payload (the real per-type payloads — task-exit,
270 // model-load, fd-close — live in the epic #608 consumer crate).
271 #[derive(Clone, Debug, PartialEq)]
272 struct ExampleExit(u8);
273
274 fn term(code: u8, by: &str) -> Terminal<ExampleExit> {
275 Terminal {
276 value: ExampleExit(code),
277 latched_by: by.to_owned(),
278 }
279 }
280
281 #[test]
282 fn latch_is_monotonic_write_once() {
283 let store: TerminalStore<String, ExampleExit> = TerminalStore::new();
284 assert!(store.latch("t1".to_owned(), term(0, "owner")));
285 // Second latch is rejected — the existing terminal is never overwritten.
286 assert!(!store.latch("t1".to_owned(), term(1, "owner")));
287 // The retained value is the FIRST (monotonic).
288 assert_eq!(store.get(&"t1".to_owned()).unwrap().value.0, 0);
289 }
290
291 #[test]
292 fn is_latched_and_get_reflect_state() {
293 let store: TerminalStore<u64, ExampleExit> = TerminalStore::new();
294 assert!(!store.is_latched(&7));
295 assert!(store.latch(7, term(0, "owner")));
296 assert!(store.is_latched(&7));
297 assert_eq!(store.get(&7).unwrap().latched_by, "owner");
298 assert!(store.get(&8).is_none());
299 }
300
301 #[test]
302 fn reap_unlatched_latches_fallback_only_for_unlatched() {
303 let store: TerminalStore<u32, ExampleExit> = TerminalStore::new();
304 // One resource already terminated with a real exit (code 0).
305 assert!(store.latch(1, term(0, "owner")));
306 // Owner dies with resources {1,2,3} unlatched-or-not.
307 let keys = [1u32, 2, 3];
308 let reaped = store.reap_unlatched(&keys, |k| Terminal {
309 // Synthetic Failed-equivalent (code 255 here) for the example.
310 value: ExampleExit(255),
311 latched_by: format!("reaper@{k}"),
312 });
313 // Only the previously-unlatched keys were reaped.
314 assert_eq!(reaped, vec![2, 3]);
315 // The already-terminal key kept its REAL terminal (monotonic).
316 assert_eq!(store.get(&1).unwrap().value.0, 0);
317 // Reaped keys got the fallback.
318 assert_eq!(store.get(&2).unwrap().value.0, 255);
319 assert_eq!(store.get(&3).unwrap().latched_by, "reaper@3");
320 }
321
322 #[test]
323 fn allow_all_authz_permits_late_read() {
324 // The plaintext-plane counterpart to deny_all_authz_blocks_late_read:
325 // AllowAllTerminalAuthz (correct for a Public EventService source, e.g.
326 // model lifecycle pre-#555) authorizes a late read of a retained value.
327 let store: TerminalStore<String, ExampleExit> = TerminalStore::new();
328 store.latch("t1".to_owned(), term(0, "owner"));
329 let authz = AllowAllTerminalAuthz;
330 assert!(authz.authorize_read(&"t1".to_owned(), "watcher"));
331 }
332
333 #[test]
334 fn deny_all_authz_blocks_late_read() {
335 // The fast path of read_then_subscribe cannot be exercised without a
336 // live EventSubscriber (tokio + moq), but the authz decision itself is
337 // synchronous — verify the fail-closed default directly.
338 let store: TerminalStore<String, ExampleExit> = TerminalStore::new();
339 store.latch("t1".to_owned(), term(0, "owner"));
340 let authz = DenyAllTerminalAuthz;
341 assert!(!authz.authorize_read(&"t1".to_owned(), "watcher"));
342 }
343
344 #[test]
345 fn store_clone_shares_authoritative_state() {
346 // The owner and a watcher hold separate handles to the SAME store.
347 let owner_store: TerminalStore<u64, ExampleExit> = TerminalStore::new();
348 let watcher_store = owner_store.clone();
349 assert!(owner_store.latch(42, term(7, "owner")));
350 // Watcher sees the owner's latch through its own handle.
351 assert!(watcher_store.is_latched(&42));
352 assert_eq!(watcher_store.get(&42).unwrap().value.0, 7);
353 }
354
355 // A full `read_then_subscribe` end-to-end test needs a running moq EventBus
356 // + tokio runtime (out of scope for a lib unit test here); the fast-path
357 // authz + monotonic-latch + reap contracts above are the unit-testable
358 // invariants. The slow-path loop is a thin `subscriber.recv().await` whose
359 // await-contract is guaranteed by EventSubscriber::recv being a real Future.
360}