Skip to main content

hyprstream_workers/runtime/
exec_mount.rs

1//! `/exec/instances/` VFS mount — projects `SandboxPool`'s active sandboxes
2//! as a Plan9 `/proc`-style tree.
3//!
4//! This is the P2 slice of epic #608 ("P9 Task/Instance Projection"); see
5//! epic #608 for the full `/exec` tree this is one branch of
6//! (`/exec/instances/<id>/...`).
7//!
8//! Layout:
9//! - `/exec/instances/`            — dynamic dir: active sandbox/instance ids
10//! - `/exec/instances/<id>/ctl`    — ctl file: write a verb to drive the
11//!   instance's lifecycle (see [`Verb`] for the grammar)
12//! - `/exec/instances/<id>/status` — read-only: current `PodSandboxState`,
13//!   live/non-blocking poll of pool state
14//! - `/exec/instances/<id>/exit`   — read-only: blocks until the instance is
15//!   terminal, then returns the terminal status. A read that starts after
16//!   the instance has already terminated returns immediately with the
17//!   retained status (read-then-subscribe, no missed completions).
18//! - `/exec/instances/<id>/ns`     — read-only: best-effort textual listing
19//!   of the sandbox's mount-prefixes/namespace, or an empty placeholder
20//! - `/exec/instances/<id>/fd/1,2` — bounded terminal stdout/stderr streams
21//!
22//! `fd/` is a bounded task-local adapter over the existing MoQ stream
23//! substrate. It deliberately exposes no socket or host-path escape hatch.
24//! `fd/0` stays absent until a backend task interface can consume it: accepting
25//! bytes into an unobserved buffer would violate the Plan 9 file contract.
26//!
27//! ## Ctl verb grammar
28//!
29//! Writes to `<id>/ctl` are UTF-8, one verb per write (Plan9-style short text
30//! commands, not a structured protocol):
31//!
32//! | verb      | effect                                                          |
33//! |-----------|------------------------------------------------------------------|
34//! | `start`   | no-op today — sandboxes are started by `SandboxPool::acquire`;   |
35//! |           | accepted for grammar completeness/forward-compat, returns ok     |
36//! |           | if the instance exists and is not already terminal.              |
37//! | `stop`    | calls `SandboxBackend::stop` (graceful stop) on the instance.    |
38//! | `kill`    | alias for `stop` today (no separate SIGKILL-style path exposed   |
39//! |           | by `SandboxBackend` yet); kept distinct in the grammar so a       |
40//! |           | future hard-kill primitive can be wired without a ctl-format     |
41//! |           | change.                                                           |
42//! | `destroy` | calls `SandboxBackend::destroy`, then removes the instance from  |
43//! |           | the pool's active map and marks it terminal for `exit`/`status`. |
44//! | `exec`    | runs an argv-only task through this mount's `SandboxBackend`      |
45//! |           | seam, then latches stdout/stderr in `fd/1` and `fd/2`.            |
46//!
47//! Unknown verbs return [`MountError::InvalidArgument`]. A verb may be
48//! followed by trailing whitespace, which is trimmed.
49
50use std::collections::HashMap;
51use std::sync::Arc;
52
53use async_trait::async_trait;
54use rand::RngCore;
55use tokio::sync::Notify;
56
57use hyprstream_rpc::latch::{Terminal, TerminalStore};
58use hyprstream_rpc::moq_stream::{MoqStreamOrigin, MoqStreamPublisher};
59use hyprstream_rpc::stream_info::Job;
60use hyprstream_rpc::streaming::StreamContext;
61use hyprstream_vfs::{DirEntry, Fid, Mount, MountError, Stat, Subject};
62// `parking_lot::Mutex` for the ctl write→read latch (interior mutability
63// through the `&Fid` that `Mount::write` receives). When this branch is
64// rebased onto a base containing #615, swap this hand-rolled latch for
65// `hyprstream_vfs::devfile::DevFileState` (same parking_lot::Mutex shape).
66use parking_lot::Mutex as PmMutex;
67
68use super::pool::SandboxPool;
69use hyprstream_rpc_std::worker_client::PodSandboxState;
70
71// ─────────────────────────────────────────────────────────────────────────────
72// Ctl verb grammar
73// ─────────────────────────────────────────────────────────────────────────────
74
75/// Verbs accepted by `/exec/instances/<id>/ctl`. See module docs for the
76/// full grammar table.
77#[derive(Clone, Debug, PartialEq, Eq)]
78enum Verb {
79    Start,
80    Stop,
81    Kill,
82    Destroy,
83    /// Execute a task through this Plan 9 projection. Arguments are already
84    /// tokenized; this is deliberately not a shell-string interface.
85    Exec(Vec<String>),
86}
87
88impl Verb {
89    fn parse(s: &str) -> Option<Self> {
90        let words: Vec<String> = s.split_whitespace().map(str::to_owned).collect();
91        match words.as_slice() {
92            [verb] if verb == "start" => Some(Self::Start),
93            [verb] if verb == "stop" => Some(Self::Stop),
94            [verb] if verb == "kill" => Some(Self::Kill),
95            [verb] if verb == "destroy" => Some(Self::Destroy),
96            [verb, command @ ..] if verb == "exec" && !command.is_empty() => {
97                Some(Self::Exec(command.to_vec()))
98            }
99            _ => None,
100        }
101    }
102
103    /// Whether this verb mutates the instance lifecycle destructively (stop/
104    /// kill/destroy). These are the verbs the MAC PEP must mediate (#1272):
105    /// `start` is constructive (it cannot remove or halt another subject's
106    /// instance), so it is not in the mediated set.
107    fn is_lifecycle_destructive(&self) -> bool {
108        matches!(self, Verb::Stop | Verb::Kill | Verb::Destroy)
109    }
110}
111
112// ─────────────────────────────────────────────────────────────────────────────
113// Lifecycle authorization seam (#1272)
114// ─────────────────────────────────────────────────────────────────────────────
115
116/// The destructive lifecycle verbs a [`LifecyclePolicy`] mediates.
117///
118/// These are the `ctl` ops that halt or remove an instance — the
119/// "lifecycle mutation" surface issue #1272 names. `Start` is deliberately
120/// absent: it is constructive, not a mutation of another subject's running
121/// instance.
122#[derive(Clone, Copy, Debug, PartialEq, Eq)]
123pub enum LifecycleVerb {
124    Stop,
125    Kill,
126    Destroy,
127}
128
129/// Fail-closed authorization seam for destructive `ctl` lifecycle verbs
130/// (#1272, epic #1267 T3).
131///
132/// `ExecMount`'s `write` path previously ignored `_caller` entirely when
133/// invoking `stop`/`kill`/`destroy`. This trait is the mediation point: when a
134/// policy is installed on the mount ([`ExecMount::new_with_lifecycle`]), every
135/// destructive verb must pass `authorize` before the backend is touched.
136///
137/// **Fail-closed contract:** returning `Err` ⇒ the verb is refused
138/// (`MountError::PermissionDenied`); there is no permissive default inside a
139/// policy. The mount's status-quo (no policy installed) is represented by
140/// `lifecycle: None`, NOT by a permissive policy — mirroring the
141/// [`hyprstream_vfs::NamespacePep`] activation posture (the separately-gated
142/// B-lane flips construction sites to armed).
143///
144/// The clearance-provenance dependency (#698) is the same as the VFS PEP's: a
145/// production policy resolves the caller's `SecurityContext` from verified
146/// credential material and applies MAC `can_access` against the instance's
147/// label. Until #698 wires that, the policy is a structural seam.
148pub trait LifecyclePolicy: Send + Sync {
149    /// Authorize `verb` on instance `id` by `caller`. `Ok(())` permits;
150    /// `Err(detail)` denies with a human-readable reason.
151    fn authorize(&self, caller: &Subject, id: &str, verb: LifecycleVerb) -> Result<(), String>;
152}
153
154/// Fail-closed policy: denies every destructive verb for every subject.
155/// Installing this arms the mediation point to deny-by-default (e.g. during
156/// the #698 dependency window).
157#[derive(Debug, Default, Clone, Copy)]
158pub struct DenyAllLifecycle;
159
160impl LifecyclePolicy for DenyAllLifecycle {
161    fn authorize(&self, _caller: &Subject, _id: &str, _verb: LifecycleVerb) -> Result<(), String> {
162        Err("lifecycle verb denied: no permissive policy".into())
163    }
164}
165
166// ─────────────────────────────────────────────────────────────────────────────
167// Terminal tracking
168// ─────────────────────────────────────────────────────────────────────────────
169
170/// Terminal payload latched into this mount's [`TerminalStore`] once an
171/// instance exits (i.e. is destroyed/stopped-for-good). Populated by the
172/// `ctl` `destroy` handler.
173#[derive(Clone, Debug)]
174struct TerminalStatus {
175    /// Final `PodSandboxState` observed before removal.
176    last_state: PodSandboxState,
177}
178
179impl TerminalStatus {
180    fn render(&self) -> String {
181        format!("exited state={:?}\n", self.last_state)
182    }
183}
184
185// ─────────────────────────────────────────────────────────────────────────────
186// Task-local fd streams (#1434)
187// ─────────────────────────────────────────────────────────────────────────────
188
189/// Hard per-fd retained-output ceiling for the shadow-CI projection.  The
190/// backing MoQ stream keeps its `Job` profile (ordered, retained, terminal);
191/// this cap bounds the Plan 9 reader's local materialization.
192const MAX_RETAINED_FD_BYTES: usize = 1024 * 1024;
193
194/// A bounded terminal byte stream projected by one Plan 9 fd.
195struct TaskFdStream {
196    state: tokio::sync::Mutex<TaskFdState>,
197    wake: Arc<Notify>,
198}
199
200struct TaskFdState {
201    bytes: Vec<u8>,
202    closed: bool,
203    truncated: bool,
204    publisher: MoqStreamPublisher,
205}
206
207impl TaskFdStream {
208    fn new(origin: &MoqStreamOrigin, instance: &str, fd: u8) -> Result<Self, MountError> {
209        let mut mac_key = [0u8; 32];
210        rand::rngs::OsRng.fill_bytes(&mut mac_key);
211        let context = StreamContext::new(
212            format!("exec-{instance}-fd-{fd}"),
213            format!("exec-{}-fd-{fd}", uuid::Uuid::new_v4()),
214            mac_key,
215            [0u8; 32],
216        )
217        .with_qos_preset::<Job>();
218        let publisher = origin
219            .publisher(&context)
220            .map_err(|e| MountError::Io(format!("create fd/{fd} MoQ stream: {e}")))?;
221        Ok(Self {
222            state: tokio::sync::Mutex::new(TaskFdState {
223                bytes: Vec::new(),
224                closed: false,
225                truncated: false,
226                publisher,
227            }),
228            wake: Arc::new(Notify::new()),
229        })
230    }
231
232    async fn append(&self, data: &[u8]) -> Result<(), MountError> {
233        let mut state = self.state.lock().await;
234        if state.closed {
235            return Err(MountError::InvalidArgument(
236                "fd stream is already closed".into(),
237            ));
238        }
239        let available = MAX_RETAINED_FD_BYTES.saturating_sub(state.bytes.len());
240        let retained = &data[..data.len().min(available)];
241        if !retained.is_empty() {
242            state
243                .publisher
244                .publish_data(retained)
245                .await
246                .map_err(|e| MountError::Io(format!("publish fd stream data: {e}")))?;
247            state.bytes.extend_from_slice(retained);
248        }
249        if retained.len() != data.len() {
250            state.truncated = true;
251        }
252        drop(state);
253        self.wake.notify_waiters();
254        Ok(())
255    }
256
257    async fn close(&self) -> Result<(), MountError> {
258        let mut state = self.state.lock().await;
259        if !state.closed {
260            if state.truncated && state.bytes.len() < MAX_RETAINED_FD_BYTES {
261                let marker = b"\n[hyprstream: output truncated]\n";
262                let room = MAX_RETAINED_FD_BYTES - state.bytes.len();
263                state
264                    .bytes
265                    .extend_from_slice(&marker[..marker.len().min(room)]);
266            }
267            state
268                .publisher
269                .complete_ref(b"exec-fd-eof")
270                .await
271                .map_err(|e| MountError::Io(format!("complete fd stream: {e}")))?;
272            state.closed = true;
273        }
274        drop(state);
275        self.wake.notify_waiters();
276        Ok(())
277    }
278
279    /// Whether this stream has already latched EOF (a prior `exec`, or a
280    /// `stop`/`destroy` through `apply_verb`). Checked by `exec_task` *before*
281    /// invoking the backend so a second exec on one instance fails fast
282    /// instead of running the command and then discarding its output.
283    async fn is_closed(&self) -> bool {
284        self.state.lock().await.closed
285    }
286
287    async fn read_until_closed(&self, offset: u64, count: u32) -> Vec<u8> {
288        loop {
289            let notified = self.wake.notified();
290            {
291                let state = self.state.lock().await;
292                let start = offset as usize;
293                if start < state.bytes.len() {
294                    let end = start.saturating_add(count as usize).min(state.bytes.len());
295                    return state.bytes[start..end].to_vec();
296                }
297                if state.closed {
298                    return Vec::new();
299                }
300            }
301            notified.await;
302        }
303    }
304}
305
306/// The single per-task stdio adapter shared by the Plan 9 task operation and
307/// fd readers. `fd/0` is intentionally absent: no backend task interface
308/// consumes interactive stdin yet, so accepting input would be a false success.
309struct TaskStdioAdapter {
310    stdout: TaskFdStream,
311    stderr: TaskFdStream,
312}
313
314/// One completed task execution as observed by both the production CRI caller
315/// and the Plan 9 fd projection. Keeping this result adjacent to the adapter
316/// makes the single-owner contract explicit: neither caller needs to invoke a
317/// backend a second time or retain a competing output buffer.
318#[derive(Debug)]
319pub struct TaskExecResult {
320    pub exit_code: i32,
321    pub stdout: Vec<u8>,
322    pub stderr: Vec<u8>,
323}
324
325// ─────────────────────────────────────────────────────────────────────────────
326// Fid types
327// ─────────────────────────────────────────────────────────────────────────────
328
329/// Which kind of file/dir a fid refers to.
330#[derive(Clone, Debug)]
331enum ExecFidKind {
332    /// `/exec/instances/` (this mount's root).
333    InstancesDir,
334    /// `/exec/instances/<id>/` itself.
335    InstanceDir(String),
336    /// `/exec/instances/<id>/ctl`.
337    Ctl(String),
338    /// `/exec/instances/<id>/status`.
339    Status(String),
340    /// `/exec/instances/<id>/exit`.
341    Exit(String),
342    /// `/exec/instances/<id>/ns`.
343    Ns(String),
344    /// `/exec/instances/<id>/fd/`.
345    FdDir(String),
346    /// `/exec/instances/<id>/fd/{0,1,2}`.
347    Fd(String, u8),
348}
349
350/// Fid state for the exec mount.
351struct ExecFid {
352    kind: ExecFidKind,
353    /// Latch for the ctl write→read pattern (verb result message). A
354    /// `parking_lot::Mutex<Vec<u8>>` provides safe interior mutability through
355    /// the `&Fid` that `Mount::read`/`Mount::write` receive — so we avoid the
356    /// `unsafe *mut` cast-through-`&` hazard. Unused for non-ctl fid kinds.
357    write_buf: PmMutex<Vec<u8>>,
358}
359
360// ─────────────────────────────────────────────────────────────────────────────
361// ExecMount
362// ─────────────────────────────────────────────────────────────────────────────
363
364/// VFS mount that projects a [`SandboxPool`]'s active sandboxes as
365/// `/exec/instances/<id>/{ctl,status,exit,ns}`.
366///
367/// Mount at `/exec/instances` in the namespace (the parent `/exec` tree —
368/// `backends/`, `classes/`, `pool/`, `sched/` — is out of scope for this
369/// issue; see the design doc §4).
370pub struct ExecMount {
371    pool: Arc<SandboxPool>,
372    /// Task-local MoQ origin for per-instance stdio. It is intentionally not
373    /// served as a new public network plane: `/exec` is the capability surface.
374    fd_origin: MoqStreamOrigin,
375    /// Per-instance fd state. Output streams use the existing MoQ publisher
376    /// contract; the Plan 9 files retain their bounded local materialization.
377    fd_streams: tokio::sync::Mutex<HashMap<String, Arc<TaskStdioAdapter>>>,
378    /// Retained terminal state (EV7, `hyprstream_rpc::latch`) for instances
379    /// that have been `destroy`ed through this mount. `latch()` is called
380    /// exactly once per instance (write-once), so a late `exit` read is
381    /// served from here immediately.
382    terminal: TerminalStore<String, TerminalStatus>,
383    /// Ids ever latched into `terminal`. `TerminalStore` doesn't expose key
384    /// enumeration (by design — it's a per-key latch/read primitive, not a
385    /// directory), so `readdir` tracks this separately to keep destroyed
386    /// instances listed alongside `pool.list_active()`.
387    terminal_ids: PmMutex<std::collections::HashSet<String>>,
388    /// Per-instance wake for an early `exit` reader blocked on a not-yet-
389    /// latched instance. `ExecMount`'s termination signal is entirely
390    /// in-process (via `apply_verb`'s `Destroy` arm), so a lightweight
391    /// `tokio::sync::Notify` per id is the read-then-subscribe "subscribe"
392    /// half here, in place of `hyprstream_rpc::latch::read_then_subscribe`'s
393    /// moq-backed `EventSubscriber` slow path (which is for cross-process
394    /// fan-out this mount doesn't need). `TerminalStore` is still the single
395    /// source of retained truth; this only wakes waiters once it's written.
396    waiters: PmMutex<HashMap<String, Arc<Notify>>>,
397    /// Optional MAC lifecycle policy over destructive `ctl` verbs (#1272).
398    /// `None` ⇒ the un-enforced status quo (the mount's `write` ignores the
399    /// caller, the gap #1272 describes); `Some` ⇒ `stop`/`kill`/`destroy`
400    /// must pass [`LifecyclePolicy::authorize`] before the backend is touched.
401    lifecycle: Option<Arc<dyn LifecyclePolicy>>,
402}
403
404impl ExecMount {
405    /// Create a new `ExecMount` over the given pool, **unenforced**.
406    ///
407    /// No lifecycle policy is installed: destructive `ctl` verbs behave as
408    /// before (the dormant posture). To arm fail-closed mediation, use
409    /// [`Self::new_with_lifecycle`].
410    pub fn new(pool: Arc<SandboxPool>) -> Self {
411        Self {
412            pool,
413            // Unsubscribed shadow state today: nothing reads from this origin
414            // (the 9P fd reads below serve the local `TaskFdStream` buffer
415            // instead). Deferred, not wired, pending epic #608's stream-plane
416            // mount — see Finding 3 of REVIEW-connie-1447-r2-k3-2026-07-30.md.
417            fd_origin: MoqStreamOrigin::standalone()
418                .with_prefix("local/exec")
419                .build(),
420            fd_streams: tokio::sync::Mutex::new(HashMap::new()),
421            terminal: TerminalStore::new(),
422            terminal_ids: PmMutex::new(std::collections::HashSet::new()),
423            waiters: PmMutex::new(HashMap::new()),
424            lifecycle: None,
425        }
426    }
427
428    /// Create a new `ExecMount` with a MAC lifecycle policy armed.
429    ///
430    /// Every destructive verb (`stop`/`kill`/`destroy`) written to
431    /// `/exec/instances/<id>/ctl` must pass `policy.authorize(caller, id, verb)`
432    /// before the backend is touched; a denial yields
433    /// [`MountError::PermissionDenied`]. Pass [`DenyAllLifecycle`] for the
434    /// fail-closed default during the #698 dependency window.
435    pub fn new_with_lifecycle(pool: Arc<SandboxPool>, policy: Arc<dyn LifecyclePolicy>) -> Self {
436        let mut mount = Self::new(pool);
437        mount.lifecycle = Some(policy);
438        mount
439    }
440
441    /// Get (or create) the `Notify` an `exit` reader waits on for `id`.
442    fn waiter_for(&self, id: &str) -> Arc<Notify> {
443        Arc::clone(
444            self.waiters
445                .lock()
446                .entry(id.to_owned())
447                .or_insert_with(|| Arc::new(Notify::new())),
448        )
449    }
450
451    async fn fd_streams_for(&self, id: &str) -> Result<Arc<TaskStdioAdapter>, MountError> {
452        let mut streams = self.fd_streams.lock().await;
453        if let Some(existing) = streams.get(id) {
454            return Ok(Arc::clone(existing));
455        }
456        let created = Arc::new(TaskStdioAdapter {
457            stdout: TaskFdStream::new(&self.fd_origin, id, 1)?,
458            stderr: TaskFdStream::new(&self.fd_origin, id, 2)?,
459        });
460        streams.insert(id.to_owned(), Arc::clone(&created));
461        Ok(created)
462    }
463
464    /// Publish one completed task execution into the instance's terminal fd
465    /// streams. The runtime calls this after `SandboxBackend::exec_sync`; the
466    /// Plan 9 readers observe the same bounded output and then EOF.
467    async fn publish_exec_result(
468        &self,
469        id: &str,
470        stdout: &[u8],
471        stderr: &[u8],
472    ) -> Result<(), MountError> {
473        if self.pool.get(id).await.is_none() && !self.terminal.is_latched(&id.to_owned()) {
474            return Err(MountError::NotFound(format!("instances/{id}")));
475        }
476        let streams = self.fd_streams_for(id).await?;
477        streams.stdout.append(stdout).await?;
478        streams.stderr.append(stderr).await?;
479        streams.stdout.close().await?;
480        streams.stderr.close().await?;
481        Ok(())
482    }
483
484    /// Execute through the selected sandbox backend and project the terminal
485    /// stdout/stderr onto this instance's fd streams. This is the task-runtime
486    /// bridge: callers do not receive a second host-path or socket interface.
487    pub async fn exec_task(
488        &self,
489        id: &str,
490        caller: &Subject,
491        command: &[String],
492        timeout_secs: u64,
493    ) -> Result<TaskExecResult, MountError> {
494        let sandbox = self
495            .pool
496            .get(id)
497            .await
498            .ok_or_else(|| MountError::NotFound(format!("instances/{id}")))?;
499        let task_subject = sandbox
500            .annotations
501            .iter()
502            .find(|annotation| annotation.key == "hyprstream.io/subject")
503            .map(|annotation| Subject::new(annotation.value.clone()))
504            .ok_or_else(|| MountError::PermissionDenied("task has no verified Subject".into()))?;
505        if &task_subject != caller {
506            return Err(MountError::PermissionDenied(
507                "caller does not own this task's Subject-scoped namespace".into(),
508            ));
509        }
510        // Fail fast, before touching the backend (#1447 r2/k3 Finding B): the
511        // per-instance fd streams are a one-shot latch, so a second exec on an
512        // instance whose streams already closed (stopped, destroyed, or a
513        // prior exec completed) must not run the command only to discard its
514        // output afterward.
515        let streams = self.fd_streams_for(id).await?;
516        if streams.stdout.is_closed().await || streams.stderr.is_closed().await {
517            return Err(MountError::InvalidArgument(
518                "instance's exec stdio is already closed (stopped, destroyed, or a prior \
519                 exec already ran); each instance supports at most one exec"
520                    .into(),
521            ));
522        }
523        let (exit_code, stdout, stderr) = self
524            .pool
525            .backend()
526            .exec_sync(&sandbox, command, timeout_secs)
527            .await
528            .map_err(|e| MountError::Io(e.to_string()))?;
529        self.publish_exec_result(id, &stdout, &stderr).await?;
530        Ok(TaskExecResult {
531            exit_code,
532            stdout,
533            stderr,
534        })
535    }
536
537    async fn close_fd_streams(&self, id: &str) -> Result<(), MountError> {
538        let streams = self.fd_streams_for(id).await?;
539        streams.stdout.close().await?;
540        streams.stderr.close().await?;
541        Ok(())
542    }
543
544    /// Resolve the verb against the pool/backend for instance `id`.
545    ///
546    /// `caller` is the [`Subject`] that wrote the `ctl` verb (#1272): when a
547    /// [`LifecyclePolicy`] is armed, destructive verbs (stop/kill/destroy)
548    /// must pass it before any backend state is touched.
549    async fn apply_verb(
550        &self,
551        id: &str,
552        verb: Verb,
553        caller: &Subject,
554    ) -> Result<String, MountError> {
555        // Instances already marked terminal (destroyed through this mount)
556        // are not present in the pool's active map any more; ctl ops on them
557        // are rejected rather than silently no-op'd.
558        if self.terminal.is_latched(&id.to_owned()) {
559            return Err(MountError::InvalidArgument(format!(
560                "instance {id} is already terminal"
561            )));
562        }
563
564        // MAC lifecycle gate (#1272): a destructive verb must be authorized by
565        // the armed policy before we touch the backend. No policy ⇒ the
566        // un-enforced status quo (the documented gap); a policy ⇒ fail-closed
567        // (its `authorize` returns Err ⇒ PermissionDenied).
568        if verb.is_lifecycle_destructive() {
569            if let Some(policy) = &self.lifecycle {
570                let lv = match verb {
571                    Verb::Stop => LifecycleVerb::Stop,
572                    Verb::Kill => LifecycleVerb::Kill,
573                    Verb::Destroy => LifecycleVerb::Destroy,
574                    // is_lifecycle_destructive() is false for Start, so this
575                    // arm is unreachable here; kept exhaustive without `expect`.
576                    Verb::Start => {
577                        return Err(MountError::InvalidArgument(
578                            "start is not a lifecycle-destructive verb".into(),
579                        ))
580                    }
581                    Verb::Exec(_) => {
582                        return Err(MountError::InvalidArgument(
583                            "exec is not a lifecycle-destructive verb".into(),
584                        ))
585                    }
586                };
587                policy
588                    .authorize(caller, id, lv)
589                    .map_err(MountError::PermissionDenied)?;
590            }
591        }
592
593        let sandbox = self
594            .pool
595            .get(id)
596            .await
597            .ok_or_else(|| MountError::NotFound(format!("instances/{id}")))?;
598
599        match verb {
600            Verb::Start => {
601                // Sandboxes are started by `SandboxPool::acquire`; there is no
602                // separate "start an existing-but-stopped instance" backend
603                // op today. Accepted for grammar completeness/forward-compat:
604                // if the instance exists and isn't terminal, report ok.
605                Ok("ok: already started\n".to_owned())
606            }
607            Verb::Stop | Verb::Kill => {
608                self.pool
609                    .backend()
610                    .stop(&sandbox)
611                    .await
612                    .map_err(|e| MountError::Io(e.to_string()))?;
613                // A stopped task can no longer produce output. Latch both
614                // reader streams to EOF so pending fd/1 and fd/2 readers wake.
615                self.close_fd_streams(id).await?;
616                Ok("ok: stopped\n".to_owned())
617            }
618            Verb::Destroy => {
619                self.pool
620                    .backend()
621                    .destroy(&sandbox)
622                    .await
623                    .map_err(|e| MountError::Io(e.to_string()))?;
624                // Best-effort removal from the pool's active map: `release`
625                // also tries to warm-pool/reset, which we don't want after an
626                // explicit destroy, so we go straight through the backend
627                // above and just drop bookkeeping here. If the id is no
628                // longer active (e.g. concurrently released) that's fine.
629                let _ = self.pool.release(id).await;
630                self.terminal.latch(
631                    id.to_owned(),
632                    Terminal {
633                        value: TerminalStatus {
634                            last_state: sandbox.state,
635                        },
636                        latched_by: "ctl:destroy".to_owned(),
637                    },
638                );
639                self.terminal_ids.lock().insert(id.to_owned());
640                self.close_fd_streams(id).await?;
641                // Wake any `exit` reader already blocked on this id.
642                self.waiter_for(id).notify_waiters();
643                Ok("ok: destroyed\n".to_owned())
644            }
645            Verb::Exec(command) => {
646                let result = self.exec_task(id, caller, &command, 300).await?;
647                Ok(format!("ok: exited {}\n", result.exit_code))
648            }
649        }
650    }
651
652    /// Live, non-blocking poll of an instance's current state for `status`.
653    async fn read_status(&self, id: &str) -> Result<String, MountError> {
654        if let Some(term) = self.terminal.get(&id.to_owned()) {
655            return Ok(format!("{:?}\n", term.value.last_state));
656        }
657        let sandbox = self
658            .pool
659            .get(id)
660            .await
661            .ok_or_else(|| MountError::NotFound(format!("instances/{id}")))?;
662        Ok(format!("{:?}\n", sandbox.state))
663    }
664
665    /// `exit`: blocks until the instance is terminal, then returns the
666    /// retained status. A read that starts after the instance already
667    /// latched terminal (a "late" reader, in read-then-subscribe terms)
668    /// returns immediately with the retained value — no missed completion.
669    /// A read that starts before (an "early" reader) blocks until `apply_verb`
670    /// latches the `Destroy` outcome and wakes this id's waiter.
671    async fn read_exit(&self, id: &str) -> Result<String, MountError> {
672        let key = id.to_owned();
673        // Fast path: already latched — serve the retained value immediately,
674        // exactly the "late reader" half of read-then-subscribe.
675        if let Some(term) = self.terminal.get(&key) {
676            return Ok(term.value.render());
677        }
678        // Confirm the instance actually exists (vs. a bogus id) before
679        // blocking on it.
680        if self.pool.get(id).await.is_none() {
681            return Err(MountError::NotFound(format!("instances/{id}")));
682        }
683        // Slow path: subscribe to this id's wake, then re-check the store —
684        // closes the race where `apply_verb` latches + notifies between the
685        // fast-path check above and the `notified()` registration below.
686        let notify = self.waiter_for(id);
687        loop {
688            let notified = notify.notified();
689            if let Some(term) = self.terminal.get(&key) {
690                return Ok(term.value.render());
691            }
692            notified.await;
693            if let Some(term) = self.terminal.get(&key) {
694                return Ok(term.value.render());
695            }
696        }
697    }
698
699    /// Best-effort textual listing of the sandbox's mount-prefixes/namespace.
700    ///
701    /// `PodSandbox` doesn't currently carry a generic namespace listing —
702    /// only a `sandbox_path` and an optional `console_socket`. We surface
703    /// what's available; backends with no accessible namespace info return
704    /// an empty/placeholder listing rather than failing.
705    async fn read_ns(&self, id: &str) -> Result<String, MountError> {
706        let sandbox = self
707            .pool
708            .get(id)
709            .await
710            .ok_or_else(|| MountError::NotFound(format!("instances/{id}")))?;
711
712        let mut lines = vec![format!("runtime={}", sandbox.runtime_handler)];
713        lines.push(format!("path={}", sandbox.sandbox_path().display()));
714        if let Some(console) = sandbox.console_socket() {
715            lines.push(format!("console={}", console.display()));
716        }
717        Ok(format!("{}\n", lines.join("\n")))
718    }
719}
720
721#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
722#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
723impl Mount for ExecMount {
724    async fn walk(&self, components: &[&str], _caller: &Subject) -> Result<Fid, MountError> {
725        // NOTE on `Subject`: per the Mount contract this mount threads the
726        // caller through every op but does not itself authorize — the
727        // MAC reference-monitor (#547) is the eventual enforcement point.
728        // `SandboxPool`/`SandboxBackend` don't take a `Subject` today, so
729        // there is nothing further to pass it to here.
730        let kind = match components {
731            [] => ExecFidKind::InstancesDir,
732            [id] => ExecFidKind::InstanceDir((*id).to_owned()),
733            [id, "ctl"] => ExecFidKind::Ctl((*id).to_owned()),
734            [id, "status"] => ExecFidKind::Status((*id).to_owned()),
735            [id, "exit"] => ExecFidKind::Exit((*id).to_owned()),
736            [id, "ns"] => ExecFidKind::Ns((*id).to_owned()),
737            [id, "fd"] => ExecFidKind::FdDir((*id).to_owned()),
738            [id, "fd", fd] => {
739                let fd = fd
740                    .parse::<u8>()
741                    .map_err(|_| MountError::NotFound(components.join("/")))?;
742                if !(1..=2).contains(&fd) {
743                    return Err(MountError::NotFound(components.join("/")));
744                }
745                ExecFidKind::Fd((*id).to_owned(), fd)
746            }
747            _ => return Err(MountError::NotFound(components.join("/"))),
748        };
749
750        // Validate that referenced instances exist (directory walk into an
751        // unknown id should 404, same as any 9P namespace).
752        let id_to_check: Option<&str> = match &kind {
753            ExecFidKind::InstanceDir(id)
754            | ExecFidKind::Ctl(id)
755            | ExecFidKind::Status(id)
756            | ExecFidKind::Exit(id)
757            | ExecFidKind::Ns(id)
758            | ExecFidKind::FdDir(id)
759            | ExecFidKind::Fd(id, _) => Some(id.as_str()),
760            ExecFidKind::InstancesDir => None,
761        };
762        if let Some(id) = id_to_check {
763            let exists =
764                self.pool.get(id).await.is_some() || self.terminal.is_latched(&id.to_owned());
765            if !exists {
766                return Err(MountError::NotFound(components.join("/")));
767            }
768        }
769
770        Ok(Fid::new(ExecFid {
771            kind,
772            write_buf: PmMutex::new(Vec::new()),
773        }))
774    }
775
776    async fn open(&self, _fid: &mut Fid, _mode: u8, _caller: &Subject) -> Result<(), MountError> {
777        Ok(())
778    }
779
780    async fn read(
781        &self,
782        fid: &Fid,
783        offset: u64,
784        count: u32,
785        _caller: &Subject,
786    ) -> Result<Vec<u8>, MountError> {
787        let inner = fid
788            .downcast_ref::<ExecFid>()
789            .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
790
791        let data = match &inner.kind {
792            ExecFidKind::Ctl(_) => {
793                // Ctl pattern: after a write, read returns the verb result.
794                inner.write_buf.lock().clone()
795            }
796            ExecFidKind::Status(id) => self.read_status(id).await?.into_bytes(),
797            ExecFidKind::Exit(id) => self.read_exit(id).await?.into_bytes(),
798            ExecFidKind::Ns(id) => self.read_ns(id).await?.into_bytes(),
799            // `read_until_closed` already applies `offset`/`count` (it slices
800            // the retained buffer at `[offset, offset+count)`), so its result
801            // must be returned directly here — falling through to the shared
802            // offset tail below would re-apply `offset` a second time and
803            // silently corrupt or premature-EOF any nonzero-offset fd read
804            // (#1447 r2/k3 Finding A).
805            ExecFidKind::Fd(id, 1) => {
806                return Ok(self
807                    .fd_streams_for(id)
808                    .await?
809                    .stdout
810                    .read_until_closed(offset, count)
811                    .await);
812            }
813            ExecFidKind::Fd(id, 2) => {
814                return Ok(self
815                    .fd_streams_for(id)
816                    .await?
817                    .stderr
818                    .read_until_closed(offset, count)
819                    .await);
820            }
821            ExecFidKind::Fd(_, _) => {
822                return Err(MountError::NotFound("invalid exec fd".into()));
823            }
824            ExecFidKind::InstancesDir | ExecFidKind::InstanceDir(_) | ExecFidKind::FdDir(_) => {
825                return Err(MountError::IsDirectory("use readdir".into()));
826            }
827        };
828
829        let start = offset as usize;
830        if start >= data.len() {
831            return Ok(Vec::new());
832        }
833        Ok(data[start..].to_vec())
834    }
835
836    async fn write(
837        &self,
838        fid: &Fid,
839        _offset: u64,
840        data: &[u8],
841        caller: &Subject,
842    ) -> Result<u32, MountError> {
843        let inner = fid
844            .downcast_ref::<ExecFid>()
845            .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
846
847        match &inner.kind {
848            ExecFidKind::Ctl(id) => {
849                let text = String::from_utf8_lossy(data);
850                let verb = Verb::parse(&text).ok_or_else(|| {
851                    MountError::InvalidArgument(format!("unknown ctl verb: {}", text.trim()))
852                })?;
853                // `caller` is now threaded into the lifecycle gate inside
854                // `apply_verb` (#1272): a destructive verb is mediated by the
855                // armed [`LifecyclePolicy`] before the backend is touched.
856                let result = self.apply_verb(id, verb, caller).await;
857                let response_bytes = match result {
858                    Ok(s) => s.into_bytes(),
859                    Err(e) => format!("error: {e}\n").into_bytes(),
860                };
861                // Ctl write→read pattern: latch the result for a subsequent
862                // read on the same fid. The `Mount::write(&self, &Fid, ..)`
863                // signature needs interior mutability; `parking_lot::Mutex`
864                // gives it safely through `&` (the same primitive
865                // `hyprstream-workers-tcl`/`-python` and `devfile::DevFileState`
866                // use on bases that contain #615), avoiding the `unsafe *mut`
867                // cast-through-`&` the pre-review draft used — that would be a
868                // data race if two futures ever held `&Fid` to one fid
869                // concurrently (e.g. a 9P server multiplexing Twrite on one fid).
870                *inner.write_buf.lock() = response_bytes;
871                Ok(data.len() as u32)
872            }
873            ExecFidKind::Fd(_, 1 | 2) => Err(MountError::NotSupported(
874                "fd/1 and fd/2 are read-only".into(),
875            )),
876            _ => Err(MountError::NotSupported("read-only".into())),
877        }
878    }
879
880    async fn readdir(&self, fid: &Fid, _caller: &Subject) -> Result<Vec<DirEntry>, MountError> {
881        let inner = fid
882            .downcast_ref::<ExecFid>()
883            .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
884
885        match &inner.kind {
886            ExecFidKind::InstancesDir => {
887                let mut ids: Vec<String> = self
888                    .pool
889                    .list_active()
890                    .await
891                    .into_iter()
892                    .map(|s| s.id)
893                    .collect();
894                // Also surface ids that have been destroyed-through-this-mount
895                // (so `exit`/`status` remain reachable for stragglers that
896                // already read "not yet terminal" and need to poll again).
897                for id in self.terminal_ids.lock().iter() {
898                    if !ids.contains(id) {
899                        ids.push(id.clone());
900                    }
901                }
902                Ok(ids
903                    .into_iter()
904                    .map(|name| DirEntry {
905                        name,
906                        is_dir: true,
907                        size: 0,
908                        stat: None,
909                    })
910                    .collect())
911            }
912            ExecFidKind::InstanceDir(_) => Ok(vec![
913                DirEntry {
914                    name: "ctl".into(),
915                    is_dir: false,
916                    size: 0,
917                    stat: None,
918                },
919                DirEntry {
920                    name: "status".into(),
921                    is_dir: false,
922                    size: 0,
923                    stat: None,
924                },
925                DirEntry {
926                    name: "exit".into(),
927                    is_dir: false,
928                    size: 0,
929                    stat: None,
930                },
931                DirEntry {
932                    name: "ns".into(),
933                    is_dir: false,
934                    size: 0,
935                    stat: None,
936                },
937                DirEntry {
938                    name: "fd".into(),
939                    is_dir: true,
940                    size: 0,
941                    stat: None,
942                },
943            ]),
944            ExecFidKind::FdDir(_) => Ok((1..=2)
945                .map(|fd| DirEntry {
946                    name: fd.to_string(),
947                    is_dir: false,
948                    size: 0,
949                    stat: None,
950                })
951                .collect()),
952            _ => Err(MountError::NotDirectory(format!("{:?}", inner.kind))),
953        }
954    }
955
956    async fn stat(&self, fid: &Fid, _caller: &Subject) -> Result<Stat, MountError> {
957        let inner = fid
958            .downcast_ref::<ExecFid>()
959            .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
960
961        const QTDIR: u8 = 0x80;
962        let (name, qtype) = match &inner.kind {
963            ExecFidKind::InstancesDir => ("instances".to_owned(), QTDIR),
964            ExecFidKind::InstanceDir(id) => (id.clone(), QTDIR),
965            ExecFidKind::Ctl(_) => ("ctl".to_owned(), 0),
966            ExecFidKind::Status(_) => ("status".to_owned(), 0),
967            ExecFidKind::Exit(_) => ("exit".to_owned(), 0),
968            ExecFidKind::Ns(_) => ("ns".to_owned(), 0),
969            ExecFidKind::FdDir(_) => ("fd".to_owned(), QTDIR),
970            ExecFidKind::Fd(_, fd) => (fd.to_string(), 0),
971        };
972
973        Ok(Stat {
974            qtype,
975            version: 0,
976            path: 0,
977            size: 0,
978            name,
979            mtime: 0,
980        })
981    }
982
983    async fn clunk(&self, _fid: Fid, _caller: &Subject) {}
984}
985
986// ─────────────────────────────────────────────────────────────────────────────
987// Tests
988// ─────────────────────────────────────────────────────────────────────────────
989
990#[cfg(test)]
991#[allow(clippy::unwrap_used, clippy::expect_used)]
992mod tests {
993    use super::*;
994    use crate::config::PoolConfig;
995    use crate::error::Result as WorkerResult;
996    use crate::runtime::backend::{SandboxBackend, SandboxHandle};
997    use crate::runtime::sandbox::PodSandbox;
998    use hyprstream_rpc_std::worker_client::{LinuxContainerResources, PodSandboxConfig};
999    use std::any::Any;
1000    use std::sync::atomic::{AtomicBool, Ordering};
1001
1002    /// Minimal in-memory fake `SandboxBackend` for exercising `ExecMount`
1003    /// without any real isolation runtime (no kata/nspawn/wasm dependency).
1004    #[derive(Debug)]
1005    struct FakeHandle;
1006    impl SandboxHandle for FakeHandle {
1007        fn as_any(&self) -> &dyn Any {
1008            self
1009        }
1010    }
1011
1012    #[derive(Default)]
1013    struct FakeBackend {
1014        /// Tracks whether `destroy` was called, for assertions.
1015        destroyed: AtomicBool,
1016        stopped: AtomicBool,
1017    }
1018
1019    #[async_trait]
1020    impl SandboxBackend for FakeBackend {
1021        fn backend_type(&self) -> &'static str {
1022            "fake"
1023        }
1024
1025        fn is_available(&self) -> bool {
1026            true
1027        }
1028
1029        async fn initialize(&self, _config: &PoolConfig) -> WorkerResult<()> {
1030            Ok(())
1031        }
1032
1033        async fn start(
1034            &self,
1035            _sandbox: &mut PodSandbox,
1036            _config: &PodSandboxConfig,
1037            _pool_config: &PoolConfig,
1038            _annotations: &HashMap<String, String>,
1039        ) -> WorkerResult<Arc<dyn SandboxHandle>> {
1040            Ok(Arc::new(FakeHandle))
1041        }
1042
1043        async fn stop(&self, _sandbox: &PodSandbox) -> WorkerResult<()> {
1044            self.stopped.store(true, Ordering::SeqCst);
1045            Ok(())
1046        }
1047
1048        async fn destroy(&self, _sandbox: &PodSandbox) -> WorkerResult<()> {
1049            self.destroyed.store(true, Ordering::SeqCst);
1050            Ok(())
1051        }
1052
1053        async fn reset(&self, _sandbox: &mut PodSandbox) -> WorkerResult<bool> {
1054            // Ephemeral: never reusable, mirrors nspawn's shape.
1055            Ok(false)
1056        }
1057
1058        async fn get_pids(&self, _sandbox: &PodSandbox) -> WorkerResult<Vec<u32>> {
1059            Ok(vec![])
1060        }
1061
1062        fn supports_exec(&self) -> bool {
1063            true
1064        }
1065
1066        async fn exec_sync(
1067            &self,
1068            _sandbox: &PodSandbox,
1069            _command: &[String],
1070            _timeout_secs: u64,
1071        ) -> WorkerResult<(i32, Vec<u8>, Vec<u8>)> {
1072            Ok((0, b"fake stdout\n".to_vec(), b"fake stderr\n".to_vec()))
1073        }
1074
1075        async fn update_resources(
1076            &self,
1077            _sandbox: &PodSandbox,
1078            _resources: &LinuxContainerResources,
1079        ) -> WorkerResult<()> {
1080            Ok(())
1081        }
1082    }
1083
1084    async fn make_pool() -> Arc<SandboxPool> {
1085        let backend: Arc<dyn SandboxBackend> = Arc::new(FakeBackend::default());
1086        let config = PoolConfig {
1087            max_sandboxes: 10,
1088            warm_pool_size: 0,
1089            ..Default::default()
1090        };
1091        Arc::new(SandboxPool::new(config, backend))
1092    }
1093
1094    fn subject() -> Subject {
1095        Subject::anonymous()
1096    }
1097
1098    /// Distinct from `subject()` above: that one is the *VFS caller* passed
1099    /// to `ExecMount`'s `Mount` trait methods (anonymous is fine there — this
1100    /// projection doesn't itself enforce per-caller authorization). This one
1101    /// is the *admission* identity `SandboxPool::acquire` requires (#525 P2)
1102    /// — must be non-anonymous, or every `pool.acquire()` call below would
1103    /// fail closed.
1104    fn admission_subject() -> Subject {
1105        Subject::new("test-user")
1106    }
1107
1108    #[tokio::test]
1109    async fn list_instances_empty() {
1110        let pool = make_pool().await;
1111        let mount = ExecMount::new(pool);
1112        let fid = mount.walk(&[], &subject()).await.unwrap();
1113        let entries = mount.readdir(&fid, &subject()).await.unwrap();
1114        assert!(entries.is_empty());
1115    }
1116
1117    #[tokio::test]
1118    async fn list_instances_after_acquire() {
1119        let pool = make_pool().await;
1120        let id = pool
1121            .acquire(&admission_subject(), &PodSandboxConfig::default())
1122            .await
1123            .unwrap();
1124        let mount = ExecMount::new(pool);
1125
1126        let fid = mount.walk(&[], &subject()).await.unwrap();
1127        let entries = mount.readdir(&fid, &subject()).await.unwrap();
1128        let names: Vec<&str> = entries.iter().map(|e| e.name.as_str()).collect();
1129        assert!(names.contains(&id.as_str()));
1130        assert!(entries.iter().all(|e| e.is_dir));
1131    }
1132
1133    #[tokio::test]
1134    async fn walk_unknown_instance_not_found() {
1135        let pool = make_pool().await;
1136        let mount = ExecMount::new(pool);
1137        let result = mount.walk(&["nonexistent", "status"], &subject()).await;
1138        assert!(matches!(result, Err(MountError::NotFound(_))));
1139    }
1140
1141    #[tokio::test]
1142    async fn read_status_reflects_pool_state() {
1143        let pool = make_pool().await;
1144        let id = pool
1145            .acquire(&admission_subject(), &PodSandboxConfig::default())
1146            .await
1147            .unwrap();
1148        let mount = ExecMount::new(pool);
1149
1150        let mut fid = mount.walk(&[&id, "status"], &subject()).await.unwrap();
1151        mount.open(&mut fid, 0, &subject()).await.unwrap();
1152        let data = mount.read(&fid, 0, 4096, &subject()).await.unwrap();
1153        let text = String::from_utf8(data).unwrap();
1154        assert!(text.contains("SandboxReady"), "got: {text}");
1155    }
1156
1157    #[tokio::test]
1158    async fn ctl_stop_invokes_backend() {
1159        let pool = make_pool().await;
1160        let id = pool
1161            .acquire(&admission_subject(), &PodSandboxConfig::default())
1162            .await
1163            .unwrap();
1164        let mount = ExecMount::new(pool);
1165
1166        let mut fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1167        mount.open(&mut fid, 2, &subject()).await.unwrap();
1168        let written = mount.write(&fid, 0, b"stop", &subject()).await.unwrap();
1169        assert_eq!(written, 4);
1170
1171        let data = mount.read(&fid, 0, 4096, &subject()).await.unwrap();
1172        let text = String::from_utf8(data).unwrap();
1173        assert!(text.starts_with("ok:"), "got: {text}");
1174    }
1175
1176    #[tokio::test]
1177    async fn ctl_unknown_verb_rejected() {
1178        let pool = make_pool().await;
1179        let id = pool
1180            .acquire(&admission_subject(), &PodSandboxConfig::default())
1181            .await
1182            .unwrap();
1183        let mount = ExecMount::new(pool);
1184
1185        let mut fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1186        mount.open(&mut fid, 2, &subject()).await.unwrap();
1187        let result = mount.write(&fid, 0, b"frobnicate", &subject()).await;
1188        assert!(matches!(result, Err(MountError::InvalidArgument(_))));
1189    }
1190
1191    /// Early waiter: a `read()` on `exit` that starts BEFORE the instance is
1192    /// terminal blocks, then unblocks with the correct status once
1193    /// `apply_verb`'s `Destroy` arm latches it from another task — the core
1194    /// EV7 read-then-subscribe guarantee (#668).
1195    #[tokio::test]
1196    async fn exit_blocks_then_unblocks_on_destroy() {
1197        let pool = make_pool().await;
1198        let id = pool
1199            .acquire(&admission_subject(), &PodSandboxConfig::default())
1200            .await
1201            .unwrap();
1202        let mount = Arc::new(ExecMount::new(pool));
1203
1204        let reader = {
1205            let mount = Arc::clone(&mount);
1206            let id = id.clone();
1207            tokio::spawn(async move {
1208                let mut fid = mount.walk(&[&id, "exit"], &subject()).await.unwrap();
1209                mount.open(&mut fid, 0, &subject()).await.unwrap();
1210                mount.read(&fid, 0, 4096, &subject()).await.unwrap()
1211            })
1212        };
1213
1214        // The reader should be blocked (not yet resolved) — give it a beat to
1215        // reach the `.await` inside `read_exit`'s slow path, then confirm it
1216        // hasn't completed on its own.
1217        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1218        assert!(
1219            !reader.is_finished(),
1220            "exit read must block before terminal"
1221        );
1222
1223        // Destroy from a separate task — this is what latches + wakes the
1224        // blocked reader above.
1225        let mut ctl_fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1226        mount.open(&mut ctl_fid, 2, &subject()).await.unwrap();
1227        mount
1228            .write(&ctl_fid, 0, b"destroy", &subject())
1229            .await
1230            .unwrap();
1231        mount.read(&ctl_fid, 0, 4096, &subject()).await.unwrap();
1232
1233        let data = tokio::time::timeout(std::time::Duration::from_secs(2), reader)
1234            .await
1235            .expect("exit read did not unblock after destroy")
1236            .unwrap();
1237        let text = String::from_utf8(data).unwrap();
1238        assert!(text.starts_with("exited"), "got: {text}");
1239    }
1240
1241    /// Late waiter: a `read()` on `exit` that starts AFTER the instance is
1242    /// already terminal is served the retained status immediately, without
1243    /// blocking (#668).
1244    #[tokio::test]
1245    async fn exit_after_destroy_reports_terminal() {
1246        let pool = make_pool().await;
1247        let id = pool
1248            .acquire(&admission_subject(), &PodSandboxConfig::default())
1249            .await
1250            .unwrap();
1251        let mount = ExecMount::new(pool);
1252
1253        // Drive destroy through ctl.
1254        let mut ctl_fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1255        mount.open(&mut ctl_fid, 2, &subject()).await.unwrap();
1256        mount
1257            .write(&ctl_fid, 0, b"destroy", &subject())
1258            .await
1259            .unwrap();
1260        let ctl_result = mount.read(&ctl_fid, 0, 4096, &subject()).await.unwrap();
1261        assert!(String::from_utf8(ctl_result).unwrap().starts_with("ok:"));
1262
1263        // exit should now report terminal status immediately (late reader —
1264        // the retained value is served without blocking on the store).
1265        let mut exit_fid = mount.walk(&[&id, "exit"], &subject()).await.unwrap();
1266        mount.open(&mut exit_fid, 0, &subject()).await.unwrap();
1267        let data = tokio::time::timeout(
1268            std::time::Duration::from_millis(200),
1269            mount.read(&exit_fid, 0, 4096, &subject()),
1270        )
1271        .await
1272        .expect("late exit read must not block")
1273        .unwrap();
1274        let text = String::from_utf8(data).unwrap();
1275        assert!(text.starts_with("exited"), "got: {text}");
1276
1277        // status should also reflect terminal state via the retained store.
1278        let mut status_fid = mount.walk(&[&id, "status"], &subject()).await.unwrap();
1279        mount.open(&mut status_fid, 0, &subject()).await.unwrap();
1280        let data = mount.read(&status_fid, 0, 4096, &subject()).await.unwrap();
1281        assert!(!data.is_empty());
1282    }
1283
1284    #[tokio::test]
1285    async fn ctl_on_terminal_instance_rejected() {
1286        let pool = make_pool().await;
1287        let id = pool
1288            .acquire(&admission_subject(), &PodSandboxConfig::default())
1289            .await
1290            .unwrap();
1291        let mount = ExecMount::new(pool);
1292
1293        let mut ctl_fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1294        mount.open(&mut ctl_fid, 2, &subject()).await.unwrap();
1295        mount
1296            .write(&ctl_fid, 0, b"destroy", &subject())
1297            .await
1298            .unwrap();
1299
1300        let result = mount.write(&ctl_fid, 0, b"stop", &subject()).await;
1301        // apply_verb returns an error result captured into the ctl response
1302        // buffer rather than propagating through `write`'s Result, so check
1303        // the *next* destroy attempt is rejected via a fresh fid instead.
1304        assert!(result.is_ok()); // write() itself always "succeeds" (buffers the error text)
1305        let data = mount.read(&ctl_fid, 0, 4096, &subject()).await.unwrap();
1306        let text = String::from_utf8(data).unwrap();
1307        assert!(text.starts_with("error:"), "got: {text}");
1308    }
1309
1310    #[tokio::test]
1311    async fn read_ns_returns_placeholder_info() {
1312        let pool = make_pool().await;
1313        let id = pool
1314            .acquire(&admission_subject(), &PodSandboxConfig::default())
1315            .await
1316            .unwrap();
1317        let mount = ExecMount::new(pool);
1318
1319        let mut fid = mount.walk(&[&id, "ns"], &subject()).await.unwrap();
1320        mount.open(&mut fid, 0, &subject()).await.unwrap();
1321        let data = mount.read(&fid, 0, 4096, &subject()).await.unwrap();
1322        let text = String::from_utf8(data).unwrap();
1323        assert!(text.contains("runtime="), "got: {text}");
1324        assert!(text.contains("path="), "got: {text}");
1325    }
1326
1327    #[tokio::test]
1328    async fn readdir_instance_dir_lists_files() {
1329        let pool = make_pool().await;
1330        let id = pool
1331            .acquire(&admission_subject(), &PodSandboxConfig::default())
1332            .await
1333            .unwrap();
1334        let mount = ExecMount::new(pool);
1335
1336        let fid = mount.walk(&[&id], &subject()).await.unwrap();
1337        let entries = mount.readdir(&fid, &subject()).await.unwrap();
1338        let names: Vec<&str> = entries.iter().map(|e| e.name.as_str()).collect();
1339        assert!(names.contains(&"ctl"));
1340        assert!(names.contains(&"status"));
1341        assert!(names.contains(&"exit"));
1342        assert!(names.contains(&"ns"));
1343        assert!(names.contains(&"fd"));
1344
1345        let fd_dir = mount.walk(&[&id, "fd"], &subject()).await.unwrap();
1346        let fd_entries = mount.readdir(&fd_dir, &subject()).await.unwrap();
1347        let fd_names: Vec<&str> = fd_entries.iter().map(|e| e.name.as_str()).collect();
1348        assert_eq!(fd_names, ["1", "2"]);
1349    }
1350
1351    #[tokio::test]
1352    async fn fd_output_blocks_until_runtime_publishes_then_latches_eof() {
1353        let pool = make_pool().await;
1354        let id = pool
1355            .acquire(&admission_subject(), &PodSandboxConfig::default())
1356            .await
1357            .unwrap();
1358        let mount = Arc::new(ExecMount::new(pool));
1359
1360        let reader = {
1361            let mount = Arc::clone(&mount);
1362            let id = id.clone();
1363            tokio::spawn(async move {
1364                let mut fid = mount.walk(&[&id, "fd", "1"], &subject()).await.unwrap();
1365                mount.open(&mut fid, 0, &subject()).await.unwrap();
1366                mount.read(&fid, 0, 4096, &subject()).await.unwrap()
1367            })
1368        };
1369        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1370        assert!(
1371            !reader.is_finished(),
1372            "fd/1 must wait for terminal task output"
1373        );
1374
1375        mount
1376            .publish_exec_result(&id, b"build output\n", b"warning output\n")
1377            .await
1378            .unwrap();
1379
1380        let stdout = tokio::time::timeout(std::time::Duration::from_secs(2), reader)
1381            .await
1382            .expect("fd/1 reader did not wake on task completion")
1383            .unwrap();
1384        assert_eq!(stdout, b"build output\n");
1385
1386        let mut stderr_fid = mount.walk(&[&id, "fd", "2"], &subject()).await.unwrap();
1387        mount.open(&mut stderr_fid, 0, &subject()).await.unwrap();
1388        assert_eq!(
1389            mount.read(&stderr_fid, 0, 4096, &subject()).await.unwrap(),
1390            b"warning output\n"
1391        );
1392        assert!(
1393            mount
1394                .read(&stderr_fid, 4096, 4096, &subject())
1395                .await
1396                .unwrap()
1397                .is_empty(),
1398            "terminal stderr must report EOF after retained bytes"
1399        );
1400    }
1401
1402    /// Regression (#1447 r2/k3 Finding A): `read_until_closed` already slices
1403    /// the retained buffer at `[offset, offset+count)`; `Mount::read`'s shared
1404    /// offset tail must NOT re-apply `offset` to that already-offset result.
1405    /// Every other fd test in this suite reads at offset 0 or at an offset
1406    /// past end-of-buffer, both of which look identical under the buggy
1407    /// double-application and the correct behavior — only a nonzero,
1408    /// in-range offset (mid-buffer, on a buffer bigger than one read) can
1409    /// distinguish them. With a double application, offset=3/count=4 over a
1410    /// 10-byte buffer computes `bytes[3..7]` ("3456") in `read_until_closed`,
1411    /// then re-applies offset 3 to that 4-byte result (`"3456"[3..]` = "6"
1412    /// only) — silently wrong data, not even an empty read.
1413    #[tokio::test]
1414    async fn fd_read_at_nonzero_offset_is_not_double_applied() {
1415        let pool = make_pool().await;
1416        let id = pool
1417            .acquire(&admission_subject(), &PodSandboxConfig::default())
1418            .await
1419            .unwrap();
1420        let mount = ExecMount::new(pool);
1421
1422        mount
1423            .publish_exec_result(&id, b"0123456789", b"")
1424            .await
1425            .unwrap();
1426
1427        let mut fid = mount.walk(&[&id, "fd", "1"], &subject()).await.unwrap();
1428        mount.open(&mut fid, 0, &subject()).await.unwrap();
1429
1430        let data = mount.read(&fid, 3, 4, &subject()).await.unwrap();
1431        assert_eq!(
1432            data, b"3456",
1433            "offset=3,count=4 over a 10-byte retained buffer must return bytes[3..7] \
1434             verbatim from read_until_closed, not re-offset a second time"
1435        );
1436    }
1437
1438    #[tokio::test]
1439    async fn ctl_exec_is_projected_to_terminal_fd_streams() {
1440        let pool = make_pool().await;
1441        let id = pool
1442            .acquire(&admission_subject(), &PodSandboxConfig::default())
1443            .await
1444            .unwrap();
1445        let mount = ExecMount::new(pool);
1446
1447        let mut ctl = mount
1448            .walk(&[&id, "ctl"], &admission_subject())
1449            .await
1450            .unwrap();
1451        mount.open(&mut ctl, 2, &admission_subject()).await.unwrap();
1452        mount
1453            .write(&ctl, 0, b"exec build", &admission_subject())
1454            .await
1455            .unwrap();
1456        assert_eq!(
1457            mount
1458                .read(&ctl, 0, 4096, &admission_subject())
1459                .await
1460                .unwrap(),
1461            b"ok: exited 0\n"
1462        );
1463        let mut stdout = mount.walk(&[&id, "fd", "1"], &subject()).await.unwrap();
1464        mount.open(&mut stdout, 0, &subject()).await.unwrap();
1465        assert_eq!(
1466            mount.read(&stdout, 0, 4096, &subject()).await.unwrap(),
1467            b"fake stdout\n"
1468        );
1469        let mut stderr = mount.walk(&[&id, "fd", "2"], &subject()).await.unwrap();
1470        mount.open(&mut stderr, 0, &subject()).await.unwrap();
1471        assert_eq!(
1472            mount.read(&stderr, 0, 4096, &subject()).await.unwrap(),
1473            b"fake stderr\n"
1474        );
1475    }
1476
1477    #[tokio::test]
1478    async fn fd_zero_is_omitted_until_the_task_runtime_consumes_stdin() {
1479        let pool = make_pool().await;
1480        let id = pool
1481            .acquire(&admission_subject(), &PodSandboxConfig::default())
1482            .await
1483            .unwrap();
1484        let mount = ExecMount::new(pool);
1485        assert!(matches!(
1486            mount.walk(&[&id, "fd", "0"], &subject()).await,
1487            Err(MountError::NotFound(_))
1488        ));
1489    }
1490
1491    #[tokio::test]
1492    async fn ctl_exec_rejects_a_different_subject() {
1493        let pool = make_pool().await;
1494        let id = pool
1495            .acquire(&admission_subject(), &PodSandboxConfig::default())
1496            .await
1497            .unwrap();
1498        let mount = ExecMount::new(pool);
1499        let other = Subject::new("other-user");
1500        let mut ctl = mount.walk(&[&id, "ctl"], &other).await.unwrap();
1501        mount.open(&mut ctl, 2, &other).await.unwrap();
1502        mount.write(&ctl, 0, b"exec build", &other).await.unwrap();
1503        let reply = mount.read(&ctl, 0, 4096, &other).await.unwrap();
1504        assert!(String::from_utf8(reply)
1505            .unwrap()
1506            .contains("caller does not own this task's Subject-scoped namespace"));
1507    }
1508
1509    #[tokio::test]
1510    async fn stopping_instance_latches_waiting_fd_reader_to_eof() {
1511        let pool = make_pool().await;
1512        let id = pool
1513            .acquire(&admission_subject(), &PodSandboxConfig::default())
1514            .await
1515            .unwrap();
1516        let mount = Arc::new(ExecMount::new(pool));
1517        let reader = {
1518            let mount = Arc::clone(&mount);
1519            let id = id.clone();
1520            tokio::spawn(async move {
1521                let mut fid = mount.walk(&[&id, "fd", "2"], &subject()).await.unwrap();
1522                mount.open(&mut fid, 0, &subject()).await.unwrap();
1523                mount.read(&fid, 0, 4096, &subject()).await.unwrap()
1524            })
1525        };
1526        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1527        assert!(
1528            !reader.is_finished(),
1529            "fd reader must wait before cancellation"
1530        );
1531
1532        let mut ctl = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1533        mount.open(&mut ctl, 2, &subject()).await.unwrap();
1534        mount.write(&ctl, 0, b"stop", &subject()).await.unwrap();
1535
1536        assert!(
1537            tokio::time::timeout(std::time::Duration::from_secs(2), reader)
1538                .await
1539                .expect("cancellation did not wake fd reader")
1540                .unwrap()
1541                .is_empty(),
1542            "cancelled stream must latch terminal EOF"
1543        );
1544    }
1545
1546    #[tokio::test]
1547    async fn stat_root_is_dir() {
1548        let pool = make_pool().await;
1549        let mount = ExecMount::new(pool);
1550        let fid = mount.walk(&[], &subject()).await.unwrap();
1551        let st = mount.stat(&fid, &subject()).await.unwrap();
1552        assert_eq!(st.qtype, 0x80);
1553        assert_eq!(st.name, "instances");
1554    }
1555
1556    /// Regression: the ctl write→latch must be safe under concurrent access
1557    /// to the same fid. The `Mount::write`/`read` signatures take `&Fid`
1558    /// (shared), so the latch must use interior mutability. Before this fix
1559    /// the mount used an `unsafe *mut` cast through `&` to mutate `write_buf`,
1560    /// which was a data race / UB if two futures holding `&Fid` were polled
1561    /// concurrently (e.g. a 9P server multiplexing Twrite on one fid). Here a
1562    /// single ctl fid is shared across `join3`-concurrent writers: under the
1563    /// old `unsafe` cast Miri flags this as UB; under the `parking_lot::Mutex`
1564    /// latch it is sound, and exactly one writer's response wins the latch.
1565    #[tokio::test]
1566    async fn ctl_concurrent_writes_to_one_fid_are_safe() {
1567        let pool = make_pool().await;
1568        let id = pool
1569            .acquire(&admission_subject(), &PodSandboxConfig::default())
1570            .await
1571            .unwrap();
1572        let mount = ExecMount::new(pool);
1573
1574        // One shared ctl fid; opened RDWR.
1575        let mut fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1576        mount.open(&mut fid, 2, &subject()).await.unwrap();
1577
1578        // Drive several concurrent writers against the *same* `&Fid`. The
1579        // contract under test is *no UB / no panic* — the final latched value
1580        // just has to be a valid ctl response (the latch update must be
1581        // atomic, not a torn write).
1582        let caller = subject();
1583        let w1 = mount.write(&fid, 0, b"stop", &caller);
1584        let w2 = mount.write(&fid, 0, b"stop", &caller);
1585        let w3 = mount.write(&fid, 0, b"kill", &caller);
1586        let (r1, r2, r3) = futures::future::join3(w1, w2, w3).await;
1587        for r in [r1, r2, r3] {
1588            r.unwrap();
1589        }
1590
1591        let out = mount.read(&fid, 0, 4096, &caller).await.unwrap();
1592        let text = String::from_utf8(out).unwrap();
1593        assert!(
1594            text.starts_with("ok:"),
1595            "expected one of the stop/kill responses latched, got: {text}"
1596        );
1597    }
1598}