1use std::collections::HashMap;
51use std::sync::Arc;
52
53use async_trait::async_trait;
54use rand::RngCore;
55use tokio::sync::Notify;
56
57use hyprstream_rpc::latch::{Terminal, TerminalStore};
58use hyprstream_rpc::moq_stream::{MoqStreamOrigin, MoqStreamPublisher};
59use hyprstream_rpc::stream_info::Job;
60use hyprstream_rpc::streaming::StreamContext;
61use hyprstream_vfs::{DirEntry, Fid, Mount, MountError, Stat, Subject};
62use parking_lot::Mutex as PmMutex;
67
68use super::pool::SandboxPool;
69use hyprstream_rpc_std::worker_client::PodSandboxState;
70
71#[derive(Clone, Debug, PartialEq, Eq)]
78enum Verb {
79 Start,
80 Stop,
81 Kill,
82 Destroy,
83 Exec(Vec<String>),
86}
87
88impl Verb {
89 fn parse(s: &str) -> Option<Self> {
90 let words: Vec<String> = s.split_whitespace().map(str::to_owned).collect();
91 match words.as_slice() {
92 [verb] if verb == "start" => Some(Self::Start),
93 [verb] if verb == "stop" => Some(Self::Stop),
94 [verb] if verb == "kill" => Some(Self::Kill),
95 [verb] if verb == "destroy" => Some(Self::Destroy),
96 [verb, command @ ..] if verb == "exec" && !command.is_empty() => {
97 Some(Self::Exec(command.to_vec()))
98 }
99 _ => None,
100 }
101 }
102
103 fn is_lifecycle_destructive(&self) -> bool {
108 matches!(self, Verb::Stop | Verb::Kill | Verb::Destroy)
109 }
110}
111
112#[derive(Clone, Copy, Debug, PartialEq, Eq)]
123pub enum LifecycleVerb {
124 Stop,
125 Kill,
126 Destroy,
127}
128
129pub trait LifecyclePolicy: Send + Sync {
149 fn authorize(&self, caller: &Subject, id: &str, verb: LifecycleVerb) -> Result<(), String>;
152}
153
154#[derive(Debug, Default, Clone, Copy)]
158pub struct DenyAllLifecycle;
159
160impl LifecyclePolicy for DenyAllLifecycle {
161 fn authorize(&self, _caller: &Subject, _id: &str, _verb: LifecycleVerb) -> Result<(), String> {
162 Err("lifecycle verb denied: no permissive policy".into())
163 }
164}
165
166#[derive(Clone, Debug)]
174struct TerminalStatus {
175 last_state: PodSandboxState,
177}
178
179impl TerminalStatus {
180 fn render(&self) -> String {
181 format!("exited state={:?}\n", self.last_state)
182 }
183}
184
185const MAX_RETAINED_FD_BYTES: usize = 1024 * 1024;
193
194struct TaskFdStream {
196 state: tokio::sync::Mutex<TaskFdState>,
197 wake: Arc<Notify>,
198}
199
200struct TaskFdState {
201 bytes: Vec<u8>,
202 closed: bool,
203 truncated: bool,
204 publisher: MoqStreamPublisher,
205}
206
207impl TaskFdStream {
208 fn new(origin: &MoqStreamOrigin, instance: &str, fd: u8) -> Result<Self, MountError> {
209 let mut mac_key = [0u8; 32];
210 rand::rngs::OsRng.fill_bytes(&mut mac_key);
211 let context = StreamContext::new(
212 format!("exec-{instance}-fd-{fd}"),
213 format!("exec-{}-fd-{fd}", uuid::Uuid::new_v4()),
214 mac_key,
215 [0u8; 32],
216 )
217 .with_qos_preset::<Job>();
218 let publisher = origin
219 .publisher(&context)
220 .map_err(|e| MountError::Io(format!("create fd/{fd} MoQ stream: {e}")))?;
221 Ok(Self {
222 state: tokio::sync::Mutex::new(TaskFdState {
223 bytes: Vec::new(),
224 closed: false,
225 truncated: false,
226 publisher,
227 }),
228 wake: Arc::new(Notify::new()),
229 })
230 }
231
232 async fn append(&self, data: &[u8]) -> Result<(), MountError> {
233 let mut state = self.state.lock().await;
234 if state.closed {
235 return Err(MountError::InvalidArgument(
236 "fd stream is already closed".into(),
237 ));
238 }
239 let available = MAX_RETAINED_FD_BYTES.saturating_sub(state.bytes.len());
240 let retained = &data[..data.len().min(available)];
241 if !retained.is_empty() {
242 state
243 .publisher
244 .publish_data(retained)
245 .await
246 .map_err(|e| MountError::Io(format!("publish fd stream data: {e}")))?;
247 state.bytes.extend_from_slice(retained);
248 }
249 if retained.len() != data.len() {
250 state.truncated = true;
251 }
252 drop(state);
253 self.wake.notify_waiters();
254 Ok(())
255 }
256
257 async fn close(&self) -> Result<(), MountError> {
258 let mut state = self.state.lock().await;
259 if !state.closed {
260 if state.truncated && state.bytes.len() < MAX_RETAINED_FD_BYTES {
261 let marker = b"\n[hyprstream: output truncated]\n";
262 let room = MAX_RETAINED_FD_BYTES - state.bytes.len();
263 state
264 .bytes
265 .extend_from_slice(&marker[..marker.len().min(room)]);
266 }
267 state
268 .publisher
269 .complete_ref(b"exec-fd-eof")
270 .await
271 .map_err(|e| MountError::Io(format!("complete fd stream: {e}")))?;
272 state.closed = true;
273 }
274 drop(state);
275 self.wake.notify_waiters();
276 Ok(())
277 }
278
279 async fn is_closed(&self) -> bool {
284 self.state.lock().await.closed
285 }
286
287 async fn read_until_closed(&self, offset: u64, count: u32) -> Vec<u8> {
288 loop {
289 let notified = self.wake.notified();
290 {
291 let state = self.state.lock().await;
292 let start = offset as usize;
293 if start < state.bytes.len() {
294 let end = start.saturating_add(count as usize).min(state.bytes.len());
295 return state.bytes[start..end].to_vec();
296 }
297 if state.closed {
298 return Vec::new();
299 }
300 }
301 notified.await;
302 }
303 }
304}
305
306struct TaskStdioAdapter {
310 stdout: TaskFdStream,
311 stderr: TaskFdStream,
312}
313
314#[derive(Debug)]
319pub struct TaskExecResult {
320 pub exit_code: i32,
321 pub stdout: Vec<u8>,
322 pub stderr: Vec<u8>,
323}
324
325#[derive(Clone, Debug)]
331enum ExecFidKind {
332 InstancesDir,
334 InstanceDir(String),
336 Ctl(String),
338 Status(String),
340 Exit(String),
342 Ns(String),
344 FdDir(String),
346 Fd(String, u8),
348}
349
350struct ExecFid {
352 kind: ExecFidKind,
353 write_buf: PmMutex<Vec<u8>>,
358}
359
360pub struct ExecMount {
371 pool: Arc<SandboxPool>,
372 fd_origin: MoqStreamOrigin,
375 fd_streams: tokio::sync::Mutex<HashMap<String, Arc<TaskStdioAdapter>>>,
378 terminal: TerminalStore<String, TerminalStatus>,
383 terminal_ids: PmMutex<std::collections::HashSet<String>>,
388 waiters: PmMutex<HashMap<String, Arc<Notify>>>,
397 lifecycle: Option<Arc<dyn LifecyclePolicy>>,
402}
403
404impl ExecMount {
405 pub fn new(pool: Arc<SandboxPool>) -> Self {
411 Self {
412 pool,
413 fd_origin: MoqStreamOrigin::standalone()
418 .with_prefix("local/exec")
419 .build(),
420 fd_streams: tokio::sync::Mutex::new(HashMap::new()),
421 terminal: TerminalStore::new(),
422 terminal_ids: PmMutex::new(std::collections::HashSet::new()),
423 waiters: PmMutex::new(HashMap::new()),
424 lifecycle: None,
425 }
426 }
427
428 pub fn new_with_lifecycle(pool: Arc<SandboxPool>, policy: Arc<dyn LifecyclePolicy>) -> Self {
436 let mut mount = Self::new(pool);
437 mount.lifecycle = Some(policy);
438 mount
439 }
440
441 fn waiter_for(&self, id: &str) -> Arc<Notify> {
443 Arc::clone(
444 self.waiters
445 .lock()
446 .entry(id.to_owned())
447 .or_insert_with(|| Arc::new(Notify::new())),
448 )
449 }
450
451 async fn fd_streams_for(&self, id: &str) -> Result<Arc<TaskStdioAdapter>, MountError> {
452 let mut streams = self.fd_streams.lock().await;
453 if let Some(existing) = streams.get(id) {
454 return Ok(Arc::clone(existing));
455 }
456 let created = Arc::new(TaskStdioAdapter {
457 stdout: TaskFdStream::new(&self.fd_origin, id, 1)?,
458 stderr: TaskFdStream::new(&self.fd_origin, id, 2)?,
459 });
460 streams.insert(id.to_owned(), Arc::clone(&created));
461 Ok(created)
462 }
463
464 async fn publish_exec_result(
468 &self,
469 id: &str,
470 stdout: &[u8],
471 stderr: &[u8],
472 ) -> Result<(), MountError> {
473 if self.pool.get(id).await.is_none() && !self.terminal.is_latched(&id.to_owned()) {
474 return Err(MountError::NotFound(format!("instances/{id}")));
475 }
476 let streams = self.fd_streams_for(id).await?;
477 streams.stdout.append(stdout).await?;
478 streams.stderr.append(stderr).await?;
479 streams.stdout.close().await?;
480 streams.stderr.close().await?;
481 Ok(())
482 }
483
484 pub async fn exec_task(
488 &self,
489 id: &str,
490 caller: &Subject,
491 command: &[String],
492 timeout_secs: u64,
493 ) -> Result<TaskExecResult, MountError> {
494 let sandbox = self
495 .pool
496 .get(id)
497 .await
498 .ok_or_else(|| MountError::NotFound(format!("instances/{id}")))?;
499 let task_subject = sandbox
500 .annotations
501 .iter()
502 .find(|annotation| annotation.key == "hyprstream.io/subject")
503 .map(|annotation| Subject::new(annotation.value.clone()))
504 .ok_or_else(|| MountError::PermissionDenied("task has no verified Subject".into()))?;
505 if &task_subject != caller {
506 return Err(MountError::PermissionDenied(
507 "caller does not own this task's Subject-scoped namespace".into(),
508 ));
509 }
510 let streams = self.fd_streams_for(id).await?;
516 if streams.stdout.is_closed().await || streams.stderr.is_closed().await {
517 return Err(MountError::InvalidArgument(
518 "instance's exec stdio is already closed (stopped, destroyed, or a prior \
519 exec already ran); each instance supports at most one exec"
520 .into(),
521 ));
522 }
523 let (exit_code, stdout, stderr) = self
524 .pool
525 .backend()
526 .exec_sync(&sandbox, command, timeout_secs)
527 .await
528 .map_err(|e| MountError::Io(e.to_string()))?;
529 self.publish_exec_result(id, &stdout, &stderr).await?;
530 Ok(TaskExecResult {
531 exit_code,
532 stdout,
533 stderr,
534 })
535 }
536
537 async fn close_fd_streams(&self, id: &str) -> Result<(), MountError> {
538 let streams = self.fd_streams_for(id).await?;
539 streams.stdout.close().await?;
540 streams.stderr.close().await?;
541 Ok(())
542 }
543
544 async fn apply_verb(
550 &self,
551 id: &str,
552 verb: Verb,
553 caller: &Subject,
554 ) -> Result<String, MountError> {
555 if self.terminal.is_latched(&id.to_owned()) {
559 return Err(MountError::InvalidArgument(format!(
560 "instance {id} is already terminal"
561 )));
562 }
563
564 if verb.is_lifecycle_destructive() {
569 if let Some(policy) = &self.lifecycle {
570 let lv = match verb {
571 Verb::Stop => LifecycleVerb::Stop,
572 Verb::Kill => LifecycleVerb::Kill,
573 Verb::Destroy => LifecycleVerb::Destroy,
574 Verb::Start => {
577 return Err(MountError::InvalidArgument(
578 "start is not a lifecycle-destructive verb".into(),
579 ))
580 }
581 Verb::Exec(_) => {
582 return Err(MountError::InvalidArgument(
583 "exec is not a lifecycle-destructive verb".into(),
584 ))
585 }
586 };
587 policy
588 .authorize(caller, id, lv)
589 .map_err(MountError::PermissionDenied)?;
590 }
591 }
592
593 let sandbox = self
594 .pool
595 .get(id)
596 .await
597 .ok_or_else(|| MountError::NotFound(format!("instances/{id}")))?;
598
599 match verb {
600 Verb::Start => {
601 Ok("ok: already started\n".to_owned())
606 }
607 Verb::Stop | Verb::Kill => {
608 self.pool
609 .backend()
610 .stop(&sandbox)
611 .await
612 .map_err(|e| MountError::Io(e.to_string()))?;
613 self.close_fd_streams(id).await?;
616 Ok("ok: stopped\n".to_owned())
617 }
618 Verb::Destroy => {
619 self.pool
620 .backend()
621 .destroy(&sandbox)
622 .await
623 .map_err(|e| MountError::Io(e.to_string()))?;
624 let _ = self.pool.release(id).await;
630 self.terminal.latch(
631 id.to_owned(),
632 Terminal {
633 value: TerminalStatus {
634 last_state: sandbox.state,
635 },
636 latched_by: "ctl:destroy".to_owned(),
637 },
638 );
639 self.terminal_ids.lock().insert(id.to_owned());
640 self.close_fd_streams(id).await?;
641 self.waiter_for(id).notify_waiters();
643 Ok("ok: destroyed\n".to_owned())
644 }
645 Verb::Exec(command) => {
646 let result = self.exec_task(id, caller, &command, 300).await?;
647 Ok(format!("ok: exited {}\n", result.exit_code))
648 }
649 }
650 }
651
652 async fn read_status(&self, id: &str) -> Result<String, MountError> {
654 if let Some(term) = self.terminal.get(&id.to_owned()) {
655 return Ok(format!("{:?}\n", term.value.last_state));
656 }
657 let sandbox = self
658 .pool
659 .get(id)
660 .await
661 .ok_or_else(|| MountError::NotFound(format!("instances/{id}")))?;
662 Ok(format!("{:?}\n", sandbox.state))
663 }
664
665 async fn read_exit(&self, id: &str) -> Result<String, MountError> {
672 let key = id.to_owned();
673 if let Some(term) = self.terminal.get(&key) {
676 return Ok(term.value.render());
677 }
678 if self.pool.get(id).await.is_none() {
681 return Err(MountError::NotFound(format!("instances/{id}")));
682 }
683 let notify = self.waiter_for(id);
687 loop {
688 let notified = notify.notified();
689 if let Some(term) = self.terminal.get(&key) {
690 return Ok(term.value.render());
691 }
692 notified.await;
693 if let Some(term) = self.terminal.get(&key) {
694 return Ok(term.value.render());
695 }
696 }
697 }
698
699 async fn read_ns(&self, id: &str) -> Result<String, MountError> {
706 let sandbox = self
707 .pool
708 .get(id)
709 .await
710 .ok_or_else(|| MountError::NotFound(format!("instances/{id}")))?;
711
712 let mut lines = vec![format!("runtime={}", sandbox.runtime_handler)];
713 lines.push(format!("path={}", sandbox.sandbox_path().display()));
714 if let Some(console) = sandbox.console_socket() {
715 lines.push(format!("console={}", console.display()));
716 }
717 Ok(format!("{}\n", lines.join("\n")))
718 }
719}
720
721#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
722#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
723impl Mount for ExecMount {
724 async fn walk(&self, components: &[&str], _caller: &Subject) -> Result<Fid, MountError> {
725 let kind = match components {
731 [] => ExecFidKind::InstancesDir,
732 [id] => ExecFidKind::InstanceDir((*id).to_owned()),
733 [id, "ctl"] => ExecFidKind::Ctl((*id).to_owned()),
734 [id, "status"] => ExecFidKind::Status((*id).to_owned()),
735 [id, "exit"] => ExecFidKind::Exit((*id).to_owned()),
736 [id, "ns"] => ExecFidKind::Ns((*id).to_owned()),
737 [id, "fd"] => ExecFidKind::FdDir((*id).to_owned()),
738 [id, "fd", fd] => {
739 let fd = fd
740 .parse::<u8>()
741 .map_err(|_| MountError::NotFound(components.join("/")))?;
742 if !(1..=2).contains(&fd) {
743 return Err(MountError::NotFound(components.join("/")));
744 }
745 ExecFidKind::Fd((*id).to_owned(), fd)
746 }
747 _ => return Err(MountError::NotFound(components.join("/"))),
748 };
749
750 let id_to_check: Option<&str> = match &kind {
753 ExecFidKind::InstanceDir(id)
754 | ExecFidKind::Ctl(id)
755 | ExecFidKind::Status(id)
756 | ExecFidKind::Exit(id)
757 | ExecFidKind::Ns(id)
758 | ExecFidKind::FdDir(id)
759 | ExecFidKind::Fd(id, _) => Some(id.as_str()),
760 ExecFidKind::InstancesDir => None,
761 };
762 if let Some(id) = id_to_check {
763 let exists =
764 self.pool.get(id).await.is_some() || self.terminal.is_latched(&id.to_owned());
765 if !exists {
766 return Err(MountError::NotFound(components.join("/")));
767 }
768 }
769
770 Ok(Fid::new(ExecFid {
771 kind,
772 write_buf: PmMutex::new(Vec::new()),
773 }))
774 }
775
776 async fn open(&self, _fid: &mut Fid, _mode: u8, _caller: &Subject) -> Result<(), MountError> {
777 Ok(())
778 }
779
780 async fn read(
781 &self,
782 fid: &Fid,
783 offset: u64,
784 count: u32,
785 _caller: &Subject,
786 ) -> Result<Vec<u8>, MountError> {
787 let inner = fid
788 .downcast_ref::<ExecFid>()
789 .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
790
791 let data = match &inner.kind {
792 ExecFidKind::Ctl(_) => {
793 inner.write_buf.lock().clone()
795 }
796 ExecFidKind::Status(id) => self.read_status(id).await?.into_bytes(),
797 ExecFidKind::Exit(id) => self.read_exit(id).await?.into_bytes(),
798 ExecFidKind::Ns(id) => self.read_ns(id).await?.into_bytes(),
799 ExecFidKind::Fd(id, 1) => {
806 return Ok(self
807 .fd_streams_for(id)
808 .await?
809 .stdout
810 .read_until_closed(offset, count)
811 .await);
812 }
813 ExecFidKind::Fd(id, 2) => {
814 return Ok(self
815 .fd_streams_for(id)
816 .await?
817 .stderr
818 .read_until_closed(offset, count)
819 .await);
820 }
821 ExecFidKind::Fd(_, _) => {
822 return Err(MountError::NotFound("invalid exec fd".into()));
823 }
824 ExecFidKind::InstancesDir | ExecFidKind::InstanceDir(_) | ExecFidKind::FdDir(_) => {
825 return Err(MountError::IsDirectory("use readdir".into()));
826 }
827 };
828
829 let start = offset as usize;
830 if start >= data.len() {
831 return Ok(Vec::new());
832 }
833 Ok(data[start..].to_vec())
834 }
835
836 async fn write(
837 &self,
838 fid: &Fid,
839 _offset: u64,
840 data: &[u8],
841 caller: &Subject,
842 ) -> Result<u32, MountError> {
843 let inner = fid
844 .downcast_ref::<ExecFid>()
845 .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
846
847 match &inner.kind {
848 ExecFidKind::Ctl(id) => {
849 let text = String::from_utf8_lossy(data);
850 let verb = Verb::parse(&text).ok_or_else(|| {
851 MountError::InvalidArgument(format!("unknown ctl verb: {}", text.trim()))
852 })?;
853 let result = self.apply_verb(id, verb, caller).await;
857 let response_bytes = match result {
858 Ok(s) => s.into_bytes(),
859 Err(e) => format!("error: {e}\n").into_bytes(),
860 };
861 *inner.write_buf.lock() = response_bytes;
871 Ok(data.len() as u32)
872 }
873 ExecFidKind::Fd(_, 1 | 2) => Err(MountError::NotSupported(
874 "fd/1 and fd/2 are read-only".into(),
875 )),
876 _ => Err(MountError::NotSupported("read-only".into())),
877 }
878 }
879
880 async fn readdir(&self, fid: &Fid, _caller: &Subject) -> Result<Vec<DirEntry>, MountError> {
881 let inner = fid
882 .downcast_ref::<ExecFid>()
883 .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
884
885 match &inner.kind {
886 ExecFidKind::InstancesDir => {
887 let mut ids: Vec<String> = self
888 .pool
889 .list_active()
890 .await
891 .into_iter()
892 .map(|s| s.id)
893 .collect();
894 for id in self.terminal_ids.lock().iter() {
898 if !ids.contains(id) {
899 ids.push(id.clone());
900 }
901 }
902 Ok(ids
903 .into_iter()
904 .map(|name| DirEntry {
905 name,
906 is_dir: true,
907 size: 0,
908 stat: None,
909 })
910 .collect())
911 }
912 ExecFidKind::InstanceDir(_) => Ok(vec![
913 DirEntry {
914 name: "ctl".into(),
915 is_dir: false,
916 size: 0,
917 stat: None,
918 },
919 DirEntry {
920 name: "status".into(),
921 is_dir: false,
922 size: 0,
923 stat: None,
924 },
925 DirEntry {
926 name: "exit".into(),
927 is_dir: false,
928 size: 0,
929 stat: None,
930 },
931 DirEntry {
932 name: "ns".into(),
933 is_dir: false,
934 size: 0,
935 stat: None,
936 },
937 DirEntry {
938 name: "fd".into(),
939 is_dir: true,
940 size: 0,
941 stat: None,
942 },
943 ]),
944 ExecFidKind::FdDir(_) => Ok((1..=2)
945 .map(|fd| DirEntry {
946 name: fd.to_string(),
947 is_dir: false,
948 size: 0,
949 stat: None,
950 })
951 .collect()),
952 _ => Err(MountError::NotDirectory(format!("{:?}", inner.kind))),
953 }
954 }
955
956 async fn stat(&self, fid: &Fid, _caller: &Subject) -> Result<Stat, MountError> {
957 let inner = fid
958 .downcast_ref::<ExecFid>()
959 .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
960
961 const QTDIR: u8 = 0x80;
962 let (name, qtype) = match &inner.kind {
963 ExecFidKind::InstancesDir => ("instances".to_owned(), QTDIR),
964 ExecFidKind::InstanceDir(id) => (id.clone(), QTDIR),
965 ExecFidKind::Ctl(_) => ("ctl".to_owned(), 0),
966 ExecFidKind::Status(_) => ("status".to_owned(), 0),
967 ExecFidKind::Exit(_) => ("exit".to_owned(), 0),
968 ExecFidKind::Ns(_) => ("ns".to_owned(), 0),
969 ExecFidKind::FdDir(_) => ("fd".to_owned(), QTDIR),
970 ExecFidKind::Fd(_, fd) => (fd.to_string(), 0),
971 };
972
973 Ok(Stat {
974 qtype,
975 version: 0,
976 path: 0,
977 size: 0,
978 name,
979 mtime: 0,
980 })
981 }
982
983 async fn clunk(&self, _fid: Fid, _caller: &Subject) {}
984}
985
986#[cfg(test)]
991#[allow(clippy::unwrap_used, clippy::expect_used)]
992mod tests {
993 use super::*;
994 use crate::config::PoolConfig;
995 use crate::error::Result as WorkerResult;
996 use crate::runtime::backend::{SandboxBackend, SandboxHandle};
997 use crate::runtime::sandbox::PodSandbox;
998 use hyprstream_rpc_std::worker_client::{LinuxContainerResources, PodSandboxConfig};
999 use std::any::Any;
1000 use std::sync::atomic::{AtomicBool, Ordering};
1001
1002 #[derive(Debug)]
1005 struct FakeHandle;
1006 impl SandboxHandle for FakeHandle {
1007 fn as_any(&self) -> &dyn Any {
1008 self
1009 }
1010 }
1011
1012 #[derive(Default)]
1013 struct FakeBackend {
1014 destroyed: AtomicBool,
1016 stopped: AtomicBool,
1017 }
1018
1019 #[async_trait]
1020 impl SandboxBackend for FakeBackend {
1021 fn backend_type(&self) -> &'static str {
1022 "fake"
1023 }
1024
1025 fn is_available(&self) -> bool {
1026 true
1027 }
1028
1029 async fn initialize(&self, _config: &PoolConfig) -> WorkerResult<()> {
1030 Ok(())
1031 }
1032
1033 async fn start(
1034 &self,
1035 _sandbox: &mut PodSandbox,
1036 _config: &PodSandboxConfig,
1037 _pool_config: &PoolConfig,
1038 _annotations: &HashMap<String, String>,
1039 ) -> WorkerResult<Arc<dyn SandboxHandle>> {
1040 Ok(Arc::new(FakeHandle))
1041 }
1042
1043 async fn stop(&self, _sandbox: &PodSandbox) -> WorkerResult<()> {
1044 self.stopped.store(true, Ordering::SeqCst);
1045 Ok(())
1046 }
1047
1048 async fn destroy(&self, _sandbox: &PodSandbox) -> WorkerResult<()> {
1049 self.destroyed.store(true, Ordering::SeqCst);
1050 Ok(())
1051 }
1052
1053 async fn reset(&self, _sandbox: &mut PodSandbox) -> WorkerResult<bool> {
1054 Ok(false)
1056 }
1057
1058 async fn get_pids(&self, _sandbox: &PodSandbox) -> WorkerResult<Vec<u32>> {
1059 Ok(vec![])
1060 }
1061
1062 fn supports_exec(&self) -> bool {
1063 true
1064 }
1065
1066 async fn exec_sync(
1067 &self,
1068 _sandbox: &PodSandbox,
1069 _command: &[String],
1070 _timeout_secs: u64,
1071 ) -> WorkerResult<(i32, Vec<u8>, Vec<u8>)> {
1072 Ok((0, b"fake stdout\n".to_vec(), b"fake stderr\n".to_vec()))
1073 }
1074
1075 async fn update_resources(
1076 &self,
1077 _sandbox: &PodSandbox,
1078 _resources: &LinuxContainerResources,
1079 ) -> WorkerResult<()> {
1080 Ok(())
1081 }
1082 }
1083
1084 async fn make_pool() -> Arc<SandboxPool> {
1085 let backend: Arc<dyn SandboxBackend> = Arc::new(FakeBackend::default());
1086 let config = PoolConfig {
1087 max_sandboxes: 10,
1088 warm_pool_size: 0,
1089 ..Default::default()
1090 };
1091 Arc::new(SandboxPool::new(config, backend))
1092 }
1093
1094 fn subject() -> Subject {
1095 Subject::anonymous()
1096 }
1097
1098 fn admission_subject() -> Subject {
1105 Subject::new("test-user")
1106 }
1107
1108 #[tokio::test]
1109 async fn list_instances_empty() {
1110 let pool = make_pool().await;
1111 let mount = ExecMount::new(pool);
1112 let fid = mount.walk(&[], &subject()).await.unwrap();
1113 let entries = mount.readdir(&fid, &subject()).await.unwrap();
1114 assert!(entries.is_empty());
1115 }
1116
1117 #[tokio::test]
1118 async fn list_instances_after_acquire() {
1119 let pool = make_pool().await;
1120 let id = pool
1121 .acquire(&admission_subject(), &PodSandboxConfig::default())
1122 .await
1123 .unwrap();
1124 let mount = ExecMount::new(pool);
1125
1126 let fid = mount.walk(&[], &subject()).await.unwrap();
1127 let entries = mount.readdir(&fid, &subject()).await.unwrap();
1128 let names: Vec<&str> = entries.iter().map(|e| e.name.as_str()).collect();
1129 assert!(names.contains(&id.as_str()));
1130 assert!(entries.iter().all(|e| e.is_dir));
1131 }
1132
1133 #[tokio::test]
1134 async fn walk_unknown_instance_not_found() {
1135 let pool = make_pool().await;
1136 let mount = ExecMount::new(pool);
1137 let result = mount.walk(&["nonexistent", "status"], &subject()).await;
1138 assert!(matches!(result, Err(MountError::NotFound(_))));
1139 }
1140
1141 #[tokio::test]
1142 async fn read_status_reflects_pool_state() {
1143 let pool = make_pool().await;
1144 let id = pool
1145 .acquire(&admission_subject(), &PodSandboxConfig::default())
1146 .await
1147 .unwrap();
1148 let mount = ExecMount::new(pool);
1149
1150 let mut fid = mount.walk(&[&id, "status"], &subject()).await.unwrap();
1151 mount.open(&mut fid, 0, &subject()).await.unwrap();
1152 let data = mount.read(&fid, 0, 4096, &subject()).await.unwrap();
1153 let text = String::from_utf8(data).unwrap();
1154 assert!(text.contains("SandboxReady"), "got: {text}");
1155 }
1156
1157 #[tokio::test]
1158 async fn ctl_stop_invokes_backend() {
1159 let pool = make_pool().await;
1160 let id = pool
1161 .acquire(&admission_subject(), &PodSandboxConfig::default())
1162 .await
1163 .unwrap();
1164 let mount = ExecMount::new(pool);
1165
1166 let mut fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1167 mount.open(&mut fid, 2, &subject()).await.unwrap();
1168 let written = mount.write(&fid, 0, b"stop", &subject()).await.unwrap();
1169 assert_eq!(written, 4);
1170
1171 let data = mount.read(&fid, 0, 4096, &subject()).await.unwrap();
1172 let text = String::from_utf8(data).unwrap();
1173 assert!(text.starts_with("ok:"), "got: {text}");
1174 }
1175
1176 #[tokio::test]
1177 async fn ctl_unknown_verb_rejected() {
1178 let pool = make_pool().await;
1179 let id = pool
1180 .acquire(&admission_subject(), &PodSandboxConfig::default())
1181 .await
1182 .unwrap();
1183 let mount = ExecMount::new(pool);
1184
1185 let mut fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1186 mount.open(&mut fid, 2, &subject()).await.unwrap();
1187 let result = mount.write(&fid, 0, b"frobnicate", &subject()).await;
1188 assert!(matches!(result, Err(MountError::InvalidArgument(_))));
1189 }
1190
1191 #[tokio::test]
1196 async fn exit_blocks_then_unblocks_on_destroy() {
1197 let pool = make_pool().await;
1198 let id = pool
1199 .acquire(&admission_subject(), &PodSandboxConfig::default())
1200 .await
1201 .unwrap();
1202 let mount = Arc::new(ExecMount::new(pool));
1203
1204 let reader = {
1205 let mount = Arc::clone(&mount);
1206 let id = id.clone();
1207 tokio::spawn(async move {
1208 let mut fid = mount.walk(&[&id, "exit"], &subject()).await.unwrap();
1209 mount.open(&mut fid, 0, &subject()).await.unwrap();
1210 mount.read(&fid, 0, 4096, &subject()).await.unwrap()
1211 })
1212 };
1213
1214 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1218 assert!(
1219 !reader.is_finished(),
1220 "exit read must block before terminal"
1221 );
1222
1223 let mut ctl_fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1226 mount.open(&mut ctl_fid, 2, &subject()).await.unwrap();
1227 mount
1228 .write(&ctl_fid, 0, b"destroy", &subject())
1229 .await
1230 .unwrap();
1231 mount.read(&ctl_fid, 0, 4096, &subject()).await.unwrap();
1232
1233 let data = tokio::time::timeout(std::time::Duration::from_secs(2), reader)
1234 .await
1235 .expect("exit read did not unblock after destroy")
1236 .unwrap();
1237 let text = String::from_utf8(data).unwrap();
1238 assert!(text.starts_with("exited"), "got: {text}");
1239 }
1240
1241 #[tokio::test]
1245 async fn exit_after_destroy_reports_terminal() {
1246 let pool = make_pool().await;
1247 let id = pool
1248 .acquire(&admission_subject(), &PodSandboxConfig::default())
1249 .await
1250 .unwrap();
1251 let mount = ExecMount::new(pool);
1252
1253 let mut ctl_fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1255 mount.open(&mut ctl_fid, 2, &subject()).await.unwrap();
1256 mount
1257 .write(&ctl_fid, 0, b"destroy", &subject())
1258 .await
1259 .unwrap();
1260 let ctl_result = mount.read(&ctl_fid, 0, 4096, &subject()).await.unwrap();
1261 assert!(String::from_utf8(ctl_result).unwrap().starts_with("ok:"));
1262
1263 let mut exit_fid = mount.walk(&[&id, "exit"], &subject()).await.unwrap();
1266 mount.open(&mut exit_fid, 0, &subject()).await.unwrap();
1267 let data = tokio::time::timeout(
1268 std::time::Duration::from_millis(200),
1269 mount.read(&exit_fid, 0, 4096, &subject()),
1270 )
1271 .await
1272 .expect("late exit read must not block")
1273 .unwrap();
1274 let text = String::from_utf8(data).unwrap();
1275 assert!(text.starts_with("exited"), "got: {text}");
1276
1277 let mut status_fid = mount.walk(&[&id, "status"], &subject()).await.unwrap();
1279 mount.open(&mut status_fid, 0, &subject()).await.unwrap();
1280 let data = mount.read(&status_fid, 0, 4096, &subject()).await.unwrap();
1281 assert!(!data.is_empty());
1282 }
1283
1284 #[tokio::test]
1285 async fn ctl_on_terminal_instance_rejected() {
1286 let pool = make_pool().await;
1287 let id = pool
1288 .acquire(&admission_subject(), &PodSandboxConfig::default())
1289 .await
1290 .unwrap();
1291 let mount = ExecMount::new(pool);
1292
1293 let mut ctl_fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1294 mount.open(&mut ctl_fid, 2, &subject()).await.unwrap();
1295 mount
1296 .write(&ctl_fid, 0, b"destroy", &subject())
1297 .await
1298 .unwrap();
1299
1300 let result = mount.write(&ctl_fid, 0, b"stop", &subject()).await;
1301 assert!(result.is_ok()); let data = mount.read(&ctl_fid, 0, 4096, &subject()).await.unwrap();
1306 let text = String::from_utf8(data).unwrap();
1307 assert!(text.starts_with("error:"), "got: {text}");
1308 }
1309
1310 #[tokio::test]
1311 async fn read_ns_returns_placeholder_info() {
1312 let pool = make_pool().await;
1313 let id = pool
1314 .acquire(&admission_subject(), &PodSandboxConfig::default())
1315 .await
1316 .unwrap();
1317 let mount = ExecMount::new(pool);
1318
1319 let mut fid = mount.walk(&[&id, "ns"], &subject()).await.unwrap();
1320 mount.open(&mut fid, 0, &subject()).await.unwrap();
1321 let data = mount.read(&fid, 0, 4096, &subject()).await.unwrap();
1322 let text = String::from_utf8(data).unwrap();
1323 assert!(text.contains("runtime="), "got: {text}");
1324 assert!(text.contains("path="), "got: {text}");
1325 }
1326
1327 #[tokio::test]
1328 async fn readdir_instance_dir_lists_files() {
1329 let pool = make_pool().await;
1330 let id = pool
1331 .acquire(&admission_subject(), &PodSandboxConfig::default())
1332 .await
1333 .unwrap();
1334 let mount = ExecMount::new(pool);
1335
1336 let fid = mount.walk(&[&id], &subject()).await.unwrap();
1337 let entries = mount.readdir(&fid, &subject()).await.unwrap();
1338 let names: Vec<&str> = entries.iter().map(|e| e.name.as_str()).collect();
1339 assert!(names.contains(&"ctl"));
1340 assert!(names.contains(&"status"));
1341 assert!(names.contains(&"exit"));
1342 assert!(names.contains(&"ns"));
1343 assert!(names.contains(&"fd"));
1344
1345 let fd_dir = mount.walk(&[&id, "fd"], &subject()).await.unwrap();
1346 let fd_entries = mount.readdir(&fd_dir, &subject()).await.unwrap();
1347 let fd_names: Vec<&str> = fd_entries.iter().map(|e| e.name.as_str()).collect();
1348 assert_eq!(fd_names, ["1", "2"]);
1349 }
1350
1351 #[tokio::test]
1352 async fn fd_output_blocks_until_runtime_publishes_then_latches_eof() {
1353 let pool = make_pool().await;
1354 let id = pool
1355 .acquire(&admission_subject(), &PodSandboxConfig::default())
1356 .await
1357 .unwrap();
1358 let mount = Arc::new(ExecMount::new(pool));
1359
1360 let reader = {
1361 let mount = Arc::clone(&mount);
1362 let id = id.clone();
1363 tokio::spawn(async move {
1364 let mut fid = mount.walk(&[&id, "fd", "1"], &subject()).await.unwrap();
1365 mount.open(&mut fid, 0, &subject()).await.unwrap();
1366 mount.read(&fid, 0, 4096, &subject()).await.unwrap()
1367 })
1368 };
1369 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1370 assert!(
1371 !reader.is_finished(),
1372 "fd/1 must wait for terminal task output"
1373 );
1374
1375 mount
1376 .publish_exec_result(&id, b"build output\n", b"warning output\n")
1377 .await
1378 .unwrap();
1379
1380 let stdout = tokio::time::timeout(std::time::Duration::from_secs(2), reader)
1381 .await
1382 .expect("fd/1 reader did not wake on task completion")
1383 .unwrap();
1384 assert_eq!(stdout, b"build output\n");
1385
1386 let mut stderr_fid = mount.walk(&[&id, "fd", "2"], &subject()).await.unwrap();
1387 mount.open(&mut stderr_fid, 0, &subject()).await.unwrap();
1388 assert_eq!(
1389 mount.read(&stderr_fid, 0, 4096, &subject()).await.unwrap(),
1390 b"warning output\n"
1391 );
1392 assert!(
1393 mount
1394 .read(&stderr_fid, 4096, 4096, &subject())
1395 .await
1396 .unwrap()
1397 .is_empty(),
1398 "terminal stderr must report EOF after retained bytes"
1399 );
1400 }
1401
1402 #[tokio::test]
1414 async fn fd_read_at_nonzero_offset_is_not_double_applied() {
1415 let pool = make_pool().await;
1416 let id = pool
1417 .acquire(&admission_subject(), &PodSandboxConfig::default())
1418 .await
1419 .unwrap();
1420 let mount = ExecMount::new(pool);
1421
1422 mount
1423 .publish_exec_result(&id, b"0123456789", b"")
1424 .await
1425 .unwrap();
1426
1427 let mut fid = mount.walk(&[&id, "fd", "1"], &subject()).await.unwrap();
1428 mount.open(&mut fid, 0, &subject()).await.unwrap();
1429
1430 let data = mount.read(&fid, 3, 4, &subject()).await.unwrap();
1431 assert_eq!(
1432 data, b"3456",
1433 "offset=3,count=4 over a 10-byte retained buffer must return bytes[3..7] \
1434 verbatim from read_until_closed, not re-offset a second time"
1435 );
1436 }
1437
1438 #[tokio::test]
1439 async fn ctl_exec_is_projected_to_terminal_fd_streams() {
1440 let pool = make_pool().await;
1441 let id = pool
1442 .acquire(&admission_subject(), &PodSandboxConfig::default())
1443 .await
1444 .unwrap();
1445 let mount = ExecMount::new(pool);
1446
1447 let mut ctl = mount
1448 .walk(&[&id, "ctl"], &admission_subject())
1449 .await
1450 .unwrap();
1451 mount.open(&mut ctl, 2, &admission_subject()).await.unwrap();
1452 mount
1453 .write(&ctl, 0, b"exec build", &admission_subject())
1454 .await
1455 .unwrap();
1456 assert_eq!(
1457 mount
1458 .read(&ctl, 0, 4096, &admission_subject())
1459 .await
1460 .unwrap(),
1461 b"ok: exited 0\n"
1462 );
1463 let mut stdout = mount.walk(&[&id, "fd", "1"], &subject()).await.unwrap();
1464 mount.open(&mut stdout, 0, &subject()).await.unwrap();
1465 assert_eq!(
1466 mount.read(&stdout, 0, 4096, &subject()).await.unwrap(),
1467 b"fake stdout\n"
1468 );
1469 let mut stderr = mount.walk(&[&id, "fd", "2"], &subject()).await.unwrap();
1470 mount.open(&mut stderr, 0, &subject()).await.unwrap();
1471 assert_eq!(
1472 mount.read(&stderr, 0, 4096, &subject()).await.unwrap(),
1473 b"fake stderr\n"
1474 );
1475 }
1476
1477 #[tokio::test]
1478 async fn fd_zero_is_omitted_until_the_task_runtime_consumes_stdin() {
1479 let pool = make_pool().await;
1480 let id = pool
1481 .acquire(&admission_subject(), &PodSandboxConfig::default())
1482 .await
1483 .unwrap();
1484 let mount = ExecMount::new(pool);
1485 assert!(matches!(
1486 mount.walk(&[&id, "fd", "0"], &subject()).await,
1487 Err(MountError::NotFound(_))
1488 ));
1489 }
1490
1491 #[tokio::test]
1492 async fn ctl_exec_rejects_a_different_subject() {
1493 let pool = make_pool().await;
1494 let id = pool
1495 .acquire(&admission_subject(), &PodSandboxConfig::default())
1496 .await
1497 .unwrap();
1498 let mount = ExecMount::new(pool);
1499 let other = Subject::new("other-user");
1500 let mut ctl = mount.walk(&[&id, "ctl"], &other).await.unwrap();
1501 mount.open(&mut ctl, 2, &other).await.unwrap();
1502 mount.write(&ctl, 0, b"exec build", &other).await.unwrap();
1503 let reply = mount.read(&ctl, 0, 4096, &other).await.unwrap();
1504 assert!(String::from_utf8(reply)
1505 .unwrap()
1506 .contains("caller does not own this task's Subject-scoped namespace"));
1507 }
1508
1509 #[tokio::test]
1510 async fn stopping_instance_latches_waiting_fd_reader_to_eof() {
1511 let pool = make_pool().await;
1512 let id = pool
1513 .acquire(&admission_subject(), &PodSandboxConfig::default())
1514 .await
1515 .unwrap();
1516 let mount = Arc::new(ExecMount::new(pool));
1517 let reader = {
1518 let mount = Arc::clone(&mount);
1519 let id = id.clone();
1520 tokio::spawn(async move {
1521 let mut fid = mount.walk(&[&id, "fd", "2"], &subject()).await.unwrap();
1522 mount.open(&mut fid, 0, &subject()).await.unwrap();
1523 mount.read(&fid, 0, 4096, &subject()).await.unwrap()
1524 })
1525 };
1526 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1527 assert!(
1528 !reader.is_finished(),
1529 "fd reader must wait before cancellation"
1530 );
1531
1532 let mut ctl = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1533 mount.open(&mut ctl, 2, &subject()).await.unwrap();
1534 mount.write(&ctl, 0, b"stop", &subject()).await.unwrap();
1535
1536 assert!(
1537 tokio::time::timeout(std::time::Duration::from_secs(2), reader)
1538 .await
1539 .expect("cancellation did not wake fd reader")
1540 .unwrap()
1541 .is_empty(),
1542 "cancelled stream must latch terminal EOF"
1543 );
1544 }
1545
1546 #[tokio::test]
1547 async fn stat_root_is_dir() {
1548 let pool = make_pool().await;
1549 let mount = ExecMount::new(pool);
1550 let fid = mount.walk(&[], &subject()).await.unwrap();
1551 let st = mount.stat(&fid, &subject()).await.unwrap();
1552 assert_eq!(st.qtype, 0x80);
1553 assert_eq!(st.name, "instances");
1554 }
1555
1556 #[tokio::test]
1566 async fn ctl_concurrent_writes_to_one_fid_are_safe() {
1567 let pool = make_pool().await;
1568 let id = pool
1569 .acquire(&admission_subject(), &PodSandboxConfig::default())
1570 .await
1571 .unwrap();
1572 let mount = ExecMount::new(pool);
1573
1574 let mut fid = mount.walk(&[&id, "ctl"], &subject()).await.unwrap();
1576 mount.open(&mut fid, 2, &subject()).await.unwrap();
1577
1578 let caller = subject();
1583 let w1 = mount.write(&fid, 0, b"stop", &caller);
1584 let w2 = mount.write(&fid, 0, b"stop", &caller);
1585 let w3 = mount.write(&fid, 0, b"kill", &caller);
1586 let (r1, r2, r3) = futures::future::join3(w1, w2, w3).await;
1587 for r in [r1, r2, r3] {
1588 r.unwrap();
1589 }
1590
1591 let out = mount.read(&fid, 0, 4096, &caller).await.unwrap();
1592 let text = String::from_utf8(out).unwrap();
1593 assert!(
1594 text.starts_with("ok:"),
1595 "expected one of the stop/kill responses latched, got: {text}"
1596 );
1597 }
1598}