1use std::collections::HashMap;
26use std::sync::Arc;
27use tokio::sync::RwLock;
28use tokio_util::sync::CancellationToken;
29
30use anyhow::Result as AnyhowResult;
31use async_trait::async_trait;
32use tracing::{debug, info, warn};
33
34use hyprstream_rpc::events::EventPublisher;
38use hyprstream_rpc::moq_stream::AnyStreamPublisher;
39use hyprstream_rpc::prelude::SigningKey;
40use hyprstream_rpc::service::{AuthorizeFn, EnvelopeContext, RequestService};
41use hyprstream_rpc::streaming::StreamChannel;
42use hyprstream_rpc::transport::TransportConfig;
43
44use crate::config::PoolConfig;
45use crate::error::{Result, WorkerError};
46use crate::events::{
47 serialize_container_started,
48 serialize_container_stopped,
49 serialize_sandbox_started,
50 serialize_sandbox_stopped,
51 ContainerStarted,
53 ContainerStopped,
54 SandboxStarted,
55 SandboxStopped,
56};
57use crate::image::ImageStore;
62use super::status::StatusResponse;
64use crate::generated::worker_client::{
65 dispatch_worker, ContainerHandler, ImageHandler, RuntimeHandler, SandboxHandler, WorkerHandler,
66};
67use hyprstream_rpc_std::worker_client::{
68 AttachRequest,
69 AuthConfig,
70 ContainerAttributes,
71 ContainerConfig,
72 ContainerFilter,
73 ContainerInfo,
74 ContainerState,
75 ContainerStats,
76 ContainerStatsFilter,
77 ContainerStatus,
78 ContainerStatusRequest,
79 ContainerStatusResponse,
80 ExecSyncResult,
81 FilesystemUsage,
82 ImageFilter,
83 ImageInfo,
84 ImageSpec,
85 ImageStatusRequest,
86 ImageStatusResult,
87 KeyValue,
88 LinuxPodSandboxStats,
89 PodSandboxAttributes,
90 PodSandboxConfig,
91 PodSandboxFilter,
93 PodSandboxInfo,
94 PodSandboxStats,
95 PodSandboxStatsFilter,
96 PodSandboxStatus,
97 PodSandboxStatusRequest,
98 PodSandboxStatusResponse,
99 RuntimeCondition,
100 RuntimeStatus,
101 StatusRequest,
102 StopContainerRequest,
103 StreamInfo,
104 Timestamp,
105 VersionInfo,
106};
107use super::container::Container;
109use super::pool::SandboxPool;
110use super::sandbox::PodSandbox;
111use super::ExecMount;
112use super::{RUNTIME_NAME, RUNTIME_VERSION};
113
114const SERVICE_NAME: &str = "worker";
116
117pub struct WorkerService {
122 sandbox_pool: Arc<SandboxPool>,
125
126 exec_mount: Arc<ExecMount>,
130
131 image_store: Option<Arc<dyn ImageStore>>,
139
140 containers: RwLock<HashMap<String, Container>>,
142
143 container_sandbox_map: RwLock<HashMap<String, String>>,
145
146 event_publisher: tokio::sync::Mutex<EventPublisher>,
148
149 active_fd_streams: Arc<RwLock<HashMap<String, ActiveFdStream>>>,
151
152 stream_channel: Arc<StreamChannel>,
154
155 transport: TransportConfig,
157 signing_key: SigningKey,
158
159 authorize_fn: Option<AuthorizeFn>,
161 expected_audience: Option<String>,
163 jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
165}
166
167impl WorkerService {
168 pub fn new(
172 pool_config: PoolConfig,
173 backend: Arc<dyn super::backend::SandboxBackend>,
174 image_store: Option<Arc<dyn ImageStore>>,
178 transport: TransportConfig,
179 signing_key: SigningKey,
180 ) -> AnyhowResult<Self> {
181 let sandbox_pool = Arc::new(SandboxPool::new(pool_config, backend));
182 let exec_mount = Arc::new(ExecMount::new(Arc::clone(&sandbox_pool)));
183
184 let event_publisher = EventPublisher::new("worker")?;
186
187 let stream_channel = Arc::new(
188 StreamChannel::new(signing_key.clone())
189 .with_reach_config(hyprstream_rpc::moq_stream::ProducerReachConfig::default()),
190 );
191 Ok(Self {
192 sandbox_pool,
193 exec_mount,
194 image_store,
195 containers: RwLock::new(HashMap::new()),
196 container_sandbox_map: RwLock::new(HashMap::new()),
197 event_publisher: tokio::sync::Mutex::new(event_publisher),
198 active_fd_streams: Arc::new(RwLock::new(HashMap::new())),
199 stream_channel,
200 transport,
201 signing_key,
202 authorize_fn: None,
203 expected_audience: None,
204 jwt_key_source: None,
205 })
206 }
207
208 pub fn set_authorize_fn(&mut self, authorize_fn: AuthorizeFn) {
210 self.authorize_fn = Some(authorize_fn);
211 }
212
213 pub fn set_expected_audience(&mut self, audience: String) {
215 self.expected_audience = Some(audience);
216 }
217
218 pub fn set_jwt_key_source(
220 &mut self,
221 src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
222 ) {
223 self.jwt_key_source = Some(src);
224 }
225
226 pub async fn initialize(&self) -> Result<()> {
228 self.sandbox_pool.initialize().await?;
229 tracing::info!("WorkerService initialized");
230 Ok(())
231 }
232
233 pub async fn shutdown(&self) -> Result<()> {
235 self.sandbox_pool.shutdown().await?;
236 tracing::info!("WorkerService shutdown complete");
237 Ok(())
238 }
239
240 pub async fn version(&self, _version: &str) -> Result<VersionInfo> {
246 Ok(VersionInfo {
247 version: "v1".to_owned(),
248 runtime_name: RUNTIME_NAME.to_owned(),
249 runtime_version: RUNTIME_VERSION.to_owned(),
250 runtime_api_version: "v1".to_owned(),
251 })
252 }
253
254 pub async fn status(&self, verbose: bool) -> Result<StatusResponse> {
256 let conditions = vec![
257 RuntimeCondition {
258 condition_type: "RuntimeReady".to_owned(),
259 status: true,
260 reason: String::new(),
261 message: "Runtime is ready".to_owned(),
262 },
263 RuntimeCondition {
264 condition_type: "NetworkReady".to_owned(),
265 status: true,
266 reason: String::new(),
267 message: "Network is ready".to_owned(),
268 },
269 ];
270
271 let mut info = HashMap::new();
272 if verbose {
273 let stats = self.sandbox_pool.stats().await;
274 info.insert("warm_pool_size".to_owned(), stats.warm_count.to_string());
275 info.insert(
276 "active_sandboxes".to_owned(),
277 stats.active_count.to_string(),
278 );
279 info.insert("max_sandboxes".to_owned(), stats.max_sandboxes.to_string());
280 }
281
282 Ok(StatusResponse {
283 status: RuntimeStatus { conditions },
284 info,
285 })
286 }
287
288 pub async fn run_pod_sandbox(
300 &self,
301 subject: &hyprstream_vfs::Subject,
302 config: &PodSandboxConfig,
303 ) -> Result<String> {
304 let sandbox_id = self.sandbox_pool.acquire(subject, config).await?;
305 tracing::info!(sandbox_id = %sandbox_id, "Created pod sandbox");
306
307 let event = SandboxStarted {
309 sandbox_id: sandbox_id.clone(),
310 metadata: serde_json::to_string(&config.metadata).unwrap_or_default(),
311 vm_pid: 0, };
313 let payload = serialize_sandbox_started(&event).unwrap_or_default();
314 self.publish_event(&sandbox_id, "started", &payload).await;
315
316 Ok(sandbox_id)
317 }
318
319 async fn publish_event(&self, entity_id: &str, event: &str, payload: &[u8]) {
323 let publisher = self.event_publisher.lock().await;
324 if let Err(e) = publisher.publish(entity_id, event, payload).await {
325 tracing::warn!(
326 entity_id = %entity_id,
327 event = %event,
328 error = %e,
329 "Failed to publish worker event"
330 );
331 }
332 }
333
334 pub async fn stop_pod_sandbox(&self, pod_sandbox_id: &str) -> Result<()> {
336 let containers_to_stop: Vec<String> = {
338 let map = self.container_sandbox_map.read().await;
339 map.iter()
340 .filter(|(_, sid)| *sid == pod_sandbox_id)
341 .map(|(cid, _)| cid.clone())
342 .collect()
343 };
344
345 for container_id in containers_to_stop {
346 if let Err(e) = self.stop_container(&container_id, 30).await {
347 tracing::warn!(
348 container_id = %container_id,
349 error = %e,
350 "Failed to stop container during sandbox stop"
351 );
352 }
353 }
354
355 tracing::info!(sandbox_id = %pod_sandbox_id, "Stopped pod sandbox");
358
359 let event = SandboxStopped {
361 sandbox_id: pod_sandbox_id.to_owned(),
362 reason: "stopped".to_owned(),
363 exit_code: 0,
364 };
365 let payload = serialize_sandbox_stopped(&event).unwrap_or_default();
366 self.publish_event(pod_sandbox_id, "stopped", &payload)
367 .await;
368
369 Ok(())
370 }
371
372 pub async fn remove_pod_sandbox(&self, pod_sandbox_id: &str) -> Result<()> {
374 let containers_to_remove: Vec<String> = {
376 let map = self.container_sandbox_map.read().await;
377 map.iter()
378 .filter(|(_, sid)| *sid == pod_sandbox_id)
379 .map(|(cid, _)| cid.clone())
380 .collect()
381 };
382
383 for container_id in containers_to_remove {
384 if let Err(e) = self.remove_container(&container_id).await {
385 tracing::warn!(
386 container_id = %container_id,
387 error = %e,
388 "Failed to remove container during sandbox removal"
389 );
390 }
391 }
392
393 self.sandbox_pool.release(pod_sandbox_id).await?;
395 tracing::info!(sandbox_id = %pod_sandbox_id, "Removed pod sandbox");
396 Ok(())
397 }
398
399 pub async fn pod_sandbox_status(
404 &self,
405 pod_sandbox_id: &str,
406 verbose: bool,
407 ) -> Result<(PodSandboxStatus, Vec<KeyValue>)> {
408 let sandbox = self
409 .sandbox_pool
410 .get(pod_sandbox_id)
411 .await
412 .ok_or_else(|| WorkerError::SandboxNotFound(pod_sandbox_id.to_owned()))?;
413
414 let mut info = Vec::new();
415 if verbose {
416 let pids = self.sandbox_pool.backend().get_pids(&sandbox).await.ok();
418 let pid_str = pids.map_or_else(
419 || "none".to_owned(),
420 |p| {
421 p.iter()
422 .map(std::string::ToString::to_string)
423 .collect::<Vec<_>>()
424 .join(",")
425 },
426 );
427 info.push(KeyValue {
428 key: "vm_pids".to_owned(),
429 value: pid_str,
430 });
431 }
432
433 Ok((PodSandboxStatus::from(&sandbox), info))
434 }
435
436 pub async fn list_pod_sandbox(&self, filter: &PodSandboxFilter) -> Result<Vec<PodSandboxInfo>> {
441 let mut sandboxes = self.sandbox_pool.list_active().await;
442
443 if !filter.id.is_empty() {
444 sandboxes.retain(|s| s.id == filter.id);
445 }
446 for kv in &filter.label_selector {
447 sandboxes.retain(|s| {
448 s.labels
449 .iter()
450 .any(|l| l.key == kv.key && l.value == kv.value)
451 });
452 }
453
454 Ok(sandboxes.iter().map(PodSandboxInfo::from).collect())
455 }
456
457 pub async fn create_container(
470 &self,
471 pod_sandbox_id: &str,
472 config: &ContainerConfig,
473 _sandbox_config: &PodSandboxConfig,
474 ) -> Result<String> {
475 let sandbox = self
477 .sandbox_pool
478 .get(pod_sandbox_id)
479 .await
480 .ok_or_else(|| WorkerError::SandboxNotFound(pod_sandbox_id.to_owned()))?;
481
482 if !sandbox.is_ready() {
483 return Err(WorkerError::SandboxInvalidState {
484 sandbox_id: pod_sandbox_id.to_owned(),
485 state: "NotReady".to_owned(),
486 expected: "Ready".to_owned(),
487 });
488 }
489
490 let container_id = uuid::Uuid::new_v4().to_string();
492
493 if !config.image.image.is_empty() {
496 tracing::debug!(
497 container_id = %container_id,
498 image = %config.image.image,
499 "Container image reference recorded (available via virtiofs)"
500 );
501 }
504
505 let container = Container::new(container_id.clone(), pod_sandbox_id.to_owned(), config);
507
508 {
510 let mut containers = self.containers.write().await;
511 containers.insert(container_id.clone(), container);
512 }
513 {
514 let mut map = self.container_sandbox_map.write().await;
515 map.insert(container_id.clone(), pod_sandbox_id.to_owned());
516 }
517
518 tracing::info!(
519 container_id = %container_id,
520 sandbox_id = %pod_sandbox_id,
521 image = %config.image.image,
522 "Created container (host-level, rootfs via virtiofs)"
523 );
524
525 Ok(container_id)
526 }
527
528 pub async fn start_container(&self, container_id: &str) -> Result<()> {
535 let sandbox_id = {
537 let map = self.container_sandbox_map.read().await;
538 map.get(container_id)
539 .cloned()
540 .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?
541 };
542
543 let sandbox = self
544 .sandbox_pool
545 .get(&sandbox_id)
546 .await
547 .ok_or_else(|| WorkerError::SandboxNotFound(sandbox_id.clone()))?;
548
549 if !sandbox.is_ready() {
551 return Err(WorkerError::SandboxInvalidState {
552 sandbox_id: sandbox_id.clone(),
553 state: "NotReady".to_owned(),
554 expected: "Ready".to_owned(),
555 });
556 }
557
558 let mut containers = self.containers.write().await;
559
560 let container = containers
561 .get_mut(container_id)
562 .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
563
564 if container.state != ContainerState::ContainerCreated {
565 return Err(WorkerError::ContainerInvalidState {
566 container_id: container_id.to_owned(),
567 state: format!("{:?}", container.state),
568 expected: "ContainerCreated".to_owned(),
569 });
570 }
571
572 if sandbox.backend_handle.is_some() {
581 tracing::debug!(
582 container_id = %container_id,
583 sandbox_id = %sandbox_id,
584 "Container filesystem accessible via sandbox's virtiofs mount"
585 );
586 }
587
588 container.state = ContainerState::ContainerRunning;
589 let image = container.image.image.clone();
590 tracing::info!(
591 container_id = %container_id,
592 sandbox_id = %sandbox_id,
593 "Started container (host-level, VM is running)"
594 );
595
596 drop(containers);
598
599 let event = ContainerStarted {
601 container_id: container_id.to_owned(),
602 sandbox_id: sandbox_id.clone(),
603 image,
604 };
605 let payload = serialize_container_started(&event).unwrap_or_default();
606 self.publish_event(container_id, "started", &payload).await;
607
608 Ok(())
609 }
610
611 pub async fn stop_container(&self, container_id: &str, _timeout: i64) -> Result<()> {
618 let mut containers = self.containers.write().await;
619
620 let container = containers
621 .get_mut(container_id)
622 .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
623
624 if container.state != ContainerState::ContainerRunning {
625 return Ok(());
627 }
628
629 container.state = ContainerState::ContainerExited;
640 container.exit_code = Some(0);
641 let sandbox_id = container.pod_sandbox_id.clone();
642 let exit_code = container.exit_code.unwrap_or(0);
643 tracing::info!(
644 container_id = %container_id,
645 "Stopped container (host-level, container marked as exited)"
646 );
647
648 drop(containers);
650
651 let event = ContainerStopped {
653 container_id: container_id.to_owned(),
654 sandbox_id,
655 exit_code,
656 reason: "stopped".to_owned(),
657 };
658 let payload = serialize_container_stopped(&event).unwrap_or_default();
659 self.publish_event(container_id, "stopped", &payload).await;
660
661 Ok(())
662 }
663
664 pub async fn remove_container(&self, container_id: &str) -> Result<()> {
666 {
668 let mut containers = self.containers.write().await;
669 let container = containers
670 .get(container_id)
671 .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
672
673 if container.state == ContainerState::ContainerRunning {
674 return Err(WorkerError::ContainerInvalidState {
675 container_id: container_id.to_owned(),
676 state: "ContainerRunning".to_owned(),
677 expected: "ContainerExited or ContainerCreated".to_owned(),
678 });
679 }
680
681 containers.remove(container_id);
682 }
683
684 {
686 let mut map = self.container_sandbox_map.write().await;
687 map.remove(container_id);
688 }
689
690 tracing::info!(container_id = %container_id, "Removed container");
691 Ok(())
692 }
693
694 pub async fn container_status(
699 &self,
700 container_id: &str,
701 verbose: bool,
702 ) -> Result<(ContainerStatus, Vec<KeyValue>)> {
703 let containers = self.containers.read().await;
704
705 let container = containers
706 .get(container_id)
707 .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
708
709 let mut info = Vec::new();
710 if verbose {
711 info.push(KeyValue {
712 key: "pid".to_owned(),
713 value: container
714 .pid
715 .map_or_else(|| "none".to_owned(), |p| p.to_string()),
716 });
717 }
718
719 Ok((ContainerStatus::from(container), info))
720 }
721
722 pub async fn list_containers(&self, filter: &ContainerFilter) -> Result<Vec<ContainerInfo>> {
727 let containers = self.containers.read().await;
728 let mut result: Vec<Container> = containers.values().cloned().collect();
729
730 if !filter.id.is_empty() {
731 result.retain(|c| c.id == filter.id);
732 }
733 if !filter.pod_sandbox_id.is_empty() {
734 result.retain(|c| c.pod_sandbox_id == filter.pod_sandbox_id);
735 }
736 for kv in &filter.label_selector {
737 result.retain(|c| {
738 c.labels
739 .iter()
740 .any(|l| l.key == kv.key && l.value == kv.value)
741 });
742 }
743
744 Ok(result.iter().map(ContainerInfo::from).collect())
745 }
746
747 pub async fn exec_sync(
756 &self,
757 caller: &hyprstream_vfs::Subject,
758 container_id: &str,
759 cmd: &[String],
760 timeout: i64,
761 ) -> Result<ExecSyncResult> {
762 let containers = self.containers.read().await;
763
764 let container = containers
765 .get(container_id)
766 .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
767
768 if container.state != ContainerState::ContainerRunning {
769 return Err(WorkerError::ContainerInvalidState {
770 container_id: container_id.to_owned(),
771 state: format!("{:?}", container.state),
772 expected: "ContainerRunning".to_owned(),
773 });
774 }
775
776 drop(containers);
777
778 let sandbox_id = self
779 .container_sandbox_map
780 .read()
781 .await
782 .get(container_id)
783 .cloned()
784 .ok_or_else(|| WorkerError::SandboxNotFound(container_id.to_owned()))?;
785 let timeout_secs = u64::try_from(timeout)
786 .map_err(|_| WorkerError::ExecFailed("exec timeout must be non-negative".to_owned()))?;
787 let result = self
788 .exec_mount
789 .exec_task(&sandbox_id, caller, cmd, timeout_secs)
790 .await
791 .map_err(|error| WorkerError::ExecFailed(error.to_string()))?;
792
793 Ok(ExecSyncResult {
794 stdout: result.stdout,
795 stderr: result.stderr,
796 exit_code: result.exit_code,
797 })
798 }
799
800 pub async fn pod_sandbox_stats(&self, pod_sandbox_id: &str) -> Result<PodSandboxStats> {
806 let sandbox = self
807 .sandbox_pool
808 .get(pod_sandbox_id)
809 .await
810 .ok_or_else(|| WorkerError::SandboxNotFound(pod_sandbox_id.to_owned()))?;
811
812 Ok(PodSandboxStats::from(&sandbox))
814 }
815
816 pub async fn list_pod_sandbox_stats(
818 &self,
819 filter: &PodSandboxStatsFilter,
820 ) -> Result<Vec<PodSandboxStats>> {
821 let sandboxes = self.sandbox_pool.list_active().await;
822 let mut results = Vec::new();
823
824 for sandbox in sandboxes {
825 if !filter.id.is_empty() && sandbox.id != filter.id {
826 continue;
827 }
828 let mut matches_labels = true;
829 for kv in &filter.label_selector {
830 if !sandbox
831 .labels
832 .iter()
833 .any(|l| l.key == kv.key && l.value == kv.value)
834 {
835 matches_labels = false;
836 break;
837 }
838 }
839 if !matches_labels {
840 continue;
841 }
842 results.push(PodSandboxStats::from(&sandbox));
843 }
844
845 Ok(results)
846 }
847
848 pub async fn container_stats(&self, container_id: &str) -> Result<ContainerStats> {
850 let (mut stats, pod_sandbox_id) = {
854 let containers = self.containers.read().await;
855 let container = containers
856 .get(container_id)
857 .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
858 (
859 ContainerStats::from(container),
860 container.pod_sandbox_id.clone(),
861 )
862 };
863
864 self.enrich_container_stats(container_id, &pod_sandbox_id, &mut stats)
865 .await;
866 Ok(stats)
867 }
868
869 async fn enrich_container_stats(
876 &self,
877 container_id: &str,
878 pod_sandbox_id: &str,
879 stats: &mut ContainerStats,
880 ) {
881 let Some(sandbox) = self.sandbox_pool.get(pod_sandbox_id).await else {
882 return;
883 };
884 match self.sandbox_pool.backend().container_stats(&sandbox).await {
885 Ok(Some((cpu, memory))) => {
886 stats.cpu = cpu;
887 stats.memory = memory;
888 }
889 Ok(None) => {}
890 Err(e) => {
891 tracing::warn!(
892 container_id = %container_id,
893 error = %e,
894 "Failed to fetch guest container stats; returning placeholder stats"
895 );
896 }
897 }
898 }
899
900 pub async fn list_container_stats(
902 &self,
903 filter: &ContainerStatsFilter,
904 ) -> Result<Vec<ContainerStats>> {
905 let snapshots: Vec<(ContainerStats, String, String)> = {
910 let containers = self.containers.read().await;
911 containers
912 .values()
913 .filter(|container| {
914 if !filter.id.is_empty() && container.id != filter.id {
915 return false;
916 }
917 if !filter.pod_sandbox_id.is_empty()
918 && container.pod_sandbox_id != filter.pod_sandbox_id
919 {
920 return false;
921 }
922 filter.label_selector.iter().all(|kv| {
923 container
924 .labels
925 .iter()
926 .any(|l| l.key == kv.key && l.value == kv.value)
927 })
928 })
929 .map(|container| {
930 (
931 ContainerStats::from(container),
932 container.id.clone(),
933 container.pod_sandbox_id.clone(),
934 )
935 })
936 .collect()
937 };
938
939 let mut results = Vec::with_capacity(snapshots.len());
940 for (mut stats, container_id, pod_sandbox_id) in snapshots {
941 self.enrich_container_stats(&container_id, &pod_sandbox_id, &mut stats)
942 .await;
943 results.push(stats);
944 }
945
946 Ok(results)
947 }
948
949 pub async fn prepare_attach(
965 &self,
966 ctx: &EnvelopeContext,
967 container_id: &str,
968 ) -> Result<(StreamInfo, hyprstream_rpc::service::Continuation)> {
969 let containers = self.containers.read().await;
971 if !containers.contains_key(container_id) {
972 return Err(anyhow::anyhow!("Container not found: {}", container_id).into());
973 }
974 drop(containers);
975
976 let client_pubkey = ctx.ephemeral_pubkey().ok_or_else(|| {
978 anyhow::anyhow!("Attach requires client ephemeral pubkey for E2E authentication")
979 })?;
980
981 let claims = ctx.claims().cloned();
983
984 let stream_ctx = self
986 .stream_channel
987 .prepare_third_party_interop_stream_with_claims(&client_pubkey, 600, claims)
988 .await
989 .map_err(|e| anyhow::anyhow!("Stream preparation failed: {}", e))?
990 .with_qos_preset::<hyprstream_rpc::stream_info::Pipe>();
991
992 let stream_id = stream_ctx.stream_id().to_owned();
993 let broadcast_path = hyprstream_rpc::moq_stream::global_moq_origin()
994 .map(|o| o.broadcast_path(stream_ctx.topic()))
995 .unwrap_or_default();
996
997 let cancel_token = stream_ctx.cancel_token().child_token();
999 self.active_fd_streams.write().await.insert(
1000 stream_id.clone(),
1001 ActiveFdStream {
1002 container_id: container_id.to_owned(),
1003 cancel_token: cancel_token.clone(),
1004 },
1005 );
1006
1007 let sandbox_id = self
1008 .container_sandbox_map
1009 .read()
1010 .await
1011 .get(container_id)
1012 .cloned();
1013 let sandbox_pool = self.sandbox_pool.clone();
1014 let container_id_owned = container_id.to_owned();
1015 let active_streams = self.active_fd_streams.clone();
1016 let stream_id_for_cleanup = stream_id.clone();
1017 let sc = Arc::clone(&self.stream_channel);
1018
1019 let stream_info = StreamInfo {
1020 stream_id,
1021 dh_public: *stream_ctx.server_pubkey(),
1022 qos: stream_ctx.qos().clone(),
1023 broadcast_path,
1024 announced_at: stream_ctx.reach(), moql_server_identity: stream_ctx.moql_server_identity(),
1026 kem_ciphertexts: Vec::new(), };
1028
1029 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
1031 tokio::spawn(async move {
1032 let container_id_for_log = container_id_owned.clone();
1033 let result = sc
1034 .run_stream(&stream_ctx, |mut publisher| async move {
1035 let result = run_fd_streaming_task(
1036 &mut publisher,
1037 container_id_owned,
1038 cancel_token,
1039 sandbox_pool,
1040 sandbox_id,
1041 )
1042 .await;
1043 (publisher, result)
1044 })
1045 .await;
1046
1047 active_streams.write().await.remove(&stream_id_for_cleanup);
1049
1050 if let Err(e) = result {
1051 warn!(container_id = %container_id_for_log, error = %e, "FD streaming task failed");
1052 }
1053 });
1054 });
1055
1056 Ok((stream_info, continuation))
1057 }
1058
1059 pub async fn detach(&self, container_id: &str) -> Result<()> {
1063 let mut active_streams = self.active_fd_streams.write().await;
1065 let streams_to_cancel: Vec<String> = active_streams
1066 .iter()
1067 .filter(|(_, active)| active.container_id == container_id)
1068 .map(|(stream_id, _)| stream_id.clone())
1069 .collect();
1070
1071 for stream_id in streams_to_cancel {
1072 if let Some(active) = active_streams.remove(&stream_id) {
1073 info!(
1074 stream_id = %stream_id,
1075 container_id = %container_id,
1076 "Cancelling FD stream"
1077 );
1078 active.cancel_token.cancel();
1079 }
1080 }
1081
1082 Ok(())
1083 }
1084}
1085
1086struct ActiveFdStream {
1088 container_id: String,
1089 cancel_token: CancellationToken,
1090}
1091
1092async fn run_fd_streaming_task(
1103 publisher: &mut AnyStreamPublisher,
1104 container_id: String,
1105 cancel_token: CancellationToken,
1106 sandbox_pool: Arc<SandboxPool>,
1107 sandbox_id: Option<String>,
1108) -> anyhow::Result<()> {
1109 use tokio::io::AsyncReadExt;
1110
1111 info!(
1112 container_id = %container_id,
1113 topic = %publisher.topic(),
1114 "Starting FD streaming task"
1115 );
1116
1117 let sandbox = if let Some(ref sid) = sandbox_id {
1118 sandbox_pool.get(sid).await
1119 } else {
1120 None
1121 };
1122
1123 let console_path = sandbox
1124 .as_ref()
1125 .and_then(|s| s.console_socket().map(std::path::Path::to_path_buf));
1126
1127 if let Some(ref path) = console_path {
1128 match tokio::net::UnixStream::connect(path).await {
1129 Ok(mut stream) => {
1130 let mut buf = vec![0u8; 4096];
1131 loop {
1132 tokio::select! {
1133 biased;
1134 _ = cancel_token.cancelled() => {
1135 info!(container_id = %container_id, "FD streaming cancelled");
1136 if !publisher.is_terminated() {
1137 publisher.publish_error("cancelled").await?;
1138 }
1139 return Ok(());
1140 }
1141 result = stream.read(&mut buf) => {
1142 match result? {
1143 0 => {
1144 info!(container_id = %container_id, "Console EOF — container exited");
1145 return Ok(());
1146 }
1147 n => {
1148 publisher.publish_data(&buf[..n]).await?;
1149 }
1150 }
1151 }
1152 }
1153 }
1154 }
1155 Err(e) => {
1156 warn!(
1157 container_id = %container_id,
1158 path = ?path,
1159 error = %e,
1160 "Failed to connect to console socket, waiting for cancellation"
1161 );
1162 cancel_token.cancelled().await;
1163 if !publisher.is_terminated() {
1164 publisher.publish_error("console unavailable").await?;
1165 }
1166 }
1167 }
1168 } else {
1169 if sandbox_id.is_some() {
1170 warn!(container_id = %container_id, "No console socket configured for sandbox");
1171 } else {
1172 warn!(container_id = %container_id, "No sandbox found for container, FD streaming idle");
1173 }
1174 cancel_token.cancelled().await;
1175 if !publisher.is_terminated() {
1176 publisher.publish_error("cancelled").await?;
1177 }
1178 }
1179 Ok(())
1180}
1181
1182impl From<&PodSandbox> for PodSandboxStatus {
1187 fn from(s: &PodSandbox) -> Self {
1188 Self {
1189 id: s.id.clone(),
1190 metadata: s.metadata.clone(),
1191 state: s.state,
1192 created_at: Timestamp {
1193 seconds: s.created_at.timestamp(),
1194 nanos: s.created_at.timestamp_subsec_nanos() as i32,
1195 },
1196 network: Default::default(),
1197 linux: Default::default(),
1198 labels: s.labels.clone(),
1199 annotations: s.annotations.clone(),
1200 runtime_handler: s.runtime_handler.clone(),
1201 }
1202 }
1203}
1204
1205impl From<&PodSandbox> for PodSandboxInfo {
1206 fn from(s: &PodSandbox) -> Self {
1207 Self {
1208 id: s.id.clone(),
1209 metadata: s.metadata.clone(),
1210 state: s.state,
1211 created_at: Timestamp {
1212 seconds: s.created_at.timestamp(),
1213 nanos: s.created_at.timestamp_subsec_nanos() as i32,
1214 },
1215 labels: s.labels.clone(),
1216 annotations: s.annotations.clone(),
1217 runtime_handler: s.runtime_handler.clone(),
1218 }
1219 }
1220}
1221
1222impl From<&PodSandbox> for PodSandboxStats {
1223 fn from(s: &PodSandbox) -> Self {
1224 Self {
1225 attributes: PodSandboxAttributes {
1226 id: s.id.clone(),
1227 metadata: s.metadata.clone(),
1228 labels: s.labels.clone(),
1229 annotations: s.annotations.clone(),
1230 },
1231 linux: LinuxPodSandboxStats::default(),
1232 }
1233 }
1234}
1235
1236impl From<&Container> for ContainerStatus {
1237 fn from(c: &Container) -> Self {
1238 Self {
1239 id: c.id.clone(),
1240 metadata: c.metadata.clone(),
1241 state: c.state,
1242 created_at: Timestamp {
1243 seconds: c.created_at.timestamp(),
1244 nanos: c.created_at.timestamp_subsec_nanos() as i32,
1245 },
1246 started_at: Timestamp {
1247 seconds: 0,
1248 nanos: 0,
1249 },
1250 finished_at: Timestamp {
1251 seconds: 0,
1252 nanos: 0,
1253 },
1254 exit_code: c.exit_code.unwrap_or(0),
1255 image: c.image.clone(),
1256 image_ref: String::new(),
1257 reason: String::new(),
1258 message: String::new(),
1259 labels: c.labels.clone(),
1260 annotations: c.annotations.clone(),
1261 mounts: Vec::new(),
1262 log_path: String::new(),
1263 }
1264 }
1265}
1266
1267impl From<&Container> for ContainerInfo {
1268 fn from(c: &Container) -> Self {
1269 Self {
1270 id: c.id.clone(),
1271 pod_sandbox_id: c.pod_sandbox_id.clone(),
1272 metadata: c.metadata.clone(),
1273 image: c.image.clone(),
1274 image_ref: String::new(),
1275 state: c.state,
1276 created_at: Timestamp {
1277 seconds: c.created_at.timestamp(),
1278 nanos: c.created_at.timestamp_subsec_nanos() as i32,
1279 },
1280 labels: c.labels.clone(),
1281 annotations: c.annotations.clone(),
1282 }
1283 }
1284}
1285
1286impl From<&Container> for ContainerStats {
1287 fn from(c: &Container) -> Self {
1288 Self {
1289 attributes: ContainerAttributes {
1290 id: c.id.clone(),
1291 metadata: c.metadata.clone(),
1292 labels: c.labels.clone(),
1293 annotations: c.annotations.clone(),
1294 },
1295 cpu: Default::default(),
1296 memory: Default::default(),
1297 writable_layer: Default::default(),
1298 }
1299 }
1300}
1301
1302use hyprstream_rpc_std::worker_client::{
1311 CreateContainerRequest, ExecSyncRequest, PullImageRequest,
1312};
1313
1314#[async_trait::async_trait(?Send)]
1315impl RuntimeHandler for WorkerService {
1316 async fn handle_version(
1317 &self,
1318 _ctx: &EnvelopeContext,
1319 _request_id: u64,
1320 value: &str,
1321 ) -> AnyhowResult<VersionInfo> {
1322 let resp = self.version(value).await?;
1323 Ok(resp)
1324 }
1325
1326 async fn handle_status(
1327 &self,
1328 _ctx: &EnvelopeContext,
1329 _request_id: u64,
1330 request: &StatusRequest,
1331 ) -> AnyhowResult<RuntimeStatus> {
1332 let resp = self.status(request.verbose).await?;
1333 Ok(resp.status)
1334 }
1335}
1336
1337#[async_trait::async_trait(?Send)]
1338impl SandboxHandler for WorkerService {
1339 async fn handle_run(
1340 &self,
1341 ctx: &EnvelopeContext,
1342 _request_id: u64,
1343 data: &PodSandboxConfig,
1344 ) -> AnyhowResult<String> {
1345 let sandbox_id = self.run_pod_sandbox(&ctx.subject(), data).await?;
1349 Ok(sandbox_id)
1350 }
1351
1352 async fn handle_stop(
1353 &self,
1354 _ctx: &EnvelopeContext,
1355 _request_id: u64,
1356 value: &str,
1357 ) -> AnyhowResult<()> {
1358 self.stop_pod_sandbox(value).await?;
1359 Ok(())
1360 }
1361
1362 async fn handle_remove(
1363 &self,
1364 _ctx: &EnvelopeContext,
1365 _request_id: u64,
1366 value: &str,
1367 ) -> AnyhowResult<()> {
1368 self.remove_pod_sandbox(value).await?;
1369 Ok(())
1370 }
1371
1372 async fn handle_status(
1373 &self,
1374 _ctx: &EnvelopeContext,
1375 _request_id: u64,
1376 request: &PodSandboxStatusRequest,
1377 ) -> AnyhowResult<PodSandboxStatusResponse> {
1378 let (status, info) = self
1379 .pod_sandbox_status(&request.pod_sandbox_id, request.verbose)
1380 .await?;
1381 Ok(PodSandboxStatusResponse { status, info })
1382 }
1383
1384 async fn handle_list(
1385 &self,
1386 _ctx: &EnvelopeContext,
1387 _request_id: u64,
1388 data: &PodSandboxFilter,
1389 ) -> AnyhowResult<Vec<PodSandboxInfo>> {
1390 let sandboxes = self.list_pod_sandbox(data).await?;
1391 Ok(sandboxes)
1392 }
1393
1394 async fn handle_stats(
1395 &self,
1396 _ctx: &EnvelopeContext,
1397 _request_id: u64,
1398 value: &str,
1399 ) -> AnyhowResult<PodSandboxStats> {
1400 let stats = self.pod_sandbox_stats(value).await?;
1401 Ok(stats)
1402 }
1403
1404 async fn handle_list_stats(
1405 &self,
1406 _ctx: &EnvelopeContext,
1407 _request_id: u64,
1408 filter: &PodSandboxStatsFilter,
1409 ) -> AnyhowResult<Vec<PodSandboxStats>> {
1410 let stats = self.list_pod_sandbox_stats(filter).await?;
1411 Ok(stats)
1412 }
1413}
1414
1415#[async_trait::async_trait(?Send)]
1416impl ContainerHandler for WorkerService {
1417 async fn handle_create(
1418 &self,
1419 _ctx: &EnvelopeContext,
1420 _request_id: u64,
1421 data: &CreateContainerRequest,
1422 ) -> AnyhowResult<String> {
1423 let container_id = self
1424 .create_container(&data.pod_sandbox_id, &data.config, &data.sandbox_config)
1425 .await?;
1426 Ok(container_id)
1427 }
1428
1429 async fn handle_start(
1430 &self,
1431 _ctx: &EnvelopeContext,
1432 _request_id: u64,
1433 value: &str,
1434 ) -> AnyhowResult<()> {
1435 self.start_container(value).await?;
1436 Ok(())
1437 }
1438
1439 async fn handle_stop(
1440 &self,
1441 _ctx: &EnvelopeContext,
1442 _request_id: u64,
1443 request: &StopContainerRequest,
1444 ) -> AnyhowResult<()> {
1445 self.stop_container(&request.container_id, request.timeout)
1446 .await?;
1447 Ok(())
1448 }
1449
1450 async fn handle_remove(
1451 &self,
1452 _ctx: &EnvelopeContext,
1453 _request_id: u64,
1454 value: &str,
1455 ) -> AnyhowResult<()> {
1456 self.remove_container(value).await?;
1457 Ok(())
1458 }
1459
1460 async fn handle_status(
1461 &self,
1462 _ctx: &EnvelopeContext,
1463 _request_id: u64,
1464 request: &ContainerStatusRequest,
1465 ) -> AnyhowResult<ContainerStatusResponse> {
1466 let (status, info) = self
1467 .container_status(&request.container_id, request.verbose)
1468 .await?;
1469 Ok(ContainerStatusResponse { status, info })
1470 }
1471
1472 async fn handle_list(
1473 &self,
1474 _ctx: &EnvelopeContext,
1475 _request_id: u64,
1476 data: &ContainerFilter,
1477 ) -> AnyhowResult<Vec<ContainerInfo>> {
1478 let containers = self.list_containers(data).await?;
1479 Ok(containers)
1480 }
1481
1482 async fn handle_stats(
1483 &self,
1484 _ctx: &EnvelopeContext,
1485 _request_id: u64,
1486 value: &str,
1487 ) -> AnyhowResult<ContainerStats> {
1488 let stats = self.container_stats(value).await?;
1489 Ok(stats)
1490 }
1491
1492 async fn handle_list_stats(
1493 &self,
1494 _ctx: &EnvelopeContext,
1495 _request_id: u64,
1496 data: &ContainerStatsFilter,
1497 ) -> AnyhowResult<Vec<ContainerStats>> {
1498 let stats = self.list_container_stats(data).await?;
1499 Ok(stats)
1500 }
1501
1502 async fn handle_exec(
1503 &self,
1504 ctx: &EnvelopeContext,
1505 _request_id: u64,
1506 data: &ExecSyncRequest,
1507 ) -> AnyhowResult<ExecSyncResult> {
1508 let resp = self
1509 .exec_sync(&ctx.subject(), &data.container_id, &data.cmd, data.timeout)
1510 .await?;
1511 Ok(resp)
1512 }
1513
1514 async fn handle_attach(
1515 &self,
1516 ctx: &EnvelopeContext,
1517 _request_id: u64,
1518 request: &AttachRequest,
1519 ) -> AnyhowResult<(StreamInfo, hyprstream_rpc::service::Continuation)> {
1520 let _ = &request.fds; let (stream_info, continuation) = self.prepare_attach(ctx, &request.container_id).await?;
1522 Ok((stream_info, continuation))
1523 }
1524
1525 async fn handle_detach(
1526 &self,
1527 _ctx: &EnvelopeContext,
1528 _request_id: u64,
1529 value: &str,
1530 ) -> AnyhowResult<()> {
1531 self.detach(value).await?;
1532 Ok(())
1533 }
1534}
1535
1536fn image_store_missing() -> anyhow::Error {
1542 anyhow::anyhow!(
1543 "CRI image operations require the `oci-image` feature (RAFS image store); \
1544 this binary was built without oci-image support"
1545 )
1546}
1547
1548#[async_trait::async_trait(?Send)]
1549impl ImageHandler for WorkerService {
1550 async fn handle_list(
1551 &self,
1552 _ctx: &EnvelopeContext,
1553 _request_id: u64,
1554 filter: &ImageFilter,
1555 ) -> AnyhowResult<Vec<ImageInfo>> {
1556 let _ = filter; let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1558 Ok(store.list_images().await?)
1559 }
1560
1561 async fn handle_status(
1562 &self,
1563 _ctx: &EnvelopeContext,
1564 _request_id: u64,
1565 request: &ImageStatusRequest,
1566 ) -> AnyhowResult<ImageStatusResult> {
1567 let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1568 Ok(store
1569 .image_status(&request.image.image, request.verbose)
1570 .await?)
1571 }
1572
1573 async fn handle_pull(
1574 &self,
1575 _ctx: &EnvelopeContext,
1576 _request_id: u64,
1577 data: &PullImageRequest,
1578 ) -> AnyhowResult<String> {
1579 let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1580 let auth = if !data.auth.username.is_empty() {
1581 Some(AuthConfig {
1582 username: data.auth.username.clone(),
1583 password: data.auth.password.clone(),
1584 auth: String::new(),
1585 server_address: String::new(),
1586 identity_token: String::new(),
1587 registry_token: String::new(),
1588 })
1589 } else {
1590 None
1591 };
1592 Ok(store
1593 .pull_with_auth(&data.image.image, auth.as_ref())
1594 .await?)
1595 }
1596
1597 async fn handle_remove(
1598 &self,
1599 _ctx: &EnvelopeContext,
1600 _request_id: u64,
1601 data: &ImageSpec,
1602 ) -> AnyhowResult<()> {
1603 let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1604 store.remove_image(&data.image).await?;
1605 Ok(())
1606 }
1607
1608 async fn handle_fs_info(
1609 &self,
1610 _ctx: &EnvelopeContext,
1611 _request_id: u64,
1612 ) -> AnyhowResult<Vec<FilesystemUsage>> {
1613 let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1614 Ok(store.fs_info().await?)
1615 }
1616}
1617
1618#[async_trait::async_trait(?Send)]
1620impl WorkerHandler for WorkerService {
1621 async fn authorize(
1622 &self,
1623 ctx: &EnvelopeContext,
1624 resource: &str,
1625 operation: &str,
1626 ) -> AnyhowResult<()> {
1627 if let Some(ref auth_fn) = self.authorize_fn {
1628 let subject = ctx.subject().to_string();
1629 let allowed = auth_fn(ctx.clone(), resource.to_owned(), operation.to_owned())
1630 .await
1631 .unwrap_or_else(|e| {
1632 warn!(
1633 "Policy check failed for {} on {}: {} - denying access",
1634 subject, resource, e
1635 );
1636 false
1637 });
1638 if allowed {
1639 Ok(())
1640 } else {
1641 anyhow::bail!(
1642 "Unauthorized: {} cannot {} on {}",
1643 subject,
1644 operation,
1645 resource
1646 )
1647 }
1648 } else {
1649 anyhow::bail!("Authorization not configured for worker service")
1651 }
1652 }
1653}
1654
1655#[async_trait(?Send)]
1660impl RequestService for WorkerService {
1661 fn decode_request_body(
1662 &self,
1663 signed_body: &[u8],
1664 ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
1665 crate::generated::worker_client::decode_worker_request_body(signed_body)
1669 }
1670
1671 fn producer_reach_config_handle(
1672 &self,
1673 ) -> Option<hyprstream_rpc::moq_stream::ProducerReachConfigHandle> {
1674 Some(self.stream_channel.reach_config_handle())
1675 }
1676
1677 fn moq_origin_handle(&self) -> Option<hyprstream_rpc::moq_stream::MoqStreamOriginHandle> {
1678 Some(self.stream_channel.moq_origin_handle())
1679 }
1680
1681 async fn handle_request(
1682 &self,
1683 ctx: &EnvelopeContext,
1684 body: &hyprstream_rpc::service::DecodedRequestBody,
1685 ) -> AnyhowResult<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
1686 debug!(
1687 "Worker request from {} (request_id={})",
1688 ctx.subject(),
1689 ctx.request_id
1690 );
1691 dispatch_worker(self, ctx, body).await
1692 }
1693
1694 fn name(&self) -> &str {
1695 SERVICE_NAME
1696 }
1697
1698 fn transport(&self) -> &TransportConfig {
1699 &self.transport
1700 }
1701
1702 fn signing_key(&self) -> SigningKey {
1703 self.signing_key.clone()
1704 }
1705
1706 fn expected_audience(&self) -> Option<&str> {
1707 self.expected_audience.as_deref()
1708 }
1709
1710 fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
1711 self.jwt_key_source.clone()
1712 }
1713}
1714
1715#[cfg(all(test, feature = "kata"))]
1718#[allow(clippy::print_stderr)]
1719mod tests {
1720 use super::*;
1721 use crate::config::ImageConfig;
1722 use crate::image::RafsStore;
1723 use hyprstream_rpc::auth::mac::{ObjectRef, SecurityContext};
1724 use hyprstream_rpc::crypto::generate_signing_keypair;
1725 use hyprstream_rpc::transport::TransportConfig;
1726 use hyprstream_rpc_std::worker_client::{ContainerConfig, PodSandboxConfig, PodSandboxState};
1727 use tempfile::TempDir;
1728
1729 struct FixtureAccessDecider;
1730
1731 impl hyprstream_9p::AccessDecider for FixtureAccessDecider {
1732 fn check(
1733 &self,
1734 _ctx: &SecurityContext,
1735 _object: ObjectRef<'_>,
1736 _action: hyprstream_9p::Action,
1737 ) -> bool {
1738 true
1739 }
1740
1741 fn audit_denial(
1742 &self,
1743 _ctx: &SecurityContext,
1744 _object: ObjectRef<'_>,
1745 _object_label: Option<hyprstream_rpc::auth::mac::SecurityLabel>,
1746 _action: hyprstream_9p::Action,
1747 _reason: hyprstream_9p::ReferenceMonitorDenyReason,
1748 ) {
1749 }
1750 }
1751
1752 async fn create_test_service(
1754 ) -> std::result::Result<(WorkerService, TempDir), Box<dyn std::error::Error>> {
1755 let _ = hyprstream_rpc::moq_event::init_global_moq_event_origin(
1761 hyprstream_rpc::moq_event::MoqEventOrigin::new(),
1762 );
1763
1764 let temp_dir = TempDir::new()?;
1765 let base_path = temp_dir.path();
1766
1767 let image_config = ImageConfig {
1768 blobs_dir: base_path.join("blobs"),
1769 bootstrap_dir: base_path.join("bootstrap"),
1770 cache_dir: base_path.join("cache"),
1771 refs_dir: base_path.join("refs"),
1772 runtime_dir: base_path.join("runtime"),
1773 dragonfly_peer: None,
1774 fuse_threads: 4,
1775 chunk_size: 1024 * 1024,
1776 lazy_loading: true,
1777 cache_size_bytes: 1024 * 1024 * 1024, };
1779
1780 let rafs_store = Arc::new(RafsStore::new(image_config.clone())?);
1781
1782 let pool_config = PoolConfig {
1783 runtime_dir: base_path.join("sandboxes"),
1784 cloud_init_dir: base_path.join("cloud-init"),
1785 vm_image: base_path.join("vm/rootfs.img"),
1786 kernel_path: base_path.join("vm/vmlinux"),
1787 ..PoolConfig::default()
1788 };
1789
1790 let transport = TransportConfig::inproc("test-worker-service");
1791 let (signing_key, _verifying_key) = generate_signing_keypair();
1792
1793 let backend: Arc<dyn crate::runtime::backend::SandboxBackend> =
1794 Arc::new(crate::runtime::kata_backend::KataBackend::new(
1795 image_config,
1796 Arc::clone(&rafs_store),
1797 Arc::new(FixtureAccessDecider),
1798 None,
1799 ));
1800 let service = WorkerService::new(
1801 pool_config,
1802 backend,
1803 Some(rafs_store),
1804 transport,
1805 signing_key,
1806 )?;
1807 Ok((service, temp_dir))
1808 }
1809
1810 #[tokio::test]
1811 async fn test_sandbox_lifecycle() -> std::result::Result<(), Box<dyn std::error::Error>> {
1812 let (service, _temp_dir) = create_test_service().await?;
1813
1814 let config = PodSandboxConfig::default();
1816 let subject = hyprstream_vfs::Subject::new("test-user");
1817 let result = service.run_pod_sandbox(&subject, &config).await;
1818
1819 match &result {
1821 Err(WorkerError::VmStartFailed(msg))
1822 if msg.contains("No such file or directory")
1823 || msg.contains("failed to create sandbox directory")
1824 || msg.contains("/run/kata")
1825 || msg.contains("launch failed") =>
1826 {
1827 eprintln!("Skipping test: cloud-hypervisor or kata runtime not available");
1828 return Ok(());
1829 }
1830 _ => {}
1831 }
1832
1833 let sandbox_id = result?;
1834
1835 let (status, _info) = service.pod_sandbox_status(&sandbox_id, false).await?;
1837 assert_eq!(status.state, PodSandboxState::SandboxReady);
1838
1839 let sandboxes = service
1841 .list_pod_sandbox(&PodSandboxFilter::default())
1842 .await?;
1843 assert_eq!(sandboxes.len(), 1);
1844
1845 service.stop_pod_sandbox(&sandbox_id).await?;
1847 service.remove_pod_sandbox(&sandbox_id).await?;
1848
1849 let sandboxes = service
1851 .list_pod_sandbox(&PodSandboxFilter::default())
1852 .await?;
1853 assert_eq!(sandboxes.len(), 0);
1854 Ok(())
1855 }
1856
1857 #[tokio::test]
1858 async fn test_container_lifecycle() -> std::result::Result<(), Box<dyn std::error::Error>> {
1859 let (service, _temp_dir) = create_test_service().await?;
1860
1861 let sandbox_config = PodSandboxConfig::default();
1863 let subject = hyprstream_vfs::Subject::new("test-user");
1864 let result = service.run_pod_sandbox(&subject, &sandbox_config).await;
1865
1866 match &result {
1868 Err(WorkerError::VmStartFailed(msg))
1869 if msg.contains("No such file or directory")
1870 || msg.contains("failed to create sandbox directory")
1871 || msg.contains("/run/kata")
1872 || msg.contains("launch failed") =>
1873 {
1874 eprintln!("Skipping test: cloud-hypervisor or kata runtime not available");
1875 return Ok(());
1876 }
1877 _ => {}
1878 }
1879
1880 let sandbox_id = result?;
1881
1882 let container_config = ContainerConfig::default();
1884 let container_id = service
1885 .create_container(&sandbox_id, &container_config, &sandbox_config)
1886 .await?;
1887
1888 let (status, _info) = service.container_status(&container_id, false).await?;
1890 assert_eq!(status.state, ContainerState::ContainerCreated);
1891
1892 service.start_container(&container_id).await?;
1894 let (status, _info) = service.container_status(&container_id, false).await?;
1895 assert_eq!(status.state, ContainerState::ContainerRunning);
1896
1897 service.stop_container(&container_id, 30).await?;
1899 let (status, _info) = service.container_status(&container_id, false).await?;
1900 assert_eq!(status.state, ContainerState::ContainerExited);
1901
1902 service.remove_container(&container_id).await?;
1904
1905 service.remove_pod_sandbox(&sandbox_id).await?;
1907 Ok(())
1908 }
1909
1910 #[tokio::test]
1911 async fn test_service_version() -> std::result::Result<(), Box<dyn std::error::Error>> {
1912 let (service, _temp_dir) = create_test_service().await?;
1913
1914 let version = service.version("v1").await?;
1916 assert_eq!(version.runtime_name, super::super::RUNTIME_NAME);
1917 assert_eq!(version.runtime_api_version, "v1");
1918 Ok(())
1919 }
1920
1921 #[tokio::test]
1922 async fn test_service_status() -> std::result::Result<(), Box<dyn std::error::Error>> {
1923 let (service, _temp_dir) = create_test_service().await?;
1924
1925 let status = service.status(true).await?;
1927 assert!(!status.status.conditions.is_empty());
1928 assert!(status.info.contains_key("warm_pool_size"));
1929 Ok(())
1930 }
1931}