Skip to main content

hyprstream_workers/runtime/
service.rs

1//! WorkerService — AGPL implementation of the canonical worker control plane.
2//!
3//! Implements the `RequestService` trait for handling CRI-aligned requests.
4//!
5//! # Transport (#350, epic #341)
6//!
7//! This is the worker **control plane**. It is transport-agnostic: the service
8//! only implements `RequestService` (Cap'n Proto wire format via
9//! `generate_rpc_server!("worker")`) and carries a `TransportConfig`. The
10//! legacy ZMQ ROUTER/DEALER stack was removed workspace-wide (#138/#167 —
11//! ZMQ/ZeroMQ is gone); the blanket `Spawnable` impl bridges this service and
12//! serves it over its *registered* transport (`serve_bridged`: Inproc / IPC-UDS
13//! / systemd-fd, plus the QUIC/Iroh dial plane). No ZMQ socket exists in this
14//! path.
15//!
16//! The **asynchronous** surfaces of the worker already ride the moq-lite
17//! streaming plane, mirroring the event-bus migration (#167): lifecycle events
18//! publish via [`EventPublisher`] (moq-backed — see `hyprstream_rpc::events`) and
19//! terminal attach/detach FD data streams via `StreamChannel` /
20//! `AnyStreamPublisher` (moq). moq is a pub/sub fan-out plane, not a
21//! request/response RPC transport, so the synchronous CRI req/rep control plane
22//! stays on the Cap'n Proto bridged transport shared by every peer RPC service
23//! (registry/model/policy/mcp).
24
25use std::collections::HashMap;
26use std::sync::Arc;
27use tokio::sync::RwLock;
28use tokio_util::sync::CancellationToken;
29
30use anyhow::Result as AnyhowResult;
31use async_trait::async_trait;
32use tracing::{debug, info, warn};
33
34// RPC service infrastructure from hyprstream-rpc: the pluggable Cap'n Proto
35// bridged transport (inproc/UDS/QUIC/iroh) + the moq streaming plane. ZMQ is
36// gone (#138/#167) — this is not a ZMQ socket API.
37use hyprstream_rpc::events::EventPublisher;
38use hyprstream_rpc::moq_stream::AnyStreamPublisher;
39use hyprstream_rpc::prelude::SigningKey;
40use hyprstream_rpc::service::{AuthorizeFn, EnvelopeContext, RequestService};
41use hyprstream_rpc::streaming::StreamChannel;
42use hyprstream_rpc::transport::TransportConfig;
43
44use crate::config::PoolConfig;
45use crate::error::{Result, WorkerError};
46use crate::events::{
47    serialize_container_started,
48    serialize_container_stopped,
49    serialize_sandbox_started,
50    serialize_sandbox_stopped,
51    // Event types and serialization helpers
52    ContainerStarted,
53    ContainerStopped,
54    SandboxStarted,
55    SandboxStopped,
56};
57// ImageStore trait seam — always compiled (feature-invariant). Concrete
58// backends (RafsStore under `oci-image`) register via inventory; the service
59// holds an `Option<Arc<dyn ImageStore>>` and dispatches CRI image ops through
60// it, with no cfg mirror (#646).
61use crate::image::ImageStore;
62// Import generated wire types (canonical OCI-aligned names)
63use super::status::StatusResponse;
64use crate::generated::worker_client::{
65    dispatch_worker, ContainerHandler, ImageHandler, RuntimeHandler, SandboxHandler, WorkerHandler,
66};
67use hyprstream_rpc_std::worker_client::{
68    AttachRequest,
69    AuthConfig,
70    ContainerAttributes,
71    ContainerConfig,
72    ContainerFilter,
73    ContainerInfo,
74    ContainerState,
75    ContainerStats,
76    ContainerStatsFilter,
77    ContainerStatus,
78    ContainerStatusRequest,
79    ContainerStatusResponse,
80    ExecSyncResult,
81    FilesystemUsage,
82    ImageFilter,
83    ImageInfo,
84    ImageSpec,
85    ImageStatusRequest,
86    ImageStatusResult,
87    KeyValue,
88    LinuxPodSandboxStats,
89    PodSandboxAttributes,
90    PodSandboxConfig,
91    // Filter + request types for handler signatures
92    PodSandboxFilter,
93    PodSandboxInfo,
94    PodSandboxStats,
95    PodSandboxStatsFilter,
96    PodSandboxStatus,
97    PodSandboxStatusRequest,
98    PodSandboxStatusResponse,
99    RuntimeCondition,
100    RuntimeStatus,
101    StatusRequest,
102    StopContainerRequest,
103    StreamInfo,
104    Timestamp,
105    VersionInfo,
106};
107// Domain entities (business logic)
108use super::container::Container;
109use super::pool::SandboxPool;
110use super::sandbox::PodSandbox;
111use super::ExecMount;
112use super::{RUNTIME_NAME, RUNTIME_VERSION};
113
114/// Service name for endpoint registry
115const SERVICE_NAME: &str = "worker";
116
117/// WorkerService handles requests from the canonical worker contract.
118///
119/// Implements the RequestService trait for integration with hyprstream's
120/// Cap'n Proto bridged RPC transport (inproc/UDS/systemd-fd + QUIC/Iroh); ZMQ is gone.
121pub struct WorkerService {
122    // Business logic
123    /// Sandbox pool for VM management
124    sandbox_pool: Arc<SandboxPool>,
125
126    /// The single task-owned execution adapter. The CRI production caller and
127    /// the Plan 9 `/exec/instances/<id>/fd/{1,2}` view use this same owner, so
128    /// command execution is never duplicated into a private stream tree.
129    exec_mount: Arc<ExecMount>,
130
131    /// Image store — the CRI image backend (`RafsStore` under `oci-image`,
132    /// registered via inventory). `None` when no image backend is compiled in;
133    /// CRI image ops then return a clear "built without oci-image support"
134    /// error. Held as `Option<Arc<dyn ImageStore>>` so the field, the `new()`
135    /// signature, AND the handler bodies are all feature-invariant — only the
136    /// concrete impl + its `inventory::submit!` (in `image/store.rs`) are
137    /// `#[cfg(feature = "oci-image")]`. No cfg mirror here (#646).
138    image_store: Option<Arc<dyn ImageStore>>,
139
140    /// Active containers (container_id -> Container)
141    containers: RwLock<HashMap<String, Container>>,
142
143    /// Container to sandbox mapping
144    container_sandbox_map: RwLock<HashMap<String, String>>,
145
146    /// Event publisher for lifecycle events (sandbox/container started/stopped)
147    event_publisher: tokio::sync::Mutex<EventPublisher>,
148
149    /// Active FD streams (stream_id -> ActiveFdStream)
150    active_fd_streams: Arc<RwLock<HashMap<String, ActiveFdStream>>>,
151
152    /// StreamChannel for authenticated, async FD streaming
153    stream_channel: Arc<StreamChannel>,
154
155    // Infrastructure (for Spawnable)
156    transport: TransportConfig,
157    signing_key: SigningKey,
158
159    /// Optional authorization callback (injected by parent crate)
160    authorize_fn: Option<AuthorizeFn>,
161    /// Expected JWT audience for token validation (RFC 8707).
162    expected_audience: Option<String>,
163    /// JWT key source for verifying JWTs (local and federated).
164    jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
165}
166
167impl WorkerService {
168    /// Create a new WorkerService with full configuration and infrastructure
169    ///
170    /// Must be called from within a tokio runtime context.
171    pub fn new(
172        pool_config: PoolConfig,
173        backend: Arc<dyn super::backend::SandboxBackend>,
174        // Image store — `Some` only when the caller built one (which today
175        // requires `oci-image` + the `rafs` registration). Feature-invariant
176        // type; pass `None` for a build without an image backend.
177        image_store: Option<Arc<dyn ImageStore>>,
178        transport: TransportConfig,
179        signing_key: SigningKey,
180    ) -> AnyhowResult<Self> {
181        let sandbox_pool = Arc::new(SandboxPool::new(pool_config, backend));
182        let exec_mount = Arc::new(ExecMount::new(Arc::clone(&sandbox_pool)));
183
184        // Create event publisher for worker lifecycle events
185        let event_publisher = EventPublisher::new("worker")?;
186
187        let stream_channel = Arc::new(
188            StreamChannel::new(signing_key.clone())
189                .with_reach_config(hyprstream_rpc::moq_stream::ProducerReachConfig::default()),
190        );
191        Ok(Self {
192            sandbox_pool,
193            exec_mount,
194            image_store,
195            containers: RwLock::new(HashMap::new()),
196            container_sandbox_map: RwLock::new(HashMap::new()),
197            event_publisher: tokio::sync::Mutex::new(event_publisher),
198            active_fd_streams: Arc::new(RwLock::new(HashMap::new())),
199            stream_channel,
200            transport,
201            signing_key,
202            authorize_fn: None,
203            expected_audience: None,
204            jwt_key_source: None,
205        })
206    }
207
208    /// Set the authorization callback for policy checks.
209    pub fn set_authorize_fn(&mut self, authorize_fn: AuthorizeFn) {
210        self.authorize_fn = Some(authorize_fn);
211    }
212
213    /// Set the expected JWT audience for token validation.
214    pub fn set_expected_audience(&mut self, audience: String) {
215        self.expected_audience = Some(audience);
216    }
217
218    /// Set the JWT key source for verifying JWTs (local and federated).
219    pub fn set_jwt_key_source(
220        &mut self,
221        src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
222    ) {
223        self.jwt_key_source = Some(src);
224    }
225
226    /// Initialize the service (start warm pool)
227    pub async fn initialize(&self) -> Result<()> {
228        self.sandbox_pool.initialize().await?;
229        tracing::info!("WorkerService initialized");
230        Ok(())
231    }
232
233    /// Shutdown the service
234    pub async fn shutdown(&self) -> Result<()> {
235        self.sandbox_pool.shutdown().await?;
236        tracing::info!("WorkerService shutdown complete");
237        Ok(())
238    }
239
240    // ─────────────────────────────────────────────────────────────────────────
241    // Runtime Information
242    // ─────────────────────────────────────────────────────────────────────────
243
244    /// Get runtime version information
245    pub async fn version(&self, _version: &str) -> Result<VersionInfo> {
246        Ok(VersionInfo {
247            version: "v1".to_owned(),
248            runtime_name: RUNTIME_NAME.to_owned(),
249            runtime_version: RUNTIME_VERSION.to_owned(),
250            runtime_api_version: "v1".to_owned(),
251        })
252    }
253
254    /// Get runtime status
255    pub async fn status(&self, verbose: bool) -> Result<StatusResponse> {
256        let conditions = vec![
257            RuntimeCondition {
258                condition_type: "RuntimeReady".to_owned(),
259                status: true,
260                reason: String::new(),
261                message: "Runtime is ready".to_owned(),
262            },
263            RuntimeCondition {
264                condition_type: "NetworkReady".to_owned(),
265                status: true,
266                reason: String::new(),
267                message: "Network is ready".to_owned(),
268            },
269        ];
270
271        let mut info = HashMap::new();
272        if verbose {
273            let stats = self.sandbox_pool.stats().await;
274            info.insert("warm_pool_size".to_owned(), stats.warm_count.to_string());
275            info.insert(
276                "active_sandboxes".to_owned(),
277                stats.active_count.to_string(),
278            );
279            info.insert("max_sandboxes".to_owned(), stats.max_sandboxes.to_string());
280        }
281
282        Ok(StatusResponse {
283            status: RuntimeStatus { conditions },
284            info,
285        })
286    }
287
288    // ─────────────────────────────────────────────────────────────────────────
289    // Pod Sandbox Lifecycle
290    // ─────────────────────────────────────────────────────────────────────────
291
292    /// Create and start a pod sandbox (Kata VM)
293    ///
294    /// `subject` is the cryptographically-verified caller identity (#525 P2
295    /// admission control) — derive it from `ctx.subject()` at the RPC
296    /// dispatch boundary (see `handle_run`), never accept a caller-asserted
297    /// subject as a plain parameter (see the MAC interface policy: "derive,
298    /// don't extend").
299    pub async fn run_pod_sandbox(
300        &self,
301        subject: &hyprstream_vfs::Subject,
302        config: &PodSandboxConfig,
303    ) -> Result<String> {
304        let sandbox_id = self.sandbox_pool.acquire(subject, config).await?;
305        tracing::info!(sandbox_id = %sandbox_id, "Created pod sandbox");
306
307        // Publish sandbox started event with structured payload
308        let event = SandboxStarted {
309            sandbox_id: sandbox_id.clone(),
310            metadata: serde_json::to_string(&config.metadata).unwrap_or_default(),
311            vm_pid: 0, // VM PID not easily accessible here; could be enhanced later
312        };
313        let payload = serialize_sandbox_started(&event).unwrap_or_default();
314        self.publish_event(&sandbox_id, "started", &payload).await;
315
316        Ok(sandbox_id)
317    }
318
319    /// Publish a worker lifecycle event (fire-and-forget)
320    ///
321    /// Topic format: worker.{entity_id}.{event_name}
322    async fn publish_event(&self, entity_id: &str, event: &str, payload: &[u8]) {
323        let publisher = self.event_publisher.lock().await;
324        if let Err(e) = publisher.publish(entity_id, event, payload).await {
325            tracing::warn!(
326                entity_id = %entity_id,
327                event = %event,
328                error = %e,
329                "Failed to publish worker event"
330            );
331        }
332    }
333
334    /// Stop a running pod sandbox
335    pub async fn stop_pod_sandbox(&self, pod_sandbox_id: &str) -> Result<()> {
336        // Stop all containers in the sandbox first
337        let containers_to_stop: Vec<String> = {
338            let map = self.container_sandbox_map.read().await;
339            map.iter()
340                .filter(|(_, sid)| *sid == pod_sandbox_id)
341                .map(|(cid, _)| cid.clone())
342                .collect()
343        };
344
345        for container_id in containers_to_stop {
346            if let Err(e) = self.stop_container(&container_id, 30).await {
347                tracing::warn!(
348                    container_id = %container_id,
349                    error = %e,
350                    "Failed to stop container during sandbox stop"
351                );
352            }
353        }
354
355        // Mark sandbox as not ready (but don't release yet)
356        // The actual VM stop happens in remove_pod_sandbox
357        tracing::info!(sandbox_id = %pod_sandbox_id, "Stopped pod sandbox");
358
359        // Publish sandbox stopped event with structured payload
360        let event = SandboxStopped {
361            sandbox_id: pod_sandbox_id.to_owned(),
362            reason: "stopped".to_owned(),
363            exit_code: 0,
364        };
365        let payload = serialize_sandbox_stopped(&event).unwrap_or_default();
366        self.publish_event(pod_sandbox_id, "stopped", &payload)
367            .await;
368
369        Ok(())
370    }
371
372    /// Remove a pod sandbox
373    pub async fn remove_pod_sandbox(&self, pod_sandbox_id: &str) -> Result<()> {
374        // Remove all containers in the sandbox
375        let containers_to_remove: Vec<String> = {
376            let map = self.container_sandbox_map.read().await;
377            map.iter()
378                .filter(|(_, sid)| *sid == pod_sandbox_id)
379                .map(|(cid, _)| cid.clone())
380                .collect()
381        };
382
383        for container_id in containers_to_remove {
384            if let Err(e) = self.remove_container(&container_id).await {
385                tracing::warn!(
386                    container_id = %container_id,
387                    error = %e,
388                    "Failed to remove container during sandbox removal"
389                );
390            }
391        }
392
393        // Release sandbox back to pool
394        self.sandbox_pool.release(pod_sandbox_id).await?;
395        tracing::info!(sandbox_id = %pod_sandbox_id, "Removed pod sandbox");
396        Ok(())
397    }
398
399    /// Get pod sandbox status
400    ///
401    /// Returns the runtime PodSandboxStatus (has ToCapnp for handler serialization)
402    /// plus verbose info as KeyValue for the wire format.
403    pub async fn pod_sandbox_status(
404        &self,
405        pod_sandbox_id: &str,
406        verbose: bool,
407    ) -> Result<(PodSandboxStatus, Vec<KeyValue>)> {
408        let sandbox = self
409            .sandbox_pool
410            .get(pod_sandbox_id)
411            .await
412            .ok_or_else(|| WorkerError::SandboxNotFound(pod_sandbox_id.to_owned()))?;
413
414        let mut info = Vec::new();
415        if verbose {
416            // Get PIDs from backend (host-level sandbox management)
417            let pids = self.sandbox_pool.backend().get_pids(&sandbox).await.ok();
418            let pid_str = pids.map_or_else(
419                || "none".to_owned(),
420                |p| {
421                    p.iter()
422                        .map(std::string::ToString::to_string)
423                        .collect::<Vec<_>>()
424                        .join(",")
425                },
426            );
427            info.push(KeyValue {
428                key: "vm_pids".to_owned(),
429                value: pid_str,
430            });
431        }
432
433        Ok((PodSandboxStatus::from(&sandbox), info))
434    }
435
436    /// List pod sandboxes matching the given filter.
437    ///
438    /// Empty string fields mean "no filter". State filtering is not supported
439    /// over the wire (Cap'n Proto enum defaults make it ambiguous).
440    pub async fn list_pod_sandbox(&self, filter: &PodSandboxFilter) -> Result<Vec<PodSandboxInfo>> {
441        let mut sandboxes = self.sandbox_pool.list_active().await;
442
443        if !filter.id.is_empty() {
444            sandboxes.retain(|s| s.id == filter.id);
445        }
446        for kv in &filter.label_selector {
447            sandboxes.retain(|s| {
448                s.labels
449                    .iter()
450                    .any(|l| l.key == kv.key && l.value == kv.value)
451            });
452        }
453
454        Ok(sandboxes.iter().map(PodSandboxInfo::from).collect())
455    }
456
457    // ─────────────────────────────────────────────────────────────────────────
458    // Container Lifecycle
459    // ─────────────────────────────────────────────────────────────────────────
460
461    /// Create a container within a pod sandbox (host-level, no guest agent)
462    ///
463    /// In host-level mode, creating a container means:
464    /// 1. Verify the sandbox (VM) exists and is ready
465    /// 2. Ensure the container image is available via RAFS
466    /// 3. Track container metadata for lifecycle management
467    ///
468    /// The container's rootfs will be accessible to the VM via virtiofs.
469    pub async fn create_container(
470        &self,
471        pod_sandbox_id: &str,
472        config: &ContainerConfig,
473        _sandbox_config: &PodSandboxConfig,
474    ) -> Result<String> {
475        // Verify sandbox exists and is ready
476        let sandbox = self
477            .sandbox_pool
478            .get(pod_sandbox_id)
479            .await
480            .ok_or_else(|| WorkerError::SandboxNotFound(pod_sandbox_id.to_owned()))?;
481
482        if !sandbox.is_ready() {
483            return Err(WorkerError::SandboxInvalidState {
484                sandbox_id: pod_sandbox_id.to_owned(),
485                state: "NotReady".to_owned(),
486                expected: "Ready".to_owned(),
487            });
488        }
489
490        // Generate container ID
491        let container_id = uuid::Uuid::new_v4().to_string();
492
493        // Ensure image is available (pull if needed via RafsStore)
494        // The image will be served to the VM via virtiofs
495        if !config.image.image.is_empty() {
496            tracing::debug!(
497                container_id = %container_id,
498                image = %config.image.image,
499                "Container image reference recorded (available via virtiofs)"
500            );
501            // Note: Image pulling is handled by RafsStore at sandbox creation time
502            // The virtiofs daemon serves the RAFS filesystem to the VM
503        }
504
505        // Create container
506        let container = Container::new(container_id.clone(), pod_sandbox_id.to_owned(), config);
507
508        // Store container
509        {
510            let mut containers = self.containers.write().await;
511            containers.insert(container_id.clone(), container);
512        }
513        {
514            let mut map = self.container_sandbox_map.write().await;
515            map.insert(container_id.clone(), pod_sandbox_id.to_owned());
516        }
517
518        tracing::info!(
519            container_id = %container_id,
520            sandbox_id = %pod_sandbox_id,
521            image = %config.image.image,
522            "Created container (host-level, rootfs via virtiofs)"
523        );
524
525        Ok(container_id)
526    }
527
528    /// Start a created container (host-level, no guest agent)
529    ///
530    /// In host-level mode, "starting" a container means:
531    /// 1. The container's image rootfs is accessible via virtiofs
532    /// 2. The VM is running and can access the filesystem
533    /// 3. No guest agent required - VM manages its own processes
534    pub async fn start_container(&self, container_id: &str) -> Result<()> {
535        // Get sandbox for this container
536        let sandbox_id = {
537            let map = self.container_sandbox_map.read().await;
538            map.get(container_id)
539                .cloned()
540                .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?
541        };
542
543        let sandbox = self
544            .sandbox_pool
545            .get(&sandbox_id)
546            .await
547            .ok_or_else(|| WorkerError::SandboxNotFound(sandbox_id.clone()))?;
548
549        // Verify sandbox is ready (VM is running)
550        if !sandbox.is_ready() {
551            return Err(WorkerError::SandboxInvalidState {
552                sandbox_id: sandbox_id.clone(),
553                state: "NotReady".to_owned(),
554                expected: "Ready".to_owned(),
555            });
556        }
557
558        let mut containers = self.containers.write().await;
559
560        let container = containers
561            .get_mut(container_id)
562            .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
563
564        if container.state != ContainerState::ContainerCreated {
565            return Err(WorkerError::ContainerInvalidState {
566                container_id: container_id.to_owned(),
567                state: format!("{:?}", container.state),
568                expected: "ContainerCreated".to_owned(),
569            });
570        }
571
572        // Host-level container start:
573        // The VM is already running (sandbox ready), virtiofs provides the rootfs.
574        // In this model, "starting" means the container is accessible.
575        // The VM's init process or workload handles actual execution.
576        //
577        // Future enhancement: Use hypervisor.add_device() to hot-add container-specific
578        // filesystems or block devices if needed.
579
580        if sandbox.backend_handle.is_some() {
581            tracing::debug!(
582                container_id = %container_id,
583                sandbox_id = %sandbox_id,
584                "Container filesystem accessible via sandbox's virtiofs mount"
585            );
586        }
587
588        container.state = ContainerState::ContainerRunning;
589        let image = container.image.image.clone();
590        tracing::info!(
591            container_id = %container_id,
592            sandbox_id = %sandbox_id,
593            "Started container (host-level, VM is running)"
594        );
595
596        // Release the lock before publishing to avoid deadlock
597        drop(containers);
598
599        // Publish container started event with structured payload
600        let event = ContainerStarted {
601            container_id: container_id.to_owned(),
602            sandbox_id: sandbox_id.clone(),
603            image,
604        };
605        let payload = serialize_container_started(&event).unwrap_or_default();
606        self.publish_event(container_id, "started", &payload).await;
607
608        Ok(())
609    }
610
611    /// Stop a running container (host-level, no guest agent)
612    ///
613    /// In host-level mode, "stopping" a container means:
614    /// 1. Mark the container as exited
615    /// 2. The VM continues running (sandbox lifecycle is separate)
616    /// 3. Optionally remove container-specific resources
617    pub async fn stop_container(&self, container_id: &str, _timeout: i64) -> Result<()> {
618        let mut containers = self.containers.write().await;
619
620        let container = containers
621            .get_mut(container_id)
622            .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
623
624        if container.state != ContainerState::ContainerRunning {
625            // Already stopped
626            return Ok(());
627        }
628
629        // Host-level container stop:
630        // Without a guest agent, we can't signal processes inside the VM.
631        // We simply mark the container as exited - the VM sandbox continues running.
632        //
633        // In a full implementation with vsock or minimal agent, we could:
634        // - Send a shutdown signal via vsock
635        // - Use a watchdog/heartbeat mechanism
636        //
637        // For now, we rely on the VM workload managing its own lifecycle.
638
639        container.state = ContainerState::ContainerExited;
640        container.exit_code = Some(0);
641        let sandbox_id = container.pod_sandbox_id.clone();
642        let exit_code = container.exit_code.unwrap_or(0);
643        tracing::info!(
644            container_id = %container_id,
645            "Stopped container (host-level, container marked as exited)"
646        );
647
648        // Release the lock before publishing to avoid deadlock
649        drop(containers);
650
651        // Publish container stopped event with structured payload
652        let event = ContainerStopped {
653            container_id: container_id.to_owned(),
654            sandbox_id,
655            exit_code,
656            reason: "stopped".to_owned(),
657        };
658        let payload = serialize_container_stopped(&event).unwrap_or_default();
659        self.publish_event(container_id, "stopped", &payload).await;
660
661        Ok(())
662    }
663
664    /// Remove a container
665    pub async fn remove_container(&self, container_id: &str) -> Result<()> {
666        // Remove from containers map
667        {
668            let mut containers = self.containers.write().await;
669            let container = containers
670                .get(container_id)
671                .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
672
673            if container.state == ContainerState::ContainerRunning {
674                return Err(WorkerError::ContainerInvalidState {
675                    container_id: container_id.to_owned(),
676                    state: "ContainerRunning".to_owned(),
677                    expected: "ContainerExited or ContainerCreated".to_owned(),
678                });
679            }
680
681            containers.remove(container_id);
682        }
683
684        // Remove from sandbox map
685        {
686            let mut map = self.container_sandbox_map.write().await;
687            map.remove(container_id);
688        }
689
690        tracing::info!(container_id = %container_id, "Removed container");
691        Ok(())
692    }
693
694    /// Get container status
695    ///
696    /// Returns the runtime ContainerStatus (has ToCapnp for handler serialization)
697    /// plus verbose info as KeyValue for the wire format.
698    pub async fn container_status(
699        &self,
700        container_id: &str,
701        verbose: bool,
702    ) -> Result<(ContainerStatus, Vec<KeyValue>)> {
703        let containers = self.containers.read().await;
704
705        let container = containers
706            .get(container_id)
707            .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
708
709        let mut info = Vec::new();
710        if verbose {
711            info.push(KeyValue {
712                key: "pid".to_owned(),
713                value: container
714                    .pid
715                    .map_or_else(|| "none".to_owned(), |p| p.to_string()),
716            });
717        }
718
719        Ok((ContainerStatus::from(container), info))
720    }
721
722    /// List containers matching the given filter.
723    ///
724    /// Empty string fields mean "no filter". State filtering is not supported
725    /// over the wire (Cap'n Proto enum defaults make it ambiguous).
726    pub async fn list_containers(&self, filter: &ContainerFilter) -> Result<Vec<ContainerInfo>> {
727        let containers = self.containers.read().await;
728        let mut result: Vec<Container> = containers.values().cloned().collect();
729
730        if !filter.id.is_empty() {
731            result.retain(|c| c.id == filter.id);
732        }
733        if !filter.pod_sandbox_id.is_empty() {
734            result.retain(|c| c.pod_sandbox_id == filter.pod_sandbox_id);
735        }
736        for kv in &filter.label_selector {
737            result.retain(|c| {
738                c.labels
739                    .iter()
740                    .any(|l| l.key == kv.key && l.value == kv.value)
741            });
742        }
743
744        Ok(result.iter().map(ContainerInfo::from).collect())
745    }
746
747    // ─────────────────────────────────────────────────────────────────────────
748    // Exec
749    // ─────────────────────────────────────────────────────────────────────────
750
751    /// Execute a command in a container through the shared task-owned stdio
752    /// adapter. The caller is the verified RPC subject, never a request field;
753    /// `ExecMount` compares it with the subject bound when the sandbox was
754    /// admitted before invoking the backend.
755    pub async fn exec_sync(
756        &self,
757        caller: &hyprstream_vfs::Subject,
758        container_id: &str,
759        cmd: &[String],
760        timeout: i64,
761    ) -> Result<ExecSyncResult> {
762        let containers = self.containers.read().await;
763
764        let container = containers
765            .get(container_id)
766            .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
767
768        if container.state != ContainerState::ContainerRunning {
769            return Err(WorkerError::ContainerInvalidState {
770                container_id: container_id.to_owned(),
771                state: format!("{:?}", container.state),
772                expected: "ContainerRunning".to_owned(),
773            });
774        }
775
776        drop(containers);
777
778        let sandbox_id = self
779            .container_sandbox_map
780            .read()
781            .await
782            .get(container_id)
783            .cloned()
784            .ok_or_else(|| WorkerError::SandboxNotFound(container_id.to_owned()))?;
785        let timeout_secs = u64::try_from(timeout)
786            .map_err(|_| WorkerError::ExecFailed("exec timeout must be non-negative".to_owned()))?;
787        let result = self
788            .exec_mount
789            .exec_task(&sandbox_id, caller, cmd, timeout_secs)
790            .await
791            .map_err(|error| WorkerError::ExecFailed(error.to_string()))?;
792
793        Ok(ExecSyncResult {
794            stdout: result.stdout,
795            stderr: result.stderr,
796            exit_code: result.exit_code,
797        })
798    }
799
800    // ─────────────────────────────────────────────────────────────────────────
801    // Stats
802    // ─────────────────────────────────────────────────────────────────────────
803
804    /// Get pod sandbox stats
805    pub async fn pod_sandbox_stats(&self, pod_sandbox_id: &str) -> Result<PodSandboxStats> {
806        let sandbox = self
807            .sandbox_pool
808            .get(pod_sandbox_id)
809            .await
810            .ok_or_else(|| WorkerError::SandboxNotFound(pod_sandbox_id.to_owned()))?;
811
812        // TODO: Get actual stats from VM
813        Ok(PodSandboxStats::from(&sandbox))
814    }
815
816    /// List pod sandbox stats matching the given filter.
817    pub async fn list_pod_sandbox_stats(
818        &self,
819        filter: &PodSandboxStatsFilter,
820    ) -> Result<Vec<PodSandboxStats>> {
821        let sandboxes = self.sandbox_pool.list_active().await;
822        let mut results = Vec::new();
823
824        for sandbox in sandboxes {
825            if !filter.id.is_empty() && sandbox.id != filter.id {
826                continue;
827            }
828            let mut matches_labels = true;
829            for kv in &filter.label_selector {
830                if !sandbox
831                    .labels
832                    .iter()
833                    .any(|l| l.key == kv.key && l.value == kv.value)
834                {
835                    matches_labels = false;
836                    break;
837                }
838            }
839            if !matches_labels {
840                continue;
841            }
842            results.push(PodSandboxStats::from(&sandbox));
843        }
844
845        Ok(results)
846    }
847
848    /// Get container stats
849    pub async fn container_stats(&self, container_id: &str) -> Result<ContainerStats> {
850        // Snapshot the container and release the lock before any backend RPC:
851        // the guest stats call awaits, and holding the `containers` read lock
852        // across it would needlessly serialise all container mutations.
853        let (mut stats, pod_sandbox_id) = {
854            let containers = self.containers.read().await;
855            let container = containers
856                .get(container_id)
857                .ok_or_else(|| WorkerError::ContainerNotFound(container_id.to_owned()))?;
858            (
859                ContainerStats::from(container),
860                container.pod_sandbox_id.clone(),
861            )
862        };
863
864        self.enrich_container_stats(container_id, &pod_sandbox_id, &mut stats)
865            .await;
866        Ok(stats)
867    }
868
869    /// Overlay real guest-sourced CPU/memory usage onto placeholder
870    /// [`ContainerStats`] when the sandbox backend can observe the guest
871    /// (e.g. Kata via the kata-agent `StatsContainer` RPC). Backends without
872    /// in-guest visibility return `None` and the placeholder values stand.
873    /// A backend error degrades to the placeholder (logged) rather than
874    /// failing the stats call.
875    async fn enrich_container_stats(
876        &self,
877        container_id: &str,
878        pod_sandbox_id: &str,
879        stats: &mut ContainerStats,
880    ) {
881        let Some(sandbox) = self.sandbox_pool.get(pod_sandbox_id).await else {
882            return;
883        };
884        match self.sandbox_pool.backend().container_stats(&sandbox).await {
885            Ok(Some((cpu, memory))) => {
886                stats.cpu = cpu;
887                stats.memory = memory;
888            }
889            Ok(None) => {}
890            Err(e) => {
891                tracing::warn!(
892                    container_id = %container_id,
893                    error = %e,
894                    "Failed to fetch guest container stats; returning placeholder stats"
895                );
896            }
897        }
898    }
899
900    /// List container stats matching the given filter.
901    pub async fn list_container_stats(
902        &self,
903        filter: &ContainerStatsFilter,
904    ) -> Result<Vec<ContainerStats>> {
905        // Collect matching (placeholder stats, container id, sandbox id)
906        // snapshots under the lock, then release it before enriching each with
907        // guest-sourced usage (the backend RPCs await and must not run under
908        // the `containers` read lock).
909        let snapshots: Vec<(ContainerStats, String, String)> = {
910            let containers = self.containers.read().await;
911            containers
912                .values()
913                .filter(|container| {
914                    if !filter.id.is_empty() && container.id != filter.id {
915                        return false;
916                    }
917                    if !filter.pod_sandbox_id.is_empty()
918                        && container.pod_sandbox_id != filter.pod_sandbox_id
919                    {
920                        return false;
921                    }
922                    filter.label_selector.iter().all(|kv| {
923                        container
924                            .labels
925                            .iter()
926                            .any(|l| l.key == kv.key && l.value == kv.value)
927                    })
928                })
929                .map(|container| {
930                    (
931                        ContainerStats::from(container),
932                        container.id.clone(),
933                        container.pod_sandbox_id.clone(),
934                    )
935                })
936                .collect()
937        };
938
939        let mut results = Vec::with_capacity(snapshots.len());
940        for (mut stats, container_id, pod_sandbox_id) in snapshots {
941            self.enrich_container_stats(&container_id, &pod_sandbox_id, &mut stats)
942                .await;
943            results.push(stats);
944        }
945
946        Ok(results)
947    }
948
949    // ═══════════════════════════════════════════════════════════════════════════════
950    // Terminal Attach/Detach
951    // ═══════════════════════════════════════════════════════════════════════════════
952
953    /// Attach to a container's terminal I/O streams (tmux-like)
954    ///
955    /// Performs DH key exchange and pre-authorization atomically via StreamChannel.
956    /// The stream is immediately ready — no separate start_fd_stream() call needed.
957    ///
958    /// # Flow
959    /// 1. Client calls attach() with ephemeral pubkey in envelope
960    /// 2. Server does DH + pre-auth + prepares streaming
961    /// 3. REP is sent with StreamInfo (stream_id, endpoint, server_pubkey)
962    /// 4. Continuation spawns the FD streaming task AFTER client receives REP
963    /// 5. Client derives topic/mac_key from DH shared secret and subscribes
964    pub async fn prepare_attach(
965        &self,
966        ctx: &EnvelopeContext,
967        container_id: &str,
968    ) -> Result<(StreamInfo, hyprstream_rpc::service::Continuation)> {
969        // Verify container exists
970        let containers = self.containers.read().await;
971        if !containers.contains_key(container_id) {
972            return Err(anyhow::anyhow!("Container not found: {}", container_id).into());
973        }
974        drop(containers);
975
976        // Extract ephemeral pubkey from envelope (required for E2E auth)
977        let client_pubkey = ctx.ephemeral_pubkey().ok_or_else(|| {
978            anyhow::anyhow!("Attach requires client ephemeral pubkey for E2E authentication")
979        })?;
980
981        // Forward user claims for StreamService subscription-time validation
982        let claims = ctx.claims().cloned();
983
984        // DH + pre-authorization via StreamChannel — atomic, no pending state
985        let stream_ctx = self
986            .stream_channel
987            .prepare_third_party_interop_stream_with_claims(&client_pubkey, 600, claims)
988            .await
989            .map_err(|e| anyhow::anyhow!("Stream preparation failed: {}", e))?
990            .with_qos_preset::<hyprstream_rpc::stream_info::Pipe>();
991
992        let stream_id = stream_ctx.stream_id().to_owned();
993        let broadcast_path = hyprstream_rpc::moq_stream::global_moq_origin()
994            .map(|o| o.broadcast_path(stream_ctx.topic()))
995            .unwrap_or_default();
996
997        // Register active stream before returning
998        let cancel_token = stream_ctx.cancel_token().child_token();
999        self.active_fd_streams.write().await.insert(
1000            stream_id.clone(),
1001            ActiveFdStream {
1002                container_id: container_id.to_owned(),
1003                cancel_token: cancel_token.clone(),
1004            },
1005        );
1006
1007        let sandbox_id = self
1008            .container_sandbox_map
1009            .read()
1010            .await
1011            .get(container_id)
1012            .cloned();
1013        let sandbox_pool = self.sandbox_pool.clone();
1014        let container_id_owned = container_id.to_owned();
1015        let active_streams = self.active_fd_streams.clone();
1016        let stream_id_for_cleanup = stream_id.clone();
1017        let sc = Arc::clone(&self.stream_channel);
1018
1019        let stream_info = StreamInfo {
1020            stream_id,
1021            dh_public: *stream_ctx.server_pubkey(),
1022            qos: stream_ctx.qos().clone(),
1023            broadcast_path,
1024            announced_at: stream_ctx.reach(), // #384: per-stream reach via ctx
1025            moql_server_identity: stream_ctx.moql_server_identity(),
1026            kem_ciphertexts: Vec::new(), // #554: classical FD stream (dh_public path), no hybrid KEM
1027        };
1028
1029        // Continuation: spawns FD streaming task AFTER REP is sent to client
1030        let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
1031            tokio::spawn(async move {
1032                let container_id_for_log = container_id_owned.clone();
1033                let result = sc
1034                    .run_stream(&stream_ctx, |mut publisher| async move {
1035                        let result = run_fd_streaming_task(
1036                            &mut publisher,
1037                            container_id_owned,
1038                            cancel_token,
1039                            sandbox_pool,
1040                            sandbox_id,
1041                        )
1042                        .await;
1043                        (publisher, result)
1044                    })
1045                    .await;
1046
1047                // Always clean up active_fd_streams entry on task exit
1048                active_streams.write().await.remove(&stream_id_for_cleanup);
1049
1050                if let Err(e) = result {
1051                    warn!(container_id = %container_id_for_log, error = %e, "FD streaming task failed");
1052                }
1053            });
1054        });
1055
1056        Ok((stream_info, continuation))
1057    }
1058
1059    /// Detach from a container's terminal I/O streams
1060    ///
1061    /// Cancels all active FD streams for the container.
1062    pub async fn detach(&self, container_id: &str) -> Result<()> {
1063        // Find and cancel all active streams for this container
1064        let mut active_streams = self.active_fd_streams.write().await;
1065        let streams_to_cancel: Vec<String> = active_streams
1066            .iter()
1067            .filter(|(_, active)| active.container_id == container_id)
1068            .map(|(stream_id, _)| stream_id.clone())
1069            .collect();
1070
1071        for stream_id in streams_to_cancel {
1072            if let Some(active) = active_streams.remove(&stream_id) {
1073                info!(
1074                    stream_id = %stream_id,
1075                    container_id = %container_id,
1076                    "Cancelling FD stream"
1077                );
1078                active.cancel_token.cancel();
1079            }
1080        }
1081
1082        Ok(())
1083    }
1084}
1085
1086/// Active FD stream with cancellation support
1087struct ActiveFdStream {
1088    container_id: String,
1089    cancel_token: CancellationToken,
1090}
1091
1092// ═══════════════════════════════════════════════════════════════════════════════
1093// FD Streaming Task
1094// ═══════════════════════════════════════════════════════════════════════════════
1095
1096/// Run the FD streaming task that bridges container I/O to StreamService.
1097///
1098/// This task:
1099/// 1. Uses an async StreamPublisher (DH-derived keys, pre-authorized)
1100/// 2. Reads from container console (vsock/serial)
1101/// 3. Publishes data via StreamPublisher with HMAC authentication
1102async fn run_fd_streaming_task(
1103    publisher: &mut AnyStreamPublisher,
1104    container_id: String,
1105    cancel_token: CancellationToken,
1106    sandbox_pool: Arc<SandboxPool>,
1107    sandbox_id: Option<String>,
1108) -> anyhow::Result<()> {
1109    use tokio::io::AsyncReadExt;
1110
1111    info!(
1112        container_id = %container_id,
1113        topic = %publisher.topic(),
1114        "Starting FD streaming task"
1115    );
1116
1117    let sandbox = if let Some(ref sid) = sandbox_id {
1118        sandbox_pool.get(sid).await
1119    } else {
1120        None
1121    };
1122
1123    let console_path = sandbox
1124        .as_ref()
1125        .and_then(|s| s.console_socket().map(std::path::Path::to_path_buf));
1126
1127    if let Some(ref path) = console_path {
1128        match tokio::net::UnixStream::connect(path).await {
1129            Ok(mut stream) => {
1130                let mut buf = vec![0u8; 4096];
1131                loop {
1132                    tokio::select! {
1133                        biased;
1134                        _ = cancel_token.cancelled() => {
1135                            info!(container_id = %container_id, "FD streaming cancelled");
1136                            if !publisher.is_terminated() {
1137                                publisher.publish_error("cancelled").await?;
1138                            }
1139                            return Ok(());
1140                        }
1141                        result = stream.read(&mut buf) => {
1142                            match result? {
1143                                0 => {
1144                                    info!(container_id = %container_id, "Console EOF — container exited");
1145                                    return Ok(());
1146                                }
1147                                n => {
1148                                    publisher.publish_data(&buf[..n]).await?;
1149                                }
1150                            }
1151                        }
1152                    }
1153                }
1154            }
1155            Err(e) => {
1156                warn!(
1157                    container_id = %container_id,
1158                    path = ?path,
1159                    error = %e,
1160                    "Failed to connect to console socket, waiting for cancellation"
1161                );
1162                cancel_token.cancelled().await;
1163                if !publisher.is_terminated() {
1164                    publisher.publish_error("console unavailable").await?;
1165                }
1166            }
1167        }
1168    } else {
1169        if sandbox_id.is_some() {
1170            warn!(container_id = %container_id, "No console socket configured for sandbox");
1171        } else {
1172            warn!(container_id = %container_id, "No sandbox found for container, FD streaming idle");
1173        }
1174        cancel_token.cancelled().await;
1175        if !publisher.is_terminated() {
1176            publisher.publish_error("cancelled").await?;
1177        }
1178    }
1179    Ok(())
1180}
1181
1182// ═══════════════════════════════════════════════════════════════════════════════
1183// Domain → wire type conversions (From impls at service boundary)
1184// ═══════════════════════════════════════════════════════════════════════════════
1185
1186impl From<&PodSandbox> for PodSandboxStatus {
1187    fn from(s: &PodSandbox) -> Self {
1188        Self {
1189            id: s.id.clone(),
1190            metadata: s.metadata.clone(),
1191            state: s.state,
1192            created_at: Timestamp {
1193                seconds: s.created_at.timestamp(),
1194                nanos: s.created_at.timestamp_subsec_nanos() as i32,
1195            },
1196            network: Default::default(),
1197            linux: Default::default(),
1198            labels: s.labels.clone(),
1199            annotations: s.annotations.clone(),
1200            runtime_handler: s.runtime_handler.clone(),
1201        }
1202    }
1203}
1204
1205impl From<&PodSandbox> for PodSandboxInfo {
1206    fn from(s: &PodSandbox) -> Self {
1207        Self {
1208            id: s.id.clone(),
1209            metadata: s.metadata.clone(),
1210            state: s.state,
1211            created_at: Timestamp {
1212                seconds: s.created_at.timestamp(),
1213                nanos: s.created_at.timestamp_subsec_nanos() as i32,
1214            },
1215            labels: s.labels.clone(),
1216            annotations: s.annotations.clone(),
1217            runtime_handler: s.runtime_handler.clone(),
1218        }
1219    }
1220}
1221
1222impl From<&PodSandbox> for PodSandboxStats {
1223    fn from(s: &PodSandbox) -> Self {
1224        Self {
1225            attributes: PodSandboxAttributes {
1226                id: s.id.clone(),
1227                metadata: s.metadata.clone(),
1228                labels: s.labels.clone(),
1229                annotations: s.annotations.clone(),
1230            },
1231            linux: LinuxPodSandboxStats::default(),
1232        }
1233    }
1234}
1235
1236impl From<&Container> for ContainerStatus {
1237    fn from(c: &Container) -> Self {
1238        Self {
1239            id: c.id.clone(),
1240            metadata: c.metadata.clone(),
1241            state: c.state,
1242            created_at: Timestamp {
1243                seconds: c.created_at.timestamp(),
1244                nanos: c.created_at.timestamp_subsec_nanos() as i32,
1245            },
1246            started_at: Timestamp {
1247                seconds: 0,
1248                nanos: 0,
1249            },
1250            finished_at: Timestamp {
1251                seconds: 0,
1252                nanos: 0,
1253            },
1254            exit_code: c.exit_code.unwrap_or(0),
1255            image: c.image.clone(),
1256            image_ref: String::new(),
1257            reason: String::new(),
1258            message: String::new(),
1259            labels: c.labels.clone(),
1260            annotations: c.annotations.clone(),
1261            mounts: Vec::new(),
1262            log_path: String::new(),
1263        }
1264    }
1265}
1266
1267impl From<&Container> for ContainerInfo {
1268    fn from(c: &Container) -> Self {
1269        Self {
1270            id: c.id.clone(),
1271            pod_sandbox_id: c.pod_sandbox_id.clone(),
1272            metadata: c.metadata.clone(),
1273            image: c.image.clone(),
1274            image_ref: String::new(),
1275            state: c.state,
1276            created_at: Timestamp {
1277                seconds: c.created_at.timestamp(),
1278                nanos: c.created_at.timestamp_subsec_nanos() as i32,
1279            },
1280            labels: c.labels.clone(),
1281            annotations: c.annotations.clone(),
1282        }
1283    }
1284}
1285
1286impl From<&Container> for ContainerStats {
1287    fn from(c: &Container) -> Self {
1288        Self {
1289            attributes: ContainerAttributes {
1290                id: c.id.clone(),
1291                metadata: c.metadata.clone(),
1292                labels: c.labels.clone(),
1293                annotations: c.annotations.clone(),
1294            },
1295            cpu: Default::default(),
1296            memory: Default::default(),
1297            writable_layer: Default::default(),
1298        }
1299    }
1300}
1301
1302// ═══════════════════════════════════════════════════════════════════════════════
1303// Generated scope handler traits (typed inner dispatch)
1304// ═══════════════════════════════════════════════════════════════════════════════
1305
1306// ═══════════════════════════════════════════════════════════════════════════════
1307// Typed Scope Handler Implementations
1308// ═══════════════════════════════════════════════════════════════════════════════
1309
1310use hyprstream_rpc_std::worker_client::{
1311    CreateContainerRequest, ExecSyncRequest, PullImageRequest,
1312};
1313
1314#[async_trait::async_trait(?Send)]
1315impl RuntimeHandler for WorkerService {
1316    async fn handle_version(
1317        &self,
1318        _ctx: &EnvelopeContext,
1319        _request_id: u64,
1320        value: &str,
1321    ) -> AnyhowResult<VersionInfo> {
1322        let resp = self.version(value).await?;
1323        Ok(resp)
1324    }
1325
1326    async fn handle_status(
1327        &self,
1328        _ctx: &EnvelopeContext,
1329        _request_id: u64,
1330        request: &StatusRequest,
1331    ) -> AnyhowResult<RuntimeStatus> {
1332        let resp = self.status(request.verbose).await?;
1333        Ok(resp.status)
1334    }
1335}
1336
1337#[async_trait::async_trait(?Send)]
1338impl SandboxHandler for WorkerService {
1339    async fn handle_run(
1340        &self,
1341        ctx: &EnvelopeContext,
1342        _request_id: u64,
1343        data: &PodSandboxConfig,
1344    ) -> AnyhowResult<String> {
1345        // #525 P2: thread the verified caller identity through to admission
1346        // control. `ctx.subject()` is derived from the verified envelope
1347        // (signer key / JWT), never caller-asserted data.
1348        let sandbox_id = self.run_pod_sandbox(&ctx.subject(), data).await?;
1349        Ok(sandbox_id)
1350    }
1351
1352    async fn handle_stop(
1353        &self,
1354        _ctx: &EnvelopeContext,
1355        _request_id: u64,
1356        value: &str,
1357    ) -> AnyhowResult<()> {
1358        self.stop_pod_sandbox(value).await?;
1359        Ok(())
1360    }
1361
1362    async fn handle_remove(
1363        &self,
1364        _ctx: &EnvelopeContext,
1365        _request_id: u64,
1366        value: &str,
1367    ) -> AnyhowResult<()> {
1368        self.remove_pod_sandbox(value).await?;
1369        Ok(())
1370    }
1371
1372    async fn handle_status(
1373        &self,
1374        _ctx: &EnvelopeContext,
1375        _request_id: u64,
1376        request: &PodSandboxStatusRequest,
1377    ) -> AnyhowResult<PodSandboxStatusResponse> {
1378        let (status, info) = self
1379            .pod_sandbox_status(&request.pod_sandbox_id, request.verbose)
1380            .await?;
1381        Ok(PodSandboxStatusResponse { status, info })
1382    }
1383
1384    async fn handle_list(
1385        &self,
1386        _ctx: &EnvelopeContext,
1387        _request_id: u64,
1388        data: &PodSandboxFilter,
1389    ) -> AnyhowResult<Vec<PodSandboxInfo>> {
1390        let sandboxes = self.list_pod_sandbox(data).await?;
1391        Ok(sandboxes)
1392    }
1393
1394    async fn handle_stats(
1395        &self,
1396        _ctx: &EnvelopeContext,
1397        _request_id: u64,
1398        value: &str,
1399    ) -> AnyhowResult<PodSandboxStats> {
1400        let stats = self.pod_sandbox_stats(value).await?;
1401        Ok(stats)
1402    }
1403
1404    async fn handle_list_stats(
1405        &self,
1406        _ctx: &EnvelopeContext,
1407        _request_id: u64,
1408        filter: &PodSandboxStatsFilter,
1409    ) -> AnyhowResult<Vec<PodSandboxStats>> {
1410        let stats = self.list_pod_sandbox_stats(filter).await?;
1411        Ok(stats)
1412    }
1413}
1414
1415#[async_trait::async_trait(?Send)]
1416impl ContainerHandler for WorkerService {
1417    async fn handle_create(
1418        &self,
1419        _ctx: &EnvelopeContext,
1420        _request_id: u64,
1421        data: &CreateContainerRequest,
1422    ) -> AnyhowResult<String> {
1423        let container_id = self
1424            .create_container(&data.pod_sandbox_id, &data.config, &data.sandbox_config)
1425            .await?;
1426        Ok(container_id)
1427    }
1428
1429    async fn handle_start(
1430        &self,
1431        _ctx: &EnvelopeContext,
1432        _request_id: u64,
1433        value: &str,
1434    ) -> AnyhowResult<()> {
1435        self.start_container(value).await?;
1436        Ok(())
1437    }
1438
1439    async fn handle_stop(
1440        &self,
1441        _ctx: &EnvelopeContext,
1442        _request_id: u64,
1443        request: &StopContainerRequest,
1444    ) -> AnyhowResult<()> {
1445        self.stop_container(&request.container_id, request.timeout)
1446            .await?;
1447        Ok(())
1448    }
1449
1450    async fn handle_remove(
1451        &self,
1452        _ctx: &EnvelopeContext,
1453        _request_id: u64,
1454        value: &str,
1455    ) -> AnyhowResult<()> {
1456        self.remove_container(value).await?;
1457        Ok(())
1458    }
1459
1460    async fn handle_status(
1461        &self,
1462        _ctx: &EnvelopeContext,
1463        _request_id: u64,
1464        request: &ContainerStatusRequest,
1465    ) -> AnyhowResult<ContainerStatusResponse> {
1466        let (status, info) = self
1467            .container_status(&request.container_id, request.verbose)
1468            .await?;
1469        Ok(ContainerStatusResponse { status, info })
1470    }
1471
1472    async fn handle_list(
1473        &self,
1474        _ctx: &EnvelopeContext,
1475        _request_id: u64,
1476        data: &ContainerFilter,
1477    ) -> AnyhowResult<Vec<ContainerInfo>> {
1478        let containers = self.list_containers(data).await?;
1479        Ok(containers)
1480    }
1481
1482    async fn handle_stats(
1483        &self,
1484        _ctx: &EnvelopeContext,
1485        _request_id: u64,
1486        value: &str,
1487    ) -> AnyhowResult<ContainerStats> {
1488        let stats = self.container_stats(value).await?;
1489        Ok(stats)
1490    }
1491
1492    async fn handle_list_stats(
1493        &self,
1494        _ctx: &EnvelopeContext,
1495        _request_id: u64,
1496        data: &ContainerStatsFilter,
1497    ) -> AnyhowResult<Vec<ContainerStats>> {
1498        let stats = self.list_container_stats(data).await?;
1499        Ok(stats)
1500    }
1501
1502    async fn handle_exec(
1503        &self,
1504        ctx: &EnvelopeContext,
1505        _request_id: u64,
1506        data: &ExecSyncRequest,
1507    ) -> AnyhowResult<ExecSyncResult> {
1508        let resp = self
1509            .exec_sync(&ctx.subject(), &data.container_id, &data.cmd, data.timeout)
1510            .await?;
1511        Ok(resp)
1512    }
1513
1514    async fn handle_attach(
1515        &self,
1516        ctx: &EnvelopeContext,
1517        _request_id: u64,
1518        request: &AttachRequest,
1519    ) -> AnyhowResult<(StreamInfo, hyprstream_rpc::service::Continuation)> {
1520        let _ = &request.fds; // fds used for future fd selection
1521        let (stream_info, continuation) = self.prepare_attach(ctx, &request.container_id).await?;
1522        Ok((stream_info, continuation))
1523    }
1524
1525    async fn handle_detach(
1526        &self,
1527        _ctx: &EnvelopeContext,
1528        _request_id: u64,
1529        value: &str,
1530    ) -> AnyhowResult<()> {
1531        self.detach(value).await?;
1532        Ok(())
1533    }
1534}
1535
1536// CRI image operations dispatch through the `ImageStore` trait object — ONE
1537// body per method, no cfg/cfg(not) mirror (#646). When no image backend is
1538// compiled in (build without `oci-image`), `self.image_store` is `None` and the
1539// single fallback error is returned. The concrete RAFS impl + its
1540// `inventory::submit!` live in `image/store.rs`, gated by `oci-image`.
1541fn image_store_missing() -> anyhow::Error {
1542    anyhow::anyhow!(
1543        "CRI image operations require the `oci-image` feature (RAFS image store); \
1544         this binary was built without oci-image support"
1545    )
1546}
1547
1548#[async_trait::async_trait(?Send)]
1549impl ImageHandler for WorkerService {
1550    async fn handle_list(
1551        &self,
1552        _ctx: &EnvelopeContext,
1553        _request_id: u64,
1554        filter: &ImageFilter,
1555    ) -> AnyhowResult<Vec<ImageInfo>> {
1556        let _ = filter; // filter not yet used
1557        let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1558        Ok(store.list_images().await?)
1559    }
1560
1561    async fn handle_status(
1562        &self,
1563        _ctx: &EnvelopeContext,
1564        _request_id: u64,
1565        request: &ImageStatusRequest,
1566    ) -> AnyhowResult<ImageStatusResult> {
1567        let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1568        Ok(store
1569            .image_status(&request.image.image, request.verbose)
1570            .await?)
1571    }
1572
1573    async fn handle_pull(
1574        &self,
1575        _ctx: &EnvelopeContext,
1576        _request_id: u64,
1577        data: &PullImageRequest,
1578    ) -> AnyhowResult<String> {
1579        let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1580        let auth = if !data.auth.username.is_empty() {
1581            Some(AuthConfig {
1582                username: data.auth.username.clone(),
1583                password: data.auth.password.clone(),
1584                auth: String::new(),
1585                server_address: String::new(),
1586                identity_token: String::new(),
1587                registry_token: String::new(),
1588            })
1589        } else {
1590            None
1591        };
1592        Ok(store
1593            .pull_with_auth(&data.image.image, auth.as_ref())
1594            .await?)
1595    }
1596
1597    async fn handle_remove(
1598        &self,
1599        _ctx: &EnvelopeContext,
1600        _request_id: u64,
1601        data: &ImageSpec,
1602    ) -> AnyhowResult<()> {
1603        let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1604        store.remove_image(&data.image).await?;
1605        Ok(())
1606    }
1607
1608    async fn handle_fs_info(
1609        &self,
1610        _ctx: &EnvelopeContext,
1611        _request_id: u64,
1612    ) -> AnyhowResult<Vec<FilesystemUsage>> {
1613        let store = self.image_store.as_ref().ok_or_else(image_store_missing)?;
1614        Ok(store.fs_info().await?)
1615    }
1616}
1617
1618/// Super-trait with authorization — all scope traits are implemented above.
1619#[async_trait::async_trait(?Send)]
1620impl WorkerHandler for WorkerService {
1621    async fn authorize(
1622        &self,
1623        ctx: &EnvelopeContext,
1624        resource: &str,
1625        operation: &str,
1626    ) -> AnyhowResult<()> {
1627        if let Some(ref auth_fn) = self.authorize_fn {
1628            let subject = ctx.subject().to_string();
1629            let allowed = auth_fn(ctx.clone(), resource.to_owned(), operation.to_owned())
1630                .await
1631                .unwrap_or_else(|e| {
1632                    warn!(
1633                        "Policy check failed for {} on {}: {} - denying access",
1634                        subject, resource, e
1635                    );
1636                    false
1637                });
1638            if allowed {
1639                Ok(())
1640            } else {
1641                anyhow::bail!(
1642                    "Unauthorized: {} cannot {} on {}",
1643                    subject,
1644                    operation,
1645                    resource
1646                )
1647            }
1648        } else {
1649            // No authorization configured — fail-closed
1650            anyhow::bail!("Authorization not configured for worker service")
1651        }
1652    }
1653}
1654
1655// ═══════════════════════════════════════════════════════════════════════════════
1656// RequestService Implementation — delegates to generated dispatch_worker
1657// ═══════════════════════════════════════════════════════════════════════════════
1658
1659#[async_trait(?Send)]
1660impl RequestService for WorkerService {
1661    fn decode_request_body(
1662        &self,
1663        signed_body: &[u8],
1664    ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
1665        // The ONE bounded decode (v16 §5.2): the generated decoder derives
1666        // the full method leaf and returns the decoded message that policy,
1667        // MAC, and dispatch below all consume.
1668        crate::generated::worker_client::decode_worker_request_body(signed_body)
1669    }
1670
1671    fn producer_reach_config_handle(
1672        &self,
1673    ) -> Option<hyprstream_rpc::moq_stream::ProducerReachConfigHandle> {
1674        Some(self.stream_channel.reach_config_handle())
1675    }
1676
1677    fn moq_origin_handle(&self) -> Option<hyprstream_rpc::moq_stream::MoqStreamOriginHandle> {
1678        Some(self.stream_channel.moq_origin_handle())
1679    }
1680
1681    async fn handle_request(
1682        &self,
1683        ctx: &EnvelopeContext,
1684        body: &hyprstream_rpc::service::DecodedRequestBody,
1685    ) -> AnyhowResult<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
1686        debug!(
1687            "Worker request from {} (request_id={})",
1688            ctx.subject(),
1689            ctx.request_id
1690        );
1691        dispatch_worker(self, ctx, body).await
1692    }
1693
1694    fn name(&self) -> &str {
1695        SERVICE_NAME
1696    }
1697
1698    fn transport(&self) -> &TransportConfig {
1699        &self.transport
1700    }
1701
1702    fn signing_key(&self) -> SigningKey {
1703        self.signing_key.clone()
1704    }
1705
1706    fn expected_audience(&self) -> Option<&str> {
1707        self.expected_audience.as_deref()
1708    }
1709
1710    fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
1711        self.jwt_key_source.clone()
1712    }
1713}
1714
1715// These tests exercise the VM lifecycle (RafsStore + KataBackend), so they only
1716// build/run under `kata`.
1717#[cfg(all(test, feature = "kata"))]
1718#[allow(clippy::print_stderr)]
1719mod tests {
1720    use super::*;
1721    use crate::config::ImageConfig;
1722    use crate::image::RafsStore;
1723    use hyprstream_rpc::auth::mac::{ObjectRef, SecurityContext};
1724    use hyprstream_rpc::crypto::generate_signing_keypair;
1725    use hyprstream_rpc::transport::TransportConfig;
1726    use hyprstream_rpc_std::worker_client::{ContainerConfig, PodSandboxConfig, PodSandboxState};
1727    use tempfile::TempDir;
1728
1729    struct FixtureAccessDecider;
1730
1731    impl hyprstream_9p::AccessDecider for FixtureAccessDecider {
1732        fn check(
1733            &self,
1734            _ctx: &SecurityContext,
1735            _object: ObjectRef<'_>,
1736            _action: hyprstream_9p::Action,
1737        ) -> bool {
1738            true
1739        }
1740
1741        fn audit_denial(
1742            &self,
1743            _ctx: &SecurityContext,
1744            _object: ObjectRef<'_>,
1745            _object_label: Option<hyprstream_rpc::auth::mac::SecurityLabel>,
1746            _action: hyprstream_9p::Action,
1747            _reason: hyprstream_9p::ReferenceMonitorDenyReason,
1748        ) {
1749        }
1750    }
1751
1752    /// Create a test WorkerService with temporary directories
1753    async fn create_test_service(
1754    ) -> std::result::Result<(WorkerService, TempDir), Box<dyn std::error::Error>> {
1755        // The moq event-bus migration (#133/#167) made WorkerService publish
1756        // lifecycle events on the process-global moq event origin; without it,
1757        // EventPublisher::new fails with "moq event bus not initialized".
1758        // Initialize an in-process origin for tests (idempotent — first caller
1759        // wins across the test binary, the rest reuse it).
1760        let _ = hyprstream_rpc::moq_event::init_global_moq_event_origin(
1761            hyprstream_rpc::moq_event::MoqEventOrigin::new(),
1762        );
1763
1764        let temp_dir = TempDir::new()?;
1765        let base_path = temp_dir.path();
1766
1767        let image_config = ImageConfig {
1768            blobs_dir: base_path.join("blobs"),
1769            bootstrap_dir: base_path.join("bootstrap"),
1770            cache_dir: base_path.join("cache"),
1771            refs_dir: base_path.join("refs"),
1772            runtime_dir: base_path.join("runtime"),
1773            dragonfly_peer: None,
1774            fuse_threads: 4,
1775            chunk_size: 1024 * 1024,
1776            lazy_loading: true,
1777            cache_size_bytes: 1024 * 1024 * 1024, // 1GB for tests
1778        };
1779
1780        let rafs_store = Arc::new(RafsStore::new(image_config.clone())?);
1781
1782        let pool_config = PoolConfig {
1783            runtime_dir: base_path.join("sandboxes"),
1784            cloud_init_dir: base_path.join("cloud-init"),
1785            vm_image: base_path.join("vm/rootfs.img"),
1786            kernel_path: base_path.join("vm/vmlinux"),
1787            ..PoolConfig::default()
1788        };
1789
1790        let transport = TransportConfig::inproc("test-worker-service");
1791        let (signing_key, _verifying_key) = generate_signing_keypair();
1792
1793        let backend: Arc<dyn crate::runtime::backend::SandboxBackend> =
1794            Arc::new(crate::runtime::kata_backend::KataBackend::new(
1795                image_config,
1796                Arc::clone(&rafs_store),
1797                Arc::new(FixtureAccessDecider),
1798                None,
1799            ));
1800        let service = WorkerService::new(
1801            pool_config,
1802            backend,
1803            Some(rafs_store),
1804            transport,
1805            signing_key,
1806        )?;
1807        Ok((service, temp_dir))
1808    }
1809
1810    #[tokio::test]
1811    async fn test_sandbox_lifecycle() -> std::result::Result<(), Box<dyn std::error::Error>> {
1812        let (service, _temp_dir) = create_test_service().await?;
1813
1814        // Create sandbox - this requires cloud-hypervisor and kata runtime directories
1815        let config = PodSandboxConfig::default();
1816        let subject = hyprstream_vfs::Subject::new("test-user");
1817        let result = service.run_pod_sandbox(&subject, &config).await;
1818
1819        // Skip test if cloud-hypervisor not installed or kata runtime not available
1820        match &result {
1821            Err(WorkerError::VmStartFailed(msg))
1822                if msg.contains("No such file or directory")
1823                    || msg.contains("failed to create sandbox directory")
1824                    || msg.contains("/run/kata")
1825                    || msg.contains("launch failed") =>
1826            {
1827                eprintln!("Skipping test: cloud-hypervisor or kata runtime not available");
1828                return Ok(());
1829            }
1830            _ => {}
1831        }
1832
1833        let sandbox_id = result?;
1834
1835        // Check status
1836        let (status, _info) = service.pod_sandbox_status(&sandbox_id, false).await?;
1837        assert_eq!(status.state, PodSandboxState::SandboxReady);
1838
1839        // List sandboxes
1840        let sandboxes = service
1841            .list_pod_sandbox(&PodSandboxFilter::default())
1842            .await?;
1843        assert_eq!(sandboxes.len(), 1);
1844
1845        // Stop and remove
1846        service.stop_pod_sandbox(&sandbox_id).await?;
1847        service.remove_pod_sandbox(&sandbox_id).await?;
1848
1849        // Should be empty
1850        let sandboxes = service
1851            .list_pod_sandbox(&PodSandboxFilter::default())
1852            .await?;
1853        assert_eq!(sandboxes.len(), 0);
1854        Ok(())
1855    }
1856
1857    #[tokio::test]
1858    async fn test_container_lifecycle() -> std::result::Result<(), Box<dyn std::error::Error>> {
1859        let (service, _temp_dir) = create_test_service().await?;
1860
1861        // Create sandbox first - this requires cloud-hypervisor and kata runtime directories
1862        let sandbox_config = PodSandboxConfig::default();
1863        let subject = hyprstream_vfs::Subject::new("test-user");
1864        let result = service.run_pod_sandbox(&subject, &sandbox_config).await;
1865
1866        // Skip test if cloud-hypervisor not installed or kata runtime not available
1867        match &result {
1868            Err(WorkerError::VmStartFailed(msg))
1869                if msg.contains("No such file or directory")
1870                    || msg.contains("failed to create sandbox directory")
1871                    || msg.contains("/run/kata")
1872                    || msg.contains("launch failed") =>
1873            {
1874                eprintln!("Skipping test: cloud-hypervisor or kata runtime not available");
1875                return Ok(());
1876            }
1877            _ => {}
1878        }
1879
1880        let sandbox_id = result?;
1881
1882        // Create container
1883        let container_config = ContainerConfig::default();
1884        let container_id = service
1885            .create_container(&sandbox_id, &container_config, &sandbox_config)
1886            .await?;
1887
1888        // Check status
1889        let (status, _info) = service.container_status(&container_id, false).await?;
1890        assert_eq!(status.state, ContainerState::ContainerCreated);
1891
1892        // Start container
1893        service.start_container(&container_id).await?;
1894        let (status, _info) = service.container_status(&container_id, false).await?;
1895        assert_eq!(status.state, ContainerState::ContainerRunning);
1896
1897        // Stop container
1898        service.stop_container(&container_id, 30).await?;
1899        let (status, _info) = service.container_status(&container_id, false).await?;
1900        assert_eq!(status.state, ContainerState::ContainerExited);
1901
1902        // Remove container
1903        service.remove_container(&container_id).await?;
1904
1905        // Cleanup
1906        service.remove_pod_sandbox(&sandbox_id).await?;
1907        Ok(())
1908    }
1909
1910    #[tokio::test]
1911    async fn test_service_version() -> std::result::Result<(), Box<dyn std::error::Error>> {
1912        let (service, _temp_dir) = create_test_service().await?;
1913
1914        // Version should work without cloud-hypervisor
1915        let version = service.version("v1").await?;
1916        assert_eq!(version.runtime_name, super::super::RUNTIME_NAME);
1917        assert_eq!(version.runtime_api_version, "v1");
1918        Ok(())
1919    }
1920
1921    #[tokio::test]
1922    async fn test_service_status() -> std::result::Result<(), Box<dyn std::error::Error>> {
1923        let (service, _temp_dir) = create_test_service().await?;
1924
1925        // Status should work without cloud-hypervisor
1926        let status = service.status(true).await?;
1927        assert!(!status.status.conditions.is_empty());
1928        assert!(status.info.contains_key("warm_pool_size"));
1929        Ok(())
1930    }
1931}