Skip to main content

hyprstream_workers/runtime/
kata_agent.rs

1//! kata-agent ttrpc/vsock client — T1-C (#344) of the Worker GA epic (#341)
2//!
3//! Implements a real client for the `kata-agent` running *inside* a Kata
4//! guest VM, so [`KataBackend::exec_sync`](super::kata_backend::KataBackend)
5//! can create/start/exec a container inside the VM instead of returning the
6//! historical "not supported" error.
7//!
8//! # Wire protocol: REAL, not hand-rolled
9//!
10//! The ttrpc request/response message types ([`protocols::agent`],
11//! [`protocols::oci`]) and the generated async ttrpc client
12//! ([`protocols::agent_ttrpc_async::AgentServiceClient`]) come from the
13//! `protocols` crate, which is vendored verbatim from the upstream
14//! `kata-containers` Rust workspace (`src/libs/protocols`, tag matching our
15//! `kata-hypervisor` dependency). It is *already* present in our dependency
16//! graph transitively (`kata-hypervisor` depends on `protocols` with the
17//! `async` ttrpc feature enabled) — this module simply adds it as a direct
18//! dependency and uses it, rather than reimplementing the kata-agent
19//! protobuf/ttrpc wire format by hand. The `CreateContainer` / `StartContainer`
20//! / `ExecProcess` / `WaitProcess` RPCs used below are the real generated
21//! client methods (`agent_ttrpc_async.rs`), framed and (de)serialized by the
22//! real `ttrpc` crate (`ttrpc::r#async::Client`).
23//!
24//! What IS hand-written here: the hybrid-vsock *transport* dialer
25//! (connecting to the guest's kata-agent listen socket and handing the
26//! connected stream to `ttrpc::r#async::Client::new`), and the mapping from our
27//! `SandboxBackend::exec_sync` shape to the agent RPC sequence. This mirrors
28//! (does not vendor) the dialer in upstream
29//! `runtime-rs/crates/agent/src/sock/{vsock,hybrid_vsock}.rs`.
30//!
31//! # Connection convention (verified against upstream kata-containers source,
32//! tag `4.0.0`, fetched into the local cargo git checkout cache — see
33//! `~/.cargo/git/checkouts/kata-containers-*/*/src/runtime-rs/crates/agent/src/sock/`)
34//!
35//! - **Cloud Hypervisor** (the only hypervisor this crate wires up — see
36//!   `kata_backend.rs`) exposes the kata-agent over *hybrid vsock*: a Unix
37//!   domain socket at `<sandbox_dir>/ch-vm.sock`
38//!   (`hypervisor::ch::utils::get_vsock_path`), where the host writes
39//!   `"connect <port>\n"` and expects a response line containing `"OK"`
40//!   before the same stream becomes the ttrpc transport
41//!   (`runtime-rs/crates/agent/src/sock/hybrid_vsock.rs::connect_helper`).
42//!   `kata_hypervisor::Hypervisor::get_agent_socket()` returns this address
43//!   as `hvsock://<path>`, matching the `Hypervisor` trait already used by
44//!   `KataBackend`.
45//! - **Firecracker/Dragonball** use real `AF_VSOCK` (`vsock://<cid>:<port>`),
46//!   dialed via `socket(AF_VSOCK) + connect(VsockAddr{cid, port})`
47//!   (`runtime-rs/crates/agent/src/sock/vsock.rs`). We don't boot these
48//!   hypervisors yet, but the dialer below supports the scheme for parity.
49//! - The kata-agent's *default* ttrpc listen port is **1024**
50//!   (`src/runtime/virtcontainers/hypervisor.go: vSockPort = 1024`), used
51//!   whenever a caller doesn't override it.
52//!
53//! # What's REAL vs what's UNVALIDATED in this pass
54//!
55//! REAL: the message types, the ttrpc client/wire framing, the hybrid-vsock
56//! handshake implementation, and the RPC sequencing
57//! (`CreateContainer` → `StartContainer` → `ExecProcess` → `WaitProcess` (→
58//! `ReadStdout`/`ReadStderr`)). Unit tests in this module run a *real*
59//! `ttrpc::r#async::Server` (the same crate a real kata-agent links against)
60//! bound to a Unix socket, with a minimal `AgentService` impl that returns
61//! canned responses — so the request encoding / response decoding is
62//! exercised through the genuine ttrpc+protobuf wire path, not mocked at the
63//! byte level.
64//!
65//! UNVALIDATED (this sandbox has no bootable kata-agent guest image): that a
66//! *real* kata-agent's `CreateContainer`/`ExecProcess` semantics (OCI bundle
67//! layout expectations, namespace/cgroup setup it performs) are satisfied by
68//! the minimal `oci::Spec` this module constructs. The `CreateContainerRequest.OCI`
69//! field needs a real OCI runtime spec (rootfs, mounts, namespaces) wired up
70//! from `PodSandbox`/container config — this module supplies a minimal
71//! placeholder `Spec` sufficient to exercise the RPC, not a complete OCI bundle
72//! translation (that's a larger follow-up, see `build_minimal_spec`).
73//!
74//! # Exit-value structuring (issue #608 / future #610)
75//!
76//! `exec_sync`'s return shape `(exit_code, stdout, stderr)` is built from
77//! `WaitProcessResponse.status` (a real terminal exit code from the guest
78//! agent) plus drained `ReadStdout`/`ReadStderr` output — i.e. the exit code
79//! is already a true terminal value by construction, not synthesized. A
80//! future `/exec/instances/<id>/exit` projection (#610) can latch directly
81//! onto the `(i32, Vec<u8>, Vec<u8>)` returned by [`KataAgentClient::exec`]
82//! without restructuring this client.
83
84use std::time::Duration;
85
86use anyhow::{anyhow, Context, Result};
87use protocols::agent_ttrpc_async::AgentServiceClient;
88use protocols::{agent, empty, oci};
89use tokio::io::{AsyncReadExt, AsyncWriteExt};
90use tokio::net::UnixStream;
91use ttrpc::r#async::transport::Socket as TtrpcSocket;
92
93/// Default kata-agent ttrpc vsock port.
94///
95/// Source: upstream kata-containers `src/runtime/virtcontainers/hypervisor.go`
96/// (`vSockPort = 1024`), tag `4.0.0` — the same tag pinned for our
97/// `kata-hypervisor`/`protocols` dependencies.
98pub const KATA_AGENT_VSOCK_PORT: u32 = 1024;
99
100/// Guest-side storage `driver` the kata-agent dispatches on to mount a
101/// virtio-fs share. Matches `kata_types::device::DRIVER_VIRTIOFS_TYPE`
102/// (`"virtio-fs"`, tag `4.0.0`), which `agent/src/storage/fs_handler.rs`'s
103/// `VirtioFsHandler` registers.
104const VIRTIOFS_STORAGE_DRIVER: &str = "virtio-fs";
105
106/// Guest-side filesystem type used to mount a virtio-fs share. The kata-agent
107/// `baremount`s the share with this `fstype` (`FS_TYPE_VIRTIO_FS = "virtiofs"`
108/// in upstream `runtime-rs/.../share_virtio_fs.rs`, tag `4.0.0`).
109const VIRTIOFS_FSTYPE: &str = "virtiofs";
110
111/// Base directory the kata-agent uses for per-container bundles inside the
112/// guest (`CONTAINER_BASE = "/run/kata-containers"` in `agent/src/rpc.rs`,
113/// tag `4.0.0`). The container's rootfs lives at `<base>/<cid>/rootfs`.
114const CONTAINER_BASE: &str = "/run/kata-containers";
115
116/// How the container's rootfs is provided to the guest.
117///
118/// hyprstream serves the **entire tenant VFS namespace** (with the rootfs
119/// mounted at `/` — RAFS lower + per-sandbox writable upper + the injected
120/// `/models` / `/deltas` / `/stream` mounts) over a **single** vhost-user-fs
121/// (virtio-fs) device attached to the hypervisor under a mount tag (see
122/// `sandbox_fs.rs` / `kata_backend::attach_share_fs`). This tells
123/// [`KataAgentClient::create_container`] to make the kata-agent mount that
124/// share, by tag, **as the container's rootfs** — so the container's
125/// filesystem *is* the tenant VFS.
126#[derive(Debug, Clone)]
127pub struct ContainerRootfs {
128    /// The virtio-fs mount tag the hypervisor attached the share under — the
129    /// `ShareFsDevice` `mount_tag` passed to
130    /// `kata_backend::KataBackend::attach_share_fs`.
131    pub virtiofs_mount_tag: String,
132}
133
134/// Guest path where the kata-agent expects (and, via `setup_bundle`, pivots
135/// into) a container's rootfs: `<CONTAINER_BASE>/<cid>/rootfs`.
136fn guest_rootfs_path(container_id: &str) -> String {
137    format!("{CONTAINER_BASE}/{container_id}/rootfs")
138}
139
140/// Build the `Storage` entry that makes the kata-agent mount the virtio-fs
141/// share (identified by `rootfs.virtiofs_mount_tag`) directly at the
142/// container's rootfs path inside the guest.
143///
144/// The agent's `VirtioFsHandler` → `common_storage_handler` → `mount_storage`
145/// `baremount`s `source` (the tag) as `fstype` (`virtiofs`) at `mount_point`,
146/// creating the destination directory first. Because we mount it **directly
147/// at `<CONTAINER_BASE>/<cid>/rootfs`**, the agent's `setup_bundle` finds the
148/// rootfs path already present (the virtio-fs mount) and uses it as-is — no
149/// bind-mount indirection, and no separate nested `passthrough/<cid>` layout
150/// (that layout is kata's multi-container *bundle* sharing; hyprstream serves
151/// exactly one rootfs per share). The tenant VFS root thereby becomes the
152/// container root. `Root.Path` in the OCI spec points at the same path (see
153/// [`build_minimal_spec`]).
154fn rootfs_storage(container_id: &str, rootfs: &ContainerRootfs) -> agent::Storage {
155    agent::Storage {
156        driver: VIRTIOFS_STORAGE_DRIVER.to_owned(),
157        source: rootfs.virtiofs_mount_tag.clone(),
158        fstype: VIRTIOFS_FSTYPE.to_owned(),
159        // `nodev`, matching upstream kata's `SHARED_DIR_VIRTIO_FS_OPTIONS`
160        // for the sandbox virtio-fs share.
161        options: vec!["nodev".to_owned()],
162        mount_point: guest_rootfs_path(container_id),
163        ..Default::default()
164    }
165}
166
167/// Default per-dial-attempt timeout.
168const DIAL_TIMEOUT: Duration = Duration::from_millis(500);
169
170/// Default overall connect timeout (covers retries).
171const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
172
173/// Default per-RPC ttrpc timeout (nanoseconds, per `ttrpc::context::with_timeout`).
174const RPC_TIMEOUT_NS: i64 = 30 * 1_000_000_000;
175
176/// A parsed kata-agent connection address.
177///
178/// Mirrors the two schemes upstream kata-containers' own
179/// `runtime-rs/crates/agent/src/sock` dialer accepts. Cloud Hypervisor (the
180/// only hypervisor `KataBackend` boots today) always returns
181/// [`AgentAddress::HybridVsock`] from `Hypervisor::get_agent_socket()`.
182#[derive(Debug, Clone, PartialEq, Eq)]
183pub enum AgentAddress {
184    /// `hvsock://<unix-socket-path>` — Cloud Hypervisor / Dragonball hybrid
185    /// vsock: a host-side Unix socket multiplexes guest vsock ports via a
186    /// `CONNECT <port>\n` text handshake.
187    HybridVsock { path: String },
188    /// `vsock://<cid>` — real `AF_VSOCK`, used by Firecracker. Not exercised
189    /// by `KataBackend` today (it only boots Cloud Hypervisor/Dragonball),
190    /// but parsed/dialable for parity with upstream.
191    Vsock { cid: u32 },
192}
193
194impl AgentAddress {
195    /// Parse a `hvsock://<path>` or `vsock://<cid>` URI, as returned by
196    /// `kata_hypervisor::Hypervisor::get_agent_socket()`.
197    pub fn parse(address: &str) -> Result<Self> {
198        if let Some(path) = address.strip_prefix("hvsock://") {
199            return Ok(AgentAddress::HybridVsock {
200                path: path.to_owned(),
201            });
202        }
203        if let Some(cid) = address.strip_prefix("vsock://") {
204            let cid: u32 = cid
205                .parse()
206                .with_context(|| format!("invalid vsock cid in {address:?}"))?;
207            return Ok(AgentAddress::Vsock { cid });
208        }
209        Err(anyhow!("unsupported kata-agent socket scheme: {address:?}"))
210    }
211}
212
213/// Dial the kata-agent's ttrpc socket and wrap its connected stream.
214///
215/// For [`AgentAddress::HybridVsock`], performs the hybrid-vsock handshake
216/// (`connect <port>\n` → response line containing `OK`) over the host Unix
217/// socket, exactly as upstream's `hybrid_vsock::connect_helper` does. For
218/// [`AgentAddress::Vsock`], opens a real `AF_VSOCK` connection via ttrpc's
219/// async transport.
220pub async fn dial(address: &AgentAddress, port: u32) -> Result<TtrpcSocket> {
221    let deadline = tokio::time::Instant::now() + CONNECT_TIMEOUT;
222    let mut last_err = None;
223
224    while tokio::time::Instant::now() < deadline {
225        let attempt = dial_once(address, port).await;
226        match attempt {
227            Ok(socket) => return Ok(socket),
228            Err(e) => {
229                last_err = Some(e);
230                tokio::time::sleep(DIAL_TIMEOUT).await;
231            }
232        }
233    }
234
235    Err(last_err.unwrap_or_else(|| anyhow!("dial timed out with no attempts made")))
236}
237
238async fn dial_once(address: &AgentAddress, port: u32) -> Result<TtrpcSocket> {
239    match address {
240        AgentAddress::HybridVsock { path } => hybrid_vsock_connect(path, port).await,
241        AgentAddress::Vsock { cid } => vsock_connect(*cid, port).await,
242    }
243}
244
245/// Hybrid-vsock handshake: connect to the host UDS, write `"connect
246/// <port>\n"`, read one response line, require it to contain `"OK"`.
247///
248/// Matches `runtime-rs/crates/agent/src/sock/hybrid_vsock.rs::connect_helper`
249/// in upstream kata-containers (tag `4.0.0`).
250async fn hybrid_vsock_connect(uds_path: &str, port: u32) -> Result<TtrpcSocket> {
251    let mut stream = UnixStream::connect(uds_path)
252        .await
253        .with_context(|| format!("connect to hybrid-vsock UDS {uds_path:?}"))?;
254
255    stream
256        .write_all(format!("connect {port}\n").as_bytes())
257        .await
258        .context("write hybrid-vsock CONNECT handshake")?;
259
260    let mut response = Vec::new();
261    let mut byte = [0u8; 1];
262    loop {
263        let n = stream
264            .read(&mut byte)
265            .await
266            .context("read hybrid-vsock handshake response")?;
267        if n == 0 || byte[0] == b'\n' {
268            break;
269        }
270        response.push(byte[0]);
271    }
272    let response = String::from_utf8_lossy(&response);
273    if !response.contains("OK") {
274        return Err(anyhow!(
275            "hybrid-vsock handshake error: malformed response {response:?}"
276        ));
277    }
278
279    Ok(TtrpcSocket::new(stream))
280}
281
282/// Real `AF_VSOCK` connect via ttrpc's Tokio transport (Firecracker/Dragonball
283/// convention: `vsock://<cid>`, default port 1024).
284async fn vsock_connect(cid: u32, port: u32) -> Result<TtrpcSocket> {
285    TtrpcSocket::connect_vsock(format!("{cid}:{port}"))
286        .await
287        .with_context(|| format!("AF_VSOCK connect to cid={cid} port={port}"))
288}
289
290/// A connected kata-agent ttrpc client.
291///
292/// Thin wrapper around the real, generated
293/// [`protocols::agent_ttrpc_async::AgentServiceClient`], scoped to the CRI
294/// subset `SandboxBackend::exec_sync` needs: create → start → exec → wait
295/// (+ drain stdout/stderr).
296pub struct KataAgentClient {
297    client: AgentServiceClient,
298    timeout_ns: i64,
299}
300
301impl KataAgentClient {
302    /// Dial the agent at `address` (kata-agent's well-known port 1024 unless
303    /// the caller has a reason to override it) and wrap it in an
304    /// `AgentServiceClient`.
305    pub async fn connect(address: &AgentAddress) -> Result<Self> {
306        Self::connect_with_port(address, KATA_AGENT_VSOCK_PORT).await
307    }
308
309    pub async fn connect_with_port(address: &AgentAddress, port: u32) -> Result<Self> {
310        let socket = dial(address, port).await?;
311        let ttrpc_client = ttrpc::r#async::Client::new(socket);
312        Ok(Self::from_ttrpc_client(ttrpc_client))
313    }
314
315    /// Wrap an already-connected `ttrpc::r#async::Client` (used directly by
316    /// tests against a local mock server, and available to callers who
317    /// already have a connected client from elsewhere).
318    pub fn from_ttrpc_client(client: ttrpc::r#async::Client) -> Self {
319        Self {
320            client: AgentServiceClient::new(client),
321            timeout_ns: RPC_TIMEOUT_NS,
322        }
323    }
324
325    fn ctx(&self) -> ttrpc::context::Context {
326        ttrpc::context::with_timeout(self.timeout_ns)
327    }
328
329    /// `CreateContainer` — minimal OCI spec wired from `container_id` +
330    /// `argv`, plus (when `rootfs` is supplied) a virtio-fs `Storage` entry
331    /// that mounts the tenant VFS share as the container's rootfs (#721).
332    ///
333    /// When `rootfs` is `None` the spec's `Root.Path` still points at
334    /// `<CONTAINER_BASE>/<cid>/rootfs`, but nothing is mounted there — the
335    /// guest must already have a rootfs at that path (the historical bare-VM
336    /// case, which fails `CreateContainer` with `No such file or directory`
337    /// precisely because it does not). Supplying `rootfs` is what makes the
338    /// tenant VFS *be* that rootfs; see [`ContainerRootfs`] / [`rootfs_storage`].
339    ///
340    /// This is still NOT a full OCI bundle translation (no devices/volumes/
341    /// namespaces from `PodSandbox`) — just enough to boot a container whose
342    /// filesystem is the hyprstream tenant VFS.
343    pub async fn create_container(
344        &self,
345        container_id: &str,
346        exec_id: &str,
347        argv: &[String],
348        rootfs: Option<&ContainerRootfs>,
349    ) -> Result<()> {
350        let storages = rootfs
351            .map(|r| vec![rootfs_storage(container_id, r)])
352            .unwrap_or_default();
353        let req = agent::CreateContainerRequest {
354            container_id: container_id.to_owned(),
355            exec_id: exec_id.to_owned(),
356            storages,
357            OCI: protobuf::MessageField::some(build_minimal_spec(container_id, argv)),
358            ..Default::default()
359        };
360        self.client
361            .create_container(self.ctx(), &req)
362            .await
363            .context("CreateContainer ttrpc call")?;
364        Ok(())
365    }
366
367    pub async fn start_container(&self, container_id: &str) -> Result<()> {
368        let req = agent::StartContainerRequest {
369            container_id: container_id.to_owned(),
370            ..Default::default()
371        };
372        self.client
373            .start_container(self.ctx(), &req)
374            .await
375            .context("StartContainer ttrpc call")?;
376        Ok(())
377    }
378
379    /// Idempotently ensure `container_id` exists and is started inside the
380    /// guest, tolerating "already exists"/"already started" ttrpc errors
381    /// from a prior call (kata-agent has no native "create or no-op" RPC, so
382    /// this papers over that at the client level — matching what a real CRI
383    /// shim does when `exec_sync` is called against a container it didn't
384    /// just create itself).
385    ///
386    /// `argv` is only used if the container needs to be created (it seeds
387    /// the OCI `Process.Args` — see [`build_minimal_spec`]); an already
388    /// existing container's original entrypoint is left untouched. `rootfs`
389    /// (the tenant VFS virtio-fs share, #721) is likewise only consumed on
390    /// the create path.
391    async fn ensure_container(
392        &self,
393        container_id: &str,
394        argv: &[String],
395        rootfs: Option<&ContainerRootfs>,
396    ) -> Result<()> {
397        let create_exec_id = format!("{container_id}-init");
398        match self
399            .create_container(container_id, &create_exec_id, argv, rootfs)
400            .await
401        {
402            Ok(()) => {}
403            Err(e) if is_already_exists(&e) => {
404                tracing::debug!(container_id, "container already exists, skipping create");
405            }
406            Err(e) => return Err(e.context("ensure_container: create_container")),
407        }
408
409        match self.start_container(container_id).await {
410            Ok(()) => Ok(()),
411            Err(e) if is_already_exists(&e) => {
412                tracing::debug!(container_id, "container already started, skipping start");
413                Ok(())
414            }
415            Err(e) => Err(e.context("ensure_container: start_container")),
416        }
417    }
418
419    /// `ExecProcess` — run `argv` inside `container_id` under a fresh
420    /// `exec_id`. Does not block for completion; pair with [`Self::wait_process`].
421    pub async fn exec_process(
422        &self,
423        container_id: &str,
424        exec_id: &str,
425        argv: &[String],
426    ) -> Result<()> {
427        let process = oci::Process {
428            Args: argv.to_vec(),
429            Cwd: "/".to_owned(),
430            Capabilities: protobuf::MessageField::some(default_capabilities()),
431            ..Default::default()
432        };
433        let req = agent::ExecProcessRequest {
434            container_id: container_id.to_owned(),
435            exec_id: exec_id.to_owned(),
436            process: protobuf::MessageField::some(process),
437            ..Default::default()
438        };
439        self.client
440            .exec_process(self.ctx(), &req)
441            .await
442            .context("ExecProcess ttrpc call")?;
443        Ok(())
444    }
445
446    /// `WaitProcess` — blocks (ttrpc-side) until the process exits, returning
447    /// its real terminal exit status from the guest agent.
448    pub async fn wait_process(&self, container_id: &str, exec_id: &str) -> Result<i32> {
449        let req = agent::WaitProcessRequest {
450            container_id: container_id.to_owned(),
451            exec_id: exec_id.to_owned(),
452            ..Default::default()
453        };
454        let resp = self
455            .client
456            .wait_process(self.ctx(), &req)
457            .await
458            .context("WaitProcess ttrpc call")?;
459        Ok(resp.status)
460    }
461
462    /// `UpdateContainer` — apply new cgroup resource limits to a running
463    /// container inside the guest.
464    ///
465    /// This is the RPC a CRI `UpdateContainerResources` maps onto: the
466    /// kata-agent rewrites the container's in-guest cgroups (CPU shares/quota/
467    /// period, memory limit, cpuset, …). It updates the container's slice of
468    /// the *existing* VM; growing the VM itself past its boot size (vCPU /
469    /// memory hotplug) is a separate hypervisor concern and not attempted
470    /// here — matching upstream kata, whose `update_container` handler drives
471    /// the guest cgroup layer, with VM resize handled independently by the
472    /// runtime's resource manager.
473    pub async fn update_container(
474        &self,
475        container_id: &str,
476        resources: oci::LinuxResources,
477    ) -> Result<()> {
478        let req = agent::UpdateContainerRequest {
479            container_id: container_id.to_owned(),
480            resources: protobuf::MessageField::some(resources),
481            ..Default::default()
482        };
483        self.client
484            .update_container(self.ctx(), &req)
485            .await
486            .context("UpdateContainer ttrpc call")?;
487        Ok(())
488    }
489
490    /// `StatsContainer` — fetch the guest cgroup (+ network) stats for a
491    /// container. The returned [`agent::StatsContainerResponse`] carries the
492    /// real in-guest cgroup counters (`cgroup_stats.cpu_stats` /
493    /// `memory_stats`); mapping them into the CRI `ContainerStats` shape is
494    /// the caller's job (see `KataBackend::container_stats`).
495    pub async fn stats_container(
496        &self,
497        container_id: &str,
498    ) -> Result<agent::StatsContainerResponse> {
499        let req = agent::StatsContainerRequest {
500            container_id: container_id.to_owned(),
501            ..Default::default()
502        };
503        let resp = self
504            .client
505            .stats_container(self.ctx(), &req)
506            .await
507            .context("StatsContainer ttrpc call")?;
508        Ok(resp)
509    }
510
511    /// Drain available stdout via repeated `ReadStdout` calls until the
512    /// agent reports no more data (empty response).
513    pub async fn read_stdout_all(&self, container_id: &str, exec_id: &str) -> Result<Vec<u8>> {
514        self.read_stream_all(container_id, exec_id, StreamKind::Stdout)
515            .await
516    }
517
518    pub async fn read_stderr_all(&self, container_id: &str, exec_id: &str) -> Result<Vec<u8>> {
519        self.read_stream_all(container_id, exec_id, StreamKind::Stderr)
520            .await
521    }
522
523    async fn read_stream_all(
524        &self,
525        container_id: &str,
526        exec_id: &str,
527        kind: StreamKind,
528    ) -> Result<Vec<u8>> {
529        const CHUNK: u32 = 4096;
530        let mut out = Vec::new();
531        loop {
532            let req = agent::ReadStreamRequest {
533                container_id: container_id.to_owned(),
534                exec_id: exec_id.to_owned(),
535                len: CHUNK,
536                ..Default::default()
537            };
538            let resp = match kind {
539                StreamKind::Stdout => self.client.read_stdout(self.ctx(), &req).await,
540                StreamKind::Stderr => self.client.read_stderr(self.ctx(), &req).await,
541            }
542            .context("ReadStdout/ReadStderr ttrpc call")?;
543            if resp.data.is_empty() {
544                break;
545            }
546            let got = resp.data.len();
547            out.extend_from_slice(&resp.data);
548            if got < CHUNK as usize {
549                break;
550            }
551        }
552        Ok(out)
553    }
554
555    /// Full ensure-container → exec → wait → drain sequence, matching the
556    /// `(exit_code, stdout, stderr)` shape `SandboxBackend::exec_sync` needs.
557    ///
558    /// Calls [`Self::ensure_container`] first (create + start, idempotent)
559    /// so callers don't need to separately track whether `container_id` has
560    /// already been created in this guest — matching the issue's "real
561    /// CRI create/start/exec_sync" scope.
562    ///
563    /// The returned `i32` is `WaitProcessResponse.status` — a real terminal
564    /// exit value from the guest agent (see module docs re: #608/#610).
565    pub async fn exec(
566        &self,
567        container_id: &str,
568        argv: &[String],
569        timeout: Duration,
570        rootfs: Option<&ContainerRootfs>,
571    ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
572        let exec_id = uuid::Uuid::new_v4().to_string();
573
574        tokio::time::timeout(timeout, async {
575            self.ensure_container(container_id, argv, rootfs).await?;
576            self.exec_process(container_id, &exec_id, argv).await?;
577            let status = self.wait_process(container_id, &exec_id).await?;
578            let stdout = self
579                .read_stdout_all(container_id, &exec_id)
580                .await
581                .unwrap_or_default();
582            let stderr = self
583                .read_stderr_all(container_id, &exec_id)
584                .await
585                .unwrap_or_default();
586            Ok::<_, anyhow::Error>((status, stdout, stderr))
587        })
588        .await
589        .map_err(|_| anyhow!("exec timed out after {timeout:?}"))?
590    }
591}
592
593#[derive(Clone, Copy)]
594enum StreamKind {
595    Stdout,
596    Stderr,
597}
598
599/// Best-effort detection of "the container/process already
600/// exists/started" ttrpc errors, used by [`KataAgentClient::ensure_container`]
601/// to make create+start idempotent.
602///
603/// Prefers the structured gRPC-style status code (`ALREADY_EXISTS = 6`, per
604/// `ttrpc.proto` — the same code space kata-agent's `CreateContainer`/
605/// `StartContainer` handlers use to reject duplicate ids upstream), falling
606/// back to a substring match on the error chain for cases where the error
607/// was wrapped before reaching here. NOTE: the exact error this would
608/// produce against a *real* kata-agent has not been observed in this sandbox
609/// (no bootable guest available) — this is the best-effort mapping a real
610/// instance is expected to need, not a verified one.
611fn is_already_exists(err: &anyhow::Error) -> bool {
612    for cause in err.chain() {
613        if let Some(ttrpc::Error::RpcStatus(status)) = cause.downcast_ref::<ttrpc::Error>() {
614            if status.code() == ttrpc::Code::ALREADY_EXISTS {
615                return true;
616            }
617        }
618    }
619    let msg = err.to_string().to_ascii_lowercase();
620    msg.contains("already exists") || msg.contains("already started")
621}
622
623/// Build a minimal `oci::Spec` sufficient to drive `CreateContainer` against
624/// a kata-agent. NOT a complete OCI bundle translation — see module docs.
625///
626/// `Root.Path` is `<CONTAINER_BASE>/<cid>/rootfs`, the guest path the agent's
627/// `setup_bundle` pivots into. When a [`ContainerRootfs`] is supplied to
628/// [`KataAgentClient::create_container`], the accompanying [`rootfs_storage`]
629/// mounts the tenant VFS virtio-fs share at exactly this path, so the tenant
630/// VFS becomes the container's filesystem (#721).
631fn build_minimal_spec(container_id: &str, argv: &[String]) -> oci::Spec {
632    oci::Spec {
633        Version: "1.0.2".to_owned(),
634        Process: protobuf::MessageField::some(oci::Process {
635            Args: argv.to_vec(),
636            Cwd: "/".to_owned(),
637            Capabilities: protobuf::MessageField::some(default_capabilities()),
638            ..Default::default()
639        }),
640        Root: protobuf::MessageField::some(oci::Root {
641            Path: guest_rootfs_path(container_id),
642            Readonly: false,
643            ..Default::default()
644        }),
645        Hostname: container_id.to_owned(),
646        ..Default::default()
647    }
648}
649
650/// Default OCI process capability set (the runc/OCI baseline). The kata-agent
651/// (rustjail) rejects `CreateContainer`/`ExecProcess` with `missing process
652/// capabilities` when `Process.Capabilities` is unset, so every container and
653/// exec process carries this standard bounding/effective/permitted set.
654fn default_capabilities() -> oci::LinuxCapabilities {
655    let caps: Vec<String> = [
656        "CAP_CHOWN",
657        "CAP_DAC_OVERRIDE",
658        "CAP_FSETID",
659        "CAP_FOWNER",
660        "CAP_MKNOD",
661        "CAP_NET_RAW",
662        "CAP_SETGID",
663        "CAP_SETUID",
664        "CAP_SETFCAP",
665        "CAP_SETPCAP",
666        "CAP_NET_BIND_SERVICE",
667        "CAP_SYS_CHROOT",
668        "CAP_KILL",
669        "CAP_AUDIT_WRITE",
670    ]
671    .iter()
672    .map(|s| (*s).to_owned())
673    .collect();
674    oci::LinuxCapabilities {
675        Bounding: caps.clone(),
676        Effective: caps.clone(),
677        Inheritable: caps.clone(),
678        Permitted: caps.clone(),
679        Ambient: caps,
680        ..Default::default()
681    }
682}
683
684// Re-export so `kata_backend.rs` doesn't need a separate `protocols`/`empty`
685// import just to satisfy trait bounds on `Empty` responses it may inspect.
686#[allow(unused_imports)]
687pub(crate) use empty::Empty as AgentEmpty;
688
689#[cfg(test)]
690#[allow(clippy::unwrap_used, clippy::expect_used)]
691mod tests {
692    use super::*;
693    use async_trait::async_trait;
694    use protocols::agent_ttrpc_async::{create_agent_service, AgentService};
695    use std::sync::atomic::{AtomicI32, Ordering};
696    use std::sync::Arc;
697    use tempfile::TempDir;
698
699    // ─────────────────────────────────────────────────────────────────────
700    // AgentAddress parsing
701    // ─────────────────────────────────────────────────────────────────────
702
703    #[test]
704    fn test_parse_hybrid_vsock() {
705        let addr = AgentAddress::parse("hvsock:///run/sandbox/ch-vm.sock").unwrap();
706        assert_eq!(
707            addr,
708            AgentAddress::HybridVsock {
709                path: "/run/sandbox/ch-vm.sock".to_owned()
710            }
711        );
712    }
713
714    #[test]
715    fn test_parse_vsock() {
716        let addr = AgentAddress::parse("vsock://42").unwrap();
717        assert_eq!(addr, AgentAddress::Vsock { cid: 42 });
718    }
719
720    #[test]
721    fn test_parse_unsupported_scheme() {
722        let err = AgentAddress::parse("tcp://127.0.0.1:1024").unwrap_err();
723        assert!(err.to_string().contains("unsupported"));
724    }
725
726    #[test]
727    fn test_parse_invalid_vsock_cid() {
728        let err = AgentAddress::parse("vsock://not-a-number").unwrap_err();
729        assert!(err.to_string().contains("invalid vsock cid"));
730    }
731
732    #[test]
733    fn test_kata_agent_vsock_port_matches_upstream_convention() {
734        // src/runtime/virtcontainers/hypervisor.go: vSockPort = 1024
735        assert_eq!(KATA_AGENT_VSOCK_PORT, 1024);
736    }
737
738    // ─────────────────────────────────────────────────────────────────────
739    // Hybrid-vsock handshake (real UDS, fake "guest" side)
740    // ─────────────────────────────────────────────────────────────────────
741
742    #[tokio::test]
743    async fn test_hybrid_vsock_handshake_ok() {
744        let dir = TempDir::new().unwrap();
745        let sock_path = dir.path().join("ch-vm.sock");
746        let listener = tokio::net::UnixListener::bind(&sock_path).unwrap();
747
748        let accept_task = tokio::spawn(async move {
749            let (mut stream, _) = listener.accept().await.unwrap();
750            let mut buf = [0u8; 64];
751            let n = stream.read(&mut buf).await.unwrap();
752            let req = String::from_utf8_lossy(&buf[..n]);
753            assert_eq!(req, "connect 1024\n");
754            stream.write_all(b"OK 1234\n").await.unwrap();
755            // keep stream alive for the duration of the test
756            tokio::time::sleep(Duration::from_millis(50)).await;
757        });
758
759        let _socket = hybrid_vsock_connect(sock_path.to_str().unwrap(), 1024)
760            .await
761            .unwrap();
762
763        accept_task.await.unwrap();
764    }
765
766    #[tokio::test]
767    async fn test_hybrid_vsock_handshake_rejects_malformed_response() {
768        let dir = TempDir::new().unwrap();
769        let sock_path = dir.path().join("ch-vm.sock");
770        let listener = tokio::net::UnixListener::bind(&sock_path).unwrap();
771
772        tokio::spawn(async move {
773            let (mut stream, _) = listener.accept().await.unwrap();
774            let mut buf = [0u8; 64];
775            let _ = stream.read(&mut buf).await.unwrap();
776            stream.write_all(b"ERROR\n").await.unwrap();
777        });
778
779        let result = hybrid_vsock_connect(sock_path.to_str().unwrap(), 1024).await;
780        assert!(result.is_err());
781        assert!(result
782            .err()
783            .unwrap()
784            .to_string()
785            .contains("handshake error"));
786    }
787
788    // ─────────────────────────────────────────────────────────────────────
789    // Real ttrpc server/client round-trip (fake AgentService, real wire format)
790    // ─────────────────────────────────────────────────────────────────────
791
792    /// A minimal `AgentService` impl that proves request decoding +
793    /// response encoding work end-to-end over the real ttrpc transport.
794    /// Records the last request seen for assertions and returns canned,
795    /// deterministic replies — this is the "mock/fake ttrpc server" the
796    /// issue asks for, built on the *real* generated service trait rather
797    /// than hand-rolled framing.
798    struct FakeAgent {
799        wait_status: AtomicI32,
800        last_create: parking_lot::Mutex<Option<agent::CreateContainerRequest>>,
801        last_exec: parking_lot::Mutex<Option<agent::ExecProcessRequest>>,
802        /// Artificial delay before `wait_process` resolves, used to
803        /// deterministically exercise `KataAgentClient::exec`'s timeout path
804        /// (a near-zero `tokio::time::timeout` duration is not reliable: a
805        /// fast local round-trip can race the timer's first poll).
806        wait_process_delay: Duration,
807        /// When set, `create_container`/`start_container` always reply with
808        /// a `Code::ALREADY_EXISTS` ttrpc status, to exercise
809        /// `KataAgentClient::ensure_container`'s idempotent-retry path.
810        already_exists: bool,
811    }
812
813    impl FakeAgent {
814        fn new(wait_status: i32) -> Self {
815            Self {
816                wait_status: AtomicI32::new(wait_status),
817                last_create: parking_lot::Mutex::new(None),
818                last_exec: parking_lot::Mutex::new(None),
819                wait_process_delay: Duration::ZERO,
820                already_exists: false,
821            }
822        }
823
824        fn with_wait_process_delay(wait_status: i32, delay: Duration) -> Self {
825            Self {
826                wait_process_delay: delay,
827                ..Self::new(wait_status)
828            }
829        }
830
831        fn always_already_exists(wait_status: i32) -> Self {
832            Self {
833                already_exists: true,
834                ..Self::new(wait_status)
835            }
836        }
837    }
838
839    fn already_exists_status() -> ::ttrpc::Error {
840        ::ttrpc::Error::RpcStatus(::ttrpc::get_status(
841            ::ttrpc::Code::ALREADY_EXISTS,
842            "already exists".to_owned(),
843        ))
844    }
845
846    #[async_trait]
847    impl AgentService for FakeAgent {
848        async fn create_container(
849            &self,
850            _ctx: &::ttrpc::r#async::TtrpcContext,
851            req: agent::CreateContainerRequest,
852        ) -> ::ttrpc::Result<empty::Empty> {
853            if self.already_exists {
854                return Err(already_exists_status());
855            }
856            *self.last_create.lock() = Some(req);
857            Ok(empty::Empty::new())
858        }
859
860        async fn start_container(
861            &self,
862            _ctx: &::ttrpc::r#async::TtrpcContext,
863            _req: agent::StartContainerRequest,
864        ) -> ::ttrpc::Result<empty::Empty> {
865            if self.already_exists {
866                return Err(already_exists_status());
867            }
868            Ok(empty::Empty::new())
869        }
870
871        async fn exec_process(
872            &self,
873            _ctx: &::ttrpc::r#async::TtrpcContext,
874            req: agent::ExecProcessRequest,
875        ) -> ::ttrpc::Result<empty::Empty> {
876            *self.last_exec.lock() = Some(req);
877            Ok(empty::Empty::new())
878        }
879
880        async fn wait_process(
881            &self,
882            _ctx: &::ttrpc::r#async::TtrpcContext,
883            _req: agent::WaitProcessRequest,
884        ) -> ::ttrpc::Result<agent::WaitProcessResponse> {
885            if !self.wait_process_delay.is_zero() {
886                tokio::time::sleep(self.wait_process_delay).await;
887            }
888            Ok(agent::WaitProcessResponse {
889                status: self.wait_status.load(Ordering::SeqCst),
890                ..Default::default()
891            })
892        }
893
894        async fn read_stdout(
895            &self,
896            _ctx: &::ttrpc::r#async::TtrpcContext,
897            _req: agent::ReadStreamRequest,
898        ) -> ::ttrpc::Result<agent::ReadStreamResponse> {
899            Ok(agent::ReadStreamResponse {
900                data: b"hello from stdout".to_vec(),
901                ..Default::default()
902            })
903        }
904
905        async fn read_stderr(
906            &self,
907            _ctx: &::ttrpc::r#async::TtrpcContext,
908            _req: agent::ReadStreamRequest,
909        ) -> ::ttrpc::Result<agent::ReadStreamResponse> {
910            Ok(agent::ReadStreamResponse {
911                data: Vec::new(),
912                ..Default::default()
913            })
914        }
915    }
916
917    /// Spin up a real `ttrpc::r#async::Server` bound to a temp Unix socket,
918    /// serving `FakeAgent`, and return a connected `KataAgentClient` plus
919    /// the server's join handle (kept alive for the test's duration) and
920    /// the `Arc<FakeAgent>` for post-call assertions.
921    async fn spin_up_fake_agent(
922        wait_status: i32,
923    ) -> (
924        KataAgentClient,
925        Arc<FakeAgent>,
926        tokio::task::JoinHandle<()>,
927        TempDir,
928    ) {
929        spin_up_fake_agent_with(FakeAgent::new(wait_status)).await
930    }
931
932    async fn spin_up_fake_agent_with(
933        fake: FakeAgent,
934    ) -> (
935        KataAgentClient,
936        Arc<FakeAgent>,
937        tokio::task::JoinHandle<()>,
938        TempDir,
939    ) {
940        let dir = TempDir::new().unwrap();
941        let sock_path = dir.path().join("agent.sock");
942        let sockaddr = format!("unix://{}", sock_path.display());
943
944        let fake = Arc::new(fake);
945        let service = create_agent_service(fake.clone() as Arc<dyn AgentService + Send + Sync>);
946
947        let mut server = ttrpc::r#async::Server::new()
948            .bind(&sockaddr)
949            .unwrap()
950            .register_service(service);
951
952        let server_task = tokio::spawn(async move {
953            let _ = server.start().await;
954            // Hold the server alive; in a real shutdown path we'd call
955            // server.shutdown().await, omitted here since the test process
956            // exit reclaims the socket.
957            std::future::pending::<()>().await;
958        });
959
960        // Give the listener a moment to bind before dialing.
961        tokio::time::sleep(Duration::from_millis(50)).await;
962
963        let ttrpc_client = ttrpc::r#async::Client::connect(&sockaddr).await.unwrap();
964        let client = KataAgentClient::from_ttrpc_client(ttrpc_client);
965
966        (client, fake, server_task, dir)
967    }
968
969    #[tokio::test]
970    async fn test_create_container_round_trip() {
971        let (client, fake, _server, _dir) = spin_up_fake_agent(0).await;
972
973        client
974            .create_container("ctr-1", "exec-1", &["echo".into(), "hi".into()], None)
975            .await
976            .unwrap();
977
978        let seen = fake.last_create.lock().clone().unwrap();
979        assert_eq!(seen.container_id, "ctr-1");
980        assert_eq!(seen.exec_id, "exec-1");
981        assert!(seen.OCI.is_some());
982        assert_eq!(
983            seen.OCI.Process.Args,
984            vec!["echo".to_owned(), "hi".to_owned()]
985        );
986        // No rootfs supplied → no virtio-fs Storage entry.
987        assert!(seen.storages.is_empty(), "no rootfs → no storages");
988        assert_eq!(
989            seen.OCI.Root.Path, "/run/kata-containers/ctr-1/rootfs",
990            "Root.Path is the guest container rootfs path"
991        );
992    }
993
994    #[tokio::test]
995    async fn test_create_container_mounts_tenant_vfs_as_rootfs() {
996        // #721: when a ContainerRootfs (tenant VFS virtio-fs mount tag) is
997        // supplied, CreateContainer carries a Storage entry that mounts that
998        // share, by tag, directly at the container's rootfs path — and
999        // Root.Path points at that same path.
1000        let (client, fake, _server, _dir) = spin_up_fake_agent(0).await;
1001
1002        let rootfs = ContainerRootfs {
1003            virtiofs_mount_tag: "hyprstream-vfs".to_owned(),
1004        };
1005        client
1006            .create_container("ctr-1", "exec-1", &["true".into()], Some(&rootfs))
1007            .await
1008            .unwrap();
1009
1010        let seen = fake.last_create.lock().clone().unwrap();
1011        assert_eq!(seen.storages.len(), 1, "exactly one rootfs Storage entry");
1012        let st = &seen.storages[0];
1013        assert_eq!(st.driver, "virtio-fs");
1014        assert_eq!(
1015            st.source, "hyprstream-vfs",
1016            "source is the virtio-fs mount tag"
1017        );
1018        assert_eq!(st.fstype, "virtiofs");
1019        assert_eq!(st.mount_point, "/run/kata-containers/ctr-1/rootfs");
1020        assert_eq!(
1021            seen.OCI.Root.Path, st.mount_point,
1022            "Root.Path must equal the virtio-fs rootfs mount point"
1023        );
1024    }
1025
1026    #[tokio::test]
1027    async fn test_exec_process_round_trip() {
1028        let (client, fake, _server, _dir) = spin_up_fake_agent(0).await;
1029
1030        client
1031            .exec_process("ctr-1", "exec-1", &["true".into()])
1032            .await
1033            .unwrap();
1034
1035        let seen = fake.last_exec.lock().clone().unwrap();
1036        assert_eq!(seen.container_id, "ctr-1");
1037        assert_eq!(seen.process.Args, vec!["true".to_owned()]);
1038    }
1039
1040    #[tokio::test]
1041    async fn test_wait_process_returns_real_status() {
1042        let (client, _fake, _server, _dir) = spin_up_fake_agent(7).await;
1043
1044        let status = client.wait_process("ctr-1", "exec-1").await.unwrap();
1045        assert_eq!(status, 7);
1046    }
1047
1048    #[tokio::test]
1049    async fn test_read_stdout_all_drains_until_empty() {
1050        let (client, _fake, _server, _dir) = spin_up_fake_agent(0).await;
1051
1052        let out = client.read_stdout_all("ctr-1", "exec-1").await.unwrap();
1053        assert_eq!(out, b"hello from stdout");
1054    }
1055
1056    #[tokio::test]
1057    async fn test_full_exec_sequence_parses_exit_stdout_stderr() {
1058        let (client, _fake, _server, _dir) = spin_up_fake_agent(42).await;
1059
1060        let (status, stdout, stderr) = client
1061            .exec(
1062                "ctr-1",
1063                &["echo".into(), "hi".into()],
1064                Duration::from_secs(5),
1065                None,
1066            )
1067            .await
1068            .unwrap();
1069
1070        assert_eq!(
1071            status, 42,
1072            "exit code must be the real WaitProcessResponse.status"
1073        );
1074        assert_eq!(stdout, b"hello from stdout");
1075        assert_eq!(stderr, b"");
1076    }
1077
1078    #[tokio::test]
1079    async fn test_exec_times_out() {
1080        // FakeAgent's wait_process deliberately stalls longer than the exec
1081        // timeout, so this deterministically exercises the timeout path
1082        // (rather than racing a near-zero `tokio::time::timeout` against a
1083        // fast local round-trip, which can flake).
1084        let fake = FakeAgent::with_wait_process_delay(0, Duration::from_secs(10));
1085        let (client, _fake, _server, _dir) = spin_up_fake_agent_with(fake).await;
1086
1087        let result = client
1088            .exec("ctr-1", &["echo".into()], Duration::from_millis(50), None)
1089            .await;
1090        assert!(
1091            result.is_err(),
1092            "exec must time out when wait_process stalls"
1093        );
1094        assert!(result.unwrap_err().to_string().contains("timed out"));
1095    }
1096
1097    #[tokio::test]
1098    async fn test_exec_tolerates_already_existing_container() {
1099        // A container that was already created+started by a prior exec_sync
1100        // call must not make a later exec_sync fail — `ensure_container`
1101        // should swallow ALREADY_EXISTS from both CreateContainer and
1102        // StartContainer and proceed straight to ExecProcess.
1103        let fake = FakeAgent::always_already_exists(5);
1104        let (client, _fake, _server, _dir) = spin_up_fake_agent_with(fake).await;
1105
1106        let (status, _stdout, _stderr) = client
1107            .exec("ctr-1", &["true".into()], Duration::from_secs(5), None)
1108            .await
1109            .expect("exec must succeed despite ALREADY_EXISTS on create/start");
1110        assert_eq!(status, 5);
1111    }
1112
1113    #[test]
1114    fn test_is_already_exists_matches_structured_status_code() {
1115        let err = anyhow::Error::new(already_exists_status());
1116        assert!(is_already_exists(&err));
1117    }
1118
1119    #[test]
1120    fn test_is_already_exists_matches_message_fallback() {
1121        let err = anyhow!("container ctr-1 already exists");
1122        assert!(is_already_exists(&err));
1123    }
1124
1125    #[test]
1126    fn test_is_already_exists_false_for_unrelated_error() {
1127        let err = anyhow!("connection refused");
1128        assert!(!is_already_exists(&err));
1129    }
1130}