hyprstream_workers/runtime/kata_agent.rs
1//! kata-agent ttrpc/vsock client — T1-C (#344) of the Worker GA epic (#341)
2//!
3//! Implements a real client for the `kata-agent` running *inside* a Kata
4//! guest VM, so [`KataBackend::exec_sync`](super::kata_backend::KataBackend)
5//! can create/start/exec a container inside the VM instead of returning the
6//! historical "not supported" error.
7//!
8//! # Wire protocol: REAL, not hand-rolled
9//!
10//! The ttrpc request/response message types ([`protocols::agent`],
11//! [`protocols::oci`]) and the generated async ttrpc client
12//! ([`protocols::agent_ttrpc_async::AgentServiceClient`]) come from the
13//! `protocols` crate, which is vendored verbatim from the upstream
14//! `kata-containers` Rust workspace (`src/libs/protocols`, tag matching our
15//! `kata-hypervisor` dependency). It is *already* present in our dependency
16//! graph transitively (`kata-hypervisor` depends on `protocols` with the
17//! `async` ttrpc feature enabled) — this module simply adds it as a direct
18//! dependency and uses it, rather than reimplementing the kata-agent
19//! protobuf/ttrpc wire format by hand. The `CreateContainer` / `StartContainer`
20//! / `ExecProcess` / `WaitProcess` RPCs used below are the real generated
21//! client methods (`agent_ttrpc_async.rs`), framed and (de)serialized by the
22//! real `ttrpc` crate (`ttrpc::r#async::Client`).
23//!
24//! What IS hand-written here: the hybrid-vsock *transport* dialer
25//! (connecting to the guest's kata-agent listen socket and handing the
26//! connected stream to `ttrpc::r#async::Client::new`), and the mapping from our
27//! `SandboxBackend::exec_sync` shape to the agent RPC sequence. This mirrors
28//! (does not vendor) the dialer in upstream
29//! `runtime-rs/crates/agent/src/sock/{vsock,hybrid_vsock}.rs`.
30//!
31//! # Connection convention (verified against upstream kata-containers source,
32//! tag `4.0.0`, fetched into the local cargo git checkout cache — see
33//! `~/.cargo/git/checkouts/kata-containers-*/*/src/runtime-rs/crates/agent/src/sock/`)
34//!
35//! - **Cloud Hypervisor** (the only hypervisor this crate wires up — see
36//! `kata_backend.rs`) exposes the kata-agent over *hybrid vsock*: a Unix
37//! domain socket at `<sandbox_dir>/ch-vm.sock`
38//! (`hypervisor::ch::utils::get_vsock_path`), where the host writes
39//! `"connect <port>\n"` and expects a response line containing `"OK"`
40//! before the same stream becomes the ttrpc transport
41//! (`runtime-rs/crates/agent/src/sock/hybrid_vsock.rs::connect_helper`).
42//! `kata_hypervisor::Hypervisor::get_agent_socket()` returns this address
43//! as `hvsock://<path>`, matching the `Hypervisor` trait already used by
44//! `KataBackend`.
45//! - **Firecracker/Dragonball** use real `AF_VSOCK` (`vsock://<cid>:<port>`),
46//! dialed via `socket(AF_VSOCK) + connect(VsockAddr{cid, port})`
47//! (`runtime-rs/crates/agent/src/sock/vsock.rs`). We don't boot these
48//! hypervisors yet, but the dialer below supports the scheme for parity.
49//! - The kata-agent's *default* ttrpc listen port is **1024**
50//! (`src/runtime/virtcontainers/hypervisor.go: vSockPort = 1024`), used
51//! whenever a caller doesn't override it.
52//!
53//! # What's REAL vs what's UNVALIDATED in this pass
54//!
55//! REAL: the message types, the ttrpc client/wire framing, the hybrid-vsock
56//! handshake implementation, and the RPC sequencing
57//! (`CreateContainer` → `StartContainer` → `ExecProcess` → `WaitProcess` (→
58//! `ReadStdout`/`ReadStderr`)). Unit tests in this module run a *real*
59//! `ttrpc::r#async::Server` (the same crate a real kata-agent links against)
60//! bound to a Unix socket, with a minimal `AgentService` impl that returns
61//! canned responses — so the request encoding / response decoding is
62//! exercised through the genuine ttrpc+protobuf wire path, not mocked at the
63//! byte level.
64//!
65//! UNVALIDATED (this sandbox has no bootable kata-agent guest image): that a
66//! *real* kata-agent's `CreateContainer`/`ExecProcess` semantics (OCI bundle
67//! layout expectations, namespace/cgroup setup it performs) are satisfied by
68//! the minimal `oci::Spec` this module constructs. The `CreateContainerRequest.OCI`
69//! field needs a real OCI runtime spec (rootfs, mounts, namespaces) wired up
70//! from `PodSandbox`/container config — this module supplies a minimal
71//! placeholder `Spec` sufficient to exercise the RPC, not a complete OCI bundle
72//! translation (that's a larger follow-up, see `build_minimal_spec`).
73//!
74//! # Exit-value structuring (issue #608 / future #610)
75//!
76//! `exec_sync`'s return shape `(exit_code, stdout, stderr)` is built from
77//! `WaitProcessResponse.status` (a real terminal exit code from the guest
78//! agent) plus drained `ReadStdout`/`ReadStderr` output — i.e. the exit code
79//! is already a true terminal value by construction, not synthesized. A
80//! future `/exec/instances/<id>/exit` projection (#610) can latch directly
81//! onto the `(i32, Vec<u8>, Vec<u8>)` returned by [`KataAgentClient::exec`]
82//! without restructuring this client.
83
84use std::time::Duration;
85
86use anyhow::{anyhow, Context, Result};
87use protocols::agent_ttrpc_async::AgentServiceClient;
88use protocols::{agent, empty, oci};
89use tokio::io::{AsyncReadExt, AsyncWriteExt};
90use tokio::net::UnixStream;
91use ttrpc::r#async::transport::Socket as TtrpcSocket;
92
93/// Default kata-agent ttrpc vsock port.
94///
95/// Source: upstream kata-containers `src/runtime/virtcontainers/hypervisor.go`
96/// (`vSockPort = 1024`), tag `4.0.0` — the same tag pinned for our
97/// `kata-hypervisor`/`protocols` dependencies.
98pub const KATA_AGENT_VSOCK_PORT: u32 = 1024;
99
100/// Guest-side storage `driver` the kata-agent dispatches on to mount a
101/// virtio-fs share. Matches `kata_types::device::DRIVER_VIRTIOFS_TYPE`
102/// (`"virtio-fs"`, tag `4.0.0`), which `agent/src/storage/fs_handler.rs`'s
103/// `VirtioFsHandler` registers.
104const VIRTIOFS_STORAGE_DRIVER: &str = "virtio-fs";
105
106/// Guest-side filesystem type used to mount a virtio-fs share. The kata-agent
107/// `baremount`s the share with this `fstype` (`FS_TYPE_VIRTIO_FS = "virtiofs"`
108/// in upstream `runtime-rs/.../share_virtio_fs.rs`, tag `4.0.0`).
109const VIRTIOFS_FSTYPE: &str = "virtiofs";
110
111/// Base directory the kata-agent uses for per-container bundles inside the
112/// guest (`CONTAINER_BASE = "/run/kata-containers"` in `agent/src/rpc.rs`,
113/// tag `4.0.0`). The container's rootfs lives at `<base>/<cid>/rootfs`.
114const CONTAINER_BASE: &str = "/run/kata-containers";
115
116/// How the container's rootfs is provided to the guest.
117///
118/// hyprstream serves the **entire tenant VFS namespace** (with the rootfs
119/// mounted at `/` — RAFS lower + per-sandbox writable upper + the injected
120/// `/models` / `/deltas` / `/stream` mounts) over a **single** vhost-user-fs
121/// (virtio-fs) device attached to the hypervisor under a mount tag (see
122/// `sandbox_fs.rs` / `kata_backend::attach_share_fs`). This tells
123/// [`KataAgentClient::create_container`] to make the kata-agent mount that
124/// share, by tag, **as the container's rootfs** — so the container's
125/// filesystem *is* the tenant VFS.
126#[derive(Debug, Clone)]
127pub struct ContainerRootfs {
128 /// The virtio-fs mount tag the hypervisor attached the share under — the
129 /// `ShareFsDevice` `mount_tag` passed to
130 /// `kata_backend::KataBackend::attach_share_fs`.
131 pub virtiofs_mount_tag: String,
132}
133
134/// Guest path where the kata-agent expects (and, via `setup_bundle`, pivots
135/// into) a container's rootfs: `<CONTAINER_BASE>/<cid>/rootfs`.
136fn guest_rootfs_path(container_id: &str) -> String {
137 format!("{CONTAINER_BASE}/{container_id}/rootfs")
138}
139
140/// Build the `Storage` entry that makes the kata-agent mount the virtio-fs
141/// share (identified by `rootfs.virtiofs_mount_tag`) directly at the
142/// container's rootfs path inside the guest.
143///
144/// The agent's `VirtioFsHandler` → `common_storage_handler` → `mount_storage`
145/// `baremount`s `source` (the tag) as `fstype` (`virtiofs`) at `mount_point`,
146/// creating the destination directory first. Because we mount it **directly
147/// at `<CONTAINER_BASE>/<cid>/rootfs`**, the agent's `setup_bundle` finds the
148/// rootfs path already present (the virtio-fs mount) and uses it as-is — no
149/// bind-mount indirection, and no separate nested `passthrough/<cid>` layout
150/// (that layout is kata's multi-container *bundle* sharing; hyprstream serves
151/// exactly one rootfs per share). The tenant VFS root thereby becomes the
152/// container root. `Root.Path` in the OCI spec points at the same path (see
153/// [`build_minimal_spec`]).
154fn rootfs_storage(container_id: &str, rootfs: &ContainerRootfs) -> agent::Storage {
155 agent::Storage {
156 driver: VIRTIOFS_STORAGE_DRIVER.to_owned(),
157 source: rootfs.virtiofs_mount_tag.clone(),
158 fstype: VIRTIOFS_FSTYPE.to_owned(),
159 // `nodev`, matching upstream kata's `SHARED_DIR_VIRTIO_FS_OPTIONS`
160 // for the sandbox virtio-fs share.
161 options: vec!["nodev".to_owned()],
162 mount_point: guest_rootfs_path(container_id),
163 ..Default::default()
164 }
165}
166
167/// Default per-dial-attempt timeout.
168const DIAL_TIMEOUT: Duration = Duration::from_millis(500);
169
170/// Default overall connect timeout (covers retries).
171const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
172
173/// Default per-RPC ttrpc timeout (nanoseconds, per `ttrpc::context::with_timeout`).
174const RPC_TIMEOUT_NS: i64 = 30 * 1_000_000_000;
175
176/// A parsed kata-agent connection address.
177///
178/// Mirrors the two schemes upstream kata-containers' own
179/// `runtime-rs/crates/agent/src/sock` dialer accepts. Cloud Hypervisor (the
180/// only hypervisor `KataBackend` boots today) always returns
181/// [`AgentAddress::HybridVsock`] from `Hypervisor::get_agent_socket()`.
182#[derive(Debug, Clone, PartialEq, Eq)]
183pub enum AgentAddress {
184 /// `hvsock://<unix-socket-path>` — Cloud Hypervisor / Dragonball hybrid
185 /// vsock: a host-side Unix socket multiplexes guest vsock ports via a
186 /// `CONNECT <port>\n` text handshake.
187 HybridVsock { path: String },
188 /// `vsock://<cid>` — real `AF_VSOCK`, used by Firecracker. Not exercised
189 /// by `KataBackend` today (it only boots Cloud Hypervisor/Dragonball),
190 /// but parsed/dialable for parity with upstream.
191 Vsock { cid: u32 },
192}
193
194impl AgentAddress {
195 /// Parse a `hvsock://<path>` or `vsock://<cid>` URI, as returned by
196 /// `kata_hypervisor::Hypervisor::get_agent_socket()`.
197 pub fn parse(address: &str) -> Result<Self> {
198 if let Some(path) = address.strip_prefix("hvsock://") {
199 return Ok(AgentAddress::HybridVsock {
200 path: path.to_owned(),
201 });
202 }
203 if let Some(cid) = address.strip_prefix("vsock://") {
204 let cid: u32 = cid
205 .parse()
206 .with_context(|| format!("invalid vsock cid in {address:?}"))?;
207 return Ok(AgentAddress::Vsock { cid });
208 }
209 Err(anyhow!("unsupported kata-agent socket scheme: {address:?}"))
210 }
211}
212
213/// Dial the kata-agent's ttrpc socket and wrap its connected stream.
214///
215/// For [`AgentAddress::HybridVsock`], performs the hybrid-vsock handshake
216/// (`connect <port>\n` → response line containing `OK`) over the host Unix
217/// socket, exactly as upstream's `hybrid_vsock::connect_helper` does. For
218/// [`AgentAddress::Vsock`], opens a real `AF_VSOCK` connection via ttrpc's
219/// async transport.
220pub async fn dial(address: &AgentAddress, port: u32) -> Result<TtrpcSocket> {
221 let deadline = tokio::time::Instant::now() + CONNECT_TIMEOUT;
222 let mut last_err = None;
223
224 while tokio::time::Instant::now() < deadline {
225 let attempt = dial_once(address, port).await;
226 match attempt {
227 Ok(socket) => return Ok(socket),
228 Err(e) => {
229 last_err = Some(e);
230 tokio::time::sleep(DIAL_TIMEOUT).await;
231 }
232 }
233 }
234
235 Err(last_err.unwrap_or_else(|| anyhow!("dial timed out with no attempts made")))
236}
237
238async fn dial_once(address: &AgentAddress, port: u32) -> Result<TtrpcSocket> {
239 match address {
240 AgentAddress::HybridVsock { path } => hybrid_vsock_connect(path, port).await,
241 AgentAddress::Vsock { cid } => vsock_connect(*cid, port).await,
242 }
243}
244
245/// Hybrid-vsock handshake: connect to the host UDS, write `"connect
246/// <port>\n"`, read one response line, require it to contain `"OK"`.
247///
248/// Matches `runtime-rs/crates/agent/src/sock/hybrid_vsock.rs::connect_helper`
249/// in upstream kata-containers (tag `4.0.0`).
250async fn hybrid_vsock_connect(uds_path: &str, port: u32) -> Result<TtrpcSocket> {
251 let mut stream = UnixStream::connect(uds_path)
252 .await
253 .with_context(|| format!("connect to hybrid-vsock UDS {uds_path:?}"))?;
254
255 stream
256 .write_all(format!("connect {port}\n").as_bytes())
257 .await
258 .context("write hybrid-vsock CONNECT handshake")?;
259
260 let mut response = Vec::new();
261 let mut byte = [0u8; 1];
262 loop {
263 let n = stream
264 .read(&mut byte)
265 .await
266 .context("read hybrid-vsock handshake response")?;
267 if n == 0 || byte[0] == b'\n' {
268 break;
269 }
270 response.push(byte[0]);
271 }
272 let response = String::from_utf8_lossy(&response);
273 if !response.contains("OK") {
274 return Err(anyhow!(
275 "hybrid-vsock handshake error: malformed response {response:?}"
276 ));
277 }
278
279 Ok(TtrpcSocket::new(stream))
280}
281
282/// Real `AF_VSOCK` connect via ttrpc's Tokio transport (Firecracker/Dragonball
283/// convention: `vsock://<cid>`, default port 1024).
284async fn vsock_connect(cid: u32, port: u32) -> Result<TtrpcSocket> {
285 TtrpcSocket::connect_vsock(format!("{cid}:{port}"))
286 .await
287 .with_context(|| format!("AF_VSOCK connect to cid={cid} port={port}"))
288}
289
290/// A connected kata-agent ttrpc client.
291///
292/// Thin wrapper around the real, generated
293/// [`protocols::agent_ttrpc_async::AgentServiceClient`], scoped to the CRI
294/// subset `SandboxBackend::exec_sync` needs: create → start → exec → wait
295/// (+ drain stdout/stderr).
296pub struct KataAgentClient {
297 client: AgentServiceClient,
298 timeout_ns: i64,
299}
300
301impl KataAgentClient {
302 /// Dial the agent at `address` (kata-agent's well-known port 1024 unless
303 /// the caller has a reason to override it) and wrap it in an
304 /// `AgentServiceClient`.
305 pub async fn connect(address: &AgentAddress) -> Result<Self> {
306 Self::connect_with_port(address, KATA_AGENT_VSOCK_PORT).await
307 }
308
309 pub async fn connect_with_port(address: &AgentAddress, port: u32) -> Result<Self> {
310 let socket = dial(address, port).await?;
311 let ttrpc_client = ttrpc::r#async::Client::new(socket);
312 Ok(Self::from_ttrpc_client(ttrpc_client))
313 }
314
315 /// Wrap an already-connected `ttrpc::r#async::Client` (used directly by
316 /// tests against a local mock server, and available to callers who
317 /// already have a connected client from elsewhere).
318 pub fn from_ttrpc_client(client: ttrpc::r#async::Client) -> Self {
319 Self {
320 client: AgentServiceClient::new(client),
321 timeout_ns: RPC_TIMEOUT_NS,
322 }
323 }
324
325 fn ctx(&self) -> ttrpc::context::Context {
326 ttrpc::context::with_timeout(self.timeout_ns)
327 }
328
329 /// `CreateContainer` — minimal OCI spec wired from `container_id` +
330 /// `argv`, plus (when `rootfs` is supplied) a virtio-fs `Storage` entry
331 /// that mounts the tenant VFS share as the container's rootfs (#721).
332 ///
333 /// When `rootfs` is `None` the spec's `Root.Path` still points at
334 /// `<CONTAINER_BASE>/<cid>/rootfs`, but nothing is mounted there — the
335 /// guest must already have a rootfs at that path (the historical bare-VM
336 /// case, which fails `CreateContainer` with `No such file or directory`
337 /// precisely because it does not). Supplying `rootfs` is what makes the
338 /// tenant VFS *be* that rootfs; see [`ContainerRootfs`] / [`rootfs_storage`].
339 ///
340 /// This is still NOT a full OCI bundle translation (no devices/volumes/
341 /// namespaces from `PodSandbox`) — just enough to boot a container whose
342 /// filesystem is the hyprstream tenant VFS.
343 pub async fn create_container(
344 &self,
345 container_id: &str,
346 exec_id: &str,
347 argv: &[String],
348 rootfs: Option<&ContainerRootfs>,
349 ) -> Result<()> {
350 let storages = rootfs
351 .map(|r| vec![rootfs_storage(container_id, r)])
352 .unwrap_or_default();
353 let req = agent::CreateContainerRequest {
354 container_id: container_id.to_owned(),
355 exec_id: exec_id.to_owned(),
356 storages,
357 OCI: protobuf::MessageField::some(build_minimal_spec(container_id, argv)),
358 ..Default::default()
359 };
360 self.client
361 .create_container(self.ctx(), &req)
362 .await
363 .context("CreateContainer ttrpc call")?;
364 Ok(())
365 }
366
367 pub async fn start_container(&self, container_id: &str) -> Result<()> {
368 let req = agent::StartContainerRequest {
369 container_id: container_id.to_owned(),
370 ..Default::default()
371 };
372 self.client
373 .start_container(self.ctx(), &req)
374 .await
375 .context("StartContainer ttrpc call")?;
376 Ok(())
377 }
378
379 /// Idempotently ensure `container_id` exists and is started inside the
380 /// guest, tolerating "already exists"/"already started" ttrpc errors
381 /// from a prior call (kata-agent has no native "create or no-op" RPC, so
382 /// this papers over that at the client level — matching what a real CRI
383 /// shim does when `exec_sync` is called against a container it didn't
384 /// just create itself).
385 ///
386 /// `argv` is only used if the container needs to be created (it seeds
387 /// the OCI `Process.Args` — see [`build_minimal_spec`]); an already
388 /// existing container's original entrypoint is left untouched. `rootfs`
389 /// (the tenant VFS virtio-fs share, #721) is likewise only consumed on
390 /// the create path.
391 async fn ensure_container(
392 &self,
393 container_id: &str,
394 argv: &[String],
395 rootfs: Option<&ContainerRootfs>,
396 ) -> Result<()> {
397 let create_exec_id = format!("{container_id}-init");
398 match self
399 .create_container(container_id, &create_exec_id, argv, rootfs)
400 .await
401 {
402 Ok(()) => {}
403 Err(e) if is_already_exists(&e) => {
404 tracing::debug!(container_id, "container already exists, skipping create");
405 }
406 Err(e) => return Err(e.context("ensure_container: create_container")),
407 }
408
409 match self.start_container(container_id).await {
410 Ok(()) => Ok(()),
411 Err(e) if is_already_exists(&e) => {
412 tracing::debug!(container_id, "container already started, skipping start");
413 Ok(())
414 }
415 Err(e) => Err(e.context("ensure_container: start_container")),
416 }
417 }
418
419 /// `ExecProcess` — run `argv` inside `container_id` under a fresh
420 /// `exec_id`. Does not block for completion; pair with [`Self::wait_process`].
421 pub async fn exec_process(
422 &self,
423 container_id: &str,
424 exec_id: &str,
425 argv: &[String],
426 ) -> Result<()> {
427 let process = oci::Process {
428 Args: argv.to_vec(),
429 Cwd: "/".to_owned(),
430 Capabilities: protobuf::MessageField::some(default_capabilities()),
431 ..Default::default()
432 };
433 let req = agent::ExecProcessRequest {
434 container_id: container_id.to_owned(),
435 exec_id: exec_id.to_owned(),
436 process: protobuf::MessageField::some(process),
437 ..Default::default()
438 };
439 self.client
440 .exec_process(self.ctx(), &req)
441 .await
442 .context("ExecProcess ttrpc call")?;
443 Ok(())
444 }
445
446 /// `WaitProcess` — blocks (ttrpc-side) until the process exits, returning
447 /// its real terminal exit status from the guest agent.
448 pub async fn wait_process(&self, container_id: &str, exec_id: &str) -> Result<i32> {
449 let req = agent::WaitProcessRequest {
450 container_id: container_id.to_owned(),
451 exec_id: exec_id.to_owned(),
452 ..Default::default()
453 };
454 let resp = self
455 .client
456 .wait_process(self.ctx(), &req)
457 .await
458 .context("WaitProcess ttrpc call")?;
459 Ok(resp.status)
460 }
461
462 /// `UpdateContainer` — apply new cgroup resource limits to a running
463 /// container inside the guest.
464 ///
465 /// This is the RPC a CRI `UpdateContainerResources` maps onto: the
466 /// kata-agent rewrites the container's in-guest cgroups (CPU shares/quota/
467 /// period, memory limit, cpuset, …). It updates the container's slice of
468 /// the *existing* VM; growing the VM itself past its boot size (vCPU /
469 /// memory hotplug) is a separate hypervisor concern and not attempted
470 /// here — matching upstream kata, whose `update_container` handler drives
471 /// the guest cgroup layer, with VM resize handled independently by the
472 /// runtime's resource manager.
473 pub async fn update_container(
474 &self,
475 container_id: &str,
476 resources: oci::LinuxResources,
477 ) -> Result<()> {
478 let req = agent::UpdateContainerRequest {
479 container_id: container_id.to_owned(),
480 resources: protobuf::MessageField::some(resources),
481 ..Default::default()
482 };
483 self.client
484 .update_container(self.ctx(), &req)
485 .await
486 .context("UpdateContainer ttrpc call")?;
487 Ok(())
488 }
489
490 /// `StatsContainer` — fetch the guest cgroup (+ network) stats for a
491 /// container. The returned [`agent::StatsContainerResponse`] carries the
492 /// real in-guest cgroup counters (`cgroup_stats.cpu_stats` /
493 /// `memory_stats`); mapping them into the CRI `ContainerStats` shape is
494 /// the caller's job (see `KataBackend::container_stats`).
495 pub async fn stats_container(
496 &self,
497 container_id: &str,
498 ) -> Result<agent::StatsContainerResponse> {
499 let req = agent::StatsContainerRequest {
500 container_id: container_id.to_owned(),
501 ..Default::default()
502 };
503 let resp = self
504 .client
505 .stats_container(self.ctx(), &req)
506 .await
507 .context("StatsContainer ttrpc call")?;
508 Ok(resp)
509 }
510
511 /// Drain available stdout via repeated `ReadStdout` calls until the
512 /// agent reports no more data (empty response).
513 pub async fn read_stdout_all(&self, container_id: &str, exec_id: &str) -> Result<Vec<u8>> {
514 self.read_stream_all(container_id, exec_id, StreamKind::Stdout)
515 .await
516 }
517
518 pub async fn read_stderr_all(&self, container_id: &str, exec_id: &str) -> Result<Vec<u8>> {
519 self.read_stream_all(container_id, exec_id, StreamKind::Stderr)
520 .await
521 }
522
523 async fn read_stream_all(
524 &self,
525 container_id: &str,
526 exec_id: &str,
527 kind: StreamKind,
528 ) -> Result<Vec<u8>> {
529 const CHUNK: u32 = 4096;
530 let mut out = Vec::new();
531 loop {
532 let req = agent::ReadStreamRequest {
533 container_id: container_id.to_owned(),
534 exec_id: exec_id.to_owned(),
535 len: CHUNK,
536 ..Default::default()
537 };
538 let resp = match kind {
539 StreamKind::Stdout => self.client.read_stdout(self.ctx(), &req).await,
540 StreamKind::Stderr => self.client.read_stderr(self.ctx(), &req).await,
541 }
542 .context("ReadStdout/ReadStderr ttrpc call")?;
543 if resp.data.is_empty() {
544 break;
545 }
546 let got = resp.data.len();
547 out.extend_from_slice(&resp.data);
548 if got < CHUNK as usize {
549 break;
550 }
551 }
552 Ok(out)
553 }
554
555 /// Full ensure-container → exec → wait → drain sequence, matching the
556 /// `(exit_code, stdout, stderr)` shape `SandboxBackend::exec_sync` needs.
557 ///
558 /// Calls [`Self::ensure_container`] first (create + start, idempotent)
559 /// so callers don't need to separately track whether `container_id` has
560 /// already been created in this guest — matching the issue's "real
561 /// CRI create/start/exec_sync" scope.
562 ///
563 /// The returned `i32` is `WaitProcessResponse.status` — a real terminal
564 /// exit value from the guest agent (see module docs re: #608/#610).
565 pub async fn exec(
566 &self,
567 container_id: &str,
568 argv: &[String],
569 timeout: Duration,
570 rootfs: Option<&ContainerRootfs>,
571 ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
572 let exec_id = uuid::Uuid::new_v4().to_string();
573
574 tokio::time::timeout(timeout, async {
575 self.ensure_container(container_id, argv, rootfs).await?;
576 self.exec_process(container_id, &exec_id, argv).await?;
577 let status = self.wait_process(container_id, &exec_id).await?;
578 let stdout = self
579 .read_stdout_all(container_id, &exec_id)
580 .await
581 .unwrap_or_default();
582 let stderr = self
583 .read_stderr_all(container_id, &exec_id)
584 .await
585 .unwrap_or_default();
586 Ok::<_, anyhow::Error>((status, stdout, stderr))
587 })
588 .await
589 .map_err(|_| anyhow!("exec timed out after {timeout:?}"))?
590 }
591}
592
593#[derive(Clone, Copy)]
594enum StreamKind {
595 Stdout,
596 Stderr,
597}
598
599/// Best-effort detection of "the container/process already
600/// exists/started" ttrpc errors, used by [`KataAgentClient::ensure_container`]
601/// to make create+start idempotent.
602///
603/// Prefers the structured gRPC-style status code (`ALREADY_EXISTS = 6`, per
604/// `ttrpc.proto` — the same code space kata-agent's `CreateContainer`/
605/// `StartContainer` handlers use to reject duplicate ids upstream), falling
606/// back to a substring match on the error chain for cases where the error
607/// was wrapped before reaching here. NOTE: the exact error this would
608/// produce against a *real* kata-agent has not been observed in this sandbox
609/// (no bootable guest available) — this is the best-effort mapping a real
610/// instance is expected to need, not a verified one.
611fn is_already_exists(err: &anyhow::Error) -> bool {
612 for cause in err.chain() {
613 if let Some(ttrpc::Error::RpcStatus(status)) = cause.downcast_ref::<ttrpc::Error>() {
614 if status.code() == ttrpc::Code::ALREADY_EXISTS {
615 return true;
616 }
617 }
618 }
619 let msg = err.to_string().to_ascii_lowercase();
620 msg.contains("already exists") || msg.contains("already started")
621}
622
623/// Build a minimal `oci::Spec` sufficient to drive `CreateContainer` against
624/// a kata-agent. NOT a complete OCI bundle translation — see module docs.
625///
626/// `Root.Path` is `<CONTAINER_BASE>/<cid>/rootfs`, the guest path the agent's
627/// `setup_bundle` pivots into. When a [`ContainerRootfs`] is supplied to
628/// [`KataAgentClient::create_container`], the accompanying [`rootfs_storage`]
629/// mounts the tenant VFS virtio-fs share at exactly this path, so the tenant
630/// VFS becomes the container's filesystem (#721).
631fn build_minimal_spec(container_id: &str, argv: &[String]) -> oci::Spec {
632 oci::Spec {
633 Version: "1.0.2".to_owned(),
634 Process: protobuf::MessageField::some(oci::Process {
635 Args: argv.to_vec(),
636 Cwd: "/".to_owned(),
637 Capabilities: protobuf::MessageField::some(default_capabilities()),
638 ..Default::default()
639 }),
640 Root: protobuf::MessageField::some(oci::Root {
641 Path: guest_rootfs_path(container_id),
642 Readonly: false,
643 ..Default::default()
644 }),
645 Hostname: container_id.to_owned(),
646 ..Default::default()
647 }
648}
649
650/// Default OCI process capability set (the runc/OCI baseline). The kata-agent
651/// (rustjail) rejects `CreateContainer`/`ExecProcess` with `missing process
652/// capabilities` when `Process.Capabilities` is unset, so every container and
653/// exec process carries this standard bounding/effective/permitted set.
654fn default_capabilities() -> oci::LinuxCapabilities {
655 let caps: Vec<String> = [
656 "CAP_CHOWN",
657 "CAP_DAC_OVERRIDE",
658 "CAP_FSETID",
659 "CAP_FOWNER",
660 "CAP_MKNOD",
661 "CAP_NET_RAW",
662 "CAP_SETGID",
663 "CAP_SETUID",
664 "CAP_SETFCAP",
665 "CAP_SETPCAP",
666 "CAP_NET_BIND_SERVICE",
667 "CAP_SYS_CHROOT",
668 "CAP_KILL",
669 "CAP_AUDIT_WRITE",
670 ]
671 .iter()
672 .map(|s| (*s).to_owned())
673 .collect();
674 oci::LinuxCapabilities {
675 Bounding: caps.clone(),
676 Effective: caps.clone(),
677 Inheritable: caps.clone(),
678 Permitted: caps.clone(),
679 Ambient: caps,
680 ..Default::default()
681 }
682}
683
684// Re-export so `kata_backend.rs` doesn't need a separate `protocols`/`empty`
685// import just to satisfy trait bounds on `Empty` responses it may inspect.
686#[allow(unused_imports)]
687pub(crate) use empty::Empty as AgentEmpty;
688
689#[cfg(test)]
690#[allow(clippy::unwrap_used, clippy::expect_used)]
691mod tests {
692 use super::*;
693 use async_trait::async_trait;
694 use protocols::agent_ttrpc_async::{create_agent_service, AgentService};
695 use std::sync::atomic::{AtomicI32, Ordering};
696 use std::sync::Arc;
697 use tempfile::TempDir;
698
699 // ─────────────────────────────────────────────────────────────────────
700 // AgentAddress parsing
701 // ─────────────────────────────────────────────────────────────────────
702
703 #[test]
704 fn test_parse_hybrid_vsock() {
705 let addr = AgentAddress::parse("hvsock:///run/sandbox/ch-vm.sock").unwrap();
706 assert_eq!(
707 addr,
708 AgentAddress::HybridVsock {
709 path: "/run/sandbox/ch-vm.sock".to_owned()
710 }
711 );
712 }
713
714 #[test]
715 fn test_parse_vsock() {
716 let addr = AgentAddress::parse("vsock://42").unwrap();
717 assert_eq!(addr, AgentAddress::Vsock { cid: 42 });
718 }
719
720 #[test]
721 fn test_parse_unsupported_scheme() {
722 let err = AgentAddress::parse("tcp://127.0.0.1:1024").unwrap_err();
723 assert!(err.to_string().contains("unsupported"));
724 }
725
726 #[test]
727 fn test_parse_invalid_vsock_cid() {
728 let err = AgentAddress::parse("vsock://not-a-number").unwrap_err();
729 assert!(err.to_string().contains("invalid vsock cid"));
730 }
731
732 #[test]
733 fn test_kata_agent_vsock_port_matches_upstream_convention() {
734 // src/runtime/virtcontainers/hypervisor.go: vSockPort = 1024
735 assert_eq!(KATA_AGENT_VSOCK_PORT, 1024);
736 }
737
738 // ─────────────────────────────────────────────────────────────────────
739 // Hybrid-vsock handshake (real UDS, fake "guest" side)
740 // ─────────────────────────────────────────────────────────────────────
741
742 #[tokio::test]
743 async fn test_hybrid_vsock_handshake_ok() {
744 let dir = TempDir::new().unwrap();
745 let sock_path = dir.path().join("ch-vm.sock");
746 let listener = tokio::net::UnixListener::bind(&sock_path).unwrap();
747
748 let accept_task = tokio::spawn(async move {
749 let (mut stream, _) = listener.accept().await.unwrap();
750 let mut buf = [0u8; 64];
751 let n = stream.read(&mut buf).await.unwrap();
752 let req = String::from_utf8_lossy(&buf[..n]);
753 assert_eq!(req, "connect 1024\n");
754 stream.write_all(b"OK 1234\n").await.unwrap();
755 // keep stream alive for the duration of the test
756 tokio::time::sleep(Duration::from_millis(50)).await;
757 });
758
759 let _socket = hybrid_vsock_connect(sock_path.to_str().unwrap(), 1024)
760 .await
761 .unwrap();
762
763 accept_task.await.unwrap();
764 }
765
766 #[tokio::test]
767 async fn test_hybrid_vsock_handshake_rejects_malformed_response() {
768 let dir = TempDir::new().unwrap();
769 let sock_path = dir.path().join("ch-vm.sock");
770 let listener = tokio::net::UnixListener::bind(&sock_path).unwrap();
771
772 tokio::spawn(async move {
773 let (mut stream, _) = listener.accept().await.unwrap();
774 let mut buf = [0u8; 64];
775 let _ = stream.read(&mut buf).await.unwrap();
776 stream.write_all(b"ERROR\n").await.unwrap();
777 });
778
779 let result = hybrid_vsock_connect(sock_path.to_str().unwrap(), 1024).await;
780 assert!(result.is_err());
781 assert!(result
782 .err()
783 .unwrap()
784 .to_string()
785 .contains("handshake error"));
786 }
787
788 // ─────────────────────────────────────────────────────────────────────
789 // Real ttrpc server/client round-trip (fake AgentService, real wire format)
790 // ─────────────────────────────────────────────────────────────────────
791
792 /// A minimal `AgentService` impl that proves request decoding +
793 /// response encoding work end-to-end over the real ttrpc transport.
794 /// Records the last request seen for assertions and returns canned,
795 /// deterministic replies — this is the "mock/fake ttrpc server" the
796 /// issue asks for, built on the *real* generated service trait rather
797 /// than hand-rolled framing.
798 struct FakeAgent {
799 wait_status: AtomicI32,
800 last_create: parking_lot::Mutex<Option<agent::CreateContainerRequest>>,
801 last_exec: parking_lot::Mutex<Option<agent::ExecProcessRequest>>,
802 /// Artificial delay before `wait_process` resolves, used to
803 /// deterministically exercise `KataAgentClient::exec`'s timeout path
804 /// (a near-zero `tokio::time::timeout` duration is not reliable: a
805 /// fast local round-trip can race the timer's first poll).
806 wait_process_delay: Duration,
807 /// When set, `create_container`/`start_container` always reply with
808 /// a `Code::ALREADY_EXISTS` ttrpc status, to exercise
809 /// `KataAgentClient::ensure_container`'s idempotent-retry path.
810 already_exists: bool,
811 }
812
813 impl FakeAgent {
814 fn new(wait_status: i32) -> Self {
815 Self {
816 wait_status: AtomicI32::new(wait_status),
817 last_create: parking_lot::Mutex::new(None),
818 last_exec: parking_lot::Mutex::new(None),
819 wait_process_delay: Duration::ZERO,
820 already_exists: false,
821 }
822 }
823
824 fn with_wait_process_delay(wait_status: i32, delay: Duration) -> Self {
825 Self {
826 wait_process_delay: delay,
827 ..Self::new(wait_status)
828 }
829 }
830
831 fn always_already_exists(wait_status: i32) -> Self {
832 Self {
833 already_exists: true,
834 ..Self::new(wait_status)
835 }
836 }
837 }
838
839 fn already_exists_status() -> ::ttrpc::Error {
840 ::ttrpc::Error::RpcStatus(::ttrpc::get_status(
841 ::ttrpc::Code::ALREADY_EXISTS,
842 "already exists".to_owned(),
843 ))
844 }
845
846 #[async_trait]
847 impl AgentService for FakeAgent {
848 async fn create_container(
849 &self,
850 _ctx: &::ttrpc::r#async::TtrpcContext,
851 req: agent::CreateContainerRequest,
852 ) -> ::ttrpc::Result<empty::Empty> {
853 if self.already_exists {
854 return Err(already_exists_status());
855 }
856 *self.last_create.lock() = Some(req);
857 Ok(empty::Empty::new())
858 }
859
860 async fn start_container(
861 &self,
862 _ctx: &::ttrpc::r#async::TtrpcContext,
863 _req: agent::StartContainerRequest,
864 ) -> ::ttrpc::Result<empty::Empty> {
865 if self.already_exists {
866 return Err(already_exists_status());
867 }
868 Ok(empty::Empty::new())
869 }
870
871 async fn exec_process(
872 &self,
873 _ctx: &::ttrpc::r#async::TtrpcContext,
874 req: agent::ExecProcessRequest,
875 ) -> ::ttrpc::Result<empty::Empty> {
876 *self.last_exec.lock() = Some(req);
877 Ok(empty::Empty::new())
878 }
879
880 async fn wait_process(
881 &self,
882 _ctx: &::ttrpc::r#async::TtrpcContext,
883 _req: agent::WaitProcessRequest,
884 ) -> ::ttrpc::Result<agent::WaitProcessResponse> {
885 if !self.wait_process_delay.is_zero() {
886 tokio::time::sleep(self.wait_process_delay).await;
887 }
888 Ok(agent::WaitProcessResponse {
889 status: self.wait_status.load(Ordering::SeqCst),
890 ..Default::default()
891 })
892 }
893
894 async fn read_stdout(
895 &self,
896 _ctx: &::ttrpc::r#async::TtrpcContext,
897 _req: agent::ReadStreamRequest,
898 ) -> ::ttrpc::Result<agent::ReadStreamResponse> {
899 Ok(agent::ReadStreamResponse {
900 data: b"hello from stdout".to_vec(),
901 ..Default::default()
902 })
903 }
904
905 async fn read_stderr(
906 &self,
907 _ctx: &::ttrpc::r#async::TtrpcContext,
908 _req: agent::ReadStreamRequest,
909 ) -> ::ttrpc::Result<agent::ReadStreamResponse> {
910 Ok(agent::ReadStreamResponse {
911 data: Vec::new(),
912 ..Default::default()
913 })
914 }
915 }
916
917 /// Spin up a real `ttrpc::r#async::Server` bound to a temp Unix socket,
918 /// serving `FakeAgent`, and return a connected `KataAgentClient` plus
919 /// the server's join handle (kept alive for the test's duration) and
920 /// the `Arc<FakeAgent>` for post-call assertions.
921 async fn spin_up_fake_agent(
922 wait_status: i32,
923 ) -> (
924 KataAgentClient,
925 Arc<FakeAgent>,
926 tokio::task::JoinHandle<()>,
927 TempDir,
928 ) {
929 spin_up_fake_agent_with(FakeAgent::new(wait_status)).await
930 }
931
932 async fn spin_up_fake_agent_with(
933 fake: FakeAgent,
934 ) -> (
935 KataAgentClient,
936 Arc<FakeAgent>,
937 tokio::task::JoinHandle<()>,
938 TempDir,
939 ) {
940 let dir = TempDir::new().unwrap();
941 let sock_path = dir.path().join("agent.sock");
942 let sockaddr = format!("unix://{}", sock_path.display());
943
944 let fake = Arc::new(fake);
945 let service = create_agent_service(fake.clone() as Arc<dyn AgentService + Send + Sync>);
946
947 let mut server = ttrpc::r#async::Server::new()
948 .bind(&sockaddr)
949 .unwrap()
950 .register_service(service);
951
952 let server_task = tokio::spawn(async move {
953 let _ = server.start().await;
954 // Hold the server alive; in a real shutdown path we'd call
955 // server.shutdown().await, omitted here since the test process
956 // exit reclaims the socket.
957 std::future::pending::<()>().await;
958 });
959
960 // Give the listener a moment to bind before dialing.
961 tokio::time::sleep(Duration::from_millis(50)).await;
962
963 let ttrpc_client = ttrpc::r#async::Client::connect(&sockaddr).await.unwrap();
964 let client = KataAgentClient::from_ttrpc_client(ttrpc_client);
965
966 (client, fake, server_task, dir)
967 }
968
969 #[tokio::test]
970 async fn test_create_container_round_trip() {
971 let (client, fake, _server, _dir) = spin_up_fake_agent(0).await;
972
973 client
974 .create_container("ctr-1", "exec-1", &["echo".into(), "hi".into()], None)
975 .await
976 .unwrap();
977
978 let seen = fake.last_create.lock().clone().unwrap();
979 assert_eq!(seen.container_id, "ctr-1");
980 assert_eq!(seen.exec_id, "exec-1");
981 assert!(seen.OCI.is_some());
982 assert_eq!(
983 seen.OCI.Process.Args,
984 vec!["echo".to_owned(), "hi".to_owned()]
985 );
986 // No rootfs supplied → no virtio-fs Storage entry.
987 assert!(seen.storages.is_empty(), "no rootfs → no storages");
988 assert_eq!(
989 seen.OCI.Root.Path, "/run/kata-containers/ctr-1/rootfs",
990 "Root.Path is the guest container rootfs path"
991 );
992 }
993
994 #[tokio::test]
995 async fn test_create_container_mounts_tenant_vfs_as_rootfs() {
996 // #721: when a ContainerRootfs (tenant VFS virtio-fs mount tag) is
997 // supplied, CreateContainer carries a Storage entry that mounts that
998 // share, by tag, directly at the container's rootfs path — and
999 // Root.Path points at that same path.
1000 let (client, fake, _server, _dir) = spin_up_fake_agent(0).await;
1001
1002 let rootfs = ContainerRootfs {
1003 virtiofs_mount_tag: "hyprstream-vfs".to_owned(),
1004 };
1005 client
1006 .create_container("ctr-1", "exec-1", &["true".into()], Some(&rootfs))
1007 .await
1008 .unwrap();
1009
1010 let seen = fake.last_create.lock().clone().unwrap();
1011 assert_eq!(seen.storages.len(), 1, "exactly one rootfs Storage entry");
1012 let st = &seen.storages[0];
1013 assert_eq!(st.driver, "virtio-fs");
1014 assert_eq!(
1015 st.source, "hyprstream-vfs",
1016 "source is the virtio-fs mount tag"
1017 );
1018 assert_eq!(st.fstype, "virtiofs");
1019 assert_eq!(st.mount_point, "/run/kata-containers/ctr-1/rootfs");
1020 assert_eq!(
1021 seen.OCI.Root.Path, st.mount_point,
1022 "Root.Path must equal the virtio-fs rootfs mount point"
1023 );
1024 }
1025
1026 #[tokio::test]
1027 async fn test_exec_process_round_trip() {
1028 let (client, fake, _server, _dir) = spin_up_fake_agent(0).await;
1029
1030 client
1031 .exec_process("ctr-1", "exec-1", &["true".into()])
1032 .await
1033 .unwrap();
1034
1035 let seen = fake.last_exec.lock().clone().unwrap();
1036 assert_eq!(seen.container_id, "ctr-1");
1037 assert_eq!(seen.process.Args, vec!["true".to_owned()]);
1038 }
1039
1040 #[tokio::test]
1041 async fn test_wait_process_returns_real_status() {
1042 let (client, _fake, _server, _dir) = spin_up_fake_agent(7).await;
1043
1044 let status = client.wait_process("ctr-1", "exec-1").await.unwrap();
1045 assert_eq!(status, 7);
1046 }
1047
1048 #[tokio::test]
1049 async fn test_read_stdout_all_drains_until_empty() {
1050 let (client, _fake, _server, _dir) = spin_up_fake_agent(0).await;
1051
1052 let out = client.read_stdout_all("ctr-1", "exec-1").await.unwrap();
1053 assert_eq!(out, b"hello from stdout");
1054 }
1055
1056 #[tokio::test]
1057 async fn test_full_exec_sequence_parses_exit_stdout_stderr() {
1058 let (client, _fake, _server, _dir) = spin_up_fake_agent(42).await;
1059
1060 let (status, stdout, stderr) = client
1061 .exec(
1062 "ctr-1",
1063 &["echo".into(), "hi".into()],
1064 Duration::from_secs(5),
1065 None,
1066 )
1067 .await
1068 .unwrap();
1069
1070 assert_eq!(
1071 status, 42,
1072 "exit code must be the real WaitProcessResponse.status"
1073 );
1074 assert_eq!(stdout, b"hello from stdout");
1075 assert_eq!(stderr, b"");
1076 }
1077
1078 #[tokio::test]
1079 async fn test_exec_times_out() {
1080 // FakeAgent's wait_process deliberately stalls longer than the exec
1081 // timeout, so this deterministically exercises the timeout path
1082 // (rather than racing a near-zero `tokio::time::timeout` against a
1083 // fast local round-trip, which can flake).
1084 let fake = FakeAgent::with_wait_process_delay(0, Duration::from_secs(10));
1085 let (client, _fake, _server, _dir) = spin_up_fake_agent_with(fake).await;
1086
1087 let result = client
1088 .exec("ctr-1", &["echo".into()], Duration::from_millis(50), None)
1089 .await;
1090 assert!(
1091 result.is_err(),
1092 "exec must time out when wait_process stalls"
1093 );
1094 assert!(result.unwrap_err().to_string().contains("timed out"));
1095 }
1096
1097 #[tokio::test]
1098 async fn test_exec_tolerates_already_existing_container() {
1099 // A container that was already created+started by a prior exec_sync
1100 // call must not make a later exec_sync fail — `ensure_container`
1101 // should swallow ALREADY_EXISTS from both CreateContainer and
1102 // StartContainer and proceed straight to ExecProcess.
1103 let fake = FakeAgent::always_already_exists(5);
1104 let (client, _fake, _server, _dir) = spin_up_fake_agent_with(fake).await;
1105
1106 let (status, _stdout, _stderr) = client
1107 .exec("ctr-1", &["true".into()], Duration::from_secs(5), None)
1108 .await
1109 .expect("exec must succeed despite ALREADY_EXISTS on create/start");
1110 assert_eq!(status, 5);
1111 }
1112
1113 #[test]
1114 fn test_is_already_exists_matches_structured_status_code() {
1115 let err = anyhow::Error::new(already_exists_status());
1116 assert!(is_already_exists(&err));
1117 }
1118
1119 #[test]
1120 fn test_is_already_exists_matches_message_fallback() {
1121 let err = anyhow!("container ctr-1 already exists");
1122 assert!(is_already_exists(&err));
1123 }
1124
1125 #[test]
1126 fn test_is_already_exists_false_for_unrelated_error() {
1127 let err = anyhow!("connection refused");
1128 assert!(!is_already_exists(&err));
1129 }
1130}