Skip to main content

hyprstream_service/service/spawner/
standalone.rs

1//! Standalone process spawner using tokio::process::Command.
2//!
3//! Spawns daemon processes directly without systemd integration. Notified
4//! launches verify readiness through a per-child sd_notify endpoint and keep
5//! ownership of every child until termination is confirmed; children are
6//! spawned with `kill_on_drop(false)` ON PURPOSE — adopted daemons are
7//! intended to outlive the launcher.
8//!
9//! Platform support: the credential-authenticated notification receiver
10//! (`SO_PASSCRED`/`SCM_CREDENTIALS`, exact child-PID matching) exists on
11//! Linux/Android only. Other targets refuse a supervised Notify request
12//! before any launch side effect; `Immediate` launches are unchanged
13//! everywhere.
14
15// The credential-authenticated notification receiver is Linux/Android-only
16// (see `ChildNotifySocket`); its imports are gated with it. Everything the
17// Immediate path and the stop/cleanup helpers use stays portable.
18#[cfg(unix)]
19use std::os::fd::FromRawFd;
20#[cfg(any(target_os = "linux", target_os = "android"))]
21use std::os::fd::{AsRawFd, OwnedFd};
22use std::process::Stdio;
23use std::sync::Arc;
24use std::time::{Duration, Instant};
25#[cfg(any(target_os = "linux", target_os = "android"))]
26use std::time::{SystemTime, UNIX_EPOCH};
27
28use dashmap::DashMap;
29use tokio::process::{Child, Command};
30use tokio::sync::Mutex;
31
32use super::{ProcessConfig, ProcessKind, ProcessReadiness, SpawnedProcess, SpawnerBackend};
33use hyprstream_rpc::error::{Result, RpcError};
34
35/// Give adopted daemons independent, pollable standard streams while draining
36/// them in the launcher. PGlite registers stdout with epoll, so `/dev/null`
37/// is not a safe sink on Linux; a pipe preserves pollability and prevents the
38/// launcher's stdout/stderr or SSH channel from being held open by the child.
39fn detached_output_stdio() -> std::io::Result<Stdio> {
40    #[cfg(unix)]
41    {
42        use std::os::fd::RawFd;
43
44        // CLOEXEC keeps the drain reader out of the adopted daemon. Fork a
45        // tiny in-process reader instead of invoking an external `cat`: the
46        // official runtime image contains only `/hyprstream`, and the drain
47        // must continue after this launcher exits.
48        let (reader, writer) = cloexec_pipe()?;
49        let drain_pid = unsafe { nix::libc::fork() };
50        match drain_pid {
51            -1 => {
52                unsafe {
53                    nix::libc::close(reader);
54                    nix::libc::close(writer);
55                }
56                Err(std::io::Error::last_os_error())
57            }
58            0 => {
59                // After fork, use only async-signal-safe libc calls. The
60                // child owns the reader and discards bytes until EOF; it
61                // never inherits the writer, so EOF is well-defined.
62                unsafe {
63                    nix::libc::close(writer);
64                    // This child does not exec, so O_CLOEXEC alone cannot
65                    // shed the launcher's SSH/session and service sockets.
66                    // Move the reader to stdin, close every other inherited
67                    // descriptor (including stdout/stderr), then drain only
68                    // through fd 0. The bounded sweep is portable across the
69                    // Unix targets supported by this crate and avoids relying
70                    // on an external close-range utility.
71                    if nix::libc::dup2(reader, 0) < 0 {
72                        nix::libc::_exit(1);
73                    }
74                    for fd in 1..=65_535 {
75                        nix::libc::close(fd);
76                    }
77                    let mut buffer = [0u8; 8192];
78                    loop {
79                        let read = nix::libc::read(0, buffer.as_mut_ptr().cast(), buffer.len());
80                        if read <= 0 {
81                            break;
82                        }
83                    }
84                    nix::libc::close(0);
85                    nix::libc::_exit(0);
86                }
87            }
88            pid => {
89                unsafe { nix::libc::close(reader) };
90                // Reap the helper while this launcher remains alive. If the
91                // launcher exits first, init adopts and reaps the helper.
92                let _ = std::thread::Builder::new()
93                    .name("hyprstream-daemon-stdio-reaper".to_owned())
94                    .spawn(move || unsafe {
95                        nix::libc::waitpid(pid, std::ptr::null_mut(), 0);
96                    });
97                let writer = unsafe { std::fs::File::from_raw_fd(writer as RawFd) };
98                Ok(Stdio::from(writer))
99            }
100        }
101    }
102    #[cfg(not(unix))]
103    {
104        Ok(Stdio::null())
105    }
106}
107
108/// Create a pipe whose descriptors are excluded from an adopted daemon after
109/// exec. `nix::unistd::pipe2` is unavailable on Apple targets in nix 0.27.1,
110/// so use the POSIX pipe plus `FD_CLOEXEC` fallback there while retaining the
111/// atomic `pipe2(O_CLOEXEC)` path on platforms that provide it.
112#[cfg(unix)]
113fn cloexec_pipe() -> std::io::Result<(std::os::fd::RawFd, std::os::fd::RawFd)> {
114    #[cfg(any(
115        target_os = "android",
116        target_os = "dragonfly",
117        target_os = "emscripten",
118        target_os = "freebsd",
119        target_os = "illumos",
120        target_os = "linux",
121        target_os = "redox",
122        target_os = "netbsd",
123        target_os = "openbsd",
124        target_os = "solaris"
125    ))]
126    {
127        Ok(nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?)
128    }
129
130    #[cfg(not(any(
131        target_os = "android",
132        target_os = "dragonfly",
133        target_os = "emscripten",
134        target_os = "freebsd",
135        target_os = "illumos",
136        target_os = "linux",
137        target_os = "redox",
138        target_os = "netbsd",
139        target_os = "openbsd",
140        target_os = "solaris"
141    )))]
142    {
143        let (reader, writer) = nix::unistd::pipe()?;
144        for fd in [reader, writer] {
145            let flags = unsafe { nix::libc::fcntl(fd, nix::libc::F_GETFD) };
146            if flags == -1
147                || unsafe {
148                    nix::libc::fcntl(fd, nix::libc::F_SETFD, flags | nix::libc::FD_CLOEXEC)
149                } == -1
150            {
151                unsafe {
152                    nix::libc::close(reader);
153                    nix::libc::close(writer);
154                }
155                return Err(std::io::Error::last_os_error());
156            }
157        }
158        Ok((reader, writer))
159    }
160}
161
162fn configure_detached_stdio(cmd: &mut Command) -> std::io::Result<()> {
163    cmd.stdin(Stdio::null())
164        .stdout(detached_output_stdio()?)
165        .stderr(detached_output_stdio()?);
166    Ok(())
167}
168
169/// One per-child sd_notify endpoint in the Linux/Android abstract namespace.
170///
171/// The launcher binds a compact random datagram address, hands it to the child as
172/// `NOTIFY_SOCKET`, and accepts `READY=1` only from the spawned child's PID
173/// (via `SO_PASSCRED` sender credentials). This is local lifecycle
174/// supervision, not a service RPC endpoint — nothing dials it and no service
175/// traffic flows through it.
176///
177/// Abstract addresses have no filesystem inode or mode and may be visible in
178/// `/proc/net/unix`. The random name limits collisions; it is not an access
179/// control or authentication mechanism. Kernel-attested sender credentials
180/// and exact child-PID matching provide the authentication boundary.
181///
182/// Platform support: the credential combination this receiver is built on —
183/// `SO_PASSCRED` plus `SCM_CREDENTIALS` for kernel-attested sender-PID
184/// matching — is Linux/Android-specific in pinned `nix 0.27.1`, so the
185/// authenticated receiver as implemented requires those targets. (The
186/// datagram creation flags themselves are available on several other Unix
187/// targets; the boundary is the credential contract, not the flags.) Other
188/// platforms refuse supervised Notify launches pre-spawn (see
189/// `spawn_notified`) instead of weakening the sender-PID check.
190#[cfg(any(target_os = "linux", target_os = "android"))]
191#[derive(Debug)]
192struct ChildNotifySocket {
193    fd: OwnedFd,
194    endpoint: String,
195}
196
197#[cfg(any(target_os = "linux", target_os = "android"))]
198impl ChildNotifySocket {
199    fn bind(_name: &str) -> Result<Self> {
200        Self::bind_with_name_source(|| format!("hypr-n-{}", uuid::Uuid::new_v4().simple()))
201    }
202
203    fn bind_with_name_source(mut next_name: impl FnMut() -> String) -> Result<Self> {
204        use nix::sys::socket::{
205            bind, setsockopt, socket, sockopt::PassCred, AddressFamily, SockFlag, SockType,
206            UnixAddr,
207        };
208
209        // A new fd and random abstract address are created for each bounded
210        // attempt. EADDRINUSE is the only retryable outcome; every other
211        // setup error fails closed before a child is spawned.
212        for _ in 0..8 {
213            let name = next_name();
214            let address = UnixAddr::new_abstract(name.as_bytes()).map_err(|e| {
215                RpcError::SpawnFailed(format!("notification endpoint name invalid: {e}"))
216            })?;
217            let fd = socket(
218                AddressFamily::Unix,
219                SockType::Datagram,
220                SockFlag::SOCK_NONBLOCK | SockFlag::SOCK_CLOEXEC,
221                None,
222            )
223            .map_err(|e| RpcError::SpawnFailed(format!("notify socket creation failed: {e}")))?;
224            setsockopt(&fd, PassCred, &true)
225                .map_err(|e| RpcError::SpawnFailed(format!("SO_PASSCRED setup failed: {e}")))?;
226            match bind(fd.as_raw_fd(), &address) {
227                Ok(()) => {
228                    return Ok(Self {
229                        fd,
230                        endpoint: format!("@{name}"),
231                    })
232                }
233                Err(nix::errno::Errno::EADDRINUSE) => continue,
234                Err(e) => {
235                    return Err(RpcError::SpawnFailed(format!(
236                        "notification endpoint bind failed: {e}"
237                    )))
238                }
239            }
240        }
241        Err(RpcError::SpawnFailed(
242            "could not bind a unique notification endpoint after 8 attempts".to_owned(),
243        ))
244    }
245
246    fn endpoint(&self) -> &str {
247        &self.endpoint
248    }
249
250    /// Non-blocking poll for a `READY=1` datagram from exactly `child_pid`.
251    ///
252    /// Returns `Ok(true)` when the child itself reported readiness,
253    /// `Ok(false)` when nothing (or a non-child/spoofed datagram) arrived, and
254    /// `Err` on a socket failure. Unrelated senders are ignored, not honored.
255    fn try_recv_ready(&self, child_pid: i32) -> Result<bool> {
256        use nix::cmsg_space;
257        use nix::sys::socket::{recvmsg, ControlMessageOwned, MsgFlags, UnixAddr, UnixCredentials};
258        use std::io::IoSliceMut;
259
260        let mut buffer = [0u8; 128];
261        // Drain a bounded batch of the nonblocking queue in one poll. A
262        // foreign sender must not be able to keep the readiness loop from
263        // returning to its child-exit/deadline checks by keeping the queue
264        // perpetually nonempty. Subsequent polls continue draining, so a
265        // genuine READY behind a finite spoof backlog is still observed.
266        const MAX_DATAGRAMS_PER_POLL: usize = 256;
267        for _ in 0..MAX_DATAGRAMS_PER_POLL {
268            // Scope the recvmsg borrows: `message` holds the iov/cmsg borrows,
269            // so extract what we need before touching the payload buffer again.
270            let (bytes, sender_pid) = {
271                let mut iov = [IoSliceMut::new(&mut buffer)];
272                let mut cmsg = cmsg_space!(UnixCredentials);
273                let message = match recvmsg::<UnixAddr>(
274                    self.fd.as_raw_fd(),
275                    &mut iov,
276                    Some(&mut cmsg),
277                    MsgFlags::empty(),
278                ) {
279                    Ok(message) => message,
280                    Err(nix::errno::Errno::EAGAIN) => return Ok(false),
281                    Err(e) => {
282                        return Err(RpcError::SpawnFailed(format!("notify recv failed: {e}")))
283                    }
284                };
285                let sender_pid = message.cmsgs().find_map(|cmsg| match cmsg {
286                    ControlMessageOwned::ScmCredentials(creds) => Some(creds.pid()),
287                    _ => None,
288                });
289                (message.bytes, sender_pid)
290            };
291
292            if sender_pid != Some(child_pid) {
293                tracing::warn!(
294                    sender = ?sender_pid,
295                    expected = child_pid,
296                    "ignoring readiness datagram from a process other than the spawned child"
297                );
298                continue;
299            }
300            if bytes > 0 && buffer[..bytes] == *b"READY=1" {
301                return Ok(true);
302            }
303        }
304        Ok(false)
305    }
306}
307
308#[cfg(any(target_os = "linux", target_os = "android"))]
309/// What the synchronous startup backstop actually observed and did.
310#[derive(Debug, Clone, Copy, PartialEq, Eq)]
311enum BackstopOutcome {
312    /// An explicit terminal wait status was observed AND the guarded
313    /// artifact was removed (or was already absent).
314    TerminatedAndCleaned,
315    /// A terminal status was observed, but the artifact removal FAILED:
316    /// the artifact remains on disk as evidence and no cleanup success is
317    /// claimed.
318    TerminatedArtifactKept,
319    /// Termination was NOT observed (kill failure, reap error, budget
320    /// exhausted with a nonterminal status, or no usable handle): nothing
321    /// is discarded and no cleanup is claimed.
322    Unconfirmed,
323}
324
325#[cfg(any(target_os = "linux", target_os = "android"))]
326/// Terminal-status classification for backstop outcome purposes: only a
327/// real exit or kill signal establishes disappearance. StillAlive,
328/// stopped, continued, and ptrace states are nonterminal and never
329/// release ownership.
330fn wait_status_is_terminal(status: &nix::sys::wait::WaitStatus) -> bool {
331    matches!(
332        status,
333        nix::sys::wait::WaitStatus::Exited(..) | nix::sys::wait::WaitStatus::Signaled(..)
334    )
335}
336
337#[cfg(any(target_os = "linux", target_os = "android"))]
338/// Startup ownership of a not-yet-adopted child (#1585).
339///
340/// Between `spawn` and readiness the launcher — not the process map — owns
341/// the child. If anything returns early or the future is cancelled while the
342/// guard is still armed, `Drop` performs a synchronous best-effort SIGKILL
343/// with a bounded reap and removes any artifact the guard owns. This is best
344/// effort within bounded budgets: if the OS kill/reap does not complete in
345/// time, the timeout is logged — it is not silently discarded, and no
346/// immortal guarantee is claimed.
347struct StartupChildGuard {
348    child: Option<Child>,
349    name: String,
350    pid_file: Option<std::path::PathBuf>,
351}
352
353#[cfg(any(target_os = "linux", target_os = "android"))]
354impl StartupChildGuard {
355    /// Kill and reap the owned child, honestly reporting every failure.
356    ///
357    /// On partial failure the guard RETAINS ownership (`self.child` stays
358    /// armed): the Drop backstop re-attempts SIGKILL and the bounded reap, so
359    /// a failed cleanup does not silently disarm fallback ownership and
360    /// orphan the child under `kill_on_drop(false)`. The guarded artifact is
361    /// removed only after confirmed termination.
362    async fn kill_and_reap(&mut self) -> std::result::Result<(), String> {
363        let mut failures = Vec::new();
364        if let Some(child) = self.child.as_mut() {
365            if let Err(e) = child.start_kill() {
366                failures.push(format!("kill failed: {e}"));
367            }
368            match tokio::time::timeout(Duration::from_secs(5), child.wait()).await {
369                Ok(Ok(_)) => {
370                    // Reaped: ownership transfer complete.
371                    self.child = None;
372                }
373                Ok(Err(e)) => failures.push(format!("reap failed: {e}")),
374                Err(_) => failures.push("reap timed out; ownership retained".to_owned()),
375            }
376        }
377        if self.child.is_none() {
378            if let Some(path) = self.pid_file.take() {
379                if let Err(e) = std::fs::remove_file(&path) {
380                    if e.kind() != std::io::ErrorKind::NotFound {
381                        failures.push(format!("pid file removal failed: {e}"));
382                    }
383                }
384            }
385        }
386        if failures.is_empty() {
387            Ok(())
388        } else {
389            Err(failures.join("; "))
390        }
391    }
392
393    /// Synchronous cancellation backstop (the async reap is unavailable in
394    /// `Drop`): SIGKILL the owned child and reap it within a bounded WNOHANG
395    /// budget. Returns the observed [`BackstopOutcome`]; ownership is
396    /// released only when an explicit TERMINAL wait status (real exit or
397    /// kill signal — or the already-reaped `ECHILD`) is observed, and the
398    /// guarded artifact is removed only when that removal actually
399    /// succeeds. Every nonterminal wait state (still-alive, stopped,
400    /// continued, ptrace) stays inside the same bounded budget and never
401    /// establishes termination. Bounded best effort, not a guarantee.
402    fn backstop_cleanup(&mut self) -> BackstopOutcome {
403        let mut terminated = false;
404        if let Some(child) = self.child.as_ref() {
405            if let Some(pid) = child.id() {
406                let raw = nix::unistd::Pid::from_raw(pid as i32);
407                match nix::sys::signal::kill(raw, nix::sys::signal::Signal::SIGKILL) {
408                    Err(nix::errno::Errno::ESRCH) => {
409                        // The process is already gone; nothing to signal.
410                        terminated = true;
411                    }
412                    Err(e) => {
413                        tracing::warn!(
414                            pid = %pid,
415                            error = %e,
416                            "startup backstop SIGKILL failed; termination not observed"
417                        );
418                    }
419                    Ok(()) => {
420                        let deadline = Instant::now() + Duration::from_secs(1);
421                        loop {
422                            match nix::sys::wait::waitpid(
423                                raw,
424                                Some(nix::sys::wait::WaitPidFlag::WNOHANG),
425                            ) {
426                                // Only an explicit terminal status — real
427                                // exit or kill signal — establishes the reap.
428                                Ok(status) if wait_status_is_terminal(&status) => {
429                                    terminated = true;
430                                    break;
431                                }
432                                // StillAlive AND nonterminal stopped/
433                                // continued/ptrace states: NOT termination;
434                                // keep waiting inside the same budget.
435                                Ok(_) => {
436                                    if Instant::now() >= deadline {
437                                        tracing::warn!(
438                                            pid = %pid,
439                                            "reap backstop budget exhausted with a nonterminal status; child may linger"
440                                        );
441                                        break;
442                                    }
443                                    std::thread::sleep(Duration::from_millis(5));
444                                }
445                                Err(nix::errno::Errno::ECHILD) => {
446                                    // No longer our child: already reaped.
447                                    terminated = true;
448                                    break;
449                                }
450                                Err(e) => {
451                                    tracing::warn!(
452                                        pid = %pid,
453                                        error = %e,
454                                        "startup backstop reap failed; termination not observed"
455                                    );
456                                    break;
457                                }
458                            }
459                        }
460                    }
461                }
462            }
463        }
464        if !terminated {
465            return BackstopOutcome::Unconfirmed;
466        }
467        self.child = None;
468        let Some(path) = self.pid_file.take() else {
469            return BackstopOutcome::TerminatedAndCleaned;
470        };
471        match std::fs::remove_file(&path) {
472            // Removed — or already absent: the artifact is gone either way.
473            Ok(()) => BackstopOutcome::TerminatedAndCleaned,
474            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
475                BackstopOutcome::TerminatedAndCleaned
476            }
477            Err(e) => {
478                tracing::warn!(
479                    name = %self.name,
480                    path = %path.display(),
481                    error = %e,
482                    "startup backstop could not remove the guarded artifact; retained on disk as evidence"
483                );
484                self.pid_file = Some(path);
485                BackstopOutcome::TerminatedArtifactKept
486            }
487        }
488    }
489}
490
491#[cfg(any(target_os = "linux", target_os = "android"))]
492impl Drop for StartupChildGuard {
493    fn drop(&mut self) {
494        if self.child.is_some() {
495            // Bounded best effort: each outcome is reported exactly as
496            // observed. The fields drop with this struct — no handle or
497            // cleanup survives an unsuccessful backstop.
498            match self.backstop_cleanup() {
499                BackstopOutcome::TerminatedAndCleaned => {
500                    tracing::warn!(
501                        name = %self.name,
502                        "startup guard dropped with the child still owned; terminated, reaped, and cleaned up"
503                    );
504                }
505                BackstopOutcome::TerminatedArtifactKept => {
506                    tracing::warn!(
507                        name = %self.name,
508                        "startup guard dropped: child terminated but artifact removal failed; artifact retained on disk as evidence"
509                    );
510                }
511                BackstopOutcome::Unconfirmed => {
512                    tracing::warn!(
513                        name = %self.name,
514                        "startup guard dropped without confirmed termination; owned artifact retained on disk as evidence; no cleanup guarantee beyond Drop"
515                    );
516                }
517            }
518        }
519    }
520}
521
522/// Standalone process spawner backend.
523///
524/// Spawns processes directly using `tokio::process::Command`.
525/// Tracks spawned processes and provides cleanup on stop.
526pub struct StandaloneBackend {
527    /// Active processes (id -> Child handle).
528    processes: DashMap<String, Arc<Mutex<Child>>>,
529}
530
531impl StandaloneBackend {
532    /// Create a new standalone backend.
533    pub fn new() -> Self {
534        Self {
535            processes: DashMap::new(),
536        }
537    }
538}
539
540impl Default for StandaloneBackend {
541    fn default() -> Self {
542        Self::new()
543    }
544}
545
546impl StandaloneBackend {
547    /// Legacy path: report success as soon as the child is spawned.
548    ///
549    /// Documented limitations (pre-existing at HEAD #1585; kept for
550    /// compatibility, NOT the required-native contract): readiness is not
551    /// verified in any way; PID publication is a direct, non-atomic write
552    /// whose failure is logged rather than rolled back; and a pre-existing
553    /// PID artifact is overwritten without a live-duplicate check, so a
554    /// recycled PID in a stale artifact can misdirect later PID-based
555    /// cleanup. The Required notified path (`spawn_notified`) is where the
556    /// checked-publication/rollback contract is enforced.
557    async fn spawn_immediate(&self, config: ProcessConfig) -> Result<SpawnedProcess> {
558        tracing::debug!(
559            name = %config.name,
560            executable = %config.executable.display(),
561            "Spawning daemon via direct process"
562        );
563
564        // Build the command
565        let mut cmd = Command::new(&config.executable);
566
567        // Add arguments
568        cmd.args(&config.args);
569
570        // Set working directory
571        if let Some(ref dir) = config.working_dir {
572            cmd.current_dir(dir);
573        }
574
575        // Set environment variables
576        for (key, value) in &config.env {
577            cmd.env(key, value);
578        }
579
580        // Disable kill on drop - daemon processes should outlive the spawner
581        configure_detached_stdio(&mut cmd).map_err(|e| {
582            RpcError::SpawnFailed(format!(
583                "failed to detach {} standard streams: {e}",
584                config.name
585            ))
586        })?;
587        cmd.kill_on_drop(false);
588
589        // Spawn the process
590        let child = cmd.spawn().map_err(|e| {
591            RpcError::SpawnFailed(format!("failed to spawn {}: {}", config.name, e))
592        })?;
593
594        // Get the PID
595        let pid = child.id().ok_or_else(|| {
596            RpcError::SpawnFailed(format!("spawned {} but no PID available", config.name))
597        })?;
598
599        // Generate unique ID
600        let id = format!("{}-{}", config.name, pid);
601
602        // Store the child handle
603        self.processes
604            .insert(id.clone(), Arc::new(Mutex::new(child)));
605
606        // Write PID file for daemon tracking
607        let pid_file = hyprstream_rpc::paths::service_pid_file(&config.name);
608        if let Some(parent) = pid_file.parent() {
609            let _ = std::fs::create_dir_all(parent);
610        }
611        if let Err(e) = std::fs::write(&pid_file, pid.to_string()) {
612            tracing::warn!("Failed to write PID file {:?}: {}", pid_file, e);
613        }
614
615        tracing::info!(
616            name = %config.name,
617            pid = %pid,
618            id = %id,
619            "Daemon spawned successfully"
620        );
621
622        Ok(SpawnedProcess::new(id, ProcessKind::Direct(pid)).with_pid_file(pid_file))
623    }
624
625    /// Notify-supervised path (#1585): the spawn reports success only after
626    /// the child's own `READY=1` datagram arrives from the child's PID; a
627    /// child exit or the hard timeout fails the spawn, the child is
628    /// terminated/reaped, and no PID artifact or live abstract notification
629    /// endpoint survives.
630    ///
631    /// Platform support: the credential-authenticated receiver
632    /// (`SO_PASSCRED`/`SCM_CREDENTIALS`, exact child-PID matching) exists on
633    /// Linux/Android only. On any other target this request fails closed
634    /// BEFORE spawning the child or binding a notification endpoint, creating
635    /// a PID artifact, or causing any other launch side effect — there is no
636    /// fallback to
637    /// `Immediate` readiness and no unauthenticated receiver.
638    #[cfg(any(target_os = "linux", target_os = "android"))]
639    async fn spawn_notified(
640        &self,
641        config: ProcessConfig,
642        timeout: Duration,
643    ) -> Result<SpawnedProcess> {
644        tracing::debug!(
645            name = %config.name,
646            executable = %config.executable.display(),
647            timeout_ms = %timeout.as_millis(),
648            "Spawning daemon via direct process with notification readiness"
649        );
650
651        // Serialize same-service launches for the WHOLE launch — duplicate
652        // precheck, publication, and any rollback run under this lock; it is
653        // acquired first so it is dropped last, after every early-return
654        // cleanup path. A held lock deterministically reports the pending
655        // duplicate.
656        let _launch_lock = ServiceLaunchLock::acquire(&config.name)?;
657        let pid_file = hyprstream_rpc::paths::service_pid_file(&config.name);
658
659        // Duplicate precheck BEFORE any side effect of this launch: no
660        // notification endpoint is created and no child is spawned for a
661        // refused launch. Only a confirmed-missing or confirmed-dead
662        // predecessor may be replaced; ambiguous artifacts fail closed.
663        match inspect_pid_file(&pid_file) {
664            PidWitness::Absent => {}
665            PidWitness::Dead(stale) => {
666                tracing::info!(
667                    name = %config.name,
668                    stale_pid = %stale,
669                    "replacing PID artifact of a confirmed-dead predecessor"
670                );
671            }
672            PidWitness::Live(existing) => {
673                return Err(RpcError::SpawnFailed(format!(
674                    "service {} launch refused: service already appears live as pid {existing}",
675                    config.name
676                )));
677            }
678            PidWitness::Ambiguous(why) => {
679                return Err(RpcError::SpawnFailed(format!(
680                    "service {} launch refused: existing PID artifact is ambiguous ({why}); refusing to replace it",
681                    config.name
682                )));
683            }
684        }
685
686        let notify = ChildNotifySocket::bind(&config.name)?;
687
688        let mut cmd = Command::new(&config.executable);
689        cmd.args(&config.args);
690        if let Some(ref dir) = config.working_dir {
691            cmd.current_dir(dir);
692        }
693        for (key, value) in &config.env {
694            cmd.env(key, value);
695        }
696        // The child learns its notification endpoint only through its own
697        // environment; the launcher's environment is untouched.
698        cmd.env("NOTIFY_SOCKET", notify.endpoint());
699        configure_detached_stdio(&mut cmd).map_err(|e| {
700            RpcError::SpawnFailed(format!(
701                "failed to detach {} standard streams: {e}",
702                config.name
703            ))
704        })?;
705        cmd.kill_on_drop(false);
706
707        let child = cmd.spawn().map_err(|e| {
708            RpcError::SpawnFailed(format!("failed to spawn {}: {}", config.name, e))
709        })?;
710        // Take ownership BEFORE any fallible branch: if the handle yields no
711        // PID, the guard must still own and reap the child (`kill_on_drop`
712        // is false by design — adopted daemons must outlive the launcher).
713        let mut guard = StartupChildGuard {
714            child: Some(child),
715            name: config.name.clone(),
716            pid_file: None,
717        };
718        // Capture the PID before any await: it must remain identifiable in
719        // every error raised after this point.
720        let Some(pid) = guard.child.as_ref().and_then(tokio::process::Child::id) else {
721            let cleanup = guard.kill_and_reap().await;
722            return Err(RpcError::SpawnFailed(format!(
723                "service {} produced no PID{}",
724                config.name,
725                cleanup_suffix(cleanup),
726            )));
727        };
728
729        // Readiness outcome; classified from the child's actual behavior, not
730        // from whether its PID is still observable afterwards.
731        let outcome = match guard.child.as_mut() {
732            Some(owned) => self.await_child_readiness(owned, &notify, timeout).await,
733            None => Ok(()),
734        };
735
736        match outcome {
737            Ok(()) => {
738                // READY arrived; reject a child that exited immediately after.
739                let Some(owned) = guard.child.as_mut() else {
740                    // Unreachable while the guard owns the child until
741                    // adoption; if it ever occurs, fail closed with rollback.
742                    let cleanup = guard.kill_and_reap().await;
743                    return Err(RpcError::SpawnFailed(format!(
744                        "service {} (pid {}) lost child ownership before the readiness recheck{}",
745                        config.name,
746                        pid,
747                        cleanup_suffix(cleanup),
748                    )));
749                };
750                match owned.try_wait() {
751                    Ok(None) => {}
752                    Ok(Some(status)) => {
753                        let cleanup = guard.kill_and_reap().await;
754                        return Err(RpcError::SpawnFailed(format!(
755                            "service {} (pid {}) exited right after reporting readiness: {status}{}",
756                            config.name,
757                            pid,
758                            cleanup_suffix(cleanup),
759                        )));
760                    }
761                    Err(e) => {
762                        let cleanup = guard.kill_and_reap().await;
763                        return Err(RpcError::SpawnFailed(format!(
764                            "service {} (pid {}) readiness recheck failed: {e}{}",
765                            config.name,
766                            pid,
767                            cleanup_suffix(cleanup),
768                        )));
769                    }
770                }
771            }
772            Err(reason) => {
773                // Terminate and reap; the PID file is never published for a
774                // child that never became ready.
775                let cleanup = guard.kill_and_reap().await;
776                return Err(RpcError::SpawnFailed(format!(
777                    "service {} (pid {}) did not become ready: {reason}{}",
778                    config.name,
779                    pid,
780                    cleanup_suffix(cleanup),
781                )));
782            }
783        }
784
785        // Atomic PID publication while the startup guard still owns the child:
786        // any failure rolls the launch back (child killed/reaped, artifact
787        // removed) instead of reporting success with untracked state. The
788        // per-service launch lock is held for this whole section, so the
789        // precheck and the publication are one deterministic unit.
790        if let Some(parent) = pid_file.parent() {
791            if let Err(e) = std::fs::create_dir_all(parent) {
792                let cleanup = guard.kill_and_reap().await;
793                return Err(RpcError::SpawnFailed(format!(
794                    "service {} (pid {}) runtime dir creation failed: {e}{}",
795                    config.name,
796                    pid,
797                    cleanup_suffix(cleanup),
798                )));
799            }
800        }
801        // Exclusive per-attempt temp artifact: `create_new` never clobbers a
802        // pre-existing file, the PID bytes are written through the RETAINED
803        // created handle (the exclusive inode ownership established at
804        // creation is never traded for a reopened pathname), and the guard
805        // is armed only for the file THIS attempt actually created.
806        let temp_pid_file = {
807            let mut created = None;
808            for _ in 0..8 {
809                let nanos = SystemTime::now()
810                    .duration_since(UNIX_EPOCH)
811                    .map(|d| d.as_nanos())
812                    .unwrap_or_default();
813                let candidate = pid_file.with_extension(format!("pid.{nanos}"));
814                match std::fs::OpenOptions::new()
815                    .write(true)
816                    .create_new(true)
817                    .open(&candidate)
818                {
819                    Ok(file) => {
820                        created = Some((candidate, file));
821                        break;
822                    }
823                    Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
824                    Err(e) => {
825                        let cleanup = guard.kill_and_reap().await;
826                        return Err(RpcError::SpawnFailed(format!(
827                            "service {} (pid {}) PID publication failed: {e}{}",
828                            config.name,
829                            pid,
830                            cleanup_suffix(cleanup),
831                        )));
832                    }
833                }
834            }
835            let Some((candidate, mut file)) = created else {
836                let cleanup = guard.kill_and_reap().await;
837                return Err(RpcError::SpawnFailed(format!(
838                    "service {} (pid {}) PID publication failed: no unique temp artifact name{}",
839                    config.name,
840                    pid,
841                    cleanup_suffix(cleanup),
842                )));
843            };
844            // The guard owns this attempt's artifact from the moment of its
845            // exclusive creation: any rollback removes it together with the
846            // confirmed reap.
847            guard.pid_file = Some(candidate.clone());
848            use std::io::Write as _;
849            if let Err(e) = file.write_all(pid.to_string().as_bytes()) {
850                let cleanup = guard.kill_and_reap().await;
851                return Err(RpcError::SpawnFailed(format!(
852                    "service {} (pid {}) PID publication failed: {e}{}",
853                    config.name,
854                    pid,
855                    cleanup_suffix(cleanup),
856                )));
857            }
858            candidate
859        };
860        if let Err(e) = std::fs::rename(&temp_pid_file, &pid_file) {
861            let cleanup = guard.kill_and_reap().await;
862            return Err(RpcError::SpawnFailed(format!(
863                "service {} (pid {}) PID publication failed: {e}{}",
864                config.name,
865                pid,
866                cleanup_suffix(cleanup),
867            )));
868        }
869
870        // Generate unique ID
871        let id = format!("{}-{}", config.name, pid);
872
873        // Transfer ownership: the process map adopts the live child and the
874        // PID file becomes `stop`'s cleanup responsibility.
875        let Some(owned) = guard.child.take() else {
876            // Unreachable while the guard owns the child until adoption. The
877            // handle is gone, so there is nothing to kill or reap; this
878            // attempt's temp artifact is removed rather than leaked.
879            if let Some(path) = guard.pid_file.take() {
880                let _ = std::fs::remove_file(&path);
881            }
882            return Err(RpcError::SpawnFailed(format!(
883                "service {} (pid {}) lost child ownership before adoption",
884                config.name, pid,
885            )));
886        };
887        guard.pid_file = None;
888        self.processes
889            .insert(id.clone(), Arc::new(Mutex::new(owned)));
890
891        tracing::info!(
892            name = %config.name,
893            pid = %pid,
894            id = %id,
895            "Daemon spawned successfully (notification readiness satisfied)"
896        );
897
898        // ChildNotifySocket::drop closes the abstract endpoint.
899        Ok(SpawnedProcess::new(id, ProcessKind::Direct(pid)).with_pid_file(pid_file))
900    }
901
902    /// Non-Linux arm of the Notify-supervised path: an explicit pre-spawn
903    /// refusal (see the Linux/Android arm's contract above). Nothing is
904    /// spawned and no notification endpoint, child guard, or PID artifact is
905    /// created; there is deliberately no fallback to `Immediate` readiness,
906    /// because a supervised launch whose sender cannot be kernel-attested
907    /// must not report success from an unauthenticated payload.
908    #[cfg(not(any(target_os = "linux", target_os = "android")))]
909    async fn spawn_notified(
910        &self,
911        _config: ProcessConfig,
912        _timeout: Duration,
913    ) -> Result<SpawnedProcess> {
914        Err(RpcError::SpawnFailed(
915            "authenticated notification readiness (SO_PASSCRED/SCM_CREDENTIALS) is \
916             supported on Linux/Android only; a supervised Notify launch is refused \
917             on this platform"
918                .to_owned(),
919        ))
920    }
921
922    /// Poll the notification endpoint against child exit and the hard timeout.
923    #[cfg(any(target_os = "linux", target_os = "android"))]
924    async fn await_child_readiness(
925        &self,
926        child: &mut Child,
927        notify: &ChildNotifySocket,
928        timeout: Duration,
929    ) -> std::result::Result<(), String> {
930        let deadline = Instant::now() + timeout;
931        let mut poll = tokio::time::interval(Duration::from_millis(20));
932        poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
933        // The first tick completes immediately; skip it so the first real
934        // poll happens one interval in.
935        poll.tick().await;
936
937        loop {
938            match notify.try_recv_ready(child.id().map(|p| p as i32).unwrap_or(-1)) {
939                Ok(true) => return Ok(()),
940                Ok(false) => {}
941                Err(e) => return Err(e.to_string()),
942            }
943            match child.try_wait() {
944                Ok(Some(status)) => {
945                    return Err(format!("exited during startup with {status}"));
946                }
947                Ok(None) => {}
948                Err(e) => return Err(format!("exit observation failed: {e}")),
949            }
950            if Instant::now() >= deadline {
951                return Err(format!(
952                    "no READY=1 within {}s (notification endpoint {})",
953                    timeout.as_secs(),
954                    notify.endpoint()
955                ));
956            }
957            let wait = poll.tick();
958            tokio::select! {
959                _ = wait => {}
960                status = child.wait() => {
961                    // Exit while sleeping between polls is still a startup
962                    // failure with the real exit status.
963                    return match status {
964                        Ok(status) => Err(format!("exited during startup with {status}")),
965                        Err(e) => Err(format!("exit observation failed: {e}")),
966                    };
967                }
968            }
969        }
970    }
971}
972
973/// A self-exited child stays visible to `kill(pid, 0)` as a zombie until its
974/// supervisor reaps it; a zombie is not a running service. Best-effort procfs
975/// inspection — if the state cannot be read the process is conservatively
976/// treated as alive (previous `kill(pid, 0)` semantics).
977#[cfg(any(target_os = "linux", target_os = "android"))]
978fn process_is_zombie(pid: i32) -> bool {
979    let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else {
980        return false;
981    };
982    // The comm field is parenthesized and may itself contain parentheses or
983    // spaces; the single-character state code follows the final ')'.
984    let Some(after_comm) = stat.rsplit_once(')').map(|(_, rest)| rest.trim_start()) else {
985        return false;
986    };
987    after_comm.starts_with('Z')
988}
989
990#[cfg(not(any(target_os = "linux", target_os = "android")))]
991fn process_is_zombie(_pid: i32) -> bool {
992    false
993}
994
995/// Format an optional cleanup failure onto a startup error: a cleanup that
996/// itself failed is part of the failure report, never discarded.
997/// Rollback-failure suffix for spawn error messages (used by the
998/// Linux/Android supervised Notify arm only).
999#[cfg(any(target_os = "linux", target_os = "android"))]
1000fn cleanup_suffix(cleanup: std::result::Result<(), String>) -> String {
1001    match cleanup {
1002        Ok(()) => String::new(),
1003        Err(failure) => format!("; cleanup: {failure}"),
1004    }
1005}
1006
1007/// Outcome of inspecting an existing PID artifact before a launch. Anything
1008/// other than a confirmed-missing or confirmed-dead predecessor is ambiguous
1009/// and must fail closed: only a verified missing/dead witness may be replaced.
1010#[cfg(any(target_os = "linux", target_os = "android"))]
1011enum PidWitness {
1012    /// No artifact: nothing to refuse, nothing to preserve.
1013    Absent,
1014    /// Parseable PID naming a non-live process: a stale witness, replaceable.
1015    Dead(u32),
1016    /// Parseable PID naming a live process: a duplicate launch must be
1017    /// refused and this witness preserved.
1018    Live(u32),
1019    /// Unreadable or unparseable: the artifact cannot vouch either way.
1020    Ambiguous(String),
1021}
1022
1023/// Inspect an existing PID artifact into a [`PidWitness`] (Linux/Android;
1024/// used by the supervised Notify precheck).
1025#[cfg(any(target_os = "linux", target_os = "android"))]
1026fn inspect_pid_file(path: &std::path::Path) -> PidWitness {
1027    match std::fs::read_to_string(path) {
1028        Err(e) if e.kind() == std::io::ErrorKind::NotFound => PidWitness::Absent,
1029        Err(e) => PidWitness::Ambiguous(format!("unreadable: {e}")),
1030        Ok(content) => match content.trim().parse::<u32>() {
1031            Err(_) => PidWitness::Ambiguous("unparseable pid".to_owned()),
1032            Ok(pid) if pid_is_live(pid) => PidWitness::Live(pid),
1033            Ok(pid) => PidWitness::Dead(pid),
1034        },
1035    }
1036}
1037
1038/// Liveness of a PID witness: present (signalable — or present but
1039/// unsignalable, which is conservatively live), and NOT a zombie — a
1040/// self-exited process awaiting reap is a stale witness.
1041fn pid_is_live(pid: u32) -> bool {
1042    let Some(pid) = i32::try_from(pid).ok().filter(|&p| p > 0) else {
1043        return false;
1044    };
1045    matches!(
1046        nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), None),
1047        Ok(()) | Err(nix::errno::Errno::EPERM)
1048    ) && !process_is_zombie(pid)
1049}
1050
1051/// Terminate an untracked daemon handle by PID: SIGTERM with a bounded wait,
1052/// then SIGKILL with a bounded POST-SIGKILL confirmation. Returns whether
1053/// death was OBSERVED (ESRCH) — callers touch artifacts only on `true`.
1054///
1055/// Known pre-existing limitation (documented, not redesigned in #1585): on
1056/// an untracked handle the PID is a bare witness, so a recycled PID after
1057/// the target's exit can in principle be signaled; tracked children are
1058/// observed through their real handle and have no such exposure.
1059async fn stop_untracked_by_pid(pid: u32, failures: &mut Vec<String>) -> bool {
1060    let Some(pid_i32) = i32::try_from(pid).ok().filter(|&p| p > 0) else {
1061        failures.push("invalid PID".to_owned());
1062        return false;
1063    };
1064    let raw = nix::unistd::Pid::from_raw(pid_i32);
1065    match nix::sys::signal::kill(raw, nix::sys::signal::Signal::SIGTERM) {
1066        Err(nix::errno::Errno::ESRCH) => return true, // already gone
1067        Err(e) => {
1068            failures.push(format!("SIGTERM failed: {e}"));
1069            return false;
1070        }
1071        Ok(()) => {}
1072    }
1073    let deadline = Instant::now() + Duration::from_secs(5);
1074    loop {
1075        match nix::sys::signal::kill(raw, None) {
1076            Err(nix::errno::Errno::ESRCH) => return true,
1077            Err(e) => {
1078                failures.push(format!("liveness check failed: {e}"));
1079                return false;
1080            }
1081            Ok(()) if Instant::now() >= deadline => break, // escalate
1082            Ok(()) => tokio::time::sleep(Duration::from_millis(50)).await,
1083        }
1084    }
1085    match nix::sys::signal::kill(raw, nix::sys::signal::Signal::SIGKILL) {
1086        Err(nix::errno::Errno::ESRCH) => return true,
1087        Err(e) => {
1088            failures.push(format!("SIGKILL failed: {e}"));
1089            return false;
1090        }
1091        Ok(()) => {}
1092    }
1093    // SIGKILL delivery is fast but not instantaneous: death must be
1094    // observed, not assumed.
1095    let kill_deadline = Instant::now() + Duration::from_secs(2);
1096    loop {
1097        match nix::sys::signal::kill(raw, None) {
1098            Err(nix::errno::Errno::ESRCH) => return true,
1099            Err(e) => {
1100                failures.push(format!("post-SIGKILL liveness check failed: {e}"));
1101                return false;
1102            }
1103            Ok(()) if Instant::now() >= kill_deadline => {
1104                failures.push("termination not confirmed after SIGKILL".to_owned());
1105                return false;
1106            }
1107            Ok(()) => tokio::time::sleep(Duration::from_millis(20)).await,
1108        }
1109    }
1110}
1111
1112impl StandaloneBackend {
1113    /// Synchronous bounded stop for a tracked direct child, run from the
1114    /// launch-transaction guard's `Drop` when the orchestration future is
1115    /// cancelled (#1585).
1116    ///
1117    /// This preserves the backend's STRONGER tracked ownership instead of
1118    /// downgrading adopted children to a blind by-PID signal: when the child
1119    /// is still in the tracking map, termination goes through the retained
1120    /// `Child` handle itself (`start_kill` + bounded `try_wait` reaping — the
1121    /// sync counterpart of the tracked branch of [`Self::stop`], same 5s
1122    /// budget), and the map entry is surrendered only on a CONFIRMED reap. A
1123    /// child no longer in the map is either already stopped by an earlier
1124    /// confirmed stop or was never this backend's: for a live untracked PID
1125    /// this operation deliberately REFUSES to signal (a recycled PID could be
1126    /// an unrelated process) and fails honestly instead; an untracked PID
1127    /// already gone (`kill(pid, None)` → ESRCH — a liveness probe, never a
1128    /// signal) is cleaned up. PID-artifact removal keeps the established
1129    /// conditional semantics — the artifact is removed only while it still
1130    /// names this child, a newer child's artifact is preserved, and an actual
1131    /// IO failure is an error, not success. Best effort within bounded
1132    /// budgets, not a guarantee.
1133    pub fn stop_tracked_child_by_handle_sync(&self, process: &SpawnedProcess) -> Result<()> {
1134        if !process.is_direct() {
1135            return Err(RpcError::InvalidOperation(
1136                "synchronous cancellation stop requires a direct (PID-tracked) process".to_owned(),
1137            ));
1138        }
1139        let pid = process.pid().unwrap_or_default();
1140
1141        // Tracked branch: the retained Child IS the ownership — no PID
1142        // signaling at all.
1143        let child_arc = self
1144            .processes
1145            .get(&process.id)
1146            .map(|entry| Arc::clone(entry.value()));
1147        if let Some(child_arc) = child_arc {
1148            let mut child = match child_arc.try_lock() {
1149                Ok(child) => child,
1150                Err(_) => {
1151                    return Err(RpcError::SpawnFailed(
1152                        "cancellation cleanup skipped: child handle lock contended; \
1153                         child remains tracked"
1154                            .to_owned(),
1155                    ));
1156                }
1157            };
1158            let mut failures: Vec<String> = Vec::new();
1159            if let Err(e) = child.start_kill() {
1160                failures.push(format!("kill failed: {e}"));
1161            }
1162            // Bounded synchronous reap (mirror of the tracked `stop` branch).
1163            let deadline = Instant::now() + Duration::from_secs(5);
1164            let mut confirmed = false;
1165            loop {
1166                match child.try_wait() {
1167                    Ok(Some(_)) => {
1168                        confirmed = true;
1169                        // Reaped: only now does the backend stop tracking.
1170                        self.processes.remove(&process.id);
1171                        break;
1172                    }
1173                    Ok(None) => {}
1174                    Err(e) => {
1175                        failures.push(format!("reap failed: {e}"));
1176                        break;
1177                    }
1178                }
1179                if Instant::now() >= deadline {
1180                    failures.push("reap timed out; child remains tracked".to_owned());
1181                    break;
1182                }
1183                std::thread::sleep(Duration::from_millis(20));
1184            }
1185            if !confirmed {
1186                failures.push("termination unconfirmed".to_owned());
1187                // Unconfirmed: the PID artifact still names a possibly-live
1188                // process and cleanup must not claim success.
1189                return Err(RpcError::SpawnFailed(format!(
1190                    "synchronous cancellation stop of {} did not fully succeed: {}",
1191                    process.id,
1192                    failures.join("; ")
1193                )));
1194            }
1195            return finish_pid_artifact_sync(process, pid, &mut failures);
1196        }
1197
1198        // Untracked branch: never signal a PID we do not track. Probe-only
1199        // liveness (kill(pid, None) is a probe, not a signal): gone → clean
1200        // the artifact; alive → fail honestly rather than risk a recycled
1201        // PID (the documented untracked exposure is not widened here).
1202        let raw = nix::unistd::Pid::from_raw(i32::try_from(pid).unwrap_or_default());
1203        match nix::sys::signal::kill(raw, None) {
1204            Err(nix::errno::Errno::ESRCH) => {
1205                let mut failures: Vec<String> = Vec::new();
1206                finish_pid_artifact_sync(process, pid, &mut failures)
1207            }
1208            Err(e) => Err(RpcError::SpawnFailed(format!(
1209                "cancellation liveness probe of untracked {} failed: {e}",
1210                process.id
1211            ))),
1212            Ok(()) => Err(RpcError::SpawnFailed(format!(
1213                "cancellation cleanup declined to signal untracked live PID {pid} ({})",
1214                process.id
1215            ))),
1216        }
1217    }
1218}
1219
1220/// Conditional PID-artifact removal after a CONFIRMED synchronous stop: the
1221/// artifact is removed only while it still names `pid`; a file naming a
1222/// different (newer) child is intentionally preserved and is NOT a failure;
1223/// an actual IO failure IS a failure — cleanup never claims clean success it
1224/// did not achieve (#1585).
1225fn finish_pid_artifact_sync(
1226    process: &SpawnedProcess,
1227    pid: u32,
1228    failures: &mut Vec<String>,
1229) -> Result<()> {
1230    let Some(pid_file) = &process.pid_file else {
1231        return Ok(());
1232    };
1233    match std::fs::read_to_string(pid_file) {
1234        Ok(content) => {
1235            if content.trim() == pid.to_string() {
1236                match std::fs::remove_file(pid_file) {
1237                    Ok(()) => Ok(()),
1238                    Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
1239                    Err(e) => {
1240                        failures.push(format!("PID file removal failed: {e}"));
1241                        Err(RpcError::SpawnFailed(format!(
1242                            "synchronous stop of {} confirmed termination but the PID \
1243                             artifact cleanup failed: {}",
1244                            process.id,
1245                            failures.join("; ")
1246                        )))
1247                    }
1248                }
1249            } else {
1250                // The artifact names a different (newer) child: preserving it
1251                // is correct, not a failure.
1252                Ok(())
1253            }
1254        }
1255        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
1256        Err(e) => Err(RpcError::SpawnFailed(format!(
1257            "synchronous stop of {} confirmed termination but the PID artifact could \
1258             not be read for conditional removal: {e}",
1259            process.id
1260        ))),
1261    }
1262}
1263
1264#[cfg(any(target_os = "linux", target_os = "android"))]
1265/// Per-service nonblocking advisory lock for notified launches (#1585).
1266///
1267/// The lock file has a STABLE path (and thus stable inode) and is NEVER
1268/// unlinked — unlinking would break flock identity between concurrent
1269/// acquirers. Holding it makes the duplicate precheck, the PID publication,
1270/// and any rollback of one launch a deterministic unit against other launches
1271/// of the same service. Release is an EXPLICIT `FlockArg::Unlock` in `Drop`
1272/// followed by normal close: `flock` lives on the open file description, so
1273/// close alone can be prolonged by a concurrently forked process's transient
1274/// duplicate of that description (pre-exec), which would transiently and
1275/// falsely refuse the next same-service launch; explicit unlock releases the
1276/// lock immediately regardless of such copies. A held lock is reported as a
1277/// pending duplicate, deterministically. Scope is honest and narrow: it
1278/// coordinates only cooperating notified launches using this facility and
1279/// makes no claim of control over noncooperating processes; the artifact
1280/// lives in the protected runtime directory.
1281struct ServiceLaunchLock {
1282    _file: std::fs::File,
1283}
1284
1285#[cfg(any(target_os = "linux", target_os = "android"))]
1286impl Drop for ServiceLaunchLock {
1287    fn drop(&mut self) {
1288        use nix::fcntl::{flock, FlockArg};
1289        use std::os::fd::AsRawFd;
1290
1291        // Explicit unlock BEFORE close (see the type docs): close alone
1292        // waits for every duplicate of this open file description to close,
1293        // which a concurrently forked child's pre-exec copy can delay
1294        // indefinitely. An unlock failure is logged honestly — the
1295        // subsequent close remains the fallback and the next acquirer may
1296        // then see a transient in-progress refusal (fail-closed), never a
1297        // duplicate acceptance.
1298        let fd = self._file.as_raw_fd();
1299        if let Err(e) = flock(fd, FlockArg::Unlock) {
1300            tracing::warn!(
1301                fd = %fd,
1302                error = %e,
1303                "launch lock explicit unlock failed; falling back to close semantics"
1304            );
1305        }
1306    }
1307}
1308
1309#[cfg(any(target_os = "linux", target_os = "android"))]
1310impl ServiceLaunchLock {
1311    fn acquire(service: &str) -> Result<Self> {
1312        use nix::fcntl::{flock, FlockArg};
1313        use std::os::fd::AsRawFd;
1314
1315        let path = hyprstream_rpc::paths::runtime_dir().join(format!("{service}.launch.lock"));
1316        if let Some(parent) = path.parent() {
1317            std::fs::create_dir_all(parent).map_err(|e| {
1318                RpcError::SpawnFailed(format!(
1319                    "service {service} launch lock dir creation failed: {e}"
1320                ))
1321            })?;
1322        }
1323        // Create-or-open WITHOUT truncation: the file's only role is existing
1324        // as a stable inode for flock.
1325        let file = std::fs::OpenOptions::new()
1326            .create(true)
1327            .append(true)
1328            .open(&path)
1329            .map_err(|e| {
1330                RpcError::SpawnFailed(format!("service {service} launch lock open failed: {e}"))
1331            })?;
1332        match flock(file.as_raw_fd(), FlockArg::LockExclusiveNonblock) {
1333            Ok(()) => Ok(Self { _file: file }),
1334            Err(nix::errno::Errno::EWOULDBLOCK) => Err(RpcError::SpawnFailed(format!(
1335                "service {service} launch refused: another launch is already in progress (launch lock held)"
1336            ))),
1337            Err(e) => Err(RpcError::SpawnFailed(format!(
1338                "service {service} launch lock failed: {e}"
1339            ))),
1340        }
1341    }
1342}
1343
1344#[async_trait::async_trait]
1345impl SpawnerBackend for StandaloneBackend {
1346    async fn spawn(&self, config: ProcessConfig) -> Result<SpawnedProcess> {
1347        match config.readiness {
1348            ProcessReadiness::Immediate => self.spawn_immediate(config).await,
1349            ProcessReadiness::Notify { timeout } => self.spawn_notified(config, timeout).await,
1350        }
1351    }
1352
1353    async fn stop(&self, process: &SpawnedProcess) -> Result<()> {
1354        let pid = match &process.kind {
1355            ProcessKind::Direct(pid) => *pid,
1356            ProcessKind::SystemdUnit(_) => {
1357                return Err(RpcError::InvalidOperation(
1358                    "StandaloneBackend cannot stop systemd units".to_owned(),
1359                ));
1360            }
1361        };
1362
1363        tracing::debug!(
1364            id = %process.id,
1365            pid = %pid,
1366            "Stopping direct process"
1367        );
1368
1369        // Clone the tracked handle out of the map before awaiting; the map
1370        // entry itself is removed ONLY after termination is CONFIRMED, so a
1371        // failed or timed-out cleanup leaves the child tracked for a
1372        // retrying stop instead of losing ownership mid-cleanup. That
1373        // retention holds only while THIS backend lives: backend drop or
1374        // process exit releases the map without killing anything, and the
1375        // bounded startup Drop cannot guarantee retained retry ownership
1376        // either — no reaper framework exists beyond this.
1377        let child_arc = self
1378            .processes
1379            .get(&process.id)
1380            .map(|entry| Arc::clone(entry.value()));
1381
1382        let mut failures: Vec<String> = Vec::new();
1383        let mut confirmed = false;
1384        if let Some(child_arc) = child_arc {
1385            let mut child = child_arc.lock().await;
1386
1387            // Forceful termination (`start_kill` sends SIGKILL — the comment
1388            // above used to promise SIGTERM; it does not).
1389            if let Err(e) = child.start_kill() {
1390                failures.push(format!("kill failed: {e}"));
1391            }
1392            match tokio::time::timeout(Duration::from_secs(5), child.wait()).await {
1393                Ok(Ok(_)) => {
1394                    confirmed = true;
1395                    // Reaped: only now does the backend stop tracking the
1396                    // child.
1397                    self.processes.remove(&process.id);
1398                }
1399                Ok(Err(e)) => failures.push(format!("reap failed: {e}")),
1400                Err(_) => failures.push("reap timed out; child remains tracked".to_owned()),
1401            }
1402        } else {
1403            // Not tracked (adopted handle, or a retry after a confirmed
1404            // earlier stop): terminate by PID, with termination OBSERVED
1405            // before any artifact is touched.
1406            tracing::debug!(
1407                id = %process.id,
1408                pid = %pid,
1409                "Process not in map, attempting direct kill by PID"
1410            );
1411            confirmed = stop_untracked_by_pid(pid, &mut failures).await;
1412        }
1413
1414        if !confirmed {
1415            // Unconfirmed termination: the PID artifact still names a
1416            // possibly-live process and success must not be reported.
1417            failures.push("termination unconfirmed".to_owned());
1418            return Err(RpcError::SpawnFailed(format!(
1419                "stop of {} did not fully succeed: {}",
1420                process.id,
1421                failures.join("; ")
1422            )));
1423        }
1424
1425        // Conditional PID artifact removal: delete only if the file still
1426        // names THIS child, so stopping an older duplicate cannot erase a
1427        // newer process's PID file.
1428        if let Some(pid_file) = &process.pid_file {
1429            match std::fs::read_to_string(pid_file) {
1430                Ok(content) => {
1431                    let same_child = content.trim() == pid.to_string();
1432                    if same_child {
1433                        if let Err(e) = std::fs::remove_file(pid_file) {
1434                            if e.kind() != std::io::ErrorKind::NotFound {
1435                                failures.push(format!("PID file removal failed: {e}"));
1436                            }
1437                        }
1438                    }
1439                }
1440                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1441                Err(e) => failures.push(format!("PID file read failed: {e}")),
1442            }
1443        }
1444
1445        if failures.is_empty() {
1446            tracing::info!(
1447                id = %process.id,
1448                "Process stopped"
1449            );
1450            Ok(())
1451        } else {
1452            Err(RpcError::SpawnFailed(format!(
1453                "stop of {} did not fully succeed: {}",
1454                process.id,
1455                failures.join("; ")
1456            )))
1457        }
1458    }
1459
1460    async fn is_running(&self, process: &SpawnedProcess) -> Result<bool> {
1461        let pid = match &process.kind {
1462            ProcessKind::Direct(pid) => *pid,
1463            ProcessKind::SystemdUnit(_) => {
1464                return Err(RpcError::InvalidOperation(
1465                    "StandaloneBackend cannot check systemd units".to_owned(),
1466                ));
1467            }
1468        };
1469
1470        let Some(pid_i32) = i32::try_from(pid).ok().filter(|&p| p > 0) else {
1471            return Ok(false); // Invalid PID means not running
1472        };
1473
1474        // A TRACKED child is observed directly: `try_wait` observes — and
1475        // reaps — the actual child. That is portable and authoritative, and
1476        // it cannot mistake a self-exited-but-unreaped child (a zombie,
1477        // still visible to `kill(pid, 0)`) for a running service.
1478        if let Some(child_arc) = self
1479            .processes
1480            .get(&process.id)
1481            .map(|entry| Arc::clone(entry.value()))
1482        {
1483            let mut child = child_arc.lock().await;
1484            return match child.try_wait() {
1485                Ok(Some(_)) => Ok(false), // exited and reaped: not running
1486                Ok(None) => Ok(true),     // still running
1487                Err(e) => {
1488                    tracing::warn!(pid = %pid, error = %e, "Error checking child status");
1489                    Ok(false)
1490                }
1491            };
1492        }
1493
1494        // Untracked daemon handle (adopted, or after a confirmed stop):
1495        // bounded legacy liveness fallback by PID signal 0, zombie-aware so
1496        // a self-exited process awaiting reap is not reported as running.
1497        let nix_pid = nix::unistd::Pid::from_raw(pid_i32);
1498        match nix::sys::signal::kill(nix_pid, None) {
1499            // Present (Ok) or present-but-unsignalable (EPERM) — but a
1500            // zombie is a self-exited child awaiting reap, not a running
1501            // service, so it must not be reported as running.
1502            Ok(()) | Err(nix::errno::Errno::EPERM) => Ok(!process_is_zombie(pid_i32)),
1503            Err(nix::errno::Errno::ESRCH) => Ok(false), // No such process
1504            Err(e) => {
1505                tracing::warn!(pid = %pid, error = %e, "Error checking process status");
1506                Ok(false)
1507            }
1508        }
1509    }
1510
1511    fn stop_tracked_child_sync(&self, process: &SpawnedProcess) -> Result<()> {
1512        Self::stop_tracked_child_by_handle_sync(self, process)
1513    }
1514
1515    fn backend_type(&self) -> &'static str {
1516        "standalone"
1517    }
1518}
1519
1520impl Drop for StandaloneBackend {
1521    fn drop(&mut self) {
1522        // Children are spawned with kill_on_drop(false) ON PURPOSE: adopted
1523        // daemons are intended to outlive the launcher. Dropping the backend
1524        // therefore releases tracking only — it does NOT kill anything, and
1525        // no cleanup is performed here.
1526        let count = self.processes.len();
1527        if count > 0 {
1528            tracing::debug!(
1529                count = count,
1530                "StandaloneBackend dropped with tracked processes (NOT killed: daemons outlive the launcher)"
1531            );
1532        }
1533    }
1534}
1535
1536#[cfg(test)]
1537mod tests {
1538    use super::*;
1539
1540    #[tokio::test]
1541    async fn test_spawn_and_stop() -> hyprstream_rpc::Result<()> {
1542        let backend = StandaloneBackend::new();
1543
1544        // Spawn a simple process (sleep)
1545        let config = ProcessConfig::new("test-sleep", "sleep").args(["100"]);
1546
1547        let result = backend.spawn(config).await;
1548
1549        match result {
1550            Ok(process) => {
1551                assert!(process.is_direct());
1552                assert!(process.pid().is_some());
1553
1554                // Check it's running
1555                let running = backend.is_running(&process).await?;
1556                assert!(running, "Process should be running");
1557
1558                // Stop it
1559                backend.stop(&process).await?;
1560
1561                // Give it a moment to die
1562                tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1563
1564                // Check it's stopped
1565                let running = backend.is_running(&process).await?;
1566                assert!(!running, "Process should be stopped");
1567            }
1568            #[allow(clippy::print_stderr)]
1569            Err(e) => {
1570                // sleep might not be available in some test environments
1571                eprintln!("Could not spawn test process: {e}");
1572            }
1573        }
1574        Ok(())
1575    }
1576
1577    #[cfg(unix)]
1578    #[tokio::test]
1579    async fn standalone_spawn_preserves_non_utf8_argument_bytes() -> anyhow::Result<()> {
1580        use std::os::unix::ffi::{OsStrExt, OsStringExt};
1581
1582        let root = tempfile::tempdir()?;
1583        let script = root.path().join("record-argument.sh");
1584        let observed = root.path().join("observed.bin");
1585        std::fs::write(&script, b"printf '%s' \"$1\" > \"$2\"\nexec sleep 30\n")?;
1586        let expected = std::ffi::OsString::from_vec(b"config-\xff.toml".to_vec());
1587        let name = format!("non-utf8-argv-{}", uuid::Uuid::new_v4().simple());
1588        let backend = StandaloneBackend::new();
1589        let config = ProcessConfig::new(&name, "/bin/sh").args([
1590            script.as_os_str().to_owned(),
1591            expected.clone(),
1592            observed.as_os_str().to_owned(),
1593        ]);
1594        let process = backend.spawn(config).await?;
1595
1596        let deadline = Instant::now() + Duration::from_secs(5);
1597        let observation = loop {
1598            match std::fs::read(&observed) {
1599                Ok(bytes) if bytes == expected.as_os_str().as_bytes() => break Ok(()),
1600                Ok(_) | Err(_) if Instant::now() < deadline => {
1601                    tokio::time::sleep(Duration::from_millis(20)).await;
1602                }
1603                Ok(bytes) => {
1604                    break Err(anyhow::anyhow!(
1605                        "child changed argument bytes: observed {bytes:?}"
1606                    ))
1607                }
1608                Err(error) => {
1609                    break Err(anyhow::anyhow!(
1610                        "child did not record its argument before the deadline: {error}"
1611                    ))
1612                }
1613            }
1614        };
1615        let stop = backend.stop(&process).await;
1616        observation?;
1617        stop?;
1618        assert!(!backend.is_running(&process).await?);
1619        assert!(
1620            !hyprstream_rpc::paths::service_pid_file(&name).exists(),
1621            "stopped child must leave no PID artifact"
1622        );
1623        Ok(())
1624    }
1625
1626    #[test]
1627    fn test_backend_type() {
1628        let backend = StandaloneBackend::new();
1629        assert_eq!(backend.backend_type(), "standalone");
1630    }
1631
1632    #[cfg(unix)]
1633    #[test]
1634    fn detached_output_pipe_is_close_on_exec() -> anyhow::Result<()> {
1635        let (reader, writer) = cloexec_pipe()?;
1636        let flags = [reader, writer]
1637            .into_iter()
1638            .map(|fd| unsafe { nix::libc::fcntl(fd, nix::libc::F_GETFD) })
1639            .collect::<Vec<_>>();
1640        unsafe {
1641            nix::libc::close(reader);
1642            nix::libc::close(writer);
1643        }
1644        assert!(flags
1645            .iter()
1646            .all(|flags| { *flags >= 0 && *flags & nix::libc::FD_CLOEXEC != 0 }));
1647        Ok(())
1648    }
1649}
1650
1651/// Linux/Android-only coverage of the credential-authenticated receiver:
1652/// positive readiness, foreign-PID rejection, timeout/early-exit rollback,
1653/// spoofed-sender refusal, and the launcher-side cancellation/rollback
1654/// ownership regressions that exercise the supervised Notify path end to
1655/// end. These tests RUN on Linux/Android — the cfg only removes them from
1656/// non-Linux builds, where the supervised Notify path itself is refused
1657/// pre-spawn.
1658#[cfg(any(target_os = "linux", target_os = "android"))]
1659#[cfg(test)]
1660mod notify_readiness_tests {
1661    #![allow(clippy::expect_used, clippy::unwrap_used)]
1662
1663    use super::*;
1664    use std::path::Path;
1665
1666    const READY_HELPER: &str = "HYPRSTREAM_NOTIFY_READY_HELPER";
1667    const PENDING_HELPER: &str = "HYPRSTREAM_LAUNCHER_PENDING_HELPER";
1668    const PENDING_MARKER: &str = "HYPRSTREAM_LAUNCHER_PENDING_MARKER";
1669    const PENDING_RELEASE: &str = "HYPRSTREAM_LAUNCHER_PENDING_RELEASE";
1670    const OBSTACLE_HELPER: &str = "HYPRSTREAM_LAUNCHER_OBSTACLE_HELPER";
1671    const OBSTACLE_PATH: &str = "HYPRSTREAM_LAUNCHER_OBSTACLE_PATH";
1672    const LONG_RUNTIME_HELPER: &str = "HYPRSTREAM_LONG_NOTIFY_RUNTIME_HELPER";
1673    const SPOOF_HELPER: &str = "HYPRSTREAM_NOTIFY_SPOOF_HELPER";
1674
1675    /// Helper child: sends READY datagrams from a distinct process PID so the
1676    /// receiver regression test exercises kernel credential filtering rather
1677    /// than a same-process sender.
1678    #[test]
1679    fn notify_spoof_helper_child() {
1680        if std::env::var_os(SPOOF_HELPER).is_none() {
1681            return;
1682        }
1683        let endpoint = std::env::var("HYPRSTREAM_NOTIFY_SPOOF_ENDPOINT").expect("endpoint");
1684        for _ in 0..128 {
1685            let _ = send_ready_to(&endpoint);
1686        }
1687        std::thread::sleep(Duration::from_secs(2));
1688    }
1689
1690    fn send_ready_to(endpoint: &str) -> anyhow::Result<()> {
1691        use nix::sys::socket::{
1692            sendto, socket, AddressFamily, MsgFlags, SockFlag, SockType, UnixAddr,
1693        };
1694
1695        let name = endpoint
1696            .strip_prefix('@')
1697            .expect("launcher notification endpoint must be abstract");
1698        let fd = socket(
1699            AddressFamily::Unix,
1700            SockType::Datagram,
1701            SockFlag::SOCK_NONBLOCK | SockFlag::SOCK_CLOEXEC,
1702            None,
1703        )?;
1704        sendto(
1705            fd.as_raw_fd(),
1706            b"READY=1",
1707            &UnixAddr::new_abstract(name.as_bytes())?,
1708            MsgFlags::MSG_DONTWAIT,
1709        )?;
1710        Ok(())
1711    }
1712
1713    /// Enqueue the genuine sender's datagram even when the foreign helper has
1714    /// filled the receiver queue. The test sender is intentionally
1715    /// nonblocking, so a full queue is a transient condition rather than a
1716    /// failed readiness send. Drain only datagrams that cannot authenticate as
1717    /// this process, then retry; the receiver still requires the kernel PID
1718    /// credential before accepting READY=1.
1719    fn send_ready_to_until_accepted(
1720        notify: &ChildNotifySocket,
1721        expected_pid: i32,
1722    ) -> anyhow::Result<()> {
1723        let deadline = Instant::now() + Duration::from_secs(2);
1724        loop {
1725            match send_ready_to(notify.endpoint()) {
1726                Ok(()) => return Ok(()),
1727                Err(error)
1728                    if error
1729                        .downcast_ref::<nix::errno::Errno>()
1730                        .is_some_and(|errno| *errno == nix::errno::Errno::EAGAIN) =>
1731                {
1732                    // Make room for the authenticated sender while retaining
1733                    // the same receiver path and sender-PID check exercised by
1734                    // the assertion below.
1735                    while notify.try_recv_ready(expected_pid)? {}
1736                    if Instant::now() >= deadline {
1737                        return Err(error);
1738                    }
1739                    std::thread::yield_now();
1740                }
1741                Err(error) => return Err(error),
1742            }
1743        }
1744    }
1745
1746    /// Helper child: reports readiness through the real sd_notify send path,
1747    /// then stays alive so the supervisor observes a genuinely RUNNING ready
1748    /// child; the supervisor's stop is the controlled shutdown.
1749    #[test]
1750    fn notify_ready_helper_child() {
1751        if std::env::var_os(READY_HELPER).is_none() {
1752            return;
1753        }
1754        crate::notify::ready().expect("notify ready send");
1755        std::thread::sleep(Duration::from_secs(30));
1756    }
1757
1758    /// Helper child: records its PID to a marker file, then stays PENDING
1759    /// (no READY, launch lock held) until a release file appears; only then
1760    /// does it report readiness and stay alive. Lets a test hold a REAL
1761    /// notified launch mid-flight across the readiness await.
1762    #[test]
1763    fn launch_pending_helper_child() {
1764        if std::env::var_os(PENDING_HELPER).is_none() {
1765            return;
1766        }
1767        let marker = std::path::PathBuf::from(std::env::var(PENDING_MARKER).expect("marker"));
1768        let release = std::path::PathBuf::from(std::env::var(PENDING_RELEASE).expect("release"));
1769        std::fs::write(&marker, std::process::id().to_string()).expect("marker write");
1770        let deadline = Instant::now() + Duration::from_secs(30);
1771        while !release.exists() {
1772            assert!(Instant::now() < deadline, "release never arrived");
1773            std::thread::sleep(Duration::from_millis(20));
1774        }
1775        crate::notify::ready().expect("notify ready send");
1776        std::thread::sleep(Duration::from_secs(30));
1777    }
1778
1779    /// Helper child: creates a publication obstacle (a directory at the
1780    /// future PID artifact path) BEFORE reporting readiness, so a test can
1781    /// exercise a post-precheck publication failure deterministically.
1782    #[test]
1783    fn publication_obstacle_helper_child() {
1784        if std::env::var_os(OBSTACLE_HELPER).is_none() {
1785            return;
1786        }
1787        let obstacle = std::path::PathBuf::from(std::env::var(OBSTACLE_PATH).expect("obstacle"));
1788        std::fs::create_dir(&obstacle).expect("obstacle dir");
1789        crate::notify::ready().expect("notify ready send");
1790        std::thread::sleep(Duration::from_secs(30));
1791    }
1792
1793    /// Assert a process is reaped (`kill(pid, 0)` says ESRCH).
1794    fn assert_pid_gone(pid: u32) {
1795        let gone = nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), None)
1796            .is_err_and(|e| e == nix::errno::Errno::ESRCH);
1797        assert!(gone, "child pid {pid} must be reaped after cleanup");
1798    }
1799
1800    /// Assert that lifecycle supervision created no legacy pathname endpoint.
1801    /// Abstract endpoints have no filesystem inode and disappear when their
1802    /// owned descriptor closes.
1803    fn assert_no_notify_inode(name: &str) {
1804        let dir = hyprstream_rpc::paths::runtime_dir();
1805        let leftovers: Vec<_> = std::fs::read_dir(&dir)
1806            .into_iter()
1807            .flatten()
1808            .flatten()
1809            .filter_map(|entry| entry.file_name().into_string().ok())
1810            .filter(|file| file.starts_with(&format!("notify-{name}-")))
1811            .collect();
1812        assert!(
1813            leftovers.is_empty(),
1814            "notification endpoints must be cleaned up, found {leftovers:?}"
1815        );
1816    }
1817
1818    /// Cancellation: a pending notified spawn aborted after the child PID is
1819    /// learnable must leave the child reaped and no artifacts behind.
1820    #[tokio::test]
1821    async fn cancelled_pending_spawn_reaps_child_and_cleans_artifacts() -> Result<()> {
1822        let runtime = hyprstream_rpc::paths::runtime_dir();
1823        std::fs::create_dir_all(&runtime).ok();
1824        let observed_pid_file = runtime.join("notify-cancel-observed.pid");
1825        let _ = std::fs::remove_file(&observed_pid_file);
1826
1827        let backend = Arc::new(StandaloneBackend::new());
1828        let config = ProcessConfig::new("notify-cancel", Path::new("/bin/sh"))
1829            .args(["-c", "echo $$ > \"$OBSERVED_PID_FILE\"; exec sleep 100"])
1830            .env("OBSERVED_PID_FILE", observed_pid_file.display().to_string())
1831            .with_notify_ready(Duration::from_secs(30));
1832
1833        // Abort the pending handshake once the child has recorded its PID.
1834        let task = tokio::spawn({
1835            let backend = Arc::clone(&backend);
1836            async move { SpawnerBackend::spawn(&*backend, config).await }
1837        });
1838        let deadline = Instant::now() + Duration::from_secs(10);
1839        let observed: u32 = loop {
1840            if let Ok(content) = std::fs::read_to_string(&observed_pid_file) {
1841                if let Ok(pid) = content.trim().parse::<u32>() {
1842                    break pid;
1843                }
1844            }
1845            assert!(
1846                Instant::now() < deadline,
1847                "child never recorded its PID for cancellation"
1848            );
1849            // Yield to the runtime: the pending spawn runs as a task on this
1850            // same (current-thread) test runtime and must be polled.
1851            tokio::time::sleep(Duration::from_millis(20)).await;
1852        };
1853        task.abort();
1854
1855        // The ownership backstop must SIGKILL + reap the child and remove the
1856        // notification endpoint; poll with a bounded budget.
1857        let gone_deadline = Instant::now() + Duration::from_secs(10);
1858        loop {
1859            let gone = nix::sys::signal::kill(nix::unistd::Pid::from_raw(observed as i32), None)
1860                .is_err_and(|e| e == nix::errno::Errno::ESRCH);
1861            if gone {
1862                break;
1863            }
1864            assert!(
1865                Instant::now() < gone_deadline,
1866                "cancelled child {observed} must be reaped"
1867            );
1868            tokio::time::sleep(Duration::from_millis(50)).await;
1869        }
1870        assert!(
1871            !hyprstream_rpc::paths::service_pid_file("notify-cancel").exists(),
1872            "cancelled spawn must leave no PID file"
1873        );
1874        assert_no_notify_inode("notify-cancel");
1875        let _ = std::fs::remove_file(&observed_pid_file);
1876        Ok(())
1877    }
1878
1879    /// Publication rollback: an unwritable PID artifact (directory squatting
1880    /// the publication path) after READY must fail the spawn, reap the ready
1881    /// child, and leave no notification endpoint.
1882    #[tokio::test]
1883    async fn ambiguous_artifact_refuses_launch_before_spawn() -> Result<()> {
1884        let unique = std::process::id();
1885        let name = format!("notify-amb-{unique}");
1886        let backend = StandaloneBackend::new();
1887        // A DIRECTORY at the PID artifact path is a read ERROR, not an
1888        // absence: the witness is ambiguous and the launch must fail closed
1889        // BEFORE any child runs — no endpoint, no child, no pid in the error.
1890        let squat = hyprstream_rpc::paths::service_pid_file(&name);
1891        std::fs::create_dir_all(&squat).expect("squat directory");
1892        let config = ProcessConfig::new(&name, Path::new("/bin/true"))
1893            .with_notify_ready(Duration::from_secs(5));
1894
1895        let error = backend
1896            .spawn(config)
1897            .await
1898            .expect_err("an ambiguous PID artifact must fail the launch closed");
1899        assert!(
1900            error.to_string().contains("ambiguous"),
1901            "ambiguous refusal expected, got: {error}"
1902        );
1903        assert!(
1904            !error.to_string().contains("(pid "),
1905            "a refused precheck must not have spawned a child, got: {error}"
1906        );
1907        assert_no_notify_inode(&name);
1908        std::fs::remove_dir(&squat).ok();
1909        Ok(())
1910    }
1911
1912    /// Post-precheck publication failure: the controlled child creates the
1913    /// publication obstacle (a directory at the future PID artifact path)
1914    /// only AFTER the duplicate precheck has passed, then reports READY — so
1915    /// the rename onto the obstacle fails deterministically after READY and
1916    /// the ready child is rolled back (reaped, no artifacts).
1917    #[tokio::test]
1918    async fn publication_failure_after_precheck_rolls_back_ready_child() -> Result<()> {
1919        let unique = std::process::id();
1920        let name = format!("notify-pubfail-{unique}");
1921        let pid_file = hyprstream_rpc::paths::service_pid_file(&name);
1922        if let Some(parent) = pid_file.parent() {
1923            std::fs::create_dir_all(parent).ok();
1924        }
1925        let backend = StandaloneBackend::new();
1926        let config = ProcessConfig::new(&name, test_binary())
1927            .args([
1928                "--exact",
1929                "service::spawner::standalone::notify_readiness_tests::publication_obstacle_helper_child",
1930                "--nocapture",
1931            ])
1932            .env(OBSTACLE_HELPER, "1")
1933            .env(OBSTACLE_PATH, pid_file.display().to_string())
1934            .with_notify_ready(Duration::from_secs(30));
1935
1936        let error = backend
1937            .spawn(config)
1938            .await
1939            .expect_err("a post-precheck obstacle must fail publication");
1940        assert!(
1941            error.to_string().contains("PID publication failed"),
1942            "publication failure expected, got: {error}"
1943        );
1944        assert_pid_gone(failure_pid(&error, &name));
1945        assert_no_notify_inode(&name);
1946        std::fs::remove_dir(&pid_file).ok();
1947        Ok(())
1948    }
1949
1950    /// Extract the observed child PID the backend embeds in spawn failures
1951    /// ("service <name> (pid <N>) ...").
1952    fn failure_pid(error: &RpcError, name: &str) -> u32 {
1953        let text = error.to_string();
1954        let marker = format!("service {name} (pid ");
1955        let start = text
1956            .find(&marker)
1957            .unwrap_or_else(|| panic!("failure must carry the observed pid: {text}"))
1958            + marker.len();
1959        let digits: String = text[start..]
1960            .chars()
1961            .take_while(char::is_ascii_digit)
1962            .collect();
1963        digits
1964            .parse()
1965            .unwrap_or_else(|_| panic!("pid digits: {text}"))
1966    }
1967
1968    fn test_binary() -> std::path::PathBuf {
1969        std::env::current_exe().expect("test binary path")
1970    }
1971
1972    #[tokio::test]
1973    async fn notified_spawn_succeeds_on_real_child_ready() -> Result<()> {
1974        if std::env::var_os(READY_HELPER).is_some() {
1975            return Ok(());
1976        }
1977        let backend = StandaloneBackend::new();
1978        let config = ProcessConfig::new("notify-ready-control", test_binary())
1979            .args([
1980                "--exact",
1981                "service::spawner::standalone::notify_readiness_tests::notify_ready_helper_child",
1982                "--nocapture",
1983            ])
1984            .env(READY_HELPER, "1")
1985            .with_notify_ready(Duration::from_secs(30));
1986        let process = backend.spawn(config).await?;
1987        assert!(process.is_direct());
1988        let pid = process.pid().expect("direct child pid");
1989        assert!(
1990            hyprstream_rpc::paths::service_pid_file("notify-ready-control").exists(),
1991            "a ready child publishes its PID file"
1992        );
1993        assert!(
1994            backend.is_running(&process).await?,
1995            "the ready child must still be alive after the spawn reports success"
1996        );
1997        // Supervised stop: reaps the child and removes its PID artifact.
1998        backend.stop(&process).await?;
1999        assert!(
2000            !backend.is_running(&process).await?,
2001            "stopped child must not be running"
2002        );
2003        assert!(
2004            !hyprstream_rpc::paths::service_pid_file("notify-ready-control").exists(),
2005            "stop must remove the published PID file"
2006        );
2007        assert_pid_gone(pid);
2008        assert_no_notify_inode("notify-ready-control");
2009        Ok(())
2010    }
2011
2012    /// A valid long instance/runtime namespace must not influence the compact
2013    /// abstract notification address. Run this scenario in a subprocess so
2014    /// its process-global path environment cannot race other libtest cases.
2015    #[test]
2016    fn notified_spawn_with_long_runtime_reaches_ready_stops_and_reaps() -> Result<()> {
2017        if std::env::var_os(LONG_RUNTIME_HELPER).is_some() {
2018            let runtime = tokio::runtime::Builder::new_current_thread()
2019                .enable_all()
2020                .build()?;
2021            return runtime.block_on(async {
2022                let name = "notify-long-runtime";
2023                let backend = StandaloneBackend::new();
2024                let config = ProcessConfig::new(name, test_binary())
2025                    .args([
2026                        "--exact",
2027                        "service::spawner::standalone::notify_readiness_tests::notify_ready_helper_child",
2028                        "--nocapture",
2029                    ])
2030                    .env(READY_HELPER, "1")
2031                    .with_notify_ready(Duration::from_secs(30));
2032                let process = backend.spawn(config).await?;
2033                let pid = process.pid().expect("direct child pid");
2034                assert!(backend.is_running(&process).await?);
2035                backend.stop(&process).await?;
2036                assert!(!backend.is_running(&process).await?);
2037                assert_pid_gone(pid);
2038                assert!(
2039                    !hyprstream_rpc::paths::service_pid_file(name).exists(),
2040                    "stopped child must leave no PID artifact"
2041                );
2042                assert_no_notify_inode(name);
2043                Ok(())
2044            });
2045        }
2046
2047        let runtime = tempfile::tempdir()?;
2048        let status = std::process::Command::new(test_binary())
2049            .args([
2050                "--exact",
2051                "service::spawner::standalone::notify_readiness_tests::notified_spawn_with_long_runtime_reaches_ready_stops_and_reaps",
2052                "--nocapture",
2053            ])
2054            .env(LONG_RUNTIME_HELPER, "1")
2055            .env("XDG_RUNTIME_DIR", runtime.path())
2056            .env(
2057                "HYPRSTREAM_INSTANCE",
2058                "production-west-instance-0000000000000000",
2059            )
2060            .status()?;
2061        assert!(
2062            status.success(),
2063            "isolated long-runtime scenario failed: {status}"
2064        );
2065        Ok(())
2066    }
2067
2068    /// A child that exits on its own (no stop issued) must be reported as
2069    /// not-running: it is a zombie — still visible to `kill(pid, 0)` — until
2070    /// its supervisor reaps it, and "running" would be a lie. A subsequent
2071    /// supervised stop must still succeed (reap + conditional PID removal).
2072    #[tokio::test]
2073    async fn is_running_reports_self_exited_child_as_stopped() -> Result<()> {
2074        let backend = StandaloneBackend::new();
2075        let config =
2076            ProcessConfig::new("notify-self-exit", Path::new("/bin/sh")).args(["-c", "exit 0"]);
2077        let process = SpawnerBackend::spawn(&backend, config).await?;
2078        let pid = process.pid().expect("direct child pid");
2079
2080        let deadline = Instant::now() + Duration::from_secs(10);
2081        loop {
2082            if !backend.is_running(&process).await? {
2083                break;
2084            }
2085            assert!(
2086                Instant::now() < deadline,
2087                "self-exited child must not be reported as running (zombie)"
2088            );
2089            // Yield: spawn runs as a task on this same current-thread runtime.
2090            tokio::time::sleep(Duration::from_millis(20)).await;
2091        }
2092
2093        // Stopping an already-exited child is still a clean, confirmed stop.
2094        backend.stop(&process).await?;
2095        assert!(!backend.is_running(&process).await?);
2096        assert_pid_gone(pid);
2097
2098        // A SECOND stop after the confirmed first is a clean no-op through
2099        // the untracked-by-PID fallback: termination is observed (ESRCH),
2100        // not assumed.
2101        backend.stop(&process).await?;
2102        assert!(!backend.is_running(&process).await?);
2103        Ok(())
2104    }
2105
2106    /// Cleanup failures are part of the error report, never swallowed
2107    /// (the discard-the-Err `unwrap_or_default` suffixes are gone).
2108    #[test]
2109    fn cleanup_failure_suffix_propagates_into_error() {
2110        assert_eq!(cleanup_suffix(Ok(())), "");
2111        assert_eq!(
2112            cleanup_suffix(Err("kill failed: boom".to_owned())),
2113            "; cleanup: kill failed: boom"
2114        );
2115    }
2116
2117    /// Explicit-unlock release (P2): `flock` lives on the open file
2118    /// description, so a close-only release can be prolonged by any retained
2119    /// duplicate of that description — exactly what a concurrently forked
2120    /// child's pre-exec descriptor copy is. A safe `dup` of the locked
2121    /// descriptor stands in for such a duplicate; after the lock is dropped,
2122    /// a fresh same-service acquisition must succeed even though the
2123    /// duplicate is still open.
2124    #[test]
2125    fn launch_lock_releases_despite_retained_description_duplicate() {
2126        let unique = std::process::id();
2127        let name = format!("notify-unlock-{unique}");
2128
2129        let held = ServiceLaunchLock::acquire(&name).expect("acquire");
2130        // Retained duplicate of the SAME open file description (RAII clone:
2131        // an assertion failure cannot leak the demonstration descriptor).
2132        let duplicate = held._file.try_clone().expect("try_clone");
2133
2134        // Explicit-unlock drop: the lock must release despite the still-open
2135        // duplicate; close alone would not achieve that.
2136        drop(held);
2137
2138        let reacquired = ServiceLaunchLock::acquire(&name)
2139            .expect("lock must release via explicit unlock despite the retained duplicate");
2140        drop(reacquired);
2141
2142        // The demonstration duplicate is closed by its own RAII drop here.
2143        drop(duplicate);
2144    }
2145
2146    /// Startup-backstop decisions: OBSERVED termination cleans the owned
2147    /// artifact (regular file and already-absent paths); an unobservable
2148    /// outcome must RETAIN the artifact as evidence and claim no success.
2149    /// The unobserved branch is exercised through the deterministically
2150    /// inducible seam (a handle that yields no PID) — no kernel kill
2151    /// failure is forced or claimed.
2152    #[tokio::test]
2153    async fn startup_backstop_cleanup_tracks_observed_termination() -> Result<()> {
2154        let dir = tempfile::tempdir()?;
2155        let artifact = dir.path().join("owned.pid");
2156
2157        // Observed path: a live child is SIGKILLed and reaped by the
2158        // backstop; the owned artifact is removed.
2159        let mut cmd = tokio::process::Command::new("/bin/sleep");
2160        cmd.arg("30");
2161        let child = cmd.spawn().expect("sleep child");
2162        let pid = child.id().expect("live pid");
2163        std::fs::write(&artifact, pid.to_string()).expect("artifact");
2164        let mut guard = StartupChildGuard {
2165            child: Some(child),
2166            name: "backstop-observed".to_owned(),
2167            pid_file: Some(artifact.clone()),
2168        };
2169        assert_eq!(
2170            guard.backstop_cleanup(),
2171            BackstopOutcome::TerminatedAndCleaned,
2172            "live-child backstop must observe termination and clean its artifact"
2173        );
2174        assert!(
2175            !artifact.exists(),
2176            "observed cleanup must remove the owned artifact"
2177        );
2178        assert_pid_gone(pid);
2179
2180        // Already-absent path: an armed artifact that does not exist is
2181        // gone either way — still a clean, observed outcome.
2182        let mut cmd = tokio::process::Command::new("/bin/sleep");
2183        cmd.arg("30");
2184        let child = cmd.spawn().expect("sleep child");
2185        let mut guard = StartupChildGuard {
2186            child: Some(child),
2187            name: "backstop-absent".to_owned(),
2188            pid_file: Some(dir.path().join("never-written.pid")),
2189        };
2190        assert_eq!(
2191            guard.backstop_cleanup(),
2192            BackstopOutcome::TerminatedAndCleaned,
2193            "a NotFound artifact must not downgrade the observed outcome"
2194        );
2195        assert!(!guard.pid_file.is_some(), "armed path must be cleared");
2196
2197        // Unobserved path: the handle yields no PID, so the backstop cannot
2198        // observe anything and must not claim cleanup.
2199        let mut exiting = tokio::process::Command::new("/bin/sh");
2200        exiting.args(["-c", "exit 0"]);
2201        let mut child = exiting.spawn().expect("exiting child");
2202        child.wait().await.expect("reaped by this wait");
2203        std::fs::write(&artifact, "0").expect("artifact");
2204        let mut guard = StartupChildGuard {
2205            child: Some(child),
2206            name: "backstop-unobserved".to_owned(),
2207            pid_file: Some(artifact.clone()),
2208        };
2209        assert_eq!(
2210            guard.backstop_cleanup(),
2211            BackstopOutcome::Unconfirmed,
2212            "an unobservable outcome must not claim cleanup"
2213        );
2214        assert!(
2215            artifact.exists(),
2216            "unobserved backstop must retain the owned artifact as evidence"
2217        );
2218        Ok(())
2219    }
2220
2221    /// Termination and artifact cleanup are separate observed facts: a
2222    /// live child is terminated and reaped, but an artifact that cannot be
2223    /// unlinked (a directory squats the armed path) must be reported as
2224    /// `TerminatedArtifactKept` and RETAINED on disk — never as a
2225    /// cleaned-up success.
2226    #[tokio::test]
2227    async fn startup_backstop_distinguishes_termination_from_failed_cleanup() -> Result<()> {
2228        let dir = tempfile::tempdir()?;
2229        // A DIRECTORY at the armed path: remove_file fails with EISDIR
2230        // (not NotFound) even after the child is genuinely reaped.
2231        let squat = dir.path().join("owned.pid");
2232        std::fs::create_dir(&squat)?;
2233
2234        let mut cmd = tokio::process::Command::new("/bin/sleep");
2235        cmd.arg("30");
2236        let child = cmd.spawn().expect("sleep child");
2237        let pid = child.id().expect("live pid");
2238        let mut guard = StartupChildGuard {
2239            child: Some(child),
2240            name: "backstop-kept".to_owned(),
2241            pid_file: Some(squat.clone()),
2242        };
2243        assert_eq!(
2244            guard.backstop_cleanup(),
2245            BackstopOutcome::TerminatedArtifactKept,
2246            "failed artifact removal must be reported separately from termination"
2247        );
2248        // Termination was observed (reaped) AND the artifact is retained.
2249        assert_pid_gone(pid);
2250        assert!(
2251            squat.is_dir(),
2252            "the failed-removal artifact must be retained on disk as evidence"
2253        );
2254        std::fs::remove_dir(&squat)?;
2255        Ok(())
2256    }
2257
2258    /// Narrow classification-helper coverage, honestly labelled: the real
2259    /// OS cannot deterministically present a nonterminal wait status under
2260    /// WNOHANG after SIGKILL, so this proves only that the classifier
2261    /// accepts exactly Exited/Signaled and rejects still-alive, stopped,
2262    /// and continued states.
2263    #[test]
2264    fn wait_status_classification_accepts_only_terminal_statuses() {
2265        use nix::sys::signal::Signal;
2266        use nix::sys::wait::WaitStatus;
2267
2268        let pid = nix::unistd::Pid::from_raw(1);
2269        assert!(wait_status_is_terminal(&WaitStatus::Exited(pid, 0)));
2270        assert!(wait_status_is_terminal(&WaitStatus::Signaled(
2271            pid,
2272            Signal::SIGKILL,
2273            false
2274        )));
2275        assert!(!wait_status_is_terminal(&WaitStatus::StillAlive));
2276        assert!(!wait_status_is_terminal(&WaitStatus::Stopped(
2277            pid,
2278            Signal::SIGSTOP
2279        )));
2280        assert!(!wait_status_is_terminal(&WaitStatus::Continued(pid)));
2281    }
2282
2283    /// UNIT-LEVEL duplicate controls (manual lock hold / manual witness; the
2284    /// END-TO-END pending→READY→live sequencing is proven by
2285    /// `duplicate_launch_sequencing_refuses_pending_then_live`): a held
2286    /// launch lock deterministically reports the pending duplicate, and a
2287    /// PID artifact naming a LIVE process refuses a new launch — never
2288    /// silently overwriting the sole witness of a live process.
2289    #[tokio::test]
2290    async fn duplicate_launch_controls_refuse_pending_and_live() -> Result<()> {
2291        let unique = std::process::id();
2292        let name = format!("notify-dup-{unique}");
2293        let backend = StandaloneBackend::new();
2294
2295        // Pending duplicate: the launch lock itself is the deterministic
2296        // witness — a held lock refuses the attempt before anything runs.
2297        let held = ServiceLaunchLock::acquire(&name)?;
2298        let pending = ProcessConfig::new(&name, Path::new("/bin/true"))
2299            .with_notify_ready(Duration::from_secs(5));
2300        let pending_error = SpawnerBackend::spawn(&backend, pending)
2301            .await
2302            .expect_err("a held launch lock must refuse a pending duplicate");
2303        assert!(
2304            pending_error.to_string().contains("already in progress"),
2305            "pending-duplicate refusal expected, got: {pending_error}"
2306        );
2307        assert_no_notify_inode(&name);
2308        drop(held);
2309
2310        // Live duplicate: a PID artifact naming a live process refuses a
2311        // second launch. The squat process is spawned (and later reaped) by
2312        // this test itself.
2313        let mut squat = std::process::Command::new("/bin/sleep")
2314            .arg("30")
2315            .spawn()
2316            .expect("squat process for live-duplicate witness");
2317        let squat_pid = squat.id();
2318        let pid_file = hyprstream_rpc::paths::service_pid_file(&name);
2319        if let Some(parent) = pid_file.parent() {
2320            std::fs::create_dir_all(parent).ok();
2321        }
2322        std::fs::write(&pid_file, squat_pid.to_string()).expect("squat pid file");
2323
2324        let ready_config = ProcessConfig::new(&name, test_binary())
2325            .args([
2326                "--exact",
2327                "service::spawner::standalone::notify_readiness_tests::notify_ready_helper_child",
2328                "--nocapture",
2329            ])
2330            .env(READY_HELPER, "1")
2331            .with_notify_ready(Duration::from_secs(30));
2332        let live_error = SpawnerBackend::spawn(&backend, ready_config)
2333            .await
2334            .expect_err("a live PID artifact must refuse a duplicate launch");
2335        assert!(
2336            live_error
2337                .to_string()
2338                .contains(&format!("already appears live as pid {squat_pid}")),
2339            "live-duplicate refusal expected, got: {live_error}"
2340        );
2341        // The refused attempt published nothing: the artifact still names
2342        // the live squat, and no notification endpoint survives.
2343        assert_eq!(
2344            std::fs::read_to_string(&pid_file)
2345                .expect("witness pid file")
2346                .trim(),
2347            squat_pid.to_string(),
2348            "live witness artifact must not be overwritten"
2349        );
2350        assert_no_notify_inode(&name);
2351
2352        // Reap the squat process and leave no fixture artifact behind.
2353        squat.kill().expect("squat kill");
2354        squat.wait().expect("squat reap");
2355        let _ = std::fs::remove_file(&pid_file);
2356        Ok(())
2357    }
2358
2359    /// REAL duplicate sequencing, end to end: a first notified launch held
2360    /// PENDING by its own child keeps the per-service launch lock held
2361    /// ACROSS the readiness await, so a second same-name launch is refused
2362    /// with no child of its own while the first survives; after release the
2363    /// first reports READY, publishes its PID artifact, and stays alive —
2364    /// and a further launch is refused against that LIVE published witness.
2365    #[tokio::test]
2366    async fn duplicate_launch_sequencing_refuses_pending_then_live() -> Result<()> {
2367        let backend = Arc::new(StandaloneBackend::new());
2368        let dir = tempfile::tempdir()?;
2369        let unique = std::process::id();
2370        let name = format!("notify-seq-{unique}");
2371        let marker = dir.path().join("pending.marker");
2372        let release = dir.path().join("pending.release");
2373
2374        let pending_config = ProcessConfig::new(&name, test_binary())
2375            .args([
2376                "--exact",
2377                "service::spawner::standalone::notify_readiness_tests::launch_pending_helper_child",
2378                "--nocapture",
2379            ])
2380            .env(PENDING_HELPER, "1")
2381            .env(PENDING_MARKER, marker.display().to_string())
2382            .env(PENDING_RELEASE, release.display().to_string())
2383            .with_notify_ready(Duration::from_secs(30));
2384
2385        // First launch: genuinely pending inside the readiness await, with
2386        // the launch lock held by this very launch.
2387        let first = tokio::spawn({
2388            let backend = Arc::clone(&backend);
2389            let config = pending_config.clone();
2390            async move { SpawnerBackend::spawn(&*backend, config).await }
2391        });
2392        let deadline = Instant::now() + Duration::from_secs(10);
2393        let first_pid: u32 = loop {
2394            if let Ok(content) = std::fs::read_to_string(&marker) {
2395                if let Ok(pid) = content.trim().parse::<u32>() {
2396                    break pid;
2397                }
2398            }
2399            assert!(
2400                Instant::now() < deadline,
2401                "first launch never went pending (no marker)"
2402            );
2403            tokio::time::sleep(Duration::from_millis(20)).await;
2404        };
2405
2406        // Second same-name launch while the first is mid-flight: refused on
2407        // the held lock, deterministically, before any child of its own.
2408        let second_error = SpawnerBackend::spawn(&*backend, pending_config.clone())
2409            .await
2410            .expect_err("a pending first launch must refuse a same-name launch");
2411        assert!(
2412            second_error.to_string().contains("already in progress"),
2413            "pending-duplicate refusal expected, got: {second_error}"
2414        );
2415        // The first process survived the rejected second call.
2416        assert!(
2417            nix::sys::signal::kill(nix::unistd::Pid::from_raw(first_pid as i32), None).is_ok(),
2418            "first launch child must survive the refused duplicate"
2419        );
2420
2421        // Release the first: it reports READY and publishes its PID artifact.
2422        std::fs::write(&release, b"release")?;
2423        let process = tokio::time::timeout(Duration::from_secs(30), first)
2424            .await
2425            .expect("first launch task must finish after release")
2426            .expect("first launch task must not panic")
2427            .expect("first launch spawn must succeed after release");
2428        assert_eq!(
2429            std::fs::read_to_string(hyprstream_rpc::paths::service_pid_file(&name))
2430                .expect("published pid file")
2431                .trim(),
2432            first_pid.to_string(),
2433            "the first launch must publish its own pid"
2434        );
2435
2436        // A further launch is refused against the LIVE published witness.
2437        let third_error = SpawnerBackend::spawn(&*backend, pending_config.clone())
2438            .await
2439            .expect_err("a live published predecessor must refuse a duplicate");
2440        assert!(
2441            third_error
2442                .to_string()
2443                .contains(&format!("already appears live as pid {first_pid}")),
2444            "live-duplicate refusal expected, got: {third_error}"
2445        );
2446
2447        // Supervised cleanup of the first; the persistent lock file survives
2448        // (stable inode, never unlinked) while the PID artifact is gone.
2449        backend.stop(&process).await?;
2450        assert_pid_gone(first_pid);
2451        assert!(
2452            !hyprstream_rpc::paths::service_pid_file(&name).exists(),
2453            "stopped first launch must leave no PID file"
2454        );
2455        assert!(
2456            hyprstream_rpc::paths::runtime_dir()
2457                .join(format!("{name}.launch.lock"))
2458                .exists(),
2459            "the launch lock file must persist (never unlinked)"
2460        );
2461        assert_no_notify_inode(&name);
2462        Ok(())
2463    }
2464
2465    #[tokio::test]
2466    async fn notified_spawn_fails_on_early_child_exit() -> Result<()> {
2467        let backend = StandaloneBackend::new();
2468        let config = ProcessConfig::new("notify-early-exit", Path::new("/bin/sh"))
2469            .args(["-c", "exit 7"])
2470            .with_notify_ready(Duration::from_secs(30));
2471        let error = backend
2472            .spawn(config)
2473            .await
2474            .expect_err("early child exit must fail the spawn");
2475        assert!(
2476            error.to_string().contains("exited during startup"),
2477            "honest exit propagation expected, got: {error}"
2478        );
2479        assert!(
2480            !hyprstream_rpc::paths::service_pid_file("notify-early-exit").exists(),
2481            "a never-ready child must not publish a PID file"
2482        );
2483        assert_pid_gone(failure_pid(&error, "notify-early-exit"));
2484        assert_no_notify_inode("notify-early-exit");
2485        Ok(())
2486    }
2487
2488    #[tokio::test]
2489    async fn notified_spawn_times_out_kills_and_cleans_up() -> Result<()> {
2490        let backend = StandaloneBackend::new();
2491        let config = ProcessConfig::new("notify-timeout", Path::new("/bin/sleep"))
2492            .args(["100"])
2493            .with_notify_ready(Duration::from_millis(300));
2494        let error = backend
2495            .spawn(config)
2496            .await
2497            .expect_err("a hung child must hit the hard readiness timeout");
2498        assert!(
2499            error.to_string().contains("no READY=1"),
2500            "timeout failure expected, got: {error}"
2501        );
2502        // Cleanup: the observed child was terminated and reaped, and no PID or
2503        // notification artifact survives.
2504        std::thread::sleep(Duration::from_millis(150));
2505        assert!(
2506            !hyprstream_rpc::paths::service_pid_file("notify-timeout").exists(),
2507            "timed-out child must leave no PID artifact"
2508        );
2509        assert_pid_gone(failure_pid(&error, "notify-timeout"));
2510        assert_no_notify_inode("notify-timeout");
2511        Ok(())
2512    }
2513
2514    #[test]
2515    fn notification_endpoint_is_compact_unique_and_abstract() -> Result<()> {
2516        let first = ChildNotifySocket::bind("a-service-name-that-is-deliberately-ignored")?;
2517        let second = ChildNotifySocket::bind("a-service-name-that-is-deliberately-ignored")?;
2518        assert!(first.endpoint().starts_with('@'));
2519        assert!(second.endpoint().starts_with('@'));
2520        assert_ne!(first.endpoint(), second.endpoint());
2521        assert!(first.endpoint().len() <= 48);
2522        Ok(())
2523    }
2524
2525    #[test]
2526    fn notification_endpoint_collision_retries_are_bounded() -> anyhow::Result<()> {
2527        use nix::sys::socket::{bind, socket, AddressFamily, SockFlag, SockType, UnixAddr};
2528
2529        let occupied_name = format!("hypr-n-collision-{}", std::process::id());
2530        let occupied = socket(
2531            AddressFamily::Unix,
2532            SockType::Datagram,
2533            SockFlag::SOCK_CLOEXEC,
2534            None,
2535        )?;
2536        bind(
2537            occupied.as_raw_fd(),
2538            &UnixAddr::new_abstract(occupied_name.as_bytes())?,
2539        )?;
2540
2541        let mut attempts = 0usize;
2542        let error = ChildNotifySocket::bind_with_name_source(|| {
2543            attempts += 1;
2544            occupied_name.clone()
2545        })
2546        .expect_err("eight collisions must fail closed");
2547        assert_eq!(attempts, 8);
2548        assert!(error.to_string().contains("after 8 attempts"));
2549        Ok(())
2550    }
2551
2552    #[tokio::test]
2553    async fn detached_stdio_drains_large_post_launch_output() -> anyhow::Result<()> {
2554        let mut cmd = Command::new("/bin/sh");
2555        cmd.args(["-c", "sleep 0.05; yes x | head -c 1048576"]);
2556        configure_detached_stdio(&mut cmd)?;
2557        let status = cmd.status().await?;
2558        assert!(
2559            status.success(),
2560            "post-launch output child must exit cleanly"
2561        );
2562        Ok(())
2563    }
2564
2565    #[tokio::test]
2566    async fn foreign_datagram_flood_cannot_extend_readiness_deadline() -> Result<()> {
2567        use std::sync::atomic::{AtomicBool, Ordering};
2568
2569        let backend = StandaloneBackend::new();
2570        let notify = ChildNotifySocket::bind("flood-deadline")?;
2571        let mut child = tokio::process::Command::new("/bin/sleep")
2572            .arg("30")
2573            .spawn()?;
2574        let endpoint = notify.endpoint().to_owned();
2575        let flooding = Arc::new(AtomicBool::new(true));
2576        let sender = std::thread::spawn({
2577            let flooding = Arc::clone(&flooding);
2578            move || {
2579                while flooding.load(Ordering::Relaxed) {
2580                    let _ = send_ready_to(&endpoint);
2581                    std::thread::sleep(Duration::from_millis(1));
2582                }
2583            }
2584        });
2585
2586        let started = Instant::now();
2587        let outcome = backend
2588            .await_child_readiness(&mut child, &notify, Duration::from_millis(200))
2589            .await;
2590        flooding.store(false, Ordering::Relaxed);
2591        sender.join().expect("flood sender thread");
2592        child.kill().await?;
2593        child.wait().await?;
2594
2595        let error = outcome.expect_err("foreign datagrams must not satisfy readiness");
2596        assert!(error.contains("no READY=1"));
2597        assert!(started.elapsed() < Duration::from_secs(2));
2598        Ok(())
2599    }
2600
2601    /// Sender-PID matching: a READY datagram from any process other than the
2602    /// expected child is ignored, not honored.
2603    #[tokio::test]
2604    async fn notify_socket_rejects_spoofed_sender() -> anyhow::Result<()> {
2605        let notify = ChildNotifySocket::bind("spoof-probe")?;
2606        let this_pid = std::process::id() as i32;
2607        // Flood from a distinct helper process, whose kernel-attested PID is
2608        // foreign to this receiver. The genuine datagram follows the flood.
2609        let mut foreign = std::process::Command::new(test_binary())
2610            .args([
2611                "--exact",
2612                "service::spawner::standalone::notify_readiness_tests::notify_spoof_helper_child",
2613                "--nocapture",
2614            ])
2615            .env(SPOOF_HELPER, "1")
2616            .env("HYPRSTREAM_NOTIFY_SPOOF_ENDPOINT", notify.endpoint())
2617            .spawn()?;
2618        std::thread::sleep(Duration::from_millis(50));
2619        // The helper deliberately fills the receiver queue with foreign
2620        // datagrams.  Drain that finite backlog before sending the genuine
2621        // datagram: send_ready_to uses a nonblocking sender, so attempting to
2622        // enqueue while the abstract socket queue is full would fail with
2623        // EAGAIN before the receiver has a chance to exercise PID matching.
2624        assert!(
2625            !notify.try_recv_ready(this_pid)?,
2626            "foreign sender must not satisfy readiness"
2627        );
2628        send_ready_to_until_accepted(&notify, this_pid)?;
2629        assert!(
2630            notify.try_recv_ready(this_pid)?,
2631            "the matching sender satisfies readiness after foreign flood"
2632        );
2633        foreign.kill()?;
2634        let _ = foreign.wait();
2635        Ok(())
2636    }
2637}
2638
2639/// Non-Linux targets only: the supervised Notify request must be refused
2640/// before any launch side effect. The configured child is an executable
2641/// sentinel — it would write a marker file if it were ever spawned — so
2642/// marker, PID-artifact, and notify-runtime-directory absence at the checked
2643/// points evidence the boundary. This cfg cannot run on Linux lanes; it
2644/// exists for actual non-Linux targets and makes no claim about them from
2645/// Linux runs.
2646#[cfg(all(unix, not(any(target_os = "linux", target_os = "android")), test))]
2647mod notify_refusal_non_linux_tests {
2648    #![allow(clippy::expect_used, clippy::unwrap_used)]
2649
2650    use super::*;
2651    use crate::service::spawner::ProcessSpawner;
2652
2653    const REFUSED_SENTINEL: &str = "HYPRSTREAM_NOTIFY_REFUSED_SENTINEL";
2654
2655    /// A Notify request on an unsupported platform is refused pre-spawn by
2656    /// the explicit unsupported branch in `spawn_notified` — that branch
2657    /// returns before any launch operation, which is the production
2658    /// guarantee. This test observes that boundary: the executable sentinel
2659    /// was not observed running, no PID artifact was published, and no
2660    /// per-launch notify runtime directory appeared at the checked points.
2661    #[tokio::test]
2662    async fn notify_request_refused_before_any_launch_side_effect() -> anyhow::Result<()> {
2663        if let Some(path) = std::env::var_os(REFUSED_SENTINEL) {
2664            // Unreachable if the boundary holds: the refusal must fire before
2665            // any child exists.
2666            std::fs::write(&path, std::process::id().to_string())?;
2667            return Ok(());
2668        }
2669
2670        let exe = std::env::current_exe()?;
2671        let dir = tempfile::tempdir()?;
2672        let marker = dir.path().join("sentinel.marker");
2673        // Unique per-run service name: unrelated runtime artifacts cannot
2674        // decide the result, and nothing existing is overwritten or removed.
2675        let name = format!("notify-refused-{}", nix::unistd::getpid());
2676        let pid_file = hyprstream_rpc::paths::service_pid_file(&name);
2677        let dir_prefix = format!("notify-{name}-");
2678
2679        let spawner = ProcessSpawner::standalone();
2680        let error = match spawner
2681            .spawn(
2682                ProcessConfig::new(&name, &exe)
2683                    .args([
2684                        "--exact",
2685                        "service::spawner::standalone::notify_refusal_non_linux_tests::notify_request_refused_before_any_launch_side_effect",
2686                        "--nocapture",
2687                    ])
2688                    .env(REFUSED_SENTINEL, marker.display().to_string())
2689                    .with_notify_ready(std::time::Duration::from_secs(30)),
2690            )
2691            .await
2692        {
2693            Ok(process) => {
2694                panic!(
2695                    "Notify must be refused on non-Linux targets; spawned pid {:?}",
2696                    process.pid()
2697                );
2698            }
2699            Err(error) => error,
2700        };
2701        let message = error.to_string();
2702        assert!(
2703            message.contains("Linux/Android"),
2704            "refusal must name the platform boundary, got: {message}"
2705        );
2706
2707        // Observed boundary: the sentinel never ran, no PID artifact was
2708        // published, and no per-launch notify runtime directory exists. An
2709        // ABSENT runtime directory is itself a valid refusal state (nothing
2710        // created it): only ErrorKind::NotFound is treated as absence; every
2711        // other read/entry error propagates.
2712        assert!(
2713            !marker.exists(),
2714            "the sentinel child must not have been spawned by the refused Notify request"
2715        );
2716        assert!(
2717            !pid_file.exists(),
2718            "a refused Notify request must not publish a PID artifact"
2719        );
2720        match std::fs::read_dir(hyprstream_rpc::paths::runtime_dir()) {
2721            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
2722            Err(e) => return Err(e.into()),
2723            Ok(entries) => {
2724                for entry in entries {
2725                    let entry = entry?;
2726                    let file_name = entry.file_name().to_string_lossy().to_string();
2727                    assert!(
2728                        !file_name.starts_with(&dir_prefix),
2729                        "a refused Notify request must not create a notify runtime \
2730                         directory, found {file_name}"
2731                    );
2732                }
2733            }
2734        }
2735        Ok(())
2736    }
2737}