Skip to main content

hyprstream_workers/workflow/
runner.rs

1//! Workflow runner — executes workflows through the VFS namespace.
2//!
3//! The WorkflowRunner is a general-purpose execution engine that:
4//! - Topologically sorts jobs by `needs:` dependencies into execution waves
5//! - Runs independent jobs in each wave concurrently via `tokio::JoinSet`
6//! - Resolves `uses:` action steps through VFS `/bin/` ctl files
7//! - Evaluates `run:` script steps via TclShell on `spawn_blocking`
8//! - Propagates CancellationToken for cooperative cancellation
9//! - Routes each job through a [`JobScheduler`] (#527) — `runs_on:` +
10//!   `resources:` decide in-proc vs. a real P2-admitted sandbox reservation
11//!   (see `workflow::scheduler`); no scheduler configured (`None`, the
12//!   default) preserves the pre-#527 in-proc-only behavior exactly.
13//! - Honors `WorkflowConfig::max_concurrent_runs` (a run-admission semaphore,
14//!   distinct from `script_semaphore`'s per-job script concurrency) and
15//!   `job_timeout_secs`/`step_timeout_secs` (#521 — wired for real, not
16//!   superseded by P2: admission governs resource capacity/quota, this
17//!   governs wall-clock run/step duration, a different axis)
18
19use std::collections::{HashMap, HashSet, VecDeque};
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use std::time::Duration;
23
24use tokio::sync::Semaphore;
25use tokio::task::JoinSet;
26use tokio_util::sync::CancellationToken;
27
28use hyprstream_workers_tcl::TclShell;
29// VFS proxy no longer needed — TclShell awaits namespace operations directly.
30use hyprstream_vfs::overlay::passthrough_rootfs_overlay;
31use hyprstream_vfs::{BindFlag, MountTarget, Namespace, Subject};
32
33use crate::config::WorkflowConfig;
34
35use super::parser::{Job, Step, Workflow};
36use super::scheduler::{JobScheduler, Placement};
37
38// ─────────────────────────────────────────────────────────────────────────────
39// Result types (local to runner, not wire-format)
40// ─────────────────────────────────────────────────────────────────────────────
41
42/// Outcome of a complete workflow run.
43#[derive(Debug, Clone)]
44pub struct WorkflowRunResult {
45    pub success: bool,
46    pub jobs: HashMap<String, JobRunResult>,
47}
48
49/// Outcome of a single job.
50#[derive(Debug, Clone)]
51pub struct JobRunResult {
52    pub name: String,
53    pub success: bool,
54    pub steps: Vec<StepRunResult>,
55}
56
57/// Outcome of a single step.
58#[derive(Debug, Clone)]
59pub struct StepRunResult {
60    pub name: String,
61    pub success: bool,
62    pub output: String,
63}
64
65/// A composed candidate worktree made available to a workflow job at `/src`.
66///
67/// The host path is canonicalized once by the trusted control plane. Each job
68/// receives a fresh OverlayFs upper over this immutable lower, so a workflow
69/// write copies up into job-local scratch and never mutates the git2db tree.
70#[derive(Clone, Debug)]
71pub struct CandidateWorktree {
72    root: Arc<PathBuf>,
73}
74
75impl CandidateWorktree {
76    pub fn from_host_path(path: impl AsRef<Path>) -> RunnerResult<Self> {
77        let root = std::fs::canonicalize(path.as_ref())
78            .map_err(|e| RunnerError::VfsError(format!("canonicalize candidate worktree: {e}")))?;
79        if !root.is_dir() {
80            return Err(RunnerError::VfsError(format!(
81                "candidate worktree is not a directory: {}",
82                root.display()
83            )));
84        }
85        Ok(Self {
86            root: Arc::new(root),
87        })
88    }
89
90    fn mount_for_job(&self) -> RunnerResult<(MountTarget, tempfile::TempDir)> {
91        let scratch = tempfile::Builder::new()
92            .prefix("hyprstream-candidate-")
93            .tempdir()
94            .map_err(|e| RunnerError::VfsError(format!("create candidate overlay scratch: {e}")))?;
95        let upper = scratch.path().join("upper");
96        let work = scratch.path().join("work");
97        std::fs::create_dir(&upper)
98            .and_then(|_| std::fs::create_dir(&work))
99            .map_err(|e| RunnerError::VfsError(format!("create candidate overlay dirs: {e}")))?;
100        let mount = passthrough_rootfs_overlay(self.root.as_ref(), &upper, &work)
101            .map_err(|e| RunnerError::VfsError(format!("compose candidate worktree mount: {e}")))?;
102        Ok((Arc::new(mount) as MountTarget, scratch))
103    }
104}
105
106// ─────────────────────────────────────────────────────────────────────────────
107// Error types
108// ─────────────────────────────────────────────────────────────────────────────
109
110/// Errors specific to workflow execution.
111#[derive(Debug, thiserror::Error)]
112pub enum RunnerError {
113    #[error("invalid action name: {0}")]
114    InvalidActionName(String),
115
116    #[error("invalid input key: {0}")]
117    InvalidInputKey(String),
118
119    #[error("invalid env var name: {0}")]
120    InvalidEnvVarName(String),
121
122    #[error("dependency cycle detected in jobs")]
123    DependencyCycle,
124
125    #[error("unknown job dependency: {0}")]
126    UnknownDependency(String),
127
128    #[error("step failed: {0}")]
129    StepFailed(String),
130
131    #[error("cancelled")]
132    Cancelled,
133
134    /// Job placement (scheduler decide + admission) failed. Carries the typed
135    /// [`WorkerError`] so callers/retry-policy can distinguish "retry later,
136    /// queue pressure" (`QueueFull`/`QueueTimeout`) from "terminal, over-quota
137    /// or unauthorized" (`AdmissionDenied`/`Unauthorized`/`AdmissionInfeasible`)
138    /// instead of a flattened string.
139    #[error("job placement failed: {0}")]
140    PlacementFailed(#[from] crate::error::WorkerError),
141
142    #[error("vfs error: {0}")]
143    VfsError(String),
144
145    #[error("tcl error: {0}")]
146    TclError(String),
147
148    #[error("join error: {0}")]
149    JoinError(String),
150}
151
152type RunnerResult<T> = std::result::Result<T, RunnerError>;
153
154// ─────────────────────────────────────────────────────────────────────────────
155// WorkflowRunner
156// ─────────────────────────────────────────────────────────────────────────────
157
158/// General-purpose workflow execution engine.
159///
160/// Resolves action steps through VFS `/bin/` ctl files and evaluates
161/// script steps via TclShell on `spawn_blocking` threads. Concurrent
162/// script-executing jobs are capped by `script_semaphore`.
163///
164/// # Scheduling (#527)
165///
166/// Each job is routed through an optional [`JobScheduler`]: `runs_on:` +
167/// `resources:` decide in-proc execution (unchanged) vs. a real sandbox
168/// reservation acquired through the #525 P2 admission engine. With no
169/// scheduler configured (the default), every job runs in-proc exactly as
170/// before this change — no regression for existing callers.
171///
172/// # Run-level concurrency + timeouts (#521)
173///
174/// `run_semaphore` bounds concurrent workflow *runs* (`WorkflowConfig::
175/// max_concurrent_runs`) — a different axis from `script_semaphore`, which
176/// bounds concurrent script-executing *jobs* within/across runs.
177/// `job_timeout`/`step_timeout` (`job_timeout_secs`/`step_timeout_secs`) wrap
178/// job and step execution in `tokio::time::timeout`; a job's own
179/// `timeout-minutes:` overrides `job_timeout` when present.
180#[derive(Clone)]
181pub struct WorkflowRunner {
182    ns: Arc<Namespace>,
183    script_semaphore: Arc<Semaphore>,
184    run_semaphore: Arc<Semaphore>,
185    job_timeout: Duration,
186    step_timeout: Duration,
187    scheduler: Option<Arc<JobScheduler>>,
188    candidate_worktree: Option<CandidateWorktree>,
189}
190
191impl WorkflowRunner {
192    /// Create a new WorkflowRunner with `WorkflowConfig::default()` timeouts/
193    /// concurrency and no job scheduler (every job runs in-proc).
194    ///
195    /// * `ns` — the root namespace (mounts for `/bin/`, `/env/`, `/srv/`, etc.)
196    /// * Default concurrency cap: 64 concurrent script-executing jobs.
197    pub fn new(ns: Arc<Namespace>) -> Self {
198        Self::with_config(ns, &WorkflowConfig::default())
199    }
200
201    /// Create a WorkflowRunner honoring `config`'s `max_concurrent_runs` +
202    /// `job_timeout_secs`/`step_timeout_secs` (#521).
203    pub fn with_config(ns: Arc<Namespace>, config: &WorkflowConfig) -> Self {
204        Self {
205            ns,
206            script_semaphore: Arc::new(Semaphore::new(64)),
207            // A zero-permit semaphore would deadlock every run forever;
208            // treat a misconfigured 0 as "at least 1" rather than a silent hang.
209            run_semaphore: Arc::new(Semaphore::new(config.max_concurrent_runs.max(1))),
210            job_timeout: Duration::from_secs(config.job_timeout_secs),
211            step_timeout: Duration::from_secs(config.step_timeout_secs),
212            scheduler: None,
213            candidate_worktree: None,
214        }
215    }
216
217    /// Attach a [`JobScheduler`] (#527): jobs whose `runs_on:` requests
218    /// isolation are placed onto its sandbox pool instead of running in-proc.
219    pub fn with_scheduler(mut self, scheduler: Arc<JobScheduler>) -> Self {
220        self.scheduler = Some(scheduler);
221        self
222    }
223
224    /// Bind a git2db-composed candidate worktree into each job namespace at
225    /// `/src`. The supplied host path is never exposed directly: every job gets
226    /// a distinct copy-up overlay and the bind is mediated as the job Subject.
227    pub fn with_candidate_worktree(mut self, candidate: CandidateWorktree) -> Self {
228        self.candidate_worktree = Some(candidate);
229        self
230    }
231
232    /// Run a complete workflow.
233    ///
234    /// Jobs are topologically sorted by `needs:` into execution waves.
235    /// Independent jobs in each wave run concurrently. The `cancel` token
236    /// is checked between waves and propagated to individual jobs.
237    ///
238    /// Bounded by `run_semaphore` (#521's `max_concurrent_runs`): a run that
239    /// can't get a permit waits here, honoring `cancel` while waiting.
240    pub async fn run(
241        &self,
242        workflow: &Workflow,
243        inputs: HashMap<String, String>,
244        subject: &Subject,
245        cancel: CancellationToken,
246    ) -> RunnerResult<WorkflowRunResult> {
247        // Validate inputs: reject _-prefixed keys (provenance injection prevention).
248        validate_inputs(&inputs)?;
249
250        // #521: bound concurrent workflow RUNS. Held for the whole run.
251        let _run_permit = tokio::select! {
252            permit = Arc::clone(&self.run_semaphore).acquire_owned() => {
253                permit.map_err(|_| RunnerError::Cancelled)?
254            }
255            _ = cancel.cancelled() => return Err(RunnerError::Cancelled),
256        };
257
258        // Topological sort jobs into execution waves.
259        let waves = topological_sort(&workflow.jobs)?;
260
261        let mut all_jobs = HashMap::new();
262        let mut all_success = true;
263
264        for wave in waves {
265            if cancel.is_cancelled() {
266                return Err(RunnerError::Cancelled);
267            }
268
269            let mut join_set = JoinSet::new();
270
271            for job_name in wave {
272                // Job names come from topological_sort which iterates jobs.keys(),
273                // so this lookup cannot fail. Continue defensively to satisfy clippy.
274                let Some(job) = workflow.jobs.get(&job_name).cloned() else {
275                    continue;
276                };
277                let ns = Arc::clone(&self.ns);
278                let sem = Arc::clone(&self.script_semaphore);
279                let scheduler = self.scheduler.clone();
280                let candidate_worktree = self.candidate_worktree.clone();
281                let job_timeout = job
282                    .timeout_minutes
283                    .map(|m| Duration::from_secs(u64::from(m) * 60))
284                    .unwrap_or(self.job_timeout);
285                let step_timeout = self.step_timeout;
286                let subject = subject.clone();
287                let cancel = cancel.clone();
288                let env = merge_env(&workflow.env, &job.env);
289                let inputs = inputs.clone();
290                let name = job_name.clone();
291
292                join_set.spawn(async move {
293                    let result = run_job(
294                        ns,
295                        sem,
296                        scheduler,
297                        candidate_worktree,
298                        &job,
299                        &name,
300                        env,
301                        &inputs,
302                        &subject,
303                        cancel,
304                        job_timeout,
305                        step_timeout,
306                    )
307                    .await;
308                    (name, result)
309                });
310            }
311
312            while let Some(result) = join_set.join_next().await {
313                match result {
314                    Ok((name, Ok(job_result))) => {
315                        if !job_result.success {
316                            all_success = false;
317                        }
318                        all_jobs.insert(name, job_result);
319                    }
320                    Ok((name, Err(e))) => {
321                        all_success = false;
322                        all_jobs.insert(
323                            name.clone(),
324                            JobRunResult {
325                                name,
326                                success: false,
327                                steps: vec![StepRunResult {
328                                    name: "<error>".to_owned(),
329                                    success: false,
330                                    output: e.to_string(),
331                                }],
332                            },
333                        );
334                    }
335                    Err(e) => {
336                        all_success = false;
337                        tracing::error!("Job task panicked: {e}");
338                    }
339                }
340            }
341
342            // If any job in this wave failed, stop executing subsequent waves.
343            if !all_success {
344                break;
345            }
346        }
347
348        Ok(WorkflowRunResult {
349            success: all_success,
350            jobs: all_jobs,
351        })
352    }
353}
354
355// ─────────────────────────────────────────────────────────────────────────────
356// Job execution
357// ─────────────────────────────────────────────────────────────────────────────
358
359/// Route `job` through the scheduler (#527), then run its steps within
360/// `job_timeout`, releasing the placement afterward regardless of outcome.
361#[allow(clippy::too_many_arguments)]
362async fn run_job(
363    ns: Arc<Namespace>,
364    semaphore: Arc<Semaphore>,
365    scheduler: Option<Arc<JobScheduler>>,
366    candidate_worktree: Option<CandidateWorktree>,
367    job: &Job,
368    job_name: &str,
369    env: HashMap<String, String>,
370    inputs: &HashMap<String, String>,
371    subject: &Subject,
372    cancel: CancellationToken,
373    job_timeout: Duration,
374    step_timeout: Duration,
375) -> RunnerResult<JobRunResult> {
376    // #527: decide + (if isolation is requested) acquire a sandbox
377    // reservation before running any steps. `None` scheduler ⇒ always
378    // in-proc (pre-#527 behavior, unchanged).
379    //
380    // The placement sits outside `job_timeout` and any step-level cancel race,
381    // so it would otherwise wait up to `queue_timeout_secs` in the admission
382    // queue even for an already-cancelled workflow. Race it against the
383    // cancellation token so a cancel aborts the queue wait promptly, and
384    // propagate the typed `WorkerError` (via `PlacementFailed`) instead of
385    // flattening it into a string.
386    let placement = match &scheduler {
387        Some(scheduler) => {
388            let place_fut = scheduler.place(job_name, &job.runs_on, &job.resources, subject);
389            tokio::select! {
390                biased;
391                _ = cancel.cancelled() => return Err(RunnerError::Cancelled),
392                result = place_fut => result?,
393            }
394        }
395        None => Placement::InProc,
396    };
397
398    // #521: bound job execution wall-clock time. A job's own
399    // `timeout-minutes:` already folded into `job_timeout` by the caller.
400    let outcome = tokio::time::timeout(
401        job_timeout,
402        run_job_steps(
403            ns,
404            semaphore,
405            candidate_worktree,
406            job,
407            job_name,
408            env,
409            inputs,
410            subject,
411            cancel,
412            step_timeout,
413        ),
414    )
415    .await;
416
417    placement.release().await;
418
419    match outcome {
420        Ok(result) => result,
421        Err(_elapsed) => Ok(JobRunResult {
422            name: job_name.to_owned(),
423            success: false,
424            steps: vec![StepRunResult {
425                name: "<timeout>".to_owned(),
426                success: false,
427                output: format!("job timed out after {}s", job_timeout.as_secs()),
428            }],
429        }),
430    }
431}
432
433/// Execute a single job's steps sequentially. For script steps (`run:`), a
434/// single TclShell is reused across all sequential steps in the same job
435/// (constructed inside `spawn_blocking`). Each step is individually bounded
436/// by `step_timeout` (#521).
437#[allow(clippy::too_many_arguments)]
438async fn run_job_steps(
439    ns: Arc<Namespace>,
440    semaphore: Arc<Semaphore>,
441    candidate_worktree: Option<CandidateWorktree>,
442    job: &Job,
443    job_name: &str,
444    env: HashMap<String, String>,
445    inputs: &HashMap<String, String>,
446    subject: &Subject,
447    cancel: CancellationToken,
448    step_timeout: Duration,
449) -> RunnerResult<JobRunResult> {
450    let mut step_results = Vec::new();
451    let mut job_success = true;
452
453    // Partition steps into action steps and script steps.
454    // We process them sequentially, but script steps share a TclShell.
455    let has_scripts = job.steps.iter().any(|s| s.run.is_some());
456
457    if has_scripts {
458        // Acquire semaphore permit for script execution.
459        let _permit = semaphore
460            .acquire()
461            .await
462            .map_err(|_| RunnerError::Cancelled)?;
463
464        // Fork namespace for script isolation: remove /config, /private, /srv.
465        let mut forked = ns.fork();
466        let _ = forked.unmount("/config");
467        let _ = forked.unmount("/private");
468        let _ = forked.unmount("/srv");
469
470        // Candidate source enters only this forked, Subject-scoped namespace.
471        // Its lower is the git2db materialization; writes copy up into a
472        // job-local overlay retained until the job completes.
473        let _candidate_scratch = if let Some(candidate) = candidate_worktree {
474            let (mount, scratch) = candidate.mount_for_job()?;
475            forked
476                .bind_mount_as("/src", mount, BindFlag::Replace, subject)
477                .await
478                .map_err(|e| RunnerError::VfsError(format!("bind candidate at /src: {e}")))?;
479            Some(scratch)
480        } else {
481            None
482        };
483
484        // Write env vars to /env/* in forked namespace.
485        // We use an in-memory mount for env vars.
486        let env_mount = Arc::new(EnvMount::new(&env, inputs));
487        forked
488            .mount("/env", env_mount)
489            .map_err(|e| RunnerError::VfsError(e.to_string()))?;
490
491        let forked = Arc::new(forked);
492
493        // Collect steps to process on the blocking thread.
494        let steps: Vec<Step> = job.steps.clone();
495        let subject_clone = subject.clone();
496        let ns_for_actions = Arc::clone(&ns);
497        let subject_for_actions = subject.clone();
498
499        // Spawn a single blocking thread with one TclShell for the job.
500        // Script steps are sent to it via channel; action steps run on async runtime.
501        // The thread runs a current-thread tokio runtime + LocalSet so TclShell
502        // (which is !Send due to Rc in molt Value) can .await VFS operations directly.
503        let (script_tx, mut script_rx) = tokio::sync::mpsc::channel::<(
504            String,                                               // script
505            HashMap<String, String>,                              // step env
506            tokio::sync::oneshot::Sender<Result<String, String>>, // reply
507        )>(1);
508
509        let shell_handle = {
510            let ns = Arc::clone(&forked);
511            let subject_clone = subject_clone.clone();
512            #[allow(clippy::expect_used)] // Runtime creation failure is unrecoverable
513            tokio::task::spawn_blocking(move || {
514                let rt = tokio::runtime::Builder::new_current_thread()
515                    .enable_all()
516                    .build()
517                    .expect("tcl runner runtime");
518                let local = tokio::task::LocalSet::new();
519                local.block_on(&rt, async {
520                    let mut shell = TclShell::new(subject_clone, ns);
521                    while let Some((script, step_env, reply)) = script_rx.recv().await {
522                        for (k, v) in &step_env {
523                            let _ = shell.set_env(k, v);
524                        }
525                        let result = shell.eval(&script).await;
526                        let _ = reply.send(result);
527                    }
528                });
529            })
530        };
531
532        for (i, step) in steps.iter().enumerate() {
533            if cancel.is_cancelled() {
534                drop(script_tx);
535                let _ = shell_handle.await;
536                return Err(RunnerError::Cancelled);
537            }
538
539            let step_name = step.name.clone().unwrap_or_else(|| format!("step-{}", i));
540
541            if let Some(ref action) = step.uses {
542                // Action step — runs on async runtime, bounded by step_timeout (#521).
543                let result = match tokio::time::timeout(
544                    step_timeout,
545                    run_action_step(action, &step.with, &ns_for_actions, &subject_for_actions),
546                )
547                .await
548                {
549                    Ok(r) => r,
550                    Err(_elapsed) => Err(RunnerError::StepFailed(format!(
551                        "step timed out after {}s",
552                        step_timeout.as_secs()
553                    ))),
554                };
555                match result {
556                    Ok(output) => {
557                        step_results.push(StepRunResult {
558                            name: step_name,
559                            success: true,
560                            output,
561                        });
562                    }
563                    Err(e) => {
564                        let failed = StepRunResult {
565                            name: step_name,
566                            success: false,
567                            output: e.to_string(),
568                        };
569                        step_results.push(failed);
570                        if !step.continue_on_error {
571                            job_success = false;
572                            break;
573                        }
574                    }
575                }
576            } else if let Some(ref script) = step.run {
577                // Script step — send to the shared TclShell on the blocking thread.
578                // Uses tokio channels: async send + oneshot reply to avoid blocking
579                // tokio worker threads.
580                let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
581                if script_tx
582                    .send((script.clone(), step.env.clone(), reply_tx))
583                    .await
584                    .is_err()
585                {
586                    step_results.push(StepRunResult {
587                        name: step_name,
588                        success: false,
589                        output: "TclShell thread exited".into(),
590                    });
591                    job_success = false;
592                    break;
593                }
594                // Bounded by step_timeout (#521); on elapsed the reply is simply
595                // dropped — the shell thread will finish the eval on its own time
596                // and move on to the next queued script (or idle if none comes).
597                match tokio::time::timeout(step_timeout, reply_rx).await {
598                    Ok(Ok(Ok(output))) => {
599                        step_results.push(StepRunResult {
600                            name: step_name,
601                            success: true,
602                            output,
603                        });
604                    }
605                    Ok(Ok(Err(e))) => {
606                        let failed = StepRunResult {
607                            name: step_name,
608                            success: false,
609                            output: e,
610                        };
611                        step_results.push(failed);
612                        if !step.continue_on_error {
613                            job_success = false;
614                            break;
615                        }
616                    }
617                    Ok(Err(_)) => {
618                        step_results.push(StepRunResult {
619                            name: step_name,
620                            success: false,
621                            output: "TclShell thread panicked".into(),
622                        });
623                        job_success = false;
624                        break;
625                    }
626                    Err(_elapsed) => {
627                        step_results.push(StepRunResult {
628                            name: step_name,
629                            success: false,
630                            output: format!("step timed out after {}s", step_timeout.as_secs()),
631                        });
632                        job_success = false;
633                        break;
634                    }
635                }
636            }
637        }
638
639        // Drop the sender to signal the shell thread to exit, then wait for it.
640        drop(script_tx);
641        let _ = shell_handle.await;
642    } else {
643        // No script steps — only action steps, run on async runtime.
644        for (i, step) in job.steps.iter().enumerate() {
645            if cancel.is_cancelled() {
646                return Err(RunnerError::Cancelled);
647            }
648
649            let step_name = step.name.clone().unwrap_or_else(|| format!("step-{}", i));
650
651            if let Some(ref action) = step.uses {
652                // Bounded by step_timeout (#521).
653                let result = match tokio::time::timeout(
654                    step_timeout,
655                    run_action_step(action, &step.with, &ns, subject),
656                )
657                .await
658                {
659                    Ok(r) => r,
660                    Err(_elapsed) => Err(RunnerError::StepFailed(format!(
661                        "step timed out after {}s",
662                        step_timeout.as_secs()
663                    ))),
664                };
665                match result {
666                    Ok(output) => {
667                        step_results.push(StepRunResult {
668                            name: step_name,
669                            success: true,
670                            output,
671                        });
672                    }
673                    Err(e) => {
674                        let failed = StepRunResult {
675                            name: step_name,
676                            success: false,
677                            output: e.to_string(),
678                        };
679                        step_results.push(failed);
680                        if !step.continue_on_error {
681                            job_success = false;
682                            break;
683                        }
684                    }
685                }
686            }
687        }
688    }
689
690    Ok(JobRunResult {
691        name: job_name.to_owned(),
692        success: job_success,
693        steps: step_results,
694    })
695}
696
697// ─────────────────────────────────────────────────────────────────────────────
698// Action step execution
699// ─────────────────────────────────────────────────────────────────────────────
700
701/// Run an action step by writing JSON of `with:` map to `/bin/{action}` ctl file.
702///
703/// `uses: solver/submit@v1` → strip `@v1`, validate name, write to `/bin/solver/submit`.
704async fn run_action_step(
705    action: &str,
706    with: &HashMap<String, String>,
707    ns: &Namespace,
708    subject: &Subject,
709) -> RunnerResult<String> {
710    // Strip @version suffix.
711    let action_name = action.split('@').next().unwrap_or(action);
712
713    // Validate action name.
714    validate_action_name(action_name)?;
715
716    // Build JSON payload from `with:` map.
717    let payload = serde_json::to_vec(with).map_err(|e| RunnerError::VfsError(e.to_string()))?;
718
719    // Write to /bin/{action} ctl file.
720    let ctl_path = format!("/bin/{action_name}");
721    let response = ns
722        .ctl(&ctl_path, &payload, subject)
723        .await
724        .map_err(|e| RunnerError::VfsError(e.to_string()))?;
725
726    Ok(String::from_utf8_lossy(&response).into_owned())
727}
728
729// ─────────────────────────────────────────────────────────────────────────────
730// Validation helpers
731// ─────────────────────────────────────────────────────────────────────────────
732
733/// Validate action name: no `..`, only alphanumeric + `/` + `-` + `_`.
734fn validate_action_name(name: &str) -> RunnerResult<()> {
735    if name.contains("..") {
736        return Err(RunnerError::InvalidActionName(format!(
737            "path traversal in action name: {name}"
738        )));
739    }
740    if name.is_empty() {
741        return Err(RunnerError::InvalidActionName("empty action name".into()));
742    }
743    for ch in name.chars() {
744        if !ch.is_alphanumeric() && ch != '/' && ch != '-' && ch != '_' {
745            return Err(RunnerError::InvalidActionName(format!(
746                "invalid character '{ch}' in action name: {name}"
747            )));
748        }
749    }
750    Ok(())
751}
752
753/// Validate input keys: reject `_`-prefixed keys (provenance injection prevention)
754/// and non-alphanumeric characters (prevent path traversal when prefixed with `INPUT_`).
755fn validate_inputs(inputs: &HashMap<String, String>) -> RunnerResult<()> {
756    for key in inputs.keys() {
757        if key.starts_with('_') {
758            return Err(RunnerError::InvalidInputKey(format!(
759                "input key must not start with '_': {key}"
760            )));
761        }
762        // Validate characters so INPUT_{key} is safe as a VFS path component.
763        for ch in key.chars() {
764            if !ch.is_alphanumeric() && ch != '_' && ch != '-' {
765                return Err(RunnerError::InvalidInputKey(format!(
766                    "invalid character '{ch}' in input key: {key}"
767                )));
768            }
769        }
770    }
771    Ok(())
772}
773
774/// Validate env var name: only alphanumeric + `_` + `-`.
775fn validate_env_var_name(name: &str) -> RunnerResult<()> {
776    if name.is_empty() {
777        return Err(RunnerError::InvalidEnvVarName("empty env var name".into()));
778    }
779    for ch in name.chars() {
780        if !ch.is_alphanumeric() && ch != '_' && ch != '-' {
781            return Err(RunnerError::InvalidEnvVarName(format!(
782                "invalid character '{ch}' in env var name: {name}"
783            )));
784        }
785    }
786    Ok(())
787}
788
789// ─────────────────────────────────────────────────────────────────────────────
790// Topological sort
791// ─────────────────────────────────────────────────────────────────────────────
792
793/// Topological sort of jobs by `needs:` into execution waves.
794///
795/// Returns a Vec of waves, where each wave is a Vec of job names that
796/// can execute concurrently. Jobs in wave N+1 depend on jobs in wave N or earlier.
797fn topological_sort(jobs: &HashMap<String, Job>) -> RunnerResult<Vec<Vec<String>>> {
798    let job_names: HashSet<&str> = jobs.keys().map(String::as_str).collect();
799
800    // Validate all dependencies exist.
801    for (name, job) in jobs {
802        if let Some(ref needs) = job.needs {
803            for dep in needs {
804                if !job_names.contains(dep.as_str()) {
805                    return Err(RunnerError::UnknownDependency(format!(
806                        "job '{name}' depends on unknown job '{dep}'"
807                    )));
808                }
809            }
810        }
811    }
812
813    // Kahn's algorithm.
814    let mut in_degree: HashMap<&str, usize> = HashMap::new();
815    let mut dependents: HashMap<&str, Vec<&str>> = HashMap::new();
816
817    for name in jobs.keys() {
818        in_degree.entry(name.as_str()).or_insert(0);
819        dependents.entry(name.as_str()).or_default();
820    }
821
822    for (name, job) in jobs {
823        if let Some(ref needs) = job.needs {
824            *in_degree.entry(name.as_str()).or_insert(0) += needs.len();
825            for dep in needs {
826                dependents
827                    .entry(dep.as_str())
828                    .or_default()
829                    .push(name.as_str());
830            }
831        }
832    }
833
834    let mut waves = Vec::new();
835    let mut queue: VecDeque<&str> = in_degree
836        .iter()
837        .filter(|(_, &deg)| deg == 0)
838        .map(|(&name, _)| name)
839        .collect();
840
841    let mut processed = 0;
842
843    while !queue.is_empty() {
844        let mut wave: Vec<String> = Vec::new();
845        let mut next_queue = VecDeque::new();
846
847        for &name in &queue {
848            wave.push(name.to_owned());
849            processed += 1;
850
851            for &dependent in dependents.get(name).unwrap_or(&Vec::new()) {
852                let Some(deg) = in_degree.get_mut(dependent) else {
853                    continue;
854                };
855                *deg -= 1;
856                if *deg == 0 {
857                    next_queue.push_back(dependent);
858                }
859            }
860        }
861
862        // Sort wave for deterministic ordering in tests.
863        wave.sort();
864        waves.push(wave);
865        queue = next_queue;
866    }
867
868    if processed != jobs.len() {
869        return Err(RunnerError::DependencyCycle);
870    }
871
872    Ok(waves)
873}
874
875// ─────────────────────────────────────────────────────────────────────────────
876// Env merge + mount
877// ─────────────────────────────────────────────────────────────────────────────
878
879/// Merge workflow-level and job-level env vars (job overrides workflow).
880fn merge_env(
881    workflow_env: &HashMap<String, String>,
882    job_env: &HashMap<String, String>,
883) -> HashMap<String, String> {
884    let mut merged = workflow_env.clone();
885    merged.extend(job_env.iter().map(|(k, v)| (k.clone(), v.clone())));
886    merged
887}
888
889/// Simple in-memory Mount that serves env vars and inputs as files.
890///
891/// Files at `/env/{KEY}` return the value as bytes.
892/// Inputs are stored with `INPUT_` prefix.
893struct EnvMount {
894    vars: HashMap<String, Vec<u8>>,
895}
896
897impl EnvMount {
898    fn new(env: &HashMap<String, String>, inputs: &HashMap<String, String>) -> Self {
899        let mut vars = HashMap::new();
900        for (k, v) in env {
901            if validate_env_var_name(k).is_ok() {
902                vars.insert(k.clone(), v.as_bytes().to_vec());
903            }
904        }
905        for (k, v) in inputs {
906            vars.insert(format!("INPUT_{k}"), v.as_bytes().to_vec());
907        }
908        Self { vars }
909    }
910}
911
912/// Fid type for EnvMount.
913struct EnvFid {
914    path: String,
915}
916
917#[async_trait::async_trait]
918impl hyprstream_vfs::Mount for EnvMount {
919    async fn walk(
920        &self,
921        components: &[&str],
922        _caller: &Subject,
923    ) -> Result<hyprstream_vfs::Fid, hyprstream_vfs::MountError> {
924        let path = components.join("/");
925        Ok(hyprstream_vfs::Fid::new(EnvFid { path }))
926    }
927
928    async fn open(
929        &self,
930        _fid: &mut hyprstream_vfs::Fid,
931        _mode: u8,
932        _caller: &Subject,
933    ) -> Result<(), hyprstream_vfs::MountError> {
934        Ok(())
935    }
936
937    async fn read(
938        &self,
939        fid: &hyprstream_vfs::Fid,
940        offset: u64,
941        _count: u32,
942        _caller: &Subject,
943    ) -> Result<Vec<u8>, hyprstream_vfs::MountError> {
944        let inner = fid
945            .downcast_ref::<EnvFid>()
946            .ok_or_else(|| hyprstream_vfs::MountError::InvalidArgument("bad fid".into()))?;
947        match self.vars.get(&inner.path) {
948            Some(data) => {
949                let start = offset as usize;
950                if start >= data.len() {
951                    Ok(vec![])
952                } else {
953                    Ok(data[start..].to_vec())
954                }
955            }
956            None => Err(hyprstream_vfs::MountError::NotFound(inner.path.clone())),
957        }
958    }
959
960    async fn write(
961        &self,
962        _fid: &hyprstream_vfs::Fid,
963        _offset: u64,
964        data: &[u8],
965        _caller: &Subject,
966    ) -> Result<u32, hyprstream_vfs::MountError> {
967        Ok(data.len() as u32)
968    }
969
970    async fn readdir(
971        &self,
972        _fid: &hyprstream_vfs::Fid,
973        _caller: &Subject,
974    ) -> Result<Vec<hyprstream_vfs::DirEntry>, hyprstream_vfs::MountError> {
975        Ok(self
976            .vars
977            .keys()
978            .map(|k| hyprstream_vfs::DirEntry {
979                name: k.clone(),
980                is_dir: false,
981                size: 0,
982                stat: None,
983            })
984            .collect())
985    }
986
987    async fn stat(
988        &self,
989        fid: &hyprstream_vfs::Fid,
990        _caller: &Subject,
991    ) -> Result<hyprstream_vfs::Stat, hyprstream_vfs::MountError> {
992        let inner = fid
993            .downcast_ref::<EnvFid>()
994            .ok_or_else(|| hyprstream_vfs::MountError::InvalidArgument("bad fid".into()))?;
995        Ok(hyprstream_vfs::Stat::unknown_qid(
996            0,
997            0,
998            inner.path.clone(),
999            0,
1000        ))
1001    }
1002
1003    async fn clunk(&self, _fid: hyprstream_vfs::Fid, _caller: &Subject) {}
1004}
1005
1006// ─────────────────────────────────────────────────────────────────────────────
1007// Tests
1008// ─────────────────────────────────────────────────────────────────────────────
1009
1010#[cfg(test)]
1011#[allow(clippy::unwrap_used, clippy::disallowed_types)]
1012mod tests {
1013    use super::*;
1014    use async_trait::async_trait;
1015    use hyprstream_vfs::{DirEntry, Fid, Mount, MountError, Stat};
1016    use std::sync::Arc;
1017
1018    /// In-memory mount that supports ctl pattern (write→read).
1019    struct MemMount {
1020        files: HashMap<String, Vec<u8>>,
1021    }
1022
1023    impl MemMount {
1024        fn new(files: Vec<(&str, &[u8])>) -> Self {
1025            Self {
1026                files: files
1027                    .into_iter()
1028                    .map(|(k, v)| (k.to_owned(), v.to_vec()))
1029                    .collect(),
1030            }
1031        }
1032    }
1033
1034    struct MemFid {
1035        path: String,
1036        write_buf: std::sync::Mutex<Vec<u8>>,
1037    }
1038
1039    #[async_trait]
1040    impl Mount for MemMount {
1041        async fn walk(&self, components: &[&str], _caller: &Subject) -> Result<Fid, MountError> {
1042            let path = components.join("/");
1043            Ok(Fid::new(MemFid {
1044                path,
1045                write_buf: std::sync::Mutex::new(Vec::new()),
1046            }))
1047        }
1048
1049        async fn open(
1050            &self,
1051            _fid: &mut Fid,
1052            _mode: u8,
1053            _caller: &Subject,
1054        ) -> Result<(), MountError> {
1055            Ok(())
1056        }
1057
1058        async fn read(
1059            &self,
1060            fid: &Fid,
1061            offset: u64,
1062            _count: u32,
1063            _caller: &Subject,
1064        ) -> Result<Vec<u8>, MountError> {
1065            let inner = fid
1066                .downcast_ref::<MemFid>()
1067                .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
1068            // If there's a write buffer (ctl pattern), return that.
1069            let wb = inner.write_buf.lock().unwrap();
1070            if !wb.is_empty() {
1071                let start = offset as usize;
1072                if start >= wb.len() {
1073                    return Ok(vec![]);
1074                }
1075                return Ok(wb[start..].to_vec());
1076            }
1077            drop(wb);
1078            match self.files.get(&inner.path) {
1079                Some(data) => {
1080                    let start = offset as usize;
1081                    if start >= data.len() {
1082                        Ok(vec![])
1083                    } else {
1084                        Ok(data[start..].to_vec())
1085                    }
1086                }
1087                None => Err(MountError::NotFound(inner.path.clone())),
1088            }
1089        }
1090
1091        async fn write(
1092            &self,
1093            fid: &Fid,
1094            _offset: u64,
1095            data: &[u8],
1096            _caller: &Subject,
1097        ) -> Result<u32, MountError> {
1098            let inner = fid
1099                .downcast_ref::<MemFid>()
1100                .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
1101            *inner.write_buf.lock().unwrap() =
1102                format!("ok: {}", String::from_utf8_lossy(data)).into_bytes();
1103            Ok(data.len() as u32)
1104        }
1105
1106        async fn readdir(&self, fid: &Fid, _caller: &Subject) -> Result<Vec<DirEntry>, MountError> {
1107            let inner = fid
1108                .downcast_ref::<MemFid>()
1109                .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
1110            let prefix = if inner.path.is_empty() {
1111                String::new()
1112            } else {
1113                format!("{}/", inner.path)
1114            };
1115            let mut entries = Vec::new();
1116            for key in self.files.keys() {
1117                if let Some(rest) = key.strip_prefix(&prefix) {
1118                    if !rest.contains('/') {
1119                        entries.push(DirEntry {
1120                            name: rest.to_owned(),
1121                            is_dir: false,
1122                            size: 0,
1123                            stat: None,
1124                        });
1125                    }
1126                } else if inner.path.is_empty() && !key.contains('/') {
1127                    entries.push(DirEntry {
1128                        name: key.clone(),
1129                        is_dir: false,
1130                        size: 0,
1131                        stat: None,
1132                    });
1133                }
1134            }
1135            Ok(entries)
1136        }
1137
1138        async fn stat(&self, fid: &Fid, _caller: &Subject) -> Result<Stat, MountError> {
1139            let inner = fid
1140                .downcast_ref::<MemFid>()
1141                .ok_or_else(|| MountError::InvalidArgument("bad fid".into()))?;
1142            Ok(Stat::unknown_qid(0, 0, inner.path.clone(), 0))
1143        }
1144
1145        async fn clunk(&self, _fid: Fid, _caller: &Subject) {}
1146    }
1147
1148    /// Ctl-pattern mount whose `write` actually awaits (`tokio::time::sleep`)
1149    /// before answering "ok" — used to make `step_timeout`/`job_timeout`
1150    /// (#521) deterministically fire in tests: a step routed through this
1151    /// mount really pends, so `tokio::time::timeout` races it for real
1152    /// instead of the inner future completing synchronously on first poll.
1153    struct SlowMount {
1154        delay: std::time::Duration,
1155    }
1156
1157    struct SlowFid;
1158
1159    #[async_trait]
1160    impl Mount for SlowMount {
1161        async fn walk(&self, _components: &[&str], _caller: &Subject) -> Result<Fid, MountError> {
1162            Ok(Fid::new(SlowFid))
1163        }
1164        async fn open(
1165            &self,
1166            _fid: &mut Fid,
1167            _mode: u8,
1168            _caller: &Subject,
1169        ) -> Result<(), MountError> {
1170            Ok(())
1171        }
1172        async fn write(
1173            &self,
1174            _fid: &Fid,
1175            _offset: u64,
1176            _data: &[u8],
1177            _caller: &Subject,
1178        ) -> Result<u32, MountError> {
1179            tokio::time::sleep(self.delay).await;
1180            Ok(0)
1181        }
1182        async fn read(
1183            &self,
1184            _fid: &Fid,
1185            offset: u64,
1186            _count: u32,
1187            _caller: &Subject,
1188        ) -> Result<Vec<u8>, MountError> {
1189            if offset == 0 {
1190                Ok(b"ok".to_vec())
1191            } else {
1192                Ok(vec![])
1193            }
1194        }
1195        async fn readdir(
1196            &self,
1197            _fid: &Fid,
1198            _caller: &Subject,
1199        ) -> Result<Vec<DirEntry>, MountError> {
1200            Ok(vec![])
1201        }
1202        async fn stat(&self, _fid: &Fid, _caller: &Subject) -> Result<Stat, MountError> {
1203            Ok(Stat::unknown_qid(0, 0, String::new(), 0))
1204        }
1205        async fn clunk(&self, _fid: Fid, _caller: &Subject) {}
1206    }
1207
1208    /// Ctl-pattern mount that tracks concurrently-in-flight `write` calls (for
1209    /// the `max_concurrent_runs` / #521 test): bumps a shared counter, records
1210    /// the observed peak, sleeps briefly (so overlapping calls actually
1211    /// overlap), then decrements.
1212    struct TrackingMount {
1213        current: Arc<std::sync::atomic::AtomicUsize>,
1214        peak: Arc<std::sync::atomic::AtomicUsize>,
1215        delay: std::time::Duration,
1216    }
1217
1218    struct TrackingFid;
1219
1220    #[async_trait]
1221    impl Mount for TrackingMount {
1222        async fn walk(&self, _components: &[&str], _caller: &Subject) -> Result<Fid, MountError> {
1223            Ok(Fid::new(TrackingFid))
1224        }
1225        async fn open(
1226            &self,
1227            _fid: &mut Fid,
1228            _mode: u8,
1229            _caller: &Subject,
1230        ) -> Result<(), MountError> {
1231            Ok(())
1232        }
1233        async fn write(
1234            &self,
1235            _fid: &Fid,
1236            _offset: u64,
1237            _data: &[u8],
1238            _caller: &Subject,
1239        ) -> Result<u32, MountError> {
1240            use std::sync::atomic::Ordering;
1241            let cur = self.current.fetch_add(1, Ordering::SeqCst) + 1;
1242            self.peak.fetch_max(cur, Ordering::SeqCst);
1243            tokio::time::sleep(self.delay).await;
1244            self.current.fetch_sub(1, Ordering::SeqCst);
1245            Ok(0)
1246        }
1247        async fn read(
1248            &self,
1249            _fid: &Fid,
1250            offset: u64,
1251            _count: u32,
1252            _caller: &Subject,
1253        ) -> Result<Vec<u8>, MountError> {
1254            if offset == 0 {
1255                Ok(b"ok".to_vec())
1256            } else {
1257                Ok(vec![])
1258            }
1259        }
1260        async fn readdir(
1261            &self,
1262            _fid: &Fid,
1263            _caller: &Subject,
1264        ) -> Result<Vec<DirEntry>, MountError> {
1265            Ok(vec![])
1266        }
1267        async fn stat(&self, _fid: &Fid, _caller: &Subject) -> Result<Stat, MountError> {
1268            Ok(Stat::unknown_qid(0, 0, String::new(), 0))
1269        }
1270        async fn clunk(&self, _fid: Fid, _caller: &Subject) {}
1271    }
1272
1273    fn test_subject() -> Subject {
1274        Subject::new("test-runner")
1275    }
1276
1277    fn make_namespace() -> Arc<Namespace> {
1278        let mut ns = Namespace::new();
1279
1280        // /bin/ with ctl files for action step testing.
1281        let bin_mount = Arc::new(MemMount::new(vec![
1282            ("solver/submit", b""),
1283            ("deploy/run", b""),
1284        ]));
1285        ns.mount("/bin", bin_mount).unwrap();
1286
1287        // /config/ mount (will be unmounted in forked namespaces).
1288        let config_mount = Arc::new(MemMount::new(vec![("secret", b"hunter2")]));
1289        ns.mount("/config", config_mount).unwrap();
1290
1291        Arc::new(ns)
1292    }
1293
1294    /// A git2db materialized tree is exposed only through a fresh job overlay:
1295    /// reads see the candidate lower while writes copy up and leave the source
1296    /// tree untouched. This exercises the actual Plan 9 mount seam used by
1297    /// `run_job_steps`, not a host-path shortcut.
1298    #[tokio::test]
1299    async fn candidate_worktree_binds_at_src_with_job_local_copy_up() {
1300        let source = tempfile::tempdir().unwrap();
1301        let source_file = source.path().join("Cargo.toml");
1302        std::fs::write(&source_file, b"[package]\nname = \"candidate\"\n").unwrap();
1303
1304        let candidate = CandidateWorktree::from_host_path(source.path()).unwrap();
1305        let (mount, _scratch) = candidate.mount_for_job().unwrap();
1306        let mut ns = Namespace::new();
1307        let subject = test_subject();
1308        ns.bind_mount_as("/src", mount, BindFlag::Replace, &subject)
1309            .await
1310            .unwrap();
1311
1312        assert_eq!(
1313            ns.cat("/src/Cargo.toml", &subject).await.unwrap(),
1314            b"[package]\nname = \"candidate\"\n"
1315        );
1316        ns.echo(
1317            "/src/Cargo.toml",
1318            b"[package]\nname = \"job-local\"\n",
1319            &subject,
1320        )
1321        .await
1322        .unwrap();
1323        assert_eq!(
1324            ns.cat("/src/Cargo.toml", &subject).await.unwrap(),
1325            b"[package]\nname = \"job-local\"\n"
1326        );
1327        assert_eq!(
1328            std::fs::read(source_file).unwrap(),
1329            b"[package]\nname = \"candidate\"\n",
1330            "candidate source lower must never be mutated by a job"
1331        );
1332        assert!(
1333            ns.cat("/etc/passwd", &subject).await.is_err(),
1334            "the composed candidate mount must not confer ambient host-path access"
1335        );
1336    }
1337
1338    fn make_namespace_with_slow_action(delay: std::time::Duration) -> Arc<Namespace> {
1339        let mut ns = Namespace::new();
1340        let bin_mount = Arc::new(SlowMount { delay });
1341        ns.mount("/bin", bin_mount).unwrap();
1342        Arc::new(ns)
1343    }
1344
1345    fn make_namespace_with_tracking_action(
1346        current: Arc<std::sync::atomic::AtomicUsize>,
1347        peak: Arc<std::sync::atomic::AtomicUsize>,
1348        delay: std::time::Duration,
1349    ) -> Arc<Namespace> {
1350        let mut ns = Namespace::new();
1351        let bin_mount = Arc::new(TrackingMount {
1352            current,
1353            peak,
1354            delay,
1355        });
1356        ns.mount("/bin", bin_mount).unwrap();
1357        Arc::new(ns)
1358    }
1359
1360    fn single_action_job(action: &str) -> Job {
1361        Job {
1362            runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1363            needs: None,
1364            env: HashMap::new(),
1365            steps: vec![Step {
1366                name: Some("step".into()),
1367                id: None,
1368                uses: Some(action.to_owned()),
1369                run: None,
1370                shell: None,
1371                working_directory: None,
1372                with: HashMap::new(),
1373                env: HashMap::new(),
1374                condition: None,
1375                continue_on_error: false,
1376            }],
1377            condition: None,
1378            timeout_minutes: None,
1379            resources: super::super::parser::JobResources::default(),
1380        }
1381    }
1382
1383    fn make_simple_workflow(jobs: HashMap<String, Job>) -> Workflow {
1384        Workflow {
1385            name: "test-workflow".to_owned(),
1386            on: super::super::parser::WorkflowTrigger::Simple("push".to_owned()),
1387            env: HashMap::new(),
1388            jobs,
1389        }
1390    }
1391
1392    // ── Topological sort tests ───────────────────────────────────────────
1393
1394    #[test]
1395    fn topo_sort_independent_jobs() {
1396        let mut jobs = HashMap::new();
1397        jobs.insert(
1398            "a".to_owned(),
1399            Job {
1400                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1401                needs: None,
1402                env: HashMap::new(),
1403                steps: vec![],
1404                condition: None,
1405                timeout_minutes: None,
1406                resources: super::super::parser::JobResources::default(),
1407            },
1408        );
1409        jobs.insert(
1410            "b".to_owned(),
1411            Job {
1412                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1413                needs: None,
1414                env: HashMap::new(),
1415                steps: vec![],
1416                condition: None,
1417                timeout_minutes: None,
1418                resources: super::super::parser::JobResources::default(),
1419            },
1420        );
1421
1422        let waves = topological_sort(&jobs).unwrap();
1423        assert_eq!(waves.len(), 1);
1424        assert_eq!(waves[0].len(), 2);
1425    }
1426
1427    #[test]
1428    fn topo_sort_with_dependency() {
1429        let mut jobs = HashMap::new();
1430        jobs.insert(
1431            "build".to_owned(),
1432            Job {
1433                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1434                needs: None,
1435                env: HashMap::new(),
1436                steps: vec![],
1437                condition: None,
1438                timeout_minutes: None,
1439                resources: super::super::parser::JobResources::default(),
1440            },
1441        );
1442        jobs.insert(
1443            "test".to_owned(),
1444            Job {
1445                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1446                needs: Some(vec!["build".to_owned()]),
1447                env: HashMap::new(),
1448                steps: vec![],
1449                condition: None,
1450                timeout_minutes: None,
1451                resources: super::super::parser::JobResources::default(),
1452            },
1453        );
1454        jobs.insert(
1455            "deploy".to_owned(),
1456            Job {
1457                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1458                needs: Some(vec!["test".to_owned()]),
1459                env: HashMap::new(),
1460                steps: vec![],
1461                condition: None,
1462                timeout_minutes: None,
1463                resources: super::super::parser::JobResources::default(),
1464            },
1465        );
1466
1467        let waves = topological_sort(&jobs).unwrap();
1468        assert_eq!(waves.len(), 3);
1469        assert_eq!(waves[0], vec!["build"]);
1470        assert_eq!(waves[1], vec!["test"]);
1471        assert_eq!(waves[2], vec!["deploy"]);
1472    }
1473
1474    #[test]
1475    fn topo_sort_cycle_detected() {
1476        let mut jobs = HashMap::new();
1477        jobs.insert(
1478            "a".to_owned(),
1479            Job {
1480                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1481                needs: Some(vec!["b".to_owned()]),
1482                env: HashMap::new(),
1483                steps: vec![],
1484                condition: None,
1485                timeout_minutes: None,
1486                resources: super::super::parser::JobResources::default(),
1487            },
1488        );
1489        jobs.insert(
1490            "b".to_owned(),
1491            Job {
1492                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1493                needs: Some(vec!["a".to_owned()]),
1494                env: HashMap::new(),
1495                steps: vec![],
1496                condition: None,
1497                timeout_minutes: None,
1498                resources: super::super::parser::JobResources::default(),
1499            },
1500        );
1501
1502        let result = topological_sort(&jobs);
1503        assert!(matches!(result, Err(RunnerError::DependencyCycle)));
1504    }
1505
1506    // ── Validation tests ─────────────────────────────────────────────────
1507
1508    #[test]
1509    fn validate_action_name_rejects_dotdot() {
1510        assert!(validate_action_name("../etc/passwd").is_err());
1511        assert!(validate_action_name("foo/../bar").is_err());
1512    }
1513
1514    #[test]
1515    fn validate_action_name_accepts_valid() {
1516        assert!(validate_action_name("solver/submit").is_ok());
1517        assert!(validate_action_name("deploy-prod").is_ok());
1518        assert!(validate_action_name("my_action").is_ok());
1519    }
1520
1521    #[test]
1522    fn validate_action_name_rejects_special_chars() {
1523        assert!(validate_action_name("foo bar").is_err());
1524        assert!(validate_action_name("foo;bar").is_err());
1525    }
1526
1527    #[test]
1528    fn validate_inputs_rejects_underscore_prefix() {
1529        let mut inputs = HashMap::new();
1530        inputs.insert("_internal".to_owned(), "value".to_owned());
1531        assert!(matches!(
1532            validate_inputs(&inputs),
1533            Err(RunnerError::InvalidInputKey(_))
1534        ));
1535    }
1536
1537    #[test]
1538    fn validate_inputs_accepts_normal_keys() {
1539        let mut inputs = HashMap::new();
1540        inputs.insert("model".to_owned(), "qwen3".to_owned());
1541        inputs.insert("temperature".to_owned(), "0.7".to_owned());
1542        assert!(validate_inputs(&inputs).is_ok());
1543    }
1544
1545    // ── Integration tests ────────────────────────────────────────────────
1546
1547    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1548    async fn two_job_workflow_with_needs() {
1549        let ns = make_namespace();
1550        let runner = WorkflowRunner::new(Arc::clone(&ns));
1551
1552        let mut jobs = HashMap::new();
1553        jobs.insert(
1554            "build".to_owned(),
1555            Job {
1556                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1557                needs: None,
1558                env: HashMap::new(),
1559                steps: vec![Step {
1560                    name: Some("build-action".into()),
1561                    id: None,
1562                    uses: Some("solver/submit".into()),
1563                    run: None,
1564                    shell: None,
1565                    working_directory: None,
1566                    with: {
1567                        let mut m = HashMap::new();
1568                        m.insert("model".into(), "qwen3".into());
1569                        m
1570                    },
1571                    env: HashMap::new(),
1572                    condition: None,
1573                    continue_on_error: false,
1574                }],
1575                condition: None,
1576                timeout_minutes: None,
1577                resources: super::super::parser::JobResources::default(),
1578            },
1579        );
1580        jobs.insert(
1581            "deploy".to_owned(),
1582            Job {
1583                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1584                needs: Some(vec!["build".to_owned()]),
1585                env: HashMap::new(),
1586                steps: vec![Step {
1587                    name: Some("deploy-action".into()),
1588                    id: None,
1589                    uses: Some("deploy/run".into()),
1590                    run: None,
1591                    shell: None,
1592                    working_directory: None,
1593                    with: HashMap::new(),
1594                    env: HashMap::new(),
1595                    condition: None,
1596                    continue_on_error: false,
1597                }],
1598                condition: None,
1599                timeout_minutes: None,
1600                resources: super::super::parser::JobResources::default(),
1601            },
1602        );
1603
1604        let workflow = make_simple_workflow(jobs);
1605        let cancel = CancellationToken::new();
1606        let result = runner
1607            .run(&workflow, HashMap::new(), &test_subject(), cancel)
1608            .await
1609            .unwrap();
1610
1611        assert!(result.success);
1612        assert_eq!(result.jobs.len(), 2);
1613        assert!(result.jobs.contains_key("build"));
1614        assert!(result.jobs.contains_key("deploy"));
1615        assert!(result.jobs["build"].success);
1616        assert!(result.jobs["deploy"].success);
1617    }
1618
1619    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1620    async fn uses_step_resolves_to_vfs_bin() {
1621        let ns = make_namespace();
1622        let runner = WorkflowRunner::new(Arc::clone(&ns));
1623
1624        let mut jobs = HashMap::new();
1625        jobs.insert(
1626            "test".to_owned(),
1627            Job {
1628                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1629                needs: None,
1630                env: HashMap::new(),
1631                steps: vec![Step {
1632                    name: Some("submit".into()),
1633                    id: None,
1634                    uses: Some("solver/submit@v1".into()),
1635                    run: None,
1636                    shell: None,
1637                    working_directory: None,
1638                    with: {
1639                        let mut m = HashMap::new();
1640                        m.insert("problem".into(), "sat".into());
1641                        m
1642                    },
1643                    env: HashMap::new(),
1644                    condition: None,
1645                    continue_on_error: false,
1646                }],
1647                condition: None,
1648                timeout_minutes: None,
1649                resources: super::super::parser::JobResources::default(),
1650            },
1651        );
1652
1653        let workflow = make_simple_workflow(jobs);
1654        let cancel = CancellationToken::new();
1655        let result = runner
1656            .run(&workflow, HashMap::new(), &test_subject(), cancel)
1657            .await
1658            .unwrap();
1659
1660        assert!(result.success);
1661        let job = &result.jobs["test"];
1662        assert!(job.success);
1663        // The MemMount ctl pattern returns "ok: {payload}".
1664        assert!(job.steps[0].output.starts_with("ok: "));
1665    }
1666
1667    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1668    async fn run_step_evaluates_tcl_on_spawn_blocking() {
1669        let ns = make_namespace();
1670        let runner = WorkflowRunner::new(Arc::clone(&ns));
1671
1672        let mut jobs = HashMap::new();
1673        jobs.insert(
1674            "script-job".to_owned(),
1675            Job {
1676                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1677                needs: None,
1678                env: HashMap::new(),
1679                steps: vec![Step {
1680                    name: Some("compute".into()),
1681                    id: None,
1682                    uses: None,
1683                    run: Some("expr {2 + 3}".into()),
1684                    shell: None,
1685                    working_directory: None,
1686                    with: HashMap::new(),
1687                    env: HashMap::new(),
1688                    condition: None,
1689                    continue_on_error: false,
1690                }],
1691                condition: None,
1692                timeout_minutes: None,
1693                resources: super::super::parser::JobResources::default(),
1694            },
1695        );
1696
1697        let workflow = make_simple_workflow(jobs);
1698        let cancel = CancellationToken::new();
1699        let result = runner
1700            .run(&workflow, HashMap::new(), &test_subject(), cancel)
1701            .await
1702            .unwrap();
1703
1704        assert!(result.success);
1705        let job = &result.jobs["script-job"];
1706        assert!(job.success);
1707        assert_eq!(job.steps[0].output, "5");
1708    }
1709
1710    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1711    async fn cancellation_token_stops_workflow() {
1712        let ns = make_namespace();
1713        let runner = WorkflowRunner::new(Arc::clone(&ns));
1714
1715        let mut jobs = HashMap::new();
1716        // Two waves: "first" runs, then "second" should be skipped.
1717        jobs.insert(
1718            "first".to_owned(),
1719            Job {
1720                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1721                needs: None,
1722                env: HashMap::new(),
1723                steps: vec![Step {
1724                    name: Some("noop".into()),
1725                    id: None,
1726                    uses: Some("solver/submit".into()),
1727                    run: None,
1728                    shell: None,
1729                    working_directory: None,
1730                    with: HashMap::new(),
1731                    env: HashMap::new(),
1732                    condition: None,
1733                    continue_on_error: false,
1734                }],
1735                condition: None,
1736                timeout_minutes: None,
1737                resources: super::super::parser::JobResources::default(),
1738            },
1739        );
1740        jobs.insert(
1741            "second".to_owned(),
1742            Job {
1743                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1744                needs: Some(vec!["first".to_owned()]),
1745                env: HashMap::new(),
1746                steps: vec![Step {
1747                    name: Some("should-not-run".into()),
1748                    id: None,
1749                    uses: Some("solver/submit".into()),
1750                    run: None,
1751                    shell: None,
1752                    working_directory: None,
1753                    with: HashMap::new(),
1754                    env: HashMap::new(),
1755                    condition: None,
1756                    continue_on_error: false,
1757                }],
1758                condition: None,
1759                timeout_minutes: None,
1760                resources: super::super::parser::JobResources::default(),
1761            },
1762        );
1763
1764        let cancel = CancellationToken::new();
1765
1766        // Cancel after first wave completes — we do this by spawning the run
1767        // and cancelling before the second wave.
1768        let workflow = make_simple_workflow(jobs);
1769
1770        // Cancel immediately — the first wave may or may not complete.
1771        cancel.cancel();
1772
1773        let result = runner
1774            .run(&workflow, HashMap::new(), &test_subject(), cancel.clone())
1775            .await;
1776
1777        // Either Cancelled error or the second job never ran.
1778        match result {
1779            Err(RunnerError::Cancelled) => { /* expected */ }
1780            Ok(r) => {
1781                // If first wave completed before cancel was checked,
1782                // the second job should not be present.
1783                assert!(!r.jobs.contains_key("second"));
1784            }
1785            Err(_) => panic!("unexpected error variant"),
1786        }
1787    }
1788
1789    #[test]
1790    fn action_name_dotdot_rejected() {
1791        assert!(validate_action_name("../../etc/passwd").is_err());
1792        assert!(validate_action_name("foo/..bar").is_err());
1793        assert!(validate_action_name("..").is_err());
1794    }
1795
1796    #[test]
1797    fn input_key_underscore_prefix_rejected() {
1798        let mut inputs = HashMap::new();
1799        inputs.insert("_provenance".into(), "injected".into());
1800        assert!(validate_inputs(&inputs).is_err());
1801
1802        // Normal key OK.
1803        let mut inputs2 = HashMap::new();
1804        inputs2.insert("normal_key".into(), "value".into());
1805        assert!(validate_inputs(&inputs2).is_ok());
1806    }
1807
1808    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1809    async fn step_failure_stops_job() {
1810        // Use a script step that fails (no VFS mount ambiguity).
1811        let ns = make_namespace();
1812        let runner = WorkflowRunner::new(Arc::clone(&ns));
1813
1814        let mut jobs = HashMap::new();
1815        jobs.insert(
1816            "fail-job".to_owned(),
1817            Job {
1818                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1819                needs: None,
1820                env: HashMap::new(),
1821                steps: vec![
1822                    Step {
1823                        name: Some("failing-step".into()),
1824                        id: None,
1825                        uses: None,
1826                        run: Some("error {intentional failure}".into()),
1827                        shell: None,
1828                        working_directory: None,
1829                        with: HashMap::new(),
1830                        env: HashMap::new(),
1831                        condition: None,
1832                        continue_on_error: false,
1833                    },
1834                    Step {
1835                        name: Some("should-not-run".into()),
1836                        id: None,
1837                        uses: None,
1838                        run: Some("expr {1 + 1}".into()),
1839                        shell: None,
1840                        working_directory: None,
1841                        with: HashMap::new(),
1842                        env: HashMap::new(),
1843                        condition: None,
1844                        continue_on_error: false,
1845                    },
1846                ],
1847                condition: None,
1848                timeout_minutes: None,
1849                resources: super::super::parser::JobResources::default(),
1850            },
1851        );
1852
1853        let workflow = make_simple_workflow(jobs);
1854        let cancel = CancellationToken::new();
1855        let result = runner
1856            .run(&workflow, HashMap::new(), &test_subject(), cancel)
1857            .await
1858            .unwrap();
1859
1860        assert!(!result.success);
1861        let job = &result.jobs["fail-job"];
1862        assert!(!job.success);
1863        // First step failed, second should not have run.
1864        assert_eq!(job.steps.len(), 1);
1865        assert!(!job.steps[0].success);
1866    }
1867
1868    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1869    async fn continue_on_error_allows_next_step() {
1870        let ns = make_namespace();
1871        let runner = WorkflowRunner::new(Arc::clone(&ns));
1872
1873        let mut jobs = HashMap::new();
1874        jobs.insert(
1875            "resilient-job".to_owned(),
1876            Job {
1877                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1878                needs: None,
1879                env: HashMap::new(),
1880                steps: vec![
1881                    Step {
1882                        name: Some("failing-step".into()),
1883                        id: None,
1884                        uses: None,
1885                        run: Some("error {intentional failure}".into()),
1886                        shell: None,
1887                        working_directory: None,
1888                        with: HashMap::new(),
1889                        env: HashMap::new(),
1890                        condition: None,
1891                        continue_on_error: true, // Should continue
1892                    },
1893                    Step {
1894                        name: Some("should-run".into()),
1895                        id: None,
1896                        uses: None,
1897                        run: Some("expr {1 + 1}".into()),
1898                        shell: None,
1899                        working_directory: None,
1900                        with: HashMap::new(),
1901                        env: HashMap::new(),
1902                        condition: None,
1903                        continue_on_error: false,
1904                    },
1905                ],
1906                condition: None,
1907                timeout_minutes: None,
1908                resources: super::super::parser::JobResources::default(),
1909            },
1910        );
1911
1912        let workflow = make_simple_workflow(jobs);
1913        let cancel = CancellationToken::new();
1914        let result = runner
1915            .run(&workflow, HashMap::new(), &test_subject(), cancel)
1916            .await
1917            .unwrap();
1918
1919        let job = &result.jobs["resilient-job"];
1920        // Both steps should have run.
1921        assert_eq!(job.steps.len(), 2);
1922        assert!(!job.steps[0].success); // First failed
1923        assert!(job.steps[1].success); // Second ran and succeeded
1924    }
1925
1926    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1927    async fn tcl_state_persists_across_steps_in_job() {
1928        // Verify that TclShell is reused across steps in the same job
1929        let ns = make_namespace();
1930        let runner = WorkflowRunner::new(Arc::clone(&ns));
1931
1932        let mut jobs = HashMap::new();
1933        jobs.insert(
1934            "state-job".to_owned(),
1935            Job {
1936                runs_on: super::super::parser::RunsOn::Label("ubuntu".into()),
1937                needs: None,
1938                env: HashMap::new(),
1939                steps: vec![
1940                    Step {
1941                        name: Some("set-var".into()),
1942                        id: None,
1943                        uses: None,
1944                        run: Some("set myvar hello".into()),
1945                        shell: None,
1946                        working_directory: None,
1947                        with: HashMap::new(),
1948                        env: HashMap::new(),
1949                        condition: None,
1950                        continue_on_error: false,
1951                    },
1952                    Step {
1953                        name: Some("read-var".into()),
1954                        id: None,
1955                        uses: None,
1956                        run: Some("set myvar".into()),
1957                        shell: None,
1958                        working_directory: None,
1959                        with: HashMap::new(),
1960                        env: HashMap::new(),
1961                        condition: None,
1962                        continue_on_error: false,
1963                    },
1964                ],
1965                condition: None,
1966                timeout_minutes: None,
1967                resources: super::super::parser::JobResources::default(),
1968            },
1969        );
1970
1971        let workflow = make_simple_workflow(jobs);
1972        let cancel = CancellationToken::new();
1973        let result = runner
1974            .run(&workflow, HashMap::new(), &test_subject(), cancel)
1975            .await
1976            .unwrap();
1977
1978        assert!(result.success);
1979        let job = &result.jobs["state-job"];
1980        assert_eq!(job.steps.len(), 2);
1981        // Second step should see the variable set by first step
1982        assert_eq!(job.steps[1].output, "hello");
1983    }
1984
1985    // ── #521: run-level concurrency + job/step timeouts ──────────────────
1986
1987    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1988    async fn step_timeout_fails_the_step_and_job() {
1989        // The mount actually sleeps 2s; step_timeout of 1s must trip first.
1990        let ns = make_namespace_with_slow_action(Duration::from_secs(2));
1991        let config = WorkflowConfig {
1992            step_timeout_secs: 1,
1993            ..Default::default()
1994        };
1995        let runner = WorkflowRunner::with_config(ns, &config);
1996
1997        let mut jobs = HashMap::new();
1998        jobs.insert("slow-job".to_owned(), single_action_job("slow/step"));
1999        let workflow = make_simple_workflow(jobs);
2000
2001        let result = runner
2002            .run(
2003                &workflow,
2004                HashMap::new(),
2005                &test_subject(),
2006                CancellationToken::new(),
2007            )
2008            .await
2009            .unwrap();
2010
2011        assert!(!result.success);
2012        let job = &result.jobs["slow-job"];
2013        assert!(!job.success);
2014        assert!(
2015            job.steps[0].output.contains("timed out"),
2016            "expected a timeout message, got: {:?}",
2017            job.steps[0].output
2018        );
2019    }
2020
2021    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2022    async fn job_timeout_fails_the_whole_job() {
2023        // step_timeout is generous; job_timeout (wrapping the whole job) is
2024        // the one that must trip.
2025        let ns = make_namespace_with_slow_action(Duration::from_secs(2));
2026        let config = WorkflowConfig {
2027            job_timeout_secs: 1,
2028            step_timeout_secs: 600,
2029            ..Default::default()
2030        };
2031        let runner = WorkflowRunner::with_config(ns, &config);
2032
2033        let mut jobs = HashMap::new();
2034        jobs.insert("slow-job".to_owned(), single_action_job("slow/step"));
2035        let workflow = make_simple_workflow(jobs);
2036
2037        let result = runner
2038            .run(
2039                &workflow,
2040                HashMap::new(),
2041                &test_subject(),
2042                CancellationToken::new(),
2043            )
2044            .await
2045            .unwrap();
2046
2047        assert!(!result.success);
2048        let job = &result.jobs["slow-job"];
2049        assert!(!job.success);
2050        assert_eq!(job.steps[0].name, "<timeout>");
2051        assert!(job.steps[0].output.contains("job timed out"));
2052    }
2053
2054    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2055    async fn max_concurrent_runs_serializes_concurrent_runs() {
2056        use std::sync::atomic::{AtomicUsize, Ordering};
2057
2058        let current = Arc::new(AtomicUsize::new(0));
2059        let peak = Arc::new(AtomicUsize::new(0));
2060        let ns = make_namespace_with_tracking_action(
2061            current.clone(),
2062            peak.clone(),
2063            Duration::from_millis(150),
2064        );
2065        let config = WorkflowConfig {
2066            max_concurrent_runs: 1,
2067            ..Default::default()
2068        };
2069        let runner = Arc::new(WorkflowRunner::with_config(ns, &config));
2070
2071        let make_workflow = || {
2072            let mut jobs = HashMap::new();
2073            jobs.insert("job".to_owned(), single_action_job("track/step"));
2074            make_simple_workflow(jobs)
2075        };
2076
2077        let r1 = {
2078            let runner = Arc::clone(&runner);
2079            let wf = make_workflow();
2080            tokio::spawn(async move {
2081                runner
2082                    .run(
2083                        &wf,
2084                        HashMap::new(),
2085                        &test_subject(),
2086                        CancellationToken::new(),
2087                    )
2088                    .await
2089            })
2090        };
2091        // Give r1 a head start so it acquires the sole run permit first.
2092        tokio::time::sleep(Duration::from_millis(30)).await;
2093        let r2 = {
2094            let runner = Arc::clone(&runner);
2095            let wf = make_workflow();
2096            tokio::spawn(async move {
2097                runner
2098                    .run(
2099                        &wf,
2100                        HashMap::new(),
2101                        &test_subject(),
2102                        CancellationToken::new(),
2103                    )
2104                    .await
2105            })
2106        };
2107
2108        let (res1, res2) = tokio::join!(r1, r2);
2109        assert!(res1.unwrap().unwrap().success);
2110        assert!(res2.unwrap().unwrap().success);
2111        assert_eq!(
2112            peak.load(Ordering::SeqCst),
2113            1,
2114            "max_concurrent_runs=1 should have serialized the two runs, never letting \
2115             both execute their job step at once"
2116        );
2117    }
2118}