Skip to main content

hyprstream_workers/workflow/
service.rs

1//! WorkflowService - RequestService implementation for workflow orchestration
2
3use std::collections::{HashMap, HashSet};
4use std::path::PathBuf;
5use std::sync::Arc;
6use tokio::sync::RwLock;
7use tokio::task::JoinHandle;
8use tokio_util::sync::CancellationToken;
9
10use anyhow::Result as AnyhowResult;
11use async_trait::async_trait;
12use hyprstream_rpc::events::EventSubscriber;
13use hyprstream_rpc::prelude::SigningKey;
14use hyprstream_rpc::service::{AuthorizeFn, EnvelopeContext, RequestService};
15use hyprstream_rpc::transport::TransportConfig;
16
17use hyprstream_vfs::Namespace;
18
19use crate::error::Result;
20use crate::events::ReceivedEvent;
21use crate::generated::workflow_client::{dispatch_workflow, WorkflowHandler};
22use hyprstream_rpc_std::workflow_client::{
23    // Request types
24    DispatchRequest,
25    JobRun as JobRunWire,
26    RunStatus as WireRunStatus,
27    StepRun as StepRunWire,
28    SubscribeRequest,
29    WorkflowDef as WorkflowDefWire,
30    WorkflowInfo,
31    WorkflowResponseVariant,
32    WorkflowRun as WorkflowRunWire,
33};
34
35use super::runner::WorkflowRunner;
36use super::scheduler::JobScheduler;
37use super::subscription::WorkflowSubscription;
38use super::triggers::{EventHandler, EventTrigger, HandlerResult};
39use super::{RunId, Workflow, WorkflowId};
40
41// ═══════════════════════════════════════════════════════════════════════════════
42// Internal domain types — NOT exposed in the public API.
43// The public WorkflowClient trait uses generated *Data types from the schema.
44// These exist for internal service state where the domain model differs
45// from the wire format (e.g., HashMap vs Vec, Option<DateTime> vs i64).
46// ═══════════════════════════════════════════════════════════════════════════════
47
48/// Internal workflow definition (domain model, not wire format)
49#[derive(Debug, Clone)]
50pub(crate) struct WorkflowDef {
51    pub path: String,
52    pub repo_id: String,
53    pub workflow: Workflow,
54    pub triggers: Vec<EventTrigger>,
55}
56
57/// Internal workflow run state
58#[derive(Debug, Clone)]
59pub(crate) struct WorkflowRun {
60    pub id: String,
61    pub workflow_id: String,
62    pub status: WireRunStatus,
63    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
64    pub completed_at: Option<chrono::DateTime<chrono::Utc>>,
65    pub jobs: HashMap<String, JobRun>,
66}
67
68/// Internal job run state
69#[derive(Debug, Clone)]
70pub(crate) struct JobRun {
71    pub name: String,
72    pub status: WireRunStatus,
73    pub steps: Vec<StepRun>,
74}
75
76/// Internal step run state
77#[derive(Debug, Clone)]
78pub(crate) struct StepRun {
79    pub name: String,
80    pub status: WireRunStatus,
81    pub exit_code: Option<i32>,
82}
83
84/// Service name for endpoint registry
85const SERVICE_NAME: &str = "workflow";
86
87/// WorkflowService handles workflow orchestration
88///
89/// Discovers workflows from repositories, subscribes to events, and routes
90/// job execution through [`WorkflowRunner`] (#527): jobs requesting
91/// isolation via `runs_on:` are scheduled onto a sandbox through the #525 P2
92/// admission engine (`SandboxPool::acquire`, see `workflow::scheduler`);
93/// others run in-process over the VFS/Tcl bridge. There is no `WorkerService`
94/// dependency here — `WorkflowRunner` talks to `SandboxPool` directly.
95pub struct WorkflowService {
96    /// Registered workflows
97    workflows: RwLock<HashMap<WorkflowId, WorkflowDef>>,
98
99    /// Active runs (Arc for sharing with spawned execution tasks)
100    runs: Arc<RwLock<HashMap<RunId, WorkflowRun>>>,
101
102    /// Per-repo workflow subscriptions (O(1) lookup)
103    repo_workflows: RwLock<HashMap<String, Vec<WorkflowSubscription>>>,
104
105    /// Registered repo paths (repo_id → filesystem root)
106    repo_paths: RwLock<HashMap<String, PathBuf>>,
107
108    /// Per-repo parse mode (#1432): a gate that loads a repo in strict mode
109    /// records it here so that later rescans (event-triggered or manual)
110    /// retain strict rather than silently dropping back to legacy.
111    /// Default for an unregistered repo is [`ParseMode::Legacy`].
112    repo_modes: RwLock<HashMap<String, super::parser::ParseMode>>,
113
114    /// Per-repo serialization locks (#1432 race fix): held from mode decision
115    /// through registry commit so that a Legacy scan cannot yield between its
116    /// mode check and commit while a Strict selection lands in between.
117    /// Acquired by `rescan_repo`, `rescan_repo_with`, and `set_repo_mode`.
118    repo_locks: RwLock<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
119
120    /// Active subscriptions (sub_id → WorkflowSubscription)
121    subscriptions: RwLock<HashMap<String, WorkflowSubscription>>,
122
123    /// Event handlers for different event types
124    handlers: RwLock<Vec<Box<dyn EventHandler>>>,
125
126    /// Background event loop handle
127    event_loop_handle: tokio::sync::Mutex<Option<JoinHandle<()>>>,
128
129    // Infrastructure (for RequestService / Spawnable)
130    /// Transport configuration
131    transport: TransportConfig,
132    /// Signing key for message authentication
133    signing_key: SigningKey,
134
135    /// Optional authorization callback (injected by parent crate)
136    authorize_fn: Option<AuthorizeFn>,
137
138    /// Expected JWT audience — fed to the envelope-verification layer so a
139    /// JWT-bearing IPC/QUIC request is validated before the policy `authorize`
140    /// callback runs (same seam as `WorkerService`). Without this, JWT-auth
141    /// requests fail at envelope verification with an audience mismatch.
142    expected_audience: Option<String>,
143
144    /// JWT verification key source (local + federated). `None` ⇒ only the
145    /// local bootstrap key is trusted; federated JWTs are rejected.
146    jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
147
148    /// Live MCP relay authorization — injected by the factory as a closure over
149    /// the global trust store so `accept_delegated_bearer` consults **current**
150    /// MCP-scope authorization at call time, not a frozen snapshot (#989 review:
151    /// a snapshot permits retired keys forever and rejects keys added after
152    /// factory start). `None` ⇒ fail-closed (no delegation accepted).
153    relay_authorizer: Option<std::sync::Arc<dyn Fn(&[u8; 32]) -> bool + Send + Sync>>,
154
155    /// Workflow execution engine
156    runner: Option<WorkflowRunner>,
157}
158
159impl WorkflowService {
160    /// Create a new WorkflowService
161    ///
162    /// # Arguments
163    ///
164    /// * `transport` - Transport configuration for service binding
165    /// * `signing_key` - Signing key for message authentication
166    pub fn new(transport: TransportConfig, signing_key: SigningKey) -> Self {
167        Self {
168            workflows: RwLock::new(HashMap::new()),
169            runs: Arc::new(RwLock::new(HashMap::new())),
170            repo_workflows: RwLock::new(HashMap::new()),
171            repo_paths: RwLock::new(HashMap::new()),
172            repo_modes: RwLock::new(HashMap::new()),
173            repo_locks: RwLock::new(HashMap::new()),
174            subscriptions: RwLock::new(HashMap::new()),
175            handlers: RwLock::new(Vec::new()),
176            event_loop_handle: tokio::sync::Mutex::new(None),
177            transport,
178            signing_key,
179            authorize_fn: None,
180            expected_audience: None,
181            jwt_key_source: None,
182            relay_authorizer: None,
183            runner: None,
184        }
185    }
186
187    /// Register the filesystem root for a repo_id.
188    ///
189    /// Once registered, `scan_repo(repo_id)` will glob
190    /// `<root>/.github/workflows/*.yml` to discover workflows.
191    pub async fn register_repo_path(&self, repo_id: impl Into<String>, path: PathBuf) {
192        let mut repo_paths = self.repo_paths.write().await;
193        repo_paths.insert(repo_id.into(), path);
194    }
195
196    /// Get-or-create the per-repo serialization lock (#1432 race fix).
197    async fn repo_lock(&self, repo_id: &str) -> Arc<tokio::sync::Mutex<()>> {
198        {
199            let read = self.repo_locks.read().await;
200            if let Some(lock) = read.get(repo_id) {
201                return Arc::clone(lock);
202            }
203        }
204        let mut write = self.repo_locks.write().await;
205        write
206            .entry(repo_id.to_owned())
207            .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
208            .clone()
209    }
210
211    /// Set the per-repo parse mode (#1432).
212    ///
213    /// Acquires the per-repo serialization lock so that mode changes are
214    /// atomic with concurrent rescans. A gate that loads a repo in strict
215    /// mode records it here so that later rescans retain strict (the default
216    /// for an unregistered repo is [`ParseMode::Legacy`]). Typically called
217    /// by
218    /// [`GitHubActionsAdapter::load_repo_with`](super::gh_adapter::GitHubActionsAdapter::load_repo_with).
219    pub async fn set_repo_mode(&self, repo_id: &str, mode: super::parser::ParseMode) {
220        let lock = self.repo_lock(repo_id).await;
221        let _guard = lock.lock().await;
222        self.set_repo_mode_locked(repo_id, mode).await;
223    }
224
225    /// Write the mode without acquiring the per-repo lock.
226    ///
227    /// Caller MUST already hold the per-repo lock (e.g. inside
228    /// `rescan_repo_with`). This avoids reentrant-deadlock.
229    async fn set_repo_mode_locked(&self, repo_id: &str, mode: super::parser::ParseMode) {
230        let mut repo_modes = self.repo_modes.write().await;
231        repo_modes.insert(repo_id.to_owned(), mode);
232    }
233
234    /// Resolve the per-repo parse mode, defaulting to [`ParseMode::Legacy`]
235    /// for repos that never opted into strict.
236    async fn repo_mode(&self, repo_id: &str) -> super::parser::ParseMode {
237        self.repo_modes
238            .read()
239            .await
240            .get(repo_id)
241            .copied()
242            .unwrap_or(super::parser::ParseMode::Legacy)
243    }
244
245    /// Evict stale registrations for a repo (#1432 fail-closed).
246    ///
247    /// Removes any `repo_id:*` registry entry absent from `fresh_ids`, so a
248    /// workflow that was rejected by strict mode, deleted, or otherwise no
249    /// longer present in the fresh scan becomes undispatchable
250    /// (`dispatch` → [`WorkflowNotFound`](crate::error::WorkerError::WorkflowNotFound)).
251    pub(crate) async fn evict_stale_for_repo(
252        &self,
253        repo_id: &str,
254        fresh_ids: &HashSet<WorkflowId>,
255    ) {
256        let stale_prefix = format!("{repo_id}:");
257        let mut workflows_lock = self.workflows.write().await;
258        let stale_keys: Vec<WorkflowId> = workflows_lock
259            .keys()
260            .filter(|id| id.starts_with(&stale_prefix) && !fresh_ids.contains(*id))
261            .cloned()
262            .collect();
263        for key in &stale_keys {
264            tracing::info!(workflow_id = %key, "Evicting stale workflow registration");
265            workflows_lock.remove(key);
266        }
267    }
268
269    /// Set the VFS namespace for workflow execution.
270    ///
271    /// This creates a WorkflowRunner that resolves actions through `/bin/`
272    /// and evaluates scripts via TclShell, using default
273    /// [`crate::config::WorkflowConfig`] timeouts/concurrency and no job
274    /// scheduler (every job runs in-proc — see [`Self::set_job_scheduler`]).
275    pub fn set_namespace(&mut self, ns: Arc<Namespace>) {
276        self.runner = Some(WorkflowRunner::new(ns));
277    }
278
279    /// Like [`Self::set_namespace`], but honors `config`'s
280    /// `max_concurrent_runs` / `job_timeout_secs` / `step_timeout_secs`
281    /// (#521) instead of defaults.
282    pub fn set_namespace_with_config(
283        &mut self,
284        ns: Arc<Namespace>,
285        config: &crate::config::WorkflowConfig,
286    ) {
287        self.runner = Some(WorkflowRunner::with_config(ns, config));
288    }
289
290    /// Attach a [`JobScheduler`] (#527) so jobs whose `runs_on:` requests
291    /// isolation are placed onto a real sandbox through the #525 P2
292    /// admission engine, instead of always running in-proc. Must be called
293    /// after `set_namespace`/`set_namespace_with_config`; a no-op (with a
294    /// warning) if no runner has been configured yet.
295    pub fn set_job_scheduler(&mut self, scheduler: Arc<JobScheduler>) {
296        match self.runner.take() {
297            Some(runner) => self.runner = Some(runner.with_scheduler(scheduler)),
298            None => {
299                tracing::warn!(
300                    "set_job_scheduler called before set_namespace — ignored; \
301                     call set_namespace/set_namespace_with_config first"
302                );
303            }
304        }
305    }
306
307    /// Set the authorization callback for policy checks.
308    pub fn set_authorize_fn(&mut self, authorize_fn: AuthorizeFn) {
309        self.authorize_fn = Some(authorize_fn);
310    }
311
312    /// Set the expected JWT audience for envelope-verification-time validation
313    /// (mirrors `WorkerService::set_expected_audience`). Fed by the factory from
314    /// the OAuth issuer URL so JWT-authenticated IPC/QUIC requests are accepted
315    /// before the per-request policy check.
316    pub fn set_expected_audience(&mut self, audience: String) {
317        self.expected_audience = Some(audience);
318    }
319
320    /// Set the JWT verification key source (local + federated). Fed by the
321    /// factory from the cluster key source.
322    pub fn set_jwt_key_source(
323        &mut self,
324        src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
325    ) {
326        self.jwt_key_source = Some(src);
327    }
328
329    /// Inject a **live** MCP relay authorizer so `accept_delegated_bearer`
330    /// consults current MCP-scope authorization at call time (not a frozen
331    /// snapshot — retired keys are rejected, new keys accepted after factory
332    /// start). The factory typically passes a closure over the global trust
333    /// store's `is_authorized(key, "mcp")`. `None` (never called) ⇒ fail-closed.
334    pub fn set_relay_authorizer(
335        &mut self,
336        authorizer: std::sync::Arc<dyn Fn(&[u8; 32]) -> bool + Send + Sync>,
337    ) {
338        self.relay_authorizer = Some(authorizer);
339    }
340
341    /// Initialize the service: scan all registered repos and register their workflows.
342    pub async fn initialize(&self) -> Result<()> {
343        let repo_ids: Vec<String> = {
344            let repo_paths = self.repo_paths.read().await;
345            repo_paths.keys().cloned().collect()
346        };
347
348        for repo_id in repo_ids {
349            match self.rescan_repo(&repo_id).await {
350                Ok(()) => {}
351                Err(e) => {
352                    tracing::warn!(repo_id = %repo_id, error = %e, "Failed to scan repo during init");
353                }
354            }
355        }
356
357        tracing::info!("WorkflowService initialized");
358        Ok(())
359    }
360
361    /// Start the event subscription loop
362    ///
363    /// Subscribes to worker events and routes them to registered handlers.
364    /// This spawns a background task that runs until `stop()` is called.
365    pub async fn start(self: Arc<Self>) -> Result<()> {
366        let mut subscriber = EventSubscriber::new()?;
367
368        // Subscribe to all event categories that can trigger or affect workflows.
369        for prefix in &["worker.", "repo.", "training.", "metrics."] {
370            subscriber.subscribe(prefix)?;
371        }
372
373        tracing::info!("WorkflowService subscribed to worker/repo/training/metrics events");
374
375        let service = self.clone();
376        let handle = tokio::spawn(async move {
377            service.event_loop(subscriber).await;
378        });
379
380        let mut event_loop_handle = self.event_loop_handle.lock().await;
381        *event_loop_handle = Some(handle);
382
383        Ok(())
384    }
385
386    /// Stop the event subscription loop
387    pub async fn stop(&self) {
388        let mut event_loop_handle = self.event_loop_handle.lock().await;
389        if let Some(handle) = event_loop_handle.take() {
390            handle.abort();
391            tracing::info!("WorkflowService event loop stopped");
392        }
393    }
394
395    /// Internal event processing loop
396    async fn event_loop(&self, mut subscriber: EventSubscriber) {
397        loop {
398            match subscriber.recv().await {
399                Ok((topic, payload)) => {
400                    if let Err(e) = self.handle_event(&topic, &payload).await {
401                        tracing::warn!(topic = %topic, error = %e, "Failed to handle event");
402                    }
403                }
404                Err(e) => {
405                    tracing::error!(error = %e, "Event subscriber error, stopping loop");
406                    break;
407                }
408            }
409        }
410    }
411
412    /// Handle an incoming event
413    ///
414    /// Creates a ReceivedEvent from the raw topic and payload,
415    /// then dispatches to registered handlers.
416    async fn handle_event(&self, topic: &str, payload: &[u8]) -> Result<()> {
417        // Parse topic and payload into ReceivedEvent (handles deserialization)
418        let event = ReceivedEvent::from_message(topic, payload);
419
420        tracing::debug!(
421            topic = %event.topic,
422            source = %event.source,
423            entity_id = %event.entity_id,
424            event_type = %event.event_type,
425            has_worker_event = event.worker_event.is_some(),
426            "Received event"
427        );
428
429        // Dispatch to handlers
430        let handlers = self.handlers.read().await;
431        for handler in handlers.iter() {
432            if handler.matches(&event) {
433                match handler.handle(&event).await {
434                    Ok(HandlerResult::Dispatch {
435                        workflow_id,
436                        inputs,
437                    }) => {
438                        tracing::info!(
439                            workflow_id = %workflow_id,
440                            trigger_event = %topic,
441                            "Dispatching workflow from event"
442                        );
443                        let event_subject = hyprstream_vfs::Subject::new("workflow-event-dispatch");
444                        if let Err(e) = self.dispatch(&workflow_id, inputs, &event_subject).await {
445                            tracing::error!(
446                                workflow_id = %workflow_id,
447                                error = %e,
448                                "Failed to dispatch workflow"
449                            );
450                        }
451                    }
452                    Ok(HandlerResult::Rescan { repo_id }) => {
453                        if let Err(e) = self.rescan_repo(&repo_id).await {
454                            tracing::error!(repo_id = %repo_id, error = %e, "Failed to rescan repo");
455                        }
456                    }
457                    Ok(HandlerResult::Ignored) => {}
458                    Err(e) => {
459                        tracing::warn!(error = %e, "Handler error");
460                    }
461                }
462            }
463        }
464
465        Ok(())
466    }
467
468    /// Register an event handler
469    pub async fn register_handler(&self, handler: Box<dyn EventHandler>) {
470        let mut handlers = self.handlers.write().await;
471        handlers.push(handler);
472    }
473
474    /// Scan a repository for `.github/workflows/*.yml` files (legacy mode).
475    ///
476    /// Generic loader entry point — parses in [`ParseMode::Legacy`], so
477    /// unknown keys are silently dropped. Used by the GitHub Actions adapter
478    /// and other non-gate callers. The merge gate must opt into strict mode
479    /// via [`Self::scan_repo_with`] (#1432 scope item 2; non-goal: changing
480    /// the default for non-gate callers).
481    ///
482    /// Uses the path registered via `register_repo_path`. Returns an empty
483    /// Vec (not an error) if no path is registered for the repo or if the
484    /// workflows directory does not exist.
485    pub(crate) async fn scan_repo(&self, repo_id: &str) -> Result<Vec<WorkflowDef>> {
486        self.scan_repo_with(repo_id, super::parser::ParseMode::Legacy)
487            .await
488    }
489
490    /// Scan a repository with an explicit [`ParseMode`] (#1432).
491    ///
492    /// In [`ParseMode::Strict`] a file with an unknown structural key
493    /// (e.g. `permissions:`, `concurrency:`, `services:`) is **skipped** with
494    /// a warn-log rather than silently loaded with dropped semantics — the
495    /// merge-gate boundary selects this mode so the gate never loads a
496    /// workflow it cannot fully model.
497    pub(crate) async fn scan_repo_with(
498        &self,
499        repo_id: &str,
500        mode: super::parser::ParseMode,
501    ) -> Result<Vec<WorkflowDef>> {
502        tracing::info!(repo_id = %repo_id, mode = ?mode, "Scanning repository for workflows");
503
504        let root = {
505            let repo_paths = self.repo_paths.read().await;
506            match repo_paths.get(repo_id) {
507                Some(p) => p.clone(),
508                None => {
509                    tracing::debug!(repo_id = %repo_id, "No path registered for repo — skipping scan");
510                    return Ok(Vec::new());
511                }
512            }
513        };
514
515        // The directory walk + file reads + YAML parsing are all blocking I/O.
516        // Run them on the blocking thread pool so we don't stall the async
517        // worker (and starve other tasks sharing this runtime).
518        let repo_id_owned = repo_id.to_owned();
519        let defs = tokio::task::spawn_blocking(move || -> Result<Vec<WorkflowDef>> {
520            let workflow_dir = root.join(".github").join("workflows");
521            if !workflow_dir.is_dir() {
522                tracing::debug!(
523                    repo_id = %repo_id_owned,
524                    dir = %workflow_dir.display(),
525                    "No .github/workflows dir found"
526                );
527                return Ok(Vec::new());
528            }
529
530            let mut defs = Vec::new();
531            let read_dir = std::fs::read_dir(&workflow_dir).map_err(|e| {
532                crate::error::WorkerError::WorkflowParseError(format!(
533                    "Failed to read workflows dir {}: {e}",
534                    workflow_dir.display()
535                ))
536            })?;
537
538            for entry in read_dir.flatten() {
539                let path = entry.path();
540                let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
541                if ext != "yml" && ext != "yaml" {
542                    continue;
543                }
544
545                let yaml = match std::fs::read_to_string(&path) {
546                    Ok(s) => s,
547                    Err(e) => {
548                        tracing::warn!(path = %path.display(), error = %e, "Failed to read workflow file");
549                        continue;
550                    }
551                };
552
553                let workflow = match Workflow::parse_with(&yaml, mode) {
554                    Ok(w) => w,
555                    Err(e) => {
556                        tracing::warn!(
557                            path = %path.display(),
558                            error = %e,
559                            "Failed to parse workflow ({:?} mode); skipping",
560                            mode,
561                        );
562                        continue;
563                    }
564                };
565
566                let rel_path = path
567                    .strip_prefix(&root)
568                    .map(|p| p.to_string_lossy().into_owned())
569                    .unwrap_or_else(|_| path.to_string_lossy().into_owned());
570
571                let triggers = extract_triggers(&workflow);
572
573                defs.push(WorkflowDef {
574                    path: rel_path,
575                    repo_id: repo_id_owned.clone(),
576                    workflow,
577                    triggers,
578                });
579            }
580
581            Ok(defs)
582        })
583        .await
584        .map_err(|e| {
585            crate::error::WorkerError::WorkflowParseError(format!("scan_repo blocking task failed: {e}"))
586        })??;
587
588        tracing::info!(repo_id = %repo_id, count = defs.len(), "Scan complete");
589        Ok(defs)
590    }
591
592    /// Register a workflow
593    pub(crate) async fn register_workflow(&self, workflow: WorkflowDef) -> Result<WorkflowId> {
594        let workflow_id = format!("{}:{}", workflow.repo_id, workflow.path);
595
596        let mut workflows = self.workflows.write().await;
597        workflows.insert(workflow_id.clone(), workflow);
598
599        tracing::info!(workflow_id = %workflow_id, "Registered workflow");
600        Ok(workflow_id)
601    }
602
603    /// List registered workflows (returns wire-format types directly)
604    pub async fn list_workflows(&self) -> Result<Vec<WorkflowInfo>> {
605        let workflows = self.workflows.read().await;
606
607        Ok(workflows
608            .iter()
609            .map(|(id, def)| WorkflowInfo {
610                id: id.clone(),
611                name: def.workflow.name.clone(),
612                path: def.path.clone(),
613                repo_id: def.repo_id.clone(),
614                enabled: true,
615            })
616            .collect())
617    }
618
619    /// Dispatch a workflow manually.
620    ///
621    /// `subject` is the execution identity (who the workflow runs as).
622    /// Rejects user-supplied inputs with `_`-prefixed keys (reserved for provenance).
623    ///
624    /// If a WorkflowRunner is configured (via `set_namespace`), the workflow
625    /// is executed asynchronously in a spawned task that updates run status.
626    /// Otherwise, the run is created in Queued state with no execution.
627    pub(crate) async fn dispatch(
628        &self,
629        workflow_id: &WorkflowId,
630        inputs: HashMap<String, String>,
631        subject: &hyprstream_vfs::Subject,
632    ) -> Result<RunId> {
633        // Reject reserved _-prefixed keys in user inputs (provenance injection prevention).
634        if let Some(bad_key) = inputs.keys().find(|k| k.starts_with('_')) {
635            return Err(crate::error::WorkerError::WorkflowParseError(format!(
636                "input keys starting with '_' are reserved: {bad_key}"
637            )));
638        }
639
640        let workflows = self.workflows.read().await;
641        let workflow_def = workflows
642            .get(workflow_id)
643            .ok_or_else(|| crate::error::WorkerError::WorkflowNotFound(workflow_id.clone()))?;
644        let workflow = workflow_def.workflow.clone();
645        drop(workflows);
646
647        let run_id = uuid::Uuid::new_v4().to_string();
648
649        let run = WorkflowRun {
650            id: run_id.clone(),
651            workflow_id: workflow_id.clone(),
652            status: WireRunStatus::Queued,
653            started_at: None,
654            completed_at: None,
655            jobs: HashMap::new(),
656        };
657
658        let mut runs = self.runs.write().await;
659        runs.insert(run_id.clone(), run);
660        drop(runs);
661
662        tracing::info!(
663            workflow_id = %workflow_id,
664            run_id = %run_id,
665            inputs = ?inputs,
666            "Dispatched workflow"
667        );
668
669        // If runner is configured, spawn execution task.
670        if let Some(ref runner) = self.runner {
671            let runner = runner.clone();
672            let subject = subject.clone();
673            let cancel = tokio_util::sync::CancellationToken::new();
674            let run_id_clone = run_id.clone();
675            let runs = self.runs.clone();
676            tokio::spawn(async move {
677                // Update status to Running.
678                if let Some(run) = runs.write().await.get_mut(&run_id_clone) {
679                    run.status = WireRunStatus::InProgress;
680                    run.started_at = Some(chrono::Utc::now());
681                }
682                let result = runner.run(&workflow, inputs, &subject, cancel).await;
683                // Update final status.
684                if let Some(run) = runs.write().await.get_mut(&run_id_clone) {
685                    match &result {
686                        Ok(wr) => {
687                            run.status = if wr.success {
688                                WireRunStatus::Success
689                            } else {
690                                WireRunStatus::Failure
691                            };
692                            tracing::info!(run_id = %run_id_clone, success = wr.success, "Workflow completed");
693                        }
694                        Err(e) => {
695                            run.status = WireRunStatus::Failure;
696                            tracing::error!(run_id = %run_id_clone, error = %e, "Workflow failed");
697                        }
698                    }
699                    run.completed_at = Some(chrono::Utc::now());
700                }
701            });
702        }
703
704        Ok(run_id)
705    }
706
707    /// Get a workflow run
708    pub(crate) async fn get_run(&self, run_id: &RunId) -> Result<WorkflowRun> {
709        let runs = self.runs.read().await;
710        runs.get(run_id)
711            .cloned()
712            .ok_or_else(|| crate::error::WorkerError::RunNotFound(run_id.clone()))
713    }
714
715    /// List runs for a workflow
716    pub(crate) async fn list_runs(&self, workflow_id: &WorkflowId) -> Result<Vec<WorkflowRun>> {
717        let runs = self.runs.read().await;
718        Ok(runs
719            .values()
720            .filter(|r| &r.workflow_id == workflow_id)
721            .cloned()
722            .collect())
723    }
724
725    /// Rescan a repository for workflow changes (generic, non-persisting).
726    ///
727    /// Resolves the per-repo parse mode and rescans in that mode, but does
728    /// **not** write the mode back to the store. This is the entry point used
729    /// by the event-triggered rescan path (`HandlerResult::Rescan`) and
730    /// `initialize`. Because it never persists, a concurrent generic rescan
731    /// that read a stale [`ParseMode::Legacy`] before a gate selected strict
732    /// cannot overwrite the newer strict selection (#1432 race fix): only the
733    /// explicit boundaries ([`Self::rescan_repo_with`], `load_repo_with`)
734    /// write the mode.
735    pub async fn rescan_repo(&self, repo_id: &str) -> Result<()> {
736        let lock = self.repo_lock(repo_id).await;
737        let _guard = lock.lock().await;
738        let mode = self.repo_mode(repo_id).await;
739        self.rescan_inner_locked(repo_id, mode).await.map(|_| ())
740    }
741
742    /// Rescan a repository with an explicit [`ParseMode`] (#1432 boundary).
743    ///
744    /// **Persists** the mode per-repo (via [`Self::set_repo_mode`]) so that
745    /// later event-triggered rescans through the generic [`Self::rescan_repo`]
746    /// retain it. Both `rescan_repo` and this method hold the per-repo
747    /// serialization lock from mode decision through registry commit, making
748    /// the scan→commit window atomic per repo (#1432 race fix: a stale Legacy
749    /// scan cannot yield between its mode check and commit while a Strict
750    /// selection lands in between).
751    pub async fn rescan_repo_with(
752        &self,
753        repo_id: &str,
754        mode: super::parser::ParseMode,
755    ) -> Result<()> {
756        self.rescan_repo_with_defs(repo_id, mode).await.map(|_| ())
757    }
758
759    /// Select a parse mode and atomically reconcile a repository, returning
760    /// the definitions that survived the scan.
761    ///
762    /// Adapter loaders use this instead of a separate `set_repo_mode` followed
763    /// by `scan_repo_with`: strict selection, scanning, stale eviction, and
764    /// registration must share the per-repo lock. Otherwise a stale Legacy
765    /// scan can register an unsupported workflow after strict selection but
766    /// before the adapter's later eviction (#1432 fail-closed boundary).
767    pub(crate) async fn rescan_repo_with_defs(
768        &self,
769        repo_id: &str,
770        mode: super::parser::ParseMode,
771    ) -> Result<Vec<WorkflowDef>> {
772        let lock = self.repo_lock(repo_id).await;
773        let _guard = lock.lock().await;
774        self.set_repo_mode_locked(repo_id, mode).await;
775        self.rescan_inner_locked(repo_id, mode).await
776    }
777
778    /// Internal scan+evict+register worker (caller holds the per-repo lock).
779    ///
780    /// The per-repo lock is the race fix: it serializes rescans so that no
781    /// concurrent mode change or commit can interleave between this scan and
782    /// its commit. The previous revalidation guard was itself TOCTOU (it
783    /// could yield between check and commit); the mutex replaces it.
784    async fn rescan_inner_locked(
785        &self,
786        repo_id: &str,
787        mode: super::parser::ParseMode,
788    ) -> Result<Vec<WorkflowDef>> {
789        let workflows = self.scan_repo_with(repo_id, mode).await?;
790
791        // Fresh set of workflow ids for this repo, per the scan.
792        let fresh_ids: HashSet<WorkflowId> = workflows
793            .iter()
794            .map(|wf| format!("{}:{}", wf.repo_id, wf.path))
795            .collect();
796
797        // Evict stale registrations absent from the fresh set (fail-closed).
798        self.evict_stale_for_repo(repo_id, &fresh_ids).await;
799
800        // Register the fresh set and rebuild subscriptions.
801        let mut repo_workflows = self.repo_workflows.write().await;
802        let mut subscriptions = Vec::new();
803
804        for wf in &workflows {
805            let wf_id = self.register_workflow(wf.clone()).await?;
806
807            for trigger in &wf.triggers {
808                let subscription = WorkflowSubscription::new(wf_id.clone(), trigger.clone());
809                subscriptions.push(subscription);
810            }
811        }
812
813        repo_workflows.insert(repo_id.to_owned(), subscriptions);
814
815        tracing::info!(repo_id = %repo_id, mode = ?mode, "Rescanned repository");
816        Ok(workflows)
817    }
818
819    /// Start a subscriber adapter. Returns a CancellationToken to stop it.
820    ///
821    /// The adapter runs in a background task that receives events and
822    /// dispatches workflows. Cancel the returned token to stop the adapter.
823    pub async fn start_adapter(
824        self: &Arc<Self>,
825        adapter: Box<dyn super::adapter::SubscriberAdapter>,
826    ) -> CancellationToken {
827        let cancel = CancellationToken::new();
828        let service = Arc::clone(self);
829        let child = cancel.child_token();
830        let name = adapter.name().to_owned();
831        tokio::spawn(async move {
832            if let Err(e) = adapter.run(service, child).await {
833                tracing::error!(adapter = %name, error = %e, "adapter failed");
834            }
835        });
836        cancel
837    }
838}
839
840// ═══════════════════════════════════════════════════════════════════════════════
841// WorkflowHandler Implementation — bridges generated types to business logic
842// ═══════════════════════════════════════════════════════════════════════════════
843
844/// Convert internal WorkflowRun to generated WorkflowRunWire
845fn to_run_data(run: &WorkflowRun) -> WorkflowRunWire {
846    WorkflowRunWire {
847        id: run.id.clone(),
848        workflow_id: run.workflow_id.clone(),
849        status: run.status,
850        started_at: run.started_at.map(|t| t.timestamp()).unwrap_or(0),
851        completed_at: run.completed_at.map(|t| t.timestamp()).unwrap_or(0),
852        jobs: run
853            .jobs
854            .values()
855            .map(|j| JobRunWire {
856                name: j.name.clone(),
857                status: j.status,
858                steps: j
859                    .steps
860                    .iter()
861                    .map(|s| StepRunWire {
862                        name: s.name.clone(),
863                        status: s.status,
864                        exit_code: s.exit_code.unwrap_or(0),
865                    })
866                    .collect(),
867            })
868            .collect(),
869    }
870}
871
872// ?Send required: capnp Reader types are !Send, and the generated dispatch_workflow
873// macro holds readers across await points. This means WorkflowService cannot use the
874// Spawnable blanket impl and must be run on a LocalSet or single-threaded runtime.
875#[async_trait::async_trait(?Send)]
876impl WorkflowHandler for WorkflowService {
877    async fn authorize(
878        &self,
879        ctx: &EnvelopeContext,
880        resource: &str,
881        operation: &str,
882    ) -> AnyhowResult<()> {
883        if let Some(ref auth_fn) = self.authorize_fn {
884            let subject = ctx.subject().to_string();
885            let allowed = auth_fn(ctx.clone(), resource.to_owned(), operation.to_owned())
886                .await
887                .unwrap_or_else(|e| {
888                    tracing::warn!(
889                        "Policy check failed for {} on {}: {} - denying access",
890                        subject,
891                        resource,
892                        e
893                    );
894                    false
895                });
896            if allowed {
897                Ok(())
898            } else {
899                anyhow::bail!(
900                    "Unauthorized: {} cannot {} on {}",
901                    subject,
902                    operation,
903                    resource
904                )
905            }
906        } else {
907            // No authorization configured — fail-closed
908            anyhow::bail!("Authorization not configured for workflow service")
909        }
910    }
911
912    async fn handle_scan_repo(
913        &self,
914        _ctx: &EnvelopeContext,
915        _request_id: u64,
916        repo_id: &str,
917    ) -> AnyhowResult<WorkflowResponseVariant> {
918        let workflows = self.scan_repo(repo_id).await?;
919        let defs = workflows
920            .iter()
921            .map(|wf| {
922                // Propagate serialization failures instead of defaulting to an
923                // empty yaml string (which would register a broken stub).
924                let yaml = serde_yaml::to_string(&wf.workflow).map_err(|e| {
925                    anyhow::anyhow!("failed to serialize workflow '{}' to YAML: {e}", wf.path)
926                })?;
927                Ok::<_, anyhow::Error>(WorkflowDefWire {
928                    path: wf.path.clone(),
929                    repo_id: wf.repo_id.clone(),
930                    name: wf.workflow.name.clone(),
931                    triggers: Vec::new(),
932                    yaml,
933                })
934            })
935            .collect::<AnyhowResult<Vec<_>>>()?;
936        Ok(WorkflowResponseVariant::ScanRepoResult(defs))
937    }
938
939    async fn handle_register(
940        &self,
941        _ctx: &EnvelopeContext,
942        _request_id: u64,
943        data: &WorkflowDefWire,
944    ) -> AnyhowResult<WorkflowResponseVariant> {
945        let workflow = if data.yaml.is_empty() {
946            // Build a minimal Workflow from wire fields when no YAML is provided.
947            Workflow {
948                name: data.name.clone(),
949                on: super::parser::WorkflowTrigger::List(Vec::new()),
950                env: HashMap::new(),
951                jobs: HashMap::new(),
952            }
953        } else {
954            // Generic RPC register path: legacy mode. Unknown keys are
955            // silently dropped (non-gate caller compatibility, #1432
956            // non-goal). The merge gate selects strict mode at its own
957            // loader boundary (`scan_repo_with`/`load_repo_with`), not here.
958            Workflow::parse(&data.yaml)?
959        };
960        let triggers = extract_triggers(&workflow);
961        let workflow_def = WorkflowDef {
962            path: data.path.clone(),
963            repo_id: data.repo_id.clone(),
964            workflow,
965            triggers,
966        };
967        let workflow_id = self.register_workflow(workflow_def).await?;
968        Ok(WorkflowResponseVariant::RegisterResult(workflow_id))
969    }
970
971    async fn handle_list(
972        &self,
973        _ctx: &EnvelopeContext,
974        _request_id: u64,
975    ) -> AnyhowResult<WorkflowResponseVariant> {
976        let workflows = self.list_workflows().await?;
977        Ok(WorkflowResponseVariant::ListResult(workflows))
978    }
979
980    async fn handle_dispatch(
981        &self,
982        ctx: &EnvelopeContext,
983        _request_id: u64,
984        request: &DispatchRequest,
985    ) -> AnyhowResult<WorkflowResponseVariant> {
986        let inputs_map: HashMap<String, String> = request
987            .inputs
988            .iter()
989            .map(|kv| (kv.key.clone(), kv.value.clone()))
990            .collect();
991        // Pass the verified `Subject` through directly — never round-trip it
992        // through `.to_string()` → `Subject::new(...)`. `Subject::anonymous()`
993        // Displays as "anonymous", so that round-trip turns an anonymous caller
994        // into a *named* subject "anonymous" (is_anonymous() == false), which
995        // would then pass the fail-closed admission check this dispatch path
996        // feeds (#527) — defeating per-subject quota + the anonymous deny.
997        let subject = ctx.subject();
998        let run_id = self
999            .dispatch(&request.workflow_id.clone(), inputs_map, &subject)
1000            .await?;
1001        Ok(WorkflowResponseVariant::DispatchResult(run_id))
1002    }
1003
1004    async fn handle_subscribe(
1005        &self,
1006        _ctx: &EnvelopeContext,
1007        _request_id: u64,
1008        request: &SubscribeRequest,
1009    ) -> AnyhowResult<WorkflowResponseVariant> {
1010        // Validate the "repo_id:path" workflow_id convention up-front. An id
1011        // without ':' would otherwise be used wholesale as the repo_id, routing
1012        // the subscription into the wrong repo bucket.
1013        let repo_id = repo_id_from_workflow_id(&request.workflow_id)
1014            .ok_or_else(|| {
1015                anyhow::anyhow!(
1016                    "invalid workflow_id '{}': expected 'repo_id:path' convention",
1017                    request.workflow_id
1018                )
1019            })?
1020            .to_owned();
1021
1022        let sub_id = format!("sub-{}-{}", request.workflow_id, uuid::Uuid::new_v4());
1023        let trigger = EventTrigger::Custom {
1024            topic: format!("workflow.{}", request.workflow_id),
1025            pattern: String::new(),
1026        };
1027        let subscription = WorkflowSubscription::new(request.workflow_id.clone(), trigger);
1028
1029        {
1030            let mut subs = self.subscriptions.write().await;
1031            subs.insert(sub_id.clone(), subscription.clone());
1032        }
1033        {
1034            let mut repo_workflows = self.repo_workflows.write().await;
1035            repo_workflows
1036                .entry(repo_id)
1037                .or_default()
1038                .push(subscription);
1039        }
1040
1041        tracing::info!(workflow_id = %request.workflow_id, sub_id = %sub_id, "Subscribed");
1042        Ok(WorkflowResponseVariant::SubscribeResult(sub_id))
1043    }
1044
1045    async fn handle_unsubscribe(
1046        &self,
1047        _ctx: &EnvelopeContext,
1048        _request_id: u64,
1049        sub_id: &str,
1050    ) -> AnyhowResult<WorkflowResponseVariant> {
1051        let removed_sub = {
1052            let mut subs = self.subscriptions.write().await;
1053            subs.remove(sub_id)
1054        };
1055        if let Some(sub) = removed_sub {
1056            // Keep repo_workflows in sync: handle_subscribe pushes the same
1057            // subscription into the repo bucket, so unsubscribe must remove it
1058            // there too (otherwise a dead entry leaks).
1059            if let Some(repo_id) = repo_id_from_workflow_id(&sub.workflow_id) {
1060                let mut repo_workflows = self.repo_workflows.write().await;
1061                if let Some(list) = repo_workflows.get_mut(repo_id) {
1062                    if let Some(pos) = list.iter().position(|s| s.workflow_id == sub.workflow_id) {
1063                        list.remove(pos);
1064                    }
1065                    if list.is_empty() {
1066                        repo_workflows.remove(repo_id);
1067                    }
1068                }
1069            }
1070            tracing::info!(sub_id = %sub_id, "Unsubscribed");
1071        } else {
1072            tracing::debug!(sub_id = %sub_id, "Unsubscribe: subscription not found");
1073        }
1074        Ok(WorkflowResponseVariant::UnsubscribeResult)
1075    }
1076
1077    async fn handle_get_run(
1078        &self,
1079        _ctx: &EnvelopeContext,
1080        _request_id: u64,
1081        run_id: &str,
1082    ) -> AnyhowResult<WorkflowResponseVariant> {
1083        let run = self.get_run(&run_id.to_owned()).await?;
1084        let data = to_run_data(&run);
1085        Ok(WorkflowResponseVariant::GetRunResult(data))
1086    }
1087
1088    async fn handle_list_runs(
1089        &self,
1090        _ctx: &EnvelopeContext,
1091        _request_id: u64,
1092        workflow_id: &str,
1093    ) -> AnyhowResult<WorkflowResponseVariant> {
1094        let runs = self.list_runs(&workflow_id.to_owned()).await?;
1095        Ok(WorkflowResponseVariant::ListRunsResult(
1096            runs.iter().map(to_run_data).collect(),
1097        ))
1098    }
1099}
1100
1101// ═══════════════════════════════════════════════════════════════════════════════
1102// RequestService Implementation — delegates to generated dispatch_workflow
1103// ═══════════════════════════════════════════════════════════════════════════════
1104
1105#[async_trait(?Send)]
1106impl RequestService for WorkflowService {
1107    fn decode_request_body(
1108        &self,
1109        signed_body: &[u8],
1110    ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
1111        // The ONE bounded decode (v16 §5.2): the generated decoder derives
1112        // the full method leaf and returns the decoded message that policy,
1113        // MAC, and dispatch below all consume.
1114        crate::generated::workflow_client::decode_workflow_request_body(signed_body)
1115    }
1116
1117    async fn handle_request(
1118        &self,
1119        ctx: &EnvelopeContext,
1120        body: &hyprstream_rpc::service::DecodedRequestBody,
1121    ) -> AnyhowResult<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
1122        tracing::debug!(
1123            "Workflow request from {} (request_id={})",
1124            ctx.subject(),
1125            ctx.request_id
1126        );
1127        dispatch_workflow(self, ctx, body).await
1128    }
1129
1130    fn name(&self) -> &str {
1131        SERVICE_NAME
1132    }
1133
1134    fn transport(&self) -> &TransportConfig {
1135        &self.transport
1136    }
1137
1138    fn signing_key(&self) -> SigningKey {
1139        self.signing_key.clone()
1140    }
1141
1142    fn expected_audience(&self) -> Option<&str> {
1143        self.expected_audience.as_deref()
1144    }
1145
1146    fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
1147        self.jwt_key_source.clone()
1148    }
1149
1150    /// Accept delegated-bearer requests by consulting the **live** MCP relay
1151    /// authorizer (#989 review). `verify_claims` calls this before policy; the
1152    /// default `false` rejects all relayed caller tokens. We delegate to the
1153    /// injected authorizer (which checks current MCP-scope authorization) —
1154    /// fail-closed when no authorizer is set.
1155    fn accept_delegated_bearer(&self, signer_pubkey: &[u8; 32]) -> bool {
1156        self.relay_authorizer
1157            .as_ref()
1158            .map(|f| f(signer_pubkey))
1159            .unwrap_or(false)
1160    }
1161}
1162
1163/// Extract the `repo_id` from a `workflow_id` following the `repo_id:path`
1164/// convention. Returns `None` when the id does not contain a `':'` separator.
1165fn repo_id_from_workflow_id(workflow_id: &str) -> Option<&str> {
1166    workflow_id.split_once(':').map(|(repo_id, _)| repo_id)
1167}
1168
1169/// Build `EventTrigger`s from a parsed `Workflow`'s `on:` field.
1170///
1171/// Maps GitHub Actions event names to the closest `EventTrigger` variant.
1172fn extract_triggers(workflow: &Workflow) -> Vec<EventTrigger> {
1173    use super::parser::WorkflowTrigger;
1174    use super::triggers::RepoEventType;
1175
1176    let event_names: Vec<String> = match &workflow.on {
1177        WorkflowTrigger::Simple(event) => vec![event.clone()],
1178        WorkflowTrigger::List(events) => events.clone(),
1179        WorkflowTrigger::Complex(map) => map.keys().cloned().collect(),
1180    };
1181
1182    event_names
1183        .iter()
1184        .flat_map(|event| match event.as_str() {
1185            "push" => vec![EventTrigger::RepositoryEvent {
1186                event_type: RepoEventType::Push,
1187                pattern: None,
1188            }],
1189            "pull_request" => vec![EventTrigger::RepositoryEvent {
1190                event_type: RepoEventType::PullRequest,
1191                pattern: None,
1192            }],
1193            // GitHub's `create` event fires for BOTH branch and tag creation. There
1194            // is no dedicated branch-create variant in RepoEventType, so emit a Tag
1195            // trigger plus a Push trigger (the closest branch-ref match) to avoid
1196            // silently dropping branch-creation events.
1197            "create" => vec![
1198                EventTrigger::RepositoryEvent {
1199                    event_type: RepoEventType::Tag,
1200                    pattern: None,
1201                },
1202                EventTrigger::RepositoryEvent {
1203                    event_type: RepoEventType::Push,
1204                    pattern: None,
1205                },
1206            ],
1207            "tag" => vec![EventTrigger::RepositoryEvent {
1208                event_type: RepoEventType::Tag,
1209                pattern: None,
1210            }],
1211            "workflow_dispatch" => vec![EventTrigger::WorkflowDispatch {
1212                inputs: std::collections::HashMap::new(),
1213            }],
1214            "schedule" | "training" => vec![EventTrigger::Custom {
1215                topic: format!("training.{event}"),
1216                pattern: String::new(),
1217            }],
1218            _ => vec![EventTrigger::Custom {
1219                topic: format!("repo.{event}"),
1220                pattern: String::new(),
1221            }],
1222        })
1223        .collect()
1224}
1225
1226#[cfg(test)]
1227mod tests {
1228    // Regression guard for the anonymous-subject fail-closed defeat (#770
1229    // review FIX 1). `handle_dispatch` derives the caller subject from
1230    // `EnvelopeContext::subject()`, which returns `Subject::anonymous()` for
1231    // an unauthenticated caller. That subject must reach `dispatch` (→ runner
1232    // → `JobScheduler::place` → `AdmissionTracker::reserve`) *untouched*:
1233    // `reserve` is fail-closed on `subject.is_anonymous()`, so a converted
1234    // subject would silently pass admission and defeat per-subject quota.
1235    //
1236    // `EnvelopeContext`'s fields are private to `hyprstream-rpc`, so an
1237    // anonymous context can't be built from this crate; this test pins the
1238    // invariant the dispatch boundary depends on instead.
1239
1240    use hyprstream_rpc::prelude::SigningKey;
1241    use hyprstream_rpc::transport::TransportConfig;
1242    use hyprstream_vfs::Subject;
1243    use rand::rngs::OsRng;
1244    use std::path::PathBuf;
1245
1246    fn test_service() -> super::WorkflowService {
1247        super::WorkflowService::new(
1248            TransportConfig::inproc("test-workflow-service"),
1249            SigningKey::generate(&mut OsRng),
1250        )
1251    }
1252
1253    /// Minimal known-subset workflow YAML (parses in both modes).
1254    const KNOWN_YAML: &str = r#"
1255name: Gate
1256on: push
1257jobs:
1258  gate:
1259    runs-on: ubuntu-latest
1260    steps:
1261      - run: cargo test
1262"#;
1263
1264    /// Same workflow with an unknown top-level key (`permissions:`). Parses
1265    /// in legacy mode (silently dropped) but is REJECTED by strict mode.
1266    const UNKNOWN_KEY_YAML: &str = r#"
1267name: Gate
1268on: push
1269permissions:
1270  contents: read
1271jobs:
1272  gate:
1273    runs-on: ubuntu-latest
1274    steps:
1275      - run: cargo test
1276"#;
1277
1278    async fn write_workflow(
1279        repo_root: &std::path::Path,
1280        yaml: &str,
1281    ) -> std::result::Result<(), Box<dyn std::error::Error>> {
1282        let dir = repo_root.join(".github").join("workflows");
1283        tokio::fs::create_dir_all(&dir).await?;
1284        tokio::fs::write(dir.join("gate.yml"), yaml).await?;
1285        Ok(())
1286    }
1287
1288    fn workflow_ids(info: &[super::WorkflowInfo]) -> Vec<String> {
1289        info.iter().map(|i| i.id.clone()).collect()
1290    }
1291
1292    type TestResult = std::result::Result<(), Box<dyn std::error::Error>>;
1293
1294    // ─── Finding B: generic loader stays legacy; gate opts in via _with ─────
1295
1296    #[tokio::test]
1297    async fn legacy_scan_accepts_unknown_keys() -> TestResult {
1298        // The generic (non-gate) loader must tolerate unknown keys — forcing
1299        // strict globally breaks legacy non-gate compatibility (#1432
1300        // non-goal).
1301        let tmp = tempfile::tempdir()?;
1302        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1303
1304        let svc = test_service();
1305        svc.register_repo_path("acme", PathBuf::from(tmp.path()))
1306            .await;
1307
1308        let defs = svc.scan_repo("acme").await?;
1309        assert_eq!(defs.len(), 1, "legacy mode loads the unknown-key workflow");
1310        Ok(())
1311    }
1312
1313    #[tokio::test]
1314    async fn strict_scan_skips_unknown_keys() -> TestResult {
1315        let tmp = tempfile::tempdir()?;
1316        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1317
1318        let svc = test_service();
1319        svc.register_repo_path("acme", PathBuf::from(tmp.path()))
1320            .await;
1321
1322        let defs = svc
1323            .scan_repo_with("acme", super::super::parser::ParseMode::Strict)
1324            .await?;
1325        assert!(
1326            defs.is_empty(),
1327            "strict mode skips the unknown-key workflow"
1328        );
1329        Ok(())
1330    }
1331
1332    // ─── Finding A: strict rescan evicts the stale registration, fail-closed ─
1333
1334    #[tokio::test]
1335    async fn strict_rescan_evicts_stale_registration() -> TestResult {
1336        // 1. Load a valid (known-subset) workflow under strict mode → registered.
1337        let tmp = tempfile::tempdir()?;
1338        write_workflow(tmp.path(), KNOWN_YAML).await?;
1339
1340        let svc = test_service();
1341        svc.register_repo_path("acme", PathBuf::from(tmp.path()))
1342            .await;
1343
1344        svc.rescan_repo_with("acme", super::super::parser::ParseMode::Strict)
1345            .await?;
1346
1347        let wf_id = String::from("acme:.github/workflows/gate.yml");
1348        let after_first = svc.list_workflows().await?;
1349        assert!(
1350            workflow_ids(&after_first).contains(&wf_id),
1351            "known-subset workflow should be registered after first scan"
1352        );
1353
1354        // 2. Mutate the file to add an unknown key. Strict rescan must now
1355        //    skip it AND evict the now-stale registration — otherwise the
1356        //    stale workflow stays dispatchable (the fail-closed gap).
1357        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1358        svc.rescan_repo_with("acme", super::super::parser::ParseMode::Strict)
1359            .await?;
1360
1361        let after_second = svc.list_workflows().await?;
1362        assert!(
1363            !workflow_ids(&after_second).contains(&wf_id),
1364            "stale registration must be evicted: a strict-rejected workflow \
1365             must not remain dispatchable"
1366        );
1367
1368        // 3. Fail-closed confirmation: dispatching the evicted id must now
1369        //    be WorkflowNotFound, not a stale run.
1370        match svc
1371            .dispatch(
1372                &wf_id,
1373                std::collections::HashMap::new(),
1374                &Subject::anonymous(),
1375            )
1376            .await
1377        {
1378            Err(crate::error::WorkerError::WorkflowNotFound(id)) => assert_eq!(id, wf_id),
1379            other => panic!("expected WorkflowNotFound for evicted workflow, got {other:?}"),
1380        }
1381        Ok(())
1382    }
1383
1384    #[tokio::test]
1385    async fn legacy_rescan_keeps_workflow_when_unknown_key_present() -> TestResult {
1386        // Legacy rescan must keep the registration (unknown keys tolerated) —
1387        // guards against accidentally evicting under the generic path.
1388        let tmp = tempfile::tempdir()?;
1389        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1390
1391        let svc = test_service();
1392        svc.register_repo_path("acme", PathBuf::from(tmp.path()))
1393            .await;
1394
1395        svc.rescan_repo("acme").await?;
1396        let wf_id = String::from("acme:.github/workflows/gate.yml");
1397        let listed = svc.list_workflows().await?;
1398        assert!(
1399            workflow_ids(&listed).contains(&wf_id),
1400            "legacy rescan must keep the unknown-key workflow registered"
1401        );
1402        Ok(())
1403    }
1404
1405    // ─── Rev 4 P1-A: strict mode retained across the generic rescan path ────
1406
1407    #[tokio::test]
1408    async fn strict_mode_retained_across_legacy_signature_rescan() -> TestResult {
1409        // The event-triggered rescan calls the generic `rescan_repo(repo)`
1410        // (no mode arg). Once a gate has loaded the repo in strict mode
1411        // (recorded via set_repo_mode), that rescan must RETAIN strict —
1412        // otherwise a push that mutates the workflow to add an unknown key
1413        // would silently reload it in legacy mode (the bypass).
1414        let tmp = tempfile::tempdir()?;
1415        write_workflow(tmp.path(), KNOWN_YAML).await?;
1416
1417        let svc = test_service();
1418        svc.register_repo_path("acme", PathBuf::from(tmp.path()))
1419            .await;
1420        // Gate loads in strict mode → records the mode per-repo.
1421        svc.set_repo_mode("acme", super::super::parser::ParseMode::Strict)
1422            .await;
1423        svc.rescan_repo("acme").await?; // registers the known workflow
1424
1425        let wf_id = String::from("acme:.github/workflows/gate.yml");
1426        assert!(workflow_ids(&svc.list_workflows().await?).contains(&wf_id));
1427
1428        // Mutate to add an unknown key, then trigger the GENERIC rescan path
1429        // (the one HandlerResult::Rescan / initialize use — no mode arg).
1430        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1431        svc.rescan_repo("acme").await?; // must still be strict → skip + evict
1432
1433        assert!(
1434            !workflow_ids(&svc.list_workflows().await?).contains(&wf_id),
1435            "strict mode must be retained across the generic rescan path; \
1436             a now-unknown-key workflow must be skipped and evicted, not \
1437             silently reloaded in legacy mode"
1438        );
1439        Ok(())
1440    }
1441
1442    #[tokio::test]
1443    async fn strict_mode_default_is_legacy_for_unregistered_repo() -> TestResult {
1444        // A repo that never opted into strict must rescan in legacy mode —
1445        // guards against the mode store accidentally defaulting to strict.
1446        let tmp = tempfile::tempdir()?;
1447        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1448
1449        let svc = test_service();
1450        svc.register_repo_path("acme", PathBuf::from(tmp.path()))
1451            .await;
1452        // No set_repo_mode call → default Legacy.
1453        svc.rescan_repo("acme").await?;
1454
1455        let wf_id = String::from("acme:.github/workflows/gate.yml");
1456        assert!(
1457            workflow_ids(&svc.list_workflows().await?).contains(&wf_id),
1458            "unregistered repo must default to legacy and keep the unknown-key workflow"
1459        );
1460        Ok(())
1461    }
1462
1463    // ─── Rev 5 P1: rescan_repo_with persists mode for later generic rescan ──
1464
1465    #[tokio::test]
1466    async fn explicit_rescan_with_strict_persists_mode_for_generic_rescan() -> TestResult {
1467        // Enter strict via rescan_repo_with DIRECTLY (not load_repo_with nor
1468        // set_repo_mode). The mode must persist so a later generic
1469        // rescan_repo(repo) — the path HandlerResult::Rescan / initialize use
1470        // — stays strict rather than falling back to legacy.
1471        let tmp = tempfile::tempdir()?;
1472        write_workflow(tmp.path(), KNOWN_YAML).await?;
1473
1474        let svc = test_service();
1475        svc.register_repo_path("acme", PathBuf::from(tmp.path()))
1476            .await;
1477
1478        // Enter strict via the explicit rescan variant only.
1479        svc.rescan_repo_with("acme", super::super::parser::ParseMode::Strict)
1480            .await?;
1481        let wf_id = String::from("acme:.github/workflows/gate.yml");
1482        assert!(workflow_ids(&svc.list_workflows().await?).contains(&wf_id));
1483
1484        // Mutate to add an unknown key, then trigger the GENERIC rescan path
1485        // (no mode arg). It must have retained strict from the prior
1486        // rescan_repo_with call.
1487        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1488        svc.rescan_repo("acme").await?;
1489
1490        assert!(
1491            !workflow_ids(&svc.list_workflows().await?).contains(&wf_id),
1492            "rescan_repo_with(Strict) must persist the mode: a later generic \
1493             rescan_repo must stay strict and skip+evict the unknown-key workflow, \
1494             not silently fall back to legacy"
1495        );
1496        Ok(())
1497    }
1498
1499    // ─── Rev 6 P1: generic rescan must NOT overwrite an explicit selection ──
1500
1501    #[tokio::test]
1502    async fn generic_rescan_does_not_overwrite_explicit_strict_selection() -> TestResult {
1503        // The generic rescan_repo(repo) resolves the stored mode and rescans
1504        // in it but must NOT persist — otherwise a generic rescan that read a
1505        // stale Legacy could clobber a newer explicit Strict selection.
1506        // Enter strict explicitly, then run a generic rescan, then verify the
1507        // mode is still strict behaviorally (a later generic rescan after an
1508        // unknown-key mutation skips+evicts).
1509        let tmp = tempfile::tempdir()?;
1510        write_workflow(tmp.path(), KNOWN_YAML).await?;
1511
1512        let svc = test_service();
1513        svc.register_repo_path("acme", PathBuf::from(tmp.path()))
1514            .await;
1515
1516        // Explicit strict selection.
1517        svc.rescan_repo_with("acme", super::super::parser::ParseMode::Strict)
1518            .await?;
1519        let wf_id = String::from("acme:.github/workflows/gate.yml");
1520        assert!(workflow_ids(&svc.list_workflows().await?).contains(&wf_id));
1521
1522        // A generic rescan in between must not downgrade the stored mode.
1523        svc.rescan_repo("acme").await?;
1524        assert!(workflow_ids(&svc.list_workflows().await?).contains(&wf_id));
1525
1526        // Now mutate to add an unknown key. A generic rescan must STILL be
1527        // strict (the intermediate generic rescan did not overwrite it).
1528        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1529        svc.rescan_repo("acme").await?;
1530        assert!(
1531            !workflow_ids(&svc.list_workflows().await?).contains(&wf_id),
1532            "generic rescan must not overwrite the explicit strict selection: \
1533             the intermediate rescan_repo(repo) should have left the stored \
1534             mode as Strict"
1535        );
1536        Ok(())
1537    }
1538
1539    #[tokio::test]
1540    async fn concurrent_generic_rescans_cannot_clobber_explicit_strict() -> TestResult {
1541        use std::sync::Arc;
1542        // Race invariant: regardless of how generic rescans interleave with
1543        // an explicit strict selection, the final stored mode must be Strict
1544        // (because only rescan_repo_with / load_repo_with persist; the generic
1545        // rescan_repo never writes). Asserted behaviorally: after the batch,
1546        // a generic rescan of an unknown-key workflow skips+evicts it.
1547        let tmp = tempfile::tempdir()?;
1548        write_workflow(tmp.path(), KNOWN_YAML).await?;
1549        let root = tmp.path().to_owned();
1550
1551        let svc = Arc::new(test_service());
1552        svc.register_repo_path("acme", PathBuf::from(&root)).await;
1553
1554        // Run an explicit strict selection concurrently with several generic
1555        // rescans. The explicit one is the only writer.
1556        let mut handles = Vec::new();
1557        handles.push({
1558            let svc = Arc::clone(&svc);
1559            tokio::spawn(async move {
1560                svc.rescan_repo_with("acme", super::super::parser::ParseMode::Strict)
1561                    .await
1562            })
1563        });
1564        for _ in 0..4 {
1565            let svc = Arc::clone(&svc);
1566            handles.push(tokio::spawn(async move { svc.rescan_repo("acme").await }));
1567        }
1568        for h in handles {
1569            h.await.map_err(|e| format!("join error: {e}"))??;
1570        }
1571
1572        // Mutate to add an unknown key; a generic rescan must be strict.
1573        write_workflow(&root, UNKNOWN_KEY_YAML).await?;
1574        svc.rescan_repo("acme").await?;
1575
1576        let wf_id = String::from("acme:.github/workflows/gate.yml");
1577        let listed = svc.list_workflows().await?;
1578        assert!(
1579            !workflow_ids(&listed).contains(&wf_id),
1580            "after concurrent generic rescans interleaved with one explicit \
1581             strict selection, the stored mode must remain Strict — the \
1582             generic rescans must not have overwritten it"
1583        );
1584        Ok(())
1585    }
1586
1587    // ─── Rev 8 P1: forced check-to-commit interleaving regression ──────────
1588
1589    #[tokio::test]
1590    async fn forced_interleaving_legacy_cannot_commit_after_strict() -> TestResult {
1591        // The per-repo mutex makes the scan→commit window atomic. A Legacy
1592        // scan that started before an explicit Strict selection cannot commit
1593        // after it: the two are serialized, and whichever runs second sees
1594        // the committed state of the first.
1595        //
1596        // This test runs both concurrently via tokio::join!, exercising both
1597        // possible orderings. Regardless of order, the registry must be strict
1598        // immediately after — no extra rescan needed.
1599        use std::sync::Arc;
1600        let tmp = tempfile::tempdir()?;
1601        write_workflow(tmp.path(), UNKNOWN_KEY_YAML).await?;
1602        let root = tmp.path().to_owned();
1603
1604        let svc = Arc::new(test_service());
1605        svc.register_repo_path("acme", PathBuf::from(&root)).await;
1606
1607        // Spawn: a generic rescan (reads Legacy by default) and an explicit
1608        // Strict selection, concurrently. The per-repo mutex serializes them.
1609        let svc_a = Arc::clone(&svc);
1610        let svc_b = Arc::clone(&svc);
1611        let (ra, rb) = tokio::join!(async { svc_a.rescan_repo("acme").await }, async {
1612            svc_b
1613                .rescan_repo_with("acme", super::super::parser::ParseMode::Strict)
1614                .await
1615        });
1616        ra?;
1617        rb?;
1618
1619        // Registry must be strict — the unknown-key workflow is NOT
1620        // registered, regardless of which scan committed first. No extra
1621        // rescan required.
1622        let wf_id = String::from("acme:.github/workflows/gate.yml");
1623        assert!(
1624            !workflow_ids(&svc.list_workflows().await?).contains(&wf_id),
1625            "after concurrent Legacy + Strict rescans, the registry must be \
1626             strict (unknown-key workflow not registered) — the per-repo mutex \
1627             prevents a stale Legacy scan from committing after Strict"
1628        );
1629        Ok(())
1630    }
1631
1632    // ─── Regression guard kept from prior revision ──────────────────────────
1633
1634    /// #989 review P2-1: the factory must feed the OAuth issuer audience (and
1635    /// cluster JWT key source) into `WorkflowService` so JWT-authenticated
1636    /// IPC/QUIC requests pass envelope verification *before* the policy
1637    /// `authorize` callback. Without the `expected_audience`/`jwt_key_source`
1638    /// `RequestService` overrides, both default to `None` and JWT auth fails
1639    /// upstream of policy. This pins the audience half of the wiring (the
1640    /// `jwt_key_source` half follows the identical field→setter→override
1641    /// pattern, compile-checked by the override above; a full round-trip would
1642    /// need a `JwtKeySource` stub, deferred).
1643    /// #989 review (68d11b838 follow-up): `accept_delegated_bearer` must consult
1644    /// a LIVE authorizer, not a frozen key snapshot. These tests inject mock
1645    /// authorizers to prove: fail-closed when unset, multi-key acceptance, non-
1646    /// MCP rejection, and — critically — that live revocation takes effect
1647    /// immediately (no stale snapshot).
1648    #[test]
1649    fn workflow_accepts_delegated_bearer_from_live_mcp_authorizer() {
1650        use super::{SigningKey, WorkflowService};
1651        use hyprstream_rpc::service::RequestService;
1652        use hyprstream_rpc::transport::TransportConfig;
1653
1654        let sk = SigningKey::from_bytes(&[0x42; 32]);
1655        let mut svc = WorkflowService::new(TransportConfig::inproc("wf-relay-test"), sk);
1656
1657        // No authorizer set → fail-closed (verify_claims would reject).
1658        assert!(!RequestService::accept_delegated_bearer(&svc, &[0u8; 32]));
1659
1660        // Inject a live authorizer accepting two MCP keys.
1661        let key_a = [0xAB; 32];
1662        let key_b = [0xCD; 32];
1663        svc.set_relay_authorizer(std::sync::Arc::new(move |pk: &[u8; 32]| {
1664            *pk == key_a || *pk == key_b
1665        }));
1666
1667        assert!(
1668            RequestService::accept_delegated_bearer(&svc, &key_a),
1669            "must accept authorized MCP key A"
1670        );
1671        assert!(
1672            RequestService::accept_delegated_bearer(&svc, &key_b),
1673            "must accept authorized MCP key B"
1674        );
1675        // Non-MCP signer rejected — independently scoped, not a wildcard.
1676        assert!(!RequestService::accept_delegated_bearer(&svc, &[0xEF; 32]));
1677        assert!(!RequestService::accept_delegated_bearer(&svc, &[0u8; 32]));
1678    }
1679
1680    /// Live revocation: the same key goes from accepted to rejected **without
1681    /// re-injecting the authorizer** — proving there is no frozen snapshot. A
1682    /// shared `AtomicBool` simulates the trust store revoking the key at runtime.
1683    #[test]
1684    fn workflow_relay_revocation_takes_effect_immediately() {
1685        use std::sync::atomic::{AtomicBool, Ordering};
1686
1687        use super::{SigningKey, WorkflowService};
1688        use hyprstream_rpc::service::RequestService;
1689        use hyprstream_rpc::transport::TransportConfig;
1690
1691        let sk = SigningKey::from_bytes(&[0x99; 32]);
1692        let mut svc = WorkflowService::new(TransportConfig::inproc("wf-revoke"), sk);
1693
1694        // Shared live-authorization flag (simulates the trust store's state).
1695        let authorized = std::sync::Arc::new(AtomicBool::new(true));
1696        let flag = authorized.clone();
1697        svc.set_relay_authorizer(std::sync::Arc::new(move |_pk: &[u8; 32]| {
1698            flag.load(Ordering::Relaxed)
1699        }));
1700
1701        let key = [0x77; 32];
1702        // While authorized → accepted.
1703        assert!(
1704            RequestService::accept_delegated_bearer(&svc, &key),
1705            "must accept while the key is live-authorized"
1706        );
1707
1708        // Simulate live revocation (trust store removes / expires the key).
1709        authorized.store(false, Ordering::Relaxed);
1710
1711        // Same key, same authorizer, same service — now rejected. No snapshot.
1712        assert!(
1713            !RequestService::accept_delegated_bearer(&svc, &key),
1714            "must reject immediately after live revocation (no frozen snapshot)"
1715        );
1716    }
1717
1718    #[test]
1719    fn jwt_expected_audience_round_trips_through_request_service() {
1720        use super::{SigningKey, WorkflowService};
1721        use hyprstream_rpc::service::RequestService;
1722        use hyprstream_rpc::transport::TransportConfig;
1723
1724        let sk = SigningKey::from_bytes(&[0x42; 32]);
1725        let mut svc = WorkflowService::new(TransportConfig::inproc("wf-plumb-test"), sk);
1726        // Default: no audience configured → a JWT-auth request would be denied
1727        // at envelope verification (the pre-fix behavior the review flagged).
1728        assert!(RequestService::expected_audience(&svc).is_none());
1729        assert!(RequestService::jwt_key_source(&svc).is_none());
1730
1731        svc.set_expected_audience("https://test.hyprstream.local".to_owned());
1732        assert_eq!(
1733            RequestService::expected_audience(&svc),
1734            Some("https://test.hyprstream.local")
1735        );
1736    }
1737
1738    #[test]
1739    fn anonymous_subject_must_not_round_trip_through_string() {
1740        // What `ctx.subject()` yields for an unauthenticated caller.
1741        let anon = Subject::anonymous();
1742        assert!(anon.is_anonymous());
1743
1744        // The broken transform `handle_dispatch` used to do:
1745        // `Subject::new(ctx.subject().to_string())`. `Subject::anonymous()`
1746        // Displays as the literal "anonymous", so re-wrapping it produces a
1747        // NAMED subject — fail-closed defeated.
1748        let broken = Subject::new(anon.to_string());
1749        assert_ne!(anon, broken);
1750        assert!(
1751            !broken.is_anonymous(),
1752            "round-tripping an anonymous subject through Display must stay \
1753             forbidden: it turns anonymous into the named subject \"anonymous\" \
1754             and bypasses fail-closed admission"
1755        );
1756
1757        // The fixed transform: pass the verified `Subject` through untouched.
1758        let fixed: Subject = anon.clone();
1759        assert!(fixed.is_anonymous());
1760        assert_eq!(anon, fixed);
1761    }
1762}