Skip to main content

hyprstream_core/services/
factories.rs

1//! Service factory functions for inventory-based registration.
2//!
3//! This module contains all `#[service_factory]` decorated functions that
4//! automatically register services with the inventory system.
5//!
6//! # Pattern
7//!
8//! Same pattern as:
9//! - `#[register_scopes]` for authorization scopes
10//! - `DriverFactory` in git2db for storage drivers
11//!
12//! # Usage
13//!
14//! ```ignore
15//! use hyprstream_rpc::service::{get_factory, ServiceContext};
16//!
17//! let ctx = ServiceContext::new(...);
18//! let factory = get_factory("policy").unwrap();
19//! let spawnable = (factory.factory)(&ctx)?;
20//! manager.spawn(spawnable).await?;
21//! ```
22
23use std::collections::HashMap;
24use std::sync::Arc;
25
26use anyhow::Context;
27use git2db::Git2DB;
28use hyprstream_rpc::moq_event::MoqEventOrigin;
29use hyprstream_rpc::prelude::*;
30use hyprstream_rpc::registry::SocketKind;
31use hyprstream_rpc::service_factory;
32use hyprstream_service::{ServiceContext, Spawnable};
33use hyprstream_vfs::{MountTarget, Namespace};
34use tokio::sync::RwLock;
35use tracing::info;
36
37use crate::auth::identity_store::credentials_dir;
38use crate::auth::PolicyManager;
39use crate::config::HyprConfig;
40use crate::services::{DiscoveryService, McpConfig, McpService, PolicyService, RegistryService};
41use hyprstream_rpc_std::policy_client::{
42    PolicyClient, RefreshServiceTokenRequest, RegisterServiceKey,
43};
44use hyprstream_rpc_std::registry_client::RegistryClient;
45
46/// Load HyprConfig, failing closed on error.
47///
48/// A config that cannot be loaded or parsed must NOT silently become the
49/// defaults: backend selection (#1257) keys off `[rds]`, so discarding a
50/// configured RDS binding would flip an RDS-bound node back to the local
51/// RocksDB backend while its AZ peers run Postgres. A missing config file is
52/// not an error (the loader treats it as optional); parse/IO failures are.
53pub(crate) fn load_config() -> anyhow::Result<HyprConfig> {
54    HyprConfig::load().map_err(|e| anyhow::anyhow!("failed to load hyprstream configuration: {e}"))
55}
56
57/// Get the JWT bound to this service instance's exact signing key.
58pub(crate) fn service_token(signing_key: &SigningKey) -> Option<String> {
59    let trust = hyprstream_service::global_trust_store();
60    trust
61        .get(&signing_key.verifying_key())
62        .and_then(|att| att.jwt)
63}
64
65fn policy_client_for_transport(
66    transport: &hyprstream_rpc::transport::TransportConfig,
67    signing_key: SigningKey,
68    policy_vk: hyprstream_rpc::crypto::VerifyingKey,
69    token: Option<String>,
70) -> anyhow::Result<PolicyClient> {
71    PolicyClient::for_local_transport_bootstrap(transport, signing_key, policy_vk, token)
72}
73
74#[cfg(feature = "postgres")]
75fn federate_dispatch_for_service(
76    policy_client: PolicyClient,
77    configured_issuer: Option<&str>,
78) -> anyhow::Result<Option<Arc<crate::services::oauth::federate_proof::DispatchAdapter>>> {
79    if hyprstream_rpc::proof::enrollment::global_enrollment_resolver().is_none() {
80        return Ok(None);
81    }
82    // This is also the RPC expected audience installed below. Without that
83    // authoritative identity, the Federate consumer must remain disabled.
84    let Some(configured_issuer) = configured_issuer else {
85        return Ok(None);
86    };
87    let Ok(host) = crate::services::oauth::federate_source::FederateHost::new(configured_issuer)
88    else {
89        tracing::warn!(
90            "Federate dispatch disabled: OAuth issuer must be an exact canonical HTTPS origin"
91        );
92        return Ok(None);
93    };
94    let adapter = crate::services::oauth::federate_proof::DispatchAdapter::from_global_enrollment(
95        policy_client,
96        host,
97    )?;
98    Ok(Some(Arc::new(adapter)))
99}
100
101/// Shared Git2DB registry instance. Lazily initialized by the first factory
102/// that needs it. Both PolicyService and RegistryService share this instance.
103static SHARED_GIT2DB: std::sync::OnceLock<Arc<RwLock<Git2DB>>> = std::sync::OnceLock::new();
104
105/// Get or initialize the shared Git2DB registry for the given models directory.
106fn get_or_init_git2db(models_dir: &std::path::Path) -> anyhow::Result<Arc<RwLock<Git2DB>>> {
107    if let Some(existing) = SHARED_GIT2DB.get() {
108        return Ok(Arc::clone(existing));
109    }
110
111    let registry = tokio::task::block_in_place(|| {
112        let rt = tokio::runtime::Handle::current();
113        rt.block_on(Git2DB::open(models_dir))
114    })
115    .context("Failed to initialize shared Git2DB registry")?;
116
117    let shared = Arc::new(RwLock::new(registry));
118    // If another thread beat us, that's fine — use theirs
119    Ok(Arc::clone(SHARED_GIT2DB.get_or_init(|| shared)))
120}
121
122/// Resolve the on-disk directory for the durable PDS record store (#910a).
123///
124/// A single RocksDB database lives here, matching `RocksDbUserStore`'s
125/// `<config_dir>/users.db` convention. The registry service (the sole
126/// publisher) opens it read-write; the discovery service (the resolver)
127/// opens it read-only — see `services::discovery::PdsRecordStore`.
128#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
129pub(crate) fn pds_store_dir(ctx: &ServiceContext) -> anyhow::Result<std::path::PathBuf> {
130    Ok(ctx.deployment_data_dir()?.join("pds-store"))
131}
132
133/// Select and open the PDS record store based on the configured backend (#1257).
134///
135/// The records-role binding is resolved from `[rds]` TOML first, then the
136/// role-scoped env vars (`HYPRSTREAM_RECORDS_URL_FILE` /
137/// `HYPRSTREAM_RECORDS_SSLROOTCERT_FILE`), then the shared credentials
138/// directory — see [`crate::config::RdsConfig::resolved_from_env`]. When a
139/// binding resolves (the deployed posture):
140/// - With `pds-postgres` feature: open against shared RDS Multi-AZ Postgres
141///   (FATAL-on-unavailable — never a silent local fallback).
142/// - Without `pds-postgres` feature: FATAL (binary not built for RDS).
143///
144/// When no binding resolves (local/workstation): open the RocksDB
145/// backend at `pds_store_dir(ctx)`.
146///
147/// `readonly` selects the resolver (`true`) vs publisher (`false`) posture.
148/// The D2 invariant (signed bytes verbatim, SQL = projection-free KV) is
149/// enforced by the Postgres backend regardless of this flag.
150#[cfg(feature = "rocksdb")]
151fn open_pds_record_store(
152    ctx: &ServiceContext,
153    readonly: bool,
154) -> anyhow::Result<crate::services::discovery::PdsRecordStore> {
155    open_pds_record_store_at(&pds_store_dir(ctx)?, readonly)
156}
157
158/// [`open_pds_record_store`] for callers that hold no `ServiceContext` (the
159/// QUIC startup gate's announcement refresh): the local-backend directory is
160/// passed explicitly and consulted only when no RDS binding resolves.
161#[cfg(feature = "rocksdb")]
162fn open_pds_record_store_at(
163    store_dir: &std::path::Path,
164    readonly: bool,
165) -> anyhow::Result<crate::services::discovery::PdsRecordStore> {
166    let rds = load_config()?.rds.resolved_from_env()?;
167    if rds.is_configured() {
168        #[cfg(feature = "pds-postgres")]
169        {
170            tracing::info!(
171                cell_id = %rds.cell_id,
172                "opening PDS record store against shared RDS Postgres ({} mode)",
173                if readonly { "read-only" } else { "read-write" }
174            );
175            return crate::services::discovery::PdsRecordStore::open_postgres(&rds, readonly);
176        }
177        #[cfg(not(feature = "pds-postgres"))]
178        {
179            anyhow::bail!(
180                "RDS is configured ([rds] section or HYPRSTREAM_RECORDS_URL_FILE) but this \
181                 binary was not built with the `pds-postgres` feature — refusing to silently \
182                 fall back to local RocksDB. Rebuild with --features pds-postgres."
183            );
184        }
185    }
186    // Local RocksDB backend (workstation / dev).
187    if readonly {
188        crate::services::discovery::PdsRecordStore::open_readonly(store_dir)
189    } else {
190        crate::services::discovery::PdsRecordStore::open(store_dir)
191    }
192}
193
194/// The lifecycle state of the checkpointed PDS store, as seen by the QUIC
195/// startup gate.
196#[derive(Debug, Clone, Copy, PartialEq, Eq)]
197pub enum PdsBootState {
198    /// ≥1 checkpoint-verified accepted state is present. The caller MUST run
199    /// [`with_checkpointed_native_announcements`] to populate announcements.
200    Populated,
201    /// Genuine first boot: the store has never held an accepted state (the
202    /// durable first-boot RocksDB key is present alongside an empty store).
203    /// QUIC may be deferred for this run; a restart is required after the
204    /// registry writes its first accepted state. No runtime auto-activation
205    /// is implemented.
206    FirstBoot,
207}
208
209/// Classify the checkpointed PDS store for the QUIC startup gate.
210///
211/// The store consulted is the CONFIGURED backend (#1257): with an RDS binding
212/// resolved, classification reads the shared Postgres store (the local
213/// directory is never consulted — accepted states and the first-boot marker
214/// live in RDS); otherwise it reads the local RocksDB store.
215///
216/// This replaces the prior emptiness check, which could not distinguish a
217/// freshly provisioned first-boot store from a steady-state store that lost
218/// its data, and which collapsed every read/decode/signature/consistency
219/// error into "first boot". The classification is:
220///
221/// - Store directory present, marker present, store empty →
222///   [`PdsBootState::FirstBoot`] (provisioned via `init-deployment-store`;
223///   the registry has not yet written an accepted state).
224/// - Store directory present, ≥1 verified accepted state →
225///   [`PdsBootState::Populated`].
226/// - Store directory present, empty, **no** marker → `Err` (possible data
227///   loss or corruption; a loud steady-state failure — never defer).
228/// - Store directory **absent** → `Err` (missing security history; run
229///   `init-deployment-store` to provision, or restore from backup).
230/// - Any read/decode/signature/consistency failure → `Err` (propagated;
231///   fail-closed at the checkpoint boundary).
232///
233/// (In RDS posture the four directory cases above apply to the shared store
234/// instead of the directory: no directory-absence case exists, and the
235/// empty-unmarked case means the shared store lost its data.)
236///
237/// Only [`PdsBootState::FirstBoot`] may enter a QUIC deferral path; every
238/// other outcome either proceeds with checkpointed announcements or fails
239/// startup.
240#[cfg(feature = "rocksdb")]
241pub fn classify_pds_store_for_quic(ctx: &ServiceContext) -> anyhow::Result<PdsBootState> {
242    if load_config()?.rds.resolved_from_env()?.is_configured() {
243        // RDS posture: lifecycle evidence lives in the shared store. The
244        // local directory is deliberately NOT consulted — reading a stale or
245        // empty local store here misclassified the deployment.
246        let acceptance_identity = hyprstream_discovery::deployment_registry_verifier()?;
247        let store =
248            open_pds_record_store(ctx, true)?.with_at9p_deployment_verifier(acceptance_identity);
249        // Propagates connection, decode, signature-verification, and
250        // consistency failures as Err — these are NEVER evidence of first boot.
251        let states = store.accepted_at9p_states()?;
252        if !states.is_empty() {
253            return Ok(PdsBootState::Populated);
254        }
255        if store.first_boot_pending()? {
256            return Ok(PdsBootState::FirstBoot);
257        }
258        anyhow::bail!(
259            "the RDS-backed PDS accepted-state store holds no verified accepted \
260             states and no first-boot provisioning marker. This indicates data loss \
261             or corruption, not first boot; refusing to defer QUIC startup. Run \
262             `init-deployment-store` to provision a fresh deployment against RDS."
263        );
264    }
265    let store_dir = pds_store_dir(ctx)?;
266    if !store_dir.exists() {
267        // A missing store is indistinguishable from deletion of security
268        // history — the checkpoint source's documented rule. First boot is an
269        // explicit provisioning step (init-deployment-store), not an absent
270        // directory. Fail closed; do not defer.
271        anyhow::bail!(
272            "PDS accepted-state store at {} does not exist. Run \
273             `init-deployment-store` to provision a fresh deployment, or \
274             restore from backup. Refusing to defer QUIC startup.",
275            store_dir.display()
276        );
277    }
278    let acceptance_identity = hyprstream_discovery::deployment_registry_verifier()?;
279    let store = crate::services::discovery::PdsRecordStore::open_readonly(&store_dir)?
280        .with_at9p_deployment_verifier(acceptance_identity);
281    // Propagates decode, signature-verification, and consistency failures
282    // (half-checkpoint mismatches, watermark/digest mismatches) as Err —
283    // these are NEVER evidence of first boot.
284    let states = store.accepted_at9p_states()?;
285    if !states.is_empty() {
286        return Ok(PdsBootState::Populated);
287    }
288    // The store exists and opens cleanly but holds no accepted states. Only
289    // the durable first-boot marker (a RocksDB key deleted atomically with
290    // the first accepted-state commit) distinguishes genuine first boot from
291    // data loss.
292    if store.first_boot_pending()? {
293        return Ok(PdsBootState::FirstBoot);
294    }
295    anyhow::bail!(
296        "PDS accepted-state store at {} exists but contains no verified accepted \
297         states and no first-boot provisioning marker. This indicates data loss \
298         or corruption, not first boot; refusing to defer QUIC startup.",
299        store_dir.display()
300    )
301}
302
303/// Fail-closed twin: without the `rocksdb` feature there is no PDS store to
304/// classify, so startup never enters a QUIC deferral path.
305#[cfg(not(feature = "rocksdb"))]
306pub fn classify_pds_store_for_quic(_ctx: &ServiceContext) -> anyhow::Result<PdsBootState> {
307    anyhow::bail!("PDS accepted-state store requires the `rocksdb` feature")
308}
309
310#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
311fn accepted_state_matches_service(
312    state: &hyprstream_pds::at9p_duplicity::AcceptedAt9pState,
313    service_name: &str,
314    verifying_key: &[u8; 32],
315) -> bool {
316    let service_id = format!("#{service_name}");
317    state
318        .current
319        .services
320        .iter()
321        .any(|entry| entry.id == service_id)
322        && state
323            .current
324            .subject_keys
325            .iter()
326            .any(|key| key.ed25519_pub.as_slice() == verifying_key)
327}
328
329/// Rebuild one running service's announcement from a fresh checkpoint read
330/// and the latest registered JWT. No stale authority is returned on failure.
331///
332/// The checkpoint read honors the configured records backend (#1257): in RDS
333/// posture it reads the shared Postgres store, so the announcement reflects
334/// cross-AZ accepted states — never an absent or stale local RocksDB.
335#[cfg(feature = "rocksdb")]
336pub fn current_native_announcement(
337    request: &mut hyprstream_service::NativeAnnouncementRequest,
338) -> anyhow::Result<hyprstream_rpc_std::discovery_client::ServiceAnnouncement> {
339    use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
340    let verifier = hyprstream_discovery::deployment_registry_verifier()?;
341    let store = open_pds_record_store_at(
342        &hyprstream_service::deployment_data_dir()?.join("pds-store"),
343        true,
344    )?
345    .with_at9p_deployment_verifier(verifier);
346    let now = chrono::Utc::now();
347    let state = store
348        .accepted_at9p_state(
349            &request.service_did.to_string(),
350            Some(&now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)),
351        )?
352        .ok_or_else(|| anyhow::anyhow!("native announcement accepted state is unavailable"))?;
353    request.refresh_from_accepted_state(&state)?;
354    let service_jwt = hyprstream_service::global_trust_store()
355        .get(&request.signing_key.verifying_key())
356        .and_then(|attestation| attestation.jwt)
357        .or_else(|| request.service_jwt.clone());
358    if let Some(jwt) = service_jwt.as_deref() {
359        let payload = jwt
360            .split('.')
361            .nth(1)
362            .ok_or_else(|| anyhow::anyhow!("native announcement JWT is malformed"))?;
363        // Do not include token/claims in diagnostics. Signature verification is
364        // still performed by Discovery; this check bounds local publication.
365        let claims: serde_json::Value = URL_SAFE_NO_PAD
366            .decode(payload)
367            .ok()
368            .and_then(|bytes| serde_json::from_slice(&bytes).ok())
369            .ok_or_else(|| anyhow::anyhow!("native announcement JWT is malformed"))?;
370        let expiry = claims["exp"]
371            .as_i64()
372            .and_then(|seconds| seconds.checked_mul(1_000))
373            .ok_or_else(|| anyhow::anyhow!("native announcement JWT has no bounded expiry"))?;
374        anyhow::ensure!(
375            expiry > now.timestamp_millis(),
376            "native announcement JWT expired"
377        );
378    }
379    Ok(hyprstream_rpc_std::discovery_client::ServiceAnnouncement {
380        service_name: request.service_name.clone(),
381        socket_kind: request.reach.socket_kind().to_owned(),
382        endpoint: request.reach.endpoint(),
383        service_jwt,
384        service_did: request.service_did.clone(),
385        capabilities: request.capabilities.clone(),
386        accepted_state_digest: request.accepted_state_digest.clone(),
387        accepted_state_epoch: request.accepted_state_epoch,
388        response_key_id: request.response_key_id.clone(),
389        request_kem_key_id: request.request_kem_key_id.clone(),
390        request_kem_recipient: request.request_kem_recipient.clone(),
391        expires_at_unix_ms: request.expires_at_unix_ms,
392    })
393}
394
395/// Fail-closed twin: no checkpoint-verifying PDS read exists without the
396/// `rocksdb` feature.
397#[cfg(not(feature = "rocksdb"))]
398pub fn current_native_announcement(
399    _request: &mut hyprstream_service::NativeAnnouncementRequest,
400) -> anyhow::Result<hyprstream_rpc_std::discovery_client::ServiceAnnouncement> {
401    anyhow::bail!("checkpointed native announcements require the `rocksdb` feature")
402}
403
404/// Whether the checkpoint announcement loop must authorize `service_name`.
405///
406/// Compatibility checkpoints may legitimately predate a `#discovery`
407/// identity: the compatibility discovery factory installs its self-publisher
408/// only under `iroh_required`, so no compatibility announcement ever consumes
409/// a `#discovery` bundle and demanding one fails startup for pre-upgrade
410/// stores. Required mode keeps the demand because its owned publication path
411/// refuses to publish without the checkpoint bundle.
412#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
413fn checkpoint_announces_service(iroh_required: bool, service_name: &str) -> bool {
414    iroh_required || service_name != "discovery"
415}
416
417/// Populate every ordinary network service announcement from a fresh
418/// checkpoint-verifying PDS read. Missing or ambiguous state fails startup
419/// before any QUIC service can bind and advertise an incomplete bundle.
420#[cfg(feature = "rocksdb")]
421pub fn with_checkpointed_native_announcements(
422    mut ctx: ServiceContext,
423    service_names: &[String],
424) -> anyhow::Result<ServiceContext> {
425    let acceptance_identity = hyprstream_discovery::deployment_registry_verifier()?;
426    let store =
427        open_pds_record_store(&ctx, true)?.with_at9p_deployment_verifier(acceptance_identity);
428    let states = store.accepted_at9p_states()?;
429    // The native-profile flag is installed during authenticated deployment
430    // bootstrap, before `ServiceContext` owns its shared QUIC config.  Read
431    // that process-wide decision here so split Discovery receives its
432    // checkpointed proof before its Event client initializer runs.
433    let iroh_required = hyprstream_discovery::native_network_required();
434    for service_name in service_names
435        .iter()
436        .filter(|name| checkpoint_announces_service(iroh_required, name))
437    {
438        let signer = ctx.service_signing_key(service_name);
439        let mut matching = states.iter().filter(|state| {
440            accepted_state_matches_service(state, service_name, signer.verifying_key().as_bytes())
441        });
442        let state = matching.next().ok_or_else(|| {
443            anyhow::anyhow!(
444                "no checkpoint-verified accepted state authorizes network service {service_name}"
445            )
446        })?;
447        anyhow::ensure!(
448            matching.next().is_none(),
449            "multiple accepted states authorize network service {service_name}"
450        );
451        let announcement = hyprstream_service::NativeServiceAnnouncement::from_accepted_state(
452            service_name,
453            &signer,
454            state,
455        )?;
456        ctx = ctx.with_native_announcement(service_name.clone(), announcement);
457    }
458    Ok(ctx)
459}
460
461/// Resolve the CA-signed JWT used to register a service's signing key.
462///
463/// Fail-closed (issue #441): returns the JWT (preferring one already in the
464/// trust store, falling back to the authoritative on-disk credential), or an
465/// ERROR naming the real cause. It never silently returns "skip" — a service
466/// that cannot produce its JWT must not come up serving signed responses.
467fn resolve_registration_jwt(
468    service_name: &str,
469    creds_dir: &std::path::Path,
470    secrets_profile: crate::auth::identity_store::SecretsProfile,
471    from_trust: Option<String>,
472) -> anyhow::Result<String> {
473    if let Some(jwt) = from_trust {
474        return Ok(jwt);
475    }
476    match crate::auth::identity_store::load_service_jwt_for_profile(
477        creds_dir,
478        service_name,
479        secrets_profile,
480    ) {
481        Ok(Some(jwt)) => Ok(jwt),
482        Ok(None) => anyhow::bail!(
483            "service '{service_name}' cannot register its signing key: \
484             no CA-signed JWT found in trust store or on disk at {}. \
485             Run 'hyprstream wizard' to provision service credentials; \
486             a service must not serve signed responses without a registered key.",
487            creds_dir.display(),
488        ),
489        Err(e) => anyhow::bail!(
490            "service '{service_name}' cannot register its signing key: \
491             failed to read CA-signed JWT from {}: {e}",
492            creds_dir.display(),
493        ),
494    }
495}
496
497/// Register this service's verifying key with the PolicyService CA.
498///
499/// Called by each non-policy factory so that peer services can resolve
500/// our pubkey via `resolveServiceKey` RPC.  No-op for PolicyService itself.
501///
502/// # Fail-closed (issue #441)
503///
504/// A service that cannot obtain its CA-signed JWT (and therefore cannot
505/// register its signing key) MUST NOT come up serving signed responses —
506/// every peer would resolve a key/JWT that disagrees with what we actually
507/// sign with, surfacing three layers away as a cryptic "Response signed by
508/// unexpected key". So registration is a hard precondition: if we cannot get
509/// a JWT, we return an error and the factory (and thus the service) fails to
510/// start, naming the real cause.
511///
512/// The authoritative source of the service JWT is on disk
513/// (`credentials/{service}/service-jwt`), written by the wizard/bootstrap
514/// manager. At process startup only the bootstrap *pubkeys* are seeded into
515/// the trust store (with `jwt: None`); the JWT itself is loaded here and
516/// seeded into the trust store so that peer-client construction
517/// (`service_token`) and the background renewal task can read it.
518fn register_service_key(
519    ctx: &ServiceContext,
520    service_name: &str,
521    signing_key: &SigningKey,
522) -> anyhow::Result<()> {
523    // Policy never registers its own key through an RPC, but its provisioned
524    // service JWT must still be loaded and seeded in EVERY transport profile.
525    // Policy resolves to the flat root/CA key (#759), so no other path ever
526    // seeds a credential onto it; without this, a compatibility-profile node's
527    // policy announcement (native_announcement_bearer) can never carry a
528    // bearer, and internal policy→discovery calls go out unauthenticated and
529    // die at the MAC dispatch PEP (NoClearance) on labeled nodes.
530
531    // The loaded config's `[secrets].path` must win: startup read the retained
532    // key and this very JWT from that directory, and the renewal task spawned
533    // below reads AND persists there every hour. A config-free fallback here
534    // made custom `--config [secrets].path` deployments work at startup and
535    // then silently skip renewal (P1, PR1585).
536    let creds_dir = match ctx.secrets_dir() {
537        Some(dir) => dir.to_path_buf(),
538        None => credentials_dir()?,
539    };
540    let secrets_profile = crate::auth::identity_store::SecretsProfile::from_env()?;
541
542    // The JWT may already be in the trust store (e.g. seeded by an earlier
543    // registration in this process); otherwise load it from disk — the
544    // authoritative location the wizard/bootstrap manager wrote it to.
545    let from_trust = {
546        let trust = hyprstream_service::global_trust_store();
547        trust
548            .get(&signing_key.verifying_key())
549            .and_then(|att| att.jwt.clone())
550    };
551    let jwt = resolve_registration_jwt(service_name, &creds_dir, secrets_profile, from_trust)?;
552
553    // Seed the loaded JWT into the trust store so that peer-client construction
554    // (`service_token`) and the background renewal task can read it. Bind it to
555    // this service's own verifying key — the key we actually sign with — so the
556    // advertised key == the actual signer (the #441 invariant).
557    {
558        let vk = signing_key.verifying_key();
559        let trust = hyprstream_service::global_trust_store();
560        let expires_at = decode_jwt_exp(&jwt).unwrap_or(0);
561        let mut att = trust
562            .get(&vk)
563            .unwrap_or_else(|| hyprstream_service::Attestation {
564                scopes: std::iter::once(service_name.to_owned()).collect(),
565                subject: None,
566                jwt: None,
567                expires_at: 0,
568                attested_by: None,
569            });
570        att.scopes.insert(service_name.to_owned());
571        att.jwt = Some(jwt.clone());
572        att.expires_at = expires_at;
573        trust.insert(vk, att);
574    }
575
576    if service_name == "policy" {
577        if ctx.iroh_required() {
578            // Policy is the native authority itself: it does not register its
579            // own key through RPC, but its provisioned service JWT still needs
580            // the same renewal task as every other native child.
581            spawn_jwt_renewal_task(
582                service_name,
583                signing_key.clone(),
584                creds_dir,
585                secrets_profile,
586                true,
587                ctx.transport("policy", SocketKind::Rep),
588            );
589        }
590        return Ok(());
591    }
592
593    if ctx.iroh_required() {
594        schedule_network_service_key_registration(service_name, signing_key.clone(), jwt.clone());
595        spawn_jwt_renewal_task(
596            service_name,
597            signing_key.clone(),
598            creds_dir,
599            secrets_profile,
600            true,
601            ctx.transport("policy", SocketKind::Rep),
602        );
603        return Ok(());
604    }
605
606    let policy_vk = hyprstream_service::global_trust_store()
607        .resolve_one("policy")
608        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
609    let policy_transport = ctx.transport("policy", SocketKind::Rep);
610    let policy_client = policy_client_for_transport(
611        &policy_transport,
612        signing_key.clone(),
613        policy_vk,
614        Some(jwt.clone()),
615    )?;
616
617    let request = RegisterServiceKey {
618        service_name: service_name.to_owned(),
619        verifying_key: signing_key.verifying_key().as_bytes().to_vec(),
620        service_jwt: jwt.clone(),
621    };
622
623    tokio::task::block_in_place(|| {
624        let rt = tokio::runtime::Handle::current();
625        rt.block_on(policy_client.register_service_key(&request))
626    })
627    .map_err(|e| anyhow::anyhow!("registerServiceKey RPC failed for '{service_name}': {e}"))?;
628
629    info!(
630        service = service_name,
631        "Registered verifying key with PolicyService"
632    );
633
634    // Spawn background JWT renewal for this service
635    spawn_jwt_renewal_task(
636        service_name,
637        signing_key.clone(),
638        creds_dir,
639        secrets_profile,
640        false,
641        policy_transport,
642    );
643
644    Ok(())
645}
646
647fn policy_client_for_deployment(
648    ctx: &ServiceContext,
649    signing_key: SigningKey,
650    policy_verifying_key: VerifyingKey,
651    token: Option<String>,
652) -> anyhow::Result<PolicyClient> {
653    if ctx.iroh_required() {
654        PolicyClient::from_provider(
655            &hyprstream_discovery::ProductionRpcClientProvider,
656            signing_key,
657            token,
658        )
659    } else {
660        // Deterministic same-host PolicyService IPC endpoint: unlike
661        // `registered_endpoint`, it is available to a separate `podman exec`
662        // process that did not start the PolicyService itself.
663        PolicyClient::for_local_transport_bootstrap(
664            &ctx.transport("policy", SocketKind::Rep),
665            signing_key,
666            policy_verifying_key,
667            token,
668        )
669    }
670}
671
672fn schedule_network_service_key_registration(
673    service_name: &str,
674    signing_key: SigningKey,
675    service_jwt: String,
676) {
677    let service_name = service_name.to_owned();
678    tokio::spawn(async move {
679        let mut delay = std::time::Duration::from_secs(2);
680        loop {
681            let attempt = async {
682                let client = PolicyClient::from_provider(
683                    &hyprstream_discovery::ProductionRpcClientProvider,
684                    signing_key.clone(),
685                    Some(service_jwt.clone()),
686                )?;
687                client
688                    .register_service_key(&RegisterServiceKey {
689                        service_name: service_name.clone(),
690                        verifying_key: signing_key.verifying_key().as_bytes().to_vec(),
691                        service_jwt: service_jwt.clone(),
692                    })
693                    .await
694                    .map_err(|error| anyhow::anyhow!(error))
695            }
696            .await;
697            match attempt {
698                Ok(()) => return,
699                Err(error) => {
700                    tracing::warn!(service = %service_name, "Policy registration over production resolver is not ready; retrying: {error}");
701                    tokio::time::sleep(delay).await;
702                    delay = delay
703                        .saturating_mul(2)
704                        .min(std::time::Duration::from_secs(30));
705                }
706            }
707        }
708    });
709}
710
711/// Decode the `exp` claim from a JWT without verifying the signature.
712///
713/// Used for local-disk JWTs that we issued ourselves — signature is verified
714/// by PolicyService; here we only need the expiry to decide whether to renew.
715fn decode_jwt_exp(jwt: &str) -> Option<i64> {
716    use base64::engine::general_purpose::URL_SAFE_NO_PAD;
717    use base64::Engine as _;
718    let payload_b64 = jwt.split('.').nth(1)?;
719    let payload = URL_SAFE_NO_PAD.decode(payload_b64).ok()?;
720    let value: serde_json::Value = serde_json::from_slice(&payload).ok()?;
721    value.get("exp")?.as_i64()
722}
723
724/// Spawn a background task that renews this service's JWT when it approaches expiry.
725///
726/// Checks hourly; renews when ≤7 days remain. Updates the global trust store so
727/// in-flight RPC calls stay authenticated, and best-effort persists the renewed
728/// JWT to disk so it survives restart. When credentials are read-only (systemd
729/// `$CREDENTIALS_DIRECTORY`), persistence uses the unit's writable state home.
730fn spawn_jwt_renewal_task(
731    service_name: &str,
732    signing_key: SigningKey,
733    credentials_dir: std::path::PathBuf,
734    secrets_profile: crate::auth::identity_store::SecretsProfile,
735    iroh_required: bool,
736    policy_transport: hyprstream_rpc::transport::TransportConfig,
737) {
738    let service_name = service_name.to_owned();
739    let Ok(runtime) = tokio::runtime::Handle::try_current() else {
740        // Factory unit tests exercise registration without a running executor;
741        // production startup always has one. Avoid panicking in the former
742        // while keeping renewal attached to the service runtime in the latter.
743        tracing::debug!(
744            service = service_name,
745            "JWT renewal deferred: no Tokio runtime"
746        );
747        return;
748    };
749    runtime.spawn(async move {
750        const CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3_600);
751        const RENEW_THRESHOLD: i64 = 7 * 24 * 3_600; // 7 days remaining
752
753        loop {
754            tokio::time::sleep(CHECK_INTERVAL).await;
755
756            let jwt = match crate::auth::identity_store::load_service_jwt_for_profile(
757                &credentials_dir,
758                &service_name,
759                secrets_profile,
760            ) {
761                Ok(Some(j)) => j,
762                _ => continue,
763            };
764
765            let expires_at = match decode_jwt_exp(&jwt) {
766                Some(exp) => exp,
767                None => continue,
768            };
769
770            let remaining = expires_at - chrono::Utc::now().timestamp();
771            if remaining > RENEW_THRESHOLD {
772                continue;
773            }
774
775            // Build a PolicyClient using current trust-store JWT
776            let (policy_vk, current_jwt) = {
777                let trust = hyprstream_service::global_trust_store();
778                let vk = match trust.resolve_one("policy") {
779                    Some(v) => v,
780                    None => {
781                        tracing::warn!(
782                            service = service_name,
783                            "policy key not in trust store; skipping JWT renewal"
784                        );
785                        continue;
786                    }
787                };
788                let svc_jwt = match trust
789                    .get(&signing_key.verifying_key())
790                    .and_then(|att| att.jwt)
791                {
792                    Some(j) => j,
793                    None => {
794                        tracing::warn!(
795                            service = service_name,
796                            "service JWT not in trust store; skipping renewal"
797                        );
798                        continue;
799                    }
800                };
801                (vk, svc_jwt)
802            };
803
804    let policy_client = match if iroh_required {
805        PolicyClient::from_provider(&hyprstream_discovery::ProductionRpcClientProvider, signing_key.clone(), Some(current_jwt))
806    } else {
807        policy_client_for_transport(
808            &policy_transport,
809            signing_key.clone(),
810            policy_vk,
811            Some(current_jwt),
812        )
813    } {
814                Ok(c) => c,
815                Err(e) => {
816                    tracing::warn!(service = service_name, error = %e, "failed to create PolicyClient; skipping JWT renewal");
817                    continue;
818                }
819            };
820            let req = RefreshServiceTokenRequest {
821                ttl_seconds: 2_592_000,
822            };
823
824            match policy_client.refresh_service_token(&req).await {
825                Ok(info) => {
826                    // Update trust store with renewed JWT
827                    let trust = hyprstream_service::global_trust_store();
828                    let vk = signing_key.verifying_key();
829                    if let Some(mut att) = trust.get(&vk) {
830                        att.jwt = Some(info.token.clone());
831                        att.expires_at = info.expires_at;
832                        trust.insert(vk, att);
833                    }
834                    // Persist so the renewed JWT survives restart (#803).
835                    if let Err(e) = crate::auth::identity_store::write_service_jwt_for_profile(
836                        &credentials_dir,
837                        &service_name,
838                        secrets_profile,
839                        &info.token,
840                    ) {
841                        tracing::error!(
842                            service = service_name,
843                            error = %e,
844                            "renewed service JWT could not be persisted; it is \
845                             process-ephemeral until the next restart re-registers"
846                        );
847                    }
848                    tracing::info!(
849                        service = service_name,
850                        expires_at = info.expires_at,
851                        "Renewed service JWT"
852                    );
853                }
854                Err(e) => {
855                    tracing::warn!(service = service_name, "JWT renewal RPC failed: {e}");
856                }
857            }
858        }
859    });
860}
861
862// ═══════════════════════════════════════════════════════════════════════════════
863// Event Service Factory
864// ═══════════════════════════════════════════════════════════════════════════════
865
866/// Factory for EventService — initializes the moq-lite event bus (#167).
867///
868/// Replaces the ZMQ XPUB/XSUB ProxyService with a `MoqEventOrigin` registered
869/// as a process global. Publishers and subscribers use the global origin directly;
870/// no forwarding proxy or thread is needed. The returned service just holds the
871/// shutdown barrier so the orchestrator tracks lifecycle correctly.
872#[service_factory("event")]
873fn create_event_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
874    info!("Creating EventService (moq-lite event bus)");
875
876    if !hyprstream_rpc::events::event_authz_installed() {
877        let config = load_config()?;
878        let sk = ctx.service_signing_key("event");
879        let authz = tokio::task::block_in_place(|| {
880            tokio::runtime::Handle::current().block_on(
881                super::native_event_policy::production_event_authz(sk, &config, "moq-event"),
882            )
883        })
884        .context("construct MoQ/event MAC PEP")?;
885        hyprstream_rpc::events::install_event_authz(authz).map_err(anyhow::Error::msg)?;
886    }
887
888    let origin = MoqEventOrigin::new();
889    hyprstream_rpc::moq_event::init_global_moq_event_origin(origin.clone());
890
891    if ctx.iroh_required() {
892        return Ok(Box::new(super::event_network::EventNetworkService::new(
893            ctx, &origin,
894        )?));
895    }
896
897    // #275: serve the event-bus origin over the well-known cross-process UDS path
898    // so OTHER service processes (worker, model, ...) can publish/subscribe events
899    // to this shared bus. In the same-process (InprocManager) deployment every
900    // service shares this global origin directly; this UDS plane is the bridge
901    // for the systemd / --ipc deployment where each service is its own process.
902    let event_moq_path = hyprstream_rpc::paths::event_socket();
903    hyprstream_rpc::moq_event::serve_event_moq_uds_background(origin, event_moq_path);
904
905    Ok(Box::new(MoqEventBarrierService::new()))
906}
907
908/// Minimal `Spawnable` that satisfies the service lifecycle contract for the
909/// moq event bus. The bus itself is a process-global `MoqEventOrigin` with no
910/// dedicated thread; this service just waits for shutdown.
911struct MoqEventBarrierService;
912
913impl MoqEventBarrierService {
914    fn new() -> Self {
915        Self
916    }
917}
918
919impl Spawnable for MoqEventBarrierService {
920    fn name(&self) -> &str {
921        "event"
922    }
923
924    fn registrations(
925        &self,
926    ) -> Vec<(
927        hyprstream_rpc::registry::SocketKind,
928        hyprstream_rpc::transport::TransportConfig,
929    )> {
930        vec![] // no ZMQ endpoints
931    }
932
933    fn run(
934        self: Box<Self>,
935        shutdown: std::sync::Arc<tokio::sync::Notify>,
936        on_ready: Option<tokio::sync::oneshot::Sender<()>>,
937    ) -> hyprstream_rpc::error::Result<()> {
938        if let Some(ready) = on_ready {
939            let _ = ready.send(());
940        }
941        // systemd Type=notify: send READY=1 so the unit reaches `active` rather
942        // than timing out (~45s) and restart-looping. These moq barrier services
943        // don't go through the RPC serve path (serve.rs::signal_ready) that
944        // normally notifies systemd, so they must signal readiness themselves —
945        // the moq origin/event-bus is already initialized in the factory before
946        // run() is called, so the service is genuinely ready here.
947        let _ = hyprstream_rpc::notify::ready();
948        let rt = tokio::runtime::Builder::new_current_thread()
949            .enable_all()
950            .build()
951            .map_err(|e| hyprstream_rpc::error::RpcError::Other(e.to_string()))?;
952        rt.block_on(shutdown.notified());
953        Ok(())
954    }
955}
956
957// ═══════════════════════════════════════════════════════════════════════════════
958// Ledger Service Factory (Phase-1 local-enforcer, #925 — `ledger` feature)
959// ═══════════════════════════════════════════════════════════════════════════════
960
961/// Factory for the Phase-1 cellular-ledger local-enforcer service (epic #922,
962/// #925). Gated behind the `ledger` cargo feature; inert unless
963/// `[ledger] enabled = true`.
964///
965/// **Follow-up wiring (clearly marked, not in this skeleton):**
966/// - The grant verifier is a [`StaticGrantVerifier`] that denies every
967///   presented grant until populated — wiring it to
968///   `hyprstream_rpc::auth::ucan` chain validation + the
969///   `ai.hyprstream.ledger.allocation` lexicon (item 1.5) is the production
970///   activation path.
971/// - The receipt sink is the [`LoggingReceiptSink`] (drains to zero, no PDS
972///   writes); the production sink writes the `ai.hyprstream.ledger.receipt`
973///   PDS records.
974/// - The live scheduler realign (`hyprstream-workers` `SandboxPool::acquire`
975///   → `LocalEnforcer::admit`) lands behind this flag once the #761
976///   group-authority decision (#921.5) is made.
977#[cfg(feature = "ledger")]
978#[service_factory("ledger", depends_on = ["policy"])]
979fn create_ledger_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
980    use crate::services::ledger::{
981        CoseCheckpointSigner, LedgerService, LoggingReceiptSink, StaticGrantVerifier,
982    };
983    use hyprstream_crypto::did_key::ed25519_to_did_key;
984    use hyprstream_ledger::{Did, MemLedger};
985    use std::sync::Arc;
986
987    info!("Creating LedgerService (Phase-1 local-enforcer, #925)");
988
989    let config = load_config()?;
990    let lcfg = config.ledger.clone();
991    if !lcfg.is_enabled() {
992        anyhow::bail!(
993            "ledger service requested but [ledger] enabled = false (the Phase-1 enforcer is opt-in)"
994        );
995    }
996
997    // Re-assert the production contract here as well as at config load: this
998    // factory is reachable from paths that construct a config programmatically
999    // rather than through `HyprConfig::validate`. The DSN field only exists in
1000    // a `postgres-ledger` build; without it there is no durable backend to
1001    // name, and production validation correctly refuses.
1002    #[cfg(feature = "postgres-ledger")]
1003    let production_dsn = config.ledger_postgres_url.as_deref();
1004    #[cfg(not(feature = "postgres-ledger"))]
1005    let production_dsn: Option<&str> = None;
1006    lcfg.validate_for_production(production_dsn)?;
1007
1008    // Cell identity = did:key over the service Ed25519 key.
1009    let ed_sk = ctx.service_signing_key("ledger");
1010    let ed_vk = ed_sk.verifying_key();
1011    let cell_identity = Did(ed25519_to_did_key(&ed_vk.to_bytes()));
1012
1013    // Register this service's verifying key with PolicyService.
1014    let _ = ctx.verifying_key();
1015
1016    // PQ (ML-DSA-65) key under the Hybrid policy. Fail-closed construction:
1017    // `require_pq_signatures` set with no key available ⇒ refuse to start the
1018    // ledger service rather than silently downgrade checkpoints to Classical.
1019    // The mint verifier is derived from the SAME key material as the checkpoint
1020    // signer, because the actor signs issuance authorizations with that signer.
1021    // Building both here keeps the two halves of the seal in lockstep — a
1022    // verifier configured more permissively than the signer would re-open the
1023    // hole the seal exists to close.
1024    let mut mint_pq_vk: Option<std::sync::Arc<hyprstream_crypto::pq::MlDsaVerifyingKey>> = None;
1025    let signer: Arc<dyn hyprstream_ledger::CheckpointSigner + Send + Sync> = if lcfg
1026        .require_pq_signatures
1027    {
1028        let secrets_dir = crate::config::HyprConfig::resolve_secrets_dir()?;
1029        let store = crate::auth::key_rotation::global_ml_dsa_key_store(&secrets_dir, &config.oauth);
1030        let pq_key = tokio::task::block_in_place(|| {
1031            tokio::runtime::Handle::current().block_on(async { store.active_key().await })
1032        });
1033        match pq_key {
1034            Some(k) => {
1035                let vk_bytes = hyprstream_crypto::pq::ml_dsa_sk_to_vk_bytes(&k);
1036                let vk = hyprstream_crypto::pq::ml_dsa_vk_from_bytes(&vk_bytes).map_err(|e| {
1037                    anyhow::anyhow!("ledger: could not derive the ML-DSA-65 verifying key: {e}")
1038                })?;
1039                mint_pq_vk = Some(std::sync::Arc::new(vk));
1040                Arc::new(CoseCheckpointSigner::hybrid(
1041                    cell_identity.clone(),
1042                    ed_sk,
1043                    (*k).clone(),
1044                ))
1045            }
1046            None => anyhow::bail!(
1047                "ledger: require_pq_signatures is set but no ML-DSA-65 key is available (fail-closed)"
1048            ),
1049        }
1050    } else {
1051        Arc::new(CoseCheckpointSigner::classical(
1052            cell_identity.clone(),
1053            ed_sk,
1054        ))
1055    };
1056
1057    // The mint authority is derived from the SAME key material the actor signs
1058    // issuance authorizations with, so the two halves of the seal cannot drift
1059    // apart. It is passed at backend construction and is immutable afterwards —
1060    // there is no setter, which is what stops a consumer holding a backend from
1061    // installing a permissive authority of its own.
1062    let mint_authority = match &mint_pq_vk {
1063        Some(pq) => Some(hyprstream_ledger::MintAuthority::hybrid(
1064            ed_vk,
1065            (**pq).clone(),
1066        )),
1067        None if lcfg.require_pq_signatures => {
1068            anyhow::bail!(
1069                "ledger: require_pq_signatures is set but no ML-DSA-65 verifying key is \
1070                 available for the mint authority (fail-closed)"
1071            )
1072        }
1073        None => Some(hyprstream_ledger::MintAuthority::classical(ed_vk)),
1074    };
1075
1076    // Backend selection (PAY-01 F8): BackendKind drives construction.
1077    // **Postgres = production (fail-closed on unavailability); Mem = dev/test.**
1078    // No silent fallback — a configured Postgres backend that cannot connect
1079    // is FATAL.
1080    let backend: Box<dyn hyprstream_ledger::LedgerBackend + Send + 'static> = match lcfg.backend {
1081        crate::services::ledger::BackendKind::Postgres => {
1082            #[cfg(feature = "postgres-ledger")]
1083            {
1084                let pg_url = config
1085                    .ledger_postgres_url
1086                    .as_deref()
1087                    .filter(|u| !u.is_empty())
1088                    .ok_or_else(|| {
1089                        anyhow::anyhow!(
1090                            "ledger: backend = Postgres but no ledger_postgres_url configured \
1091                         (FATAL — production requires a durable backend)"
1092                        )
1093                    })?;
1094                let pg_config = hyprstream_ledger::postgres::PostgresConfig {
1095                    url: pg_url.to_owned(),
1096                    pool_size: config.ledger_postgres_pool_size.unwrap_or(4),
1097                };
1098                let pg = hyprstream_ledger::postgres::PostgresLedger::connect(
1099                    pg_config,
1100                    cell_identity.clone(),
1101                    mint_authority,
1102                )
1103                .map_err(|e| {
1104                    anyhow::anyhow!(
1105                        "ledger: PostgresLedger::connect FAILED (FATAL — no silent fallback): {e}"
1106                    )
1107                })?;
1108                info!("Ledger backend: PostgresLedger (production durable)");
1109                Box::new(pg)
1110            }
1111            #[cfg(not(feature = "postgres-ledger"))]
1112            {
1113                anyhow::bail!(
1114                    "ledger: backend = Postgres but postgres-ledger feature is not compiled \
1115                     (FATAL — rebuild with --features postgres-ledger for production)"
1116                );
1117            }
1118        }
1119        crate::services::ledger::BackendKind::Mem => {
1120            // A volatile backend cannot hold a money ledger. In production this
1121            // is fatal rather than a warning: silently accounting into memory
1122            // that vanishes on restart is worse than refusing to start.
1123            if lcfg.is_production() {
1124                anyhow::bail!(
1125                    "ledger: backend = Mem is not permitted in production (FATAL — \
1126                     all ledger state would be lost on restart). Set [ledger] backend = \"postgres\" \
1127                     with a ledger_postgres_url, or unset the production mode to run dev/test."
1128                );
1129            }
1130            info!("Ledger backend: MemLedger (dev/test only — all state is volatile)");
1131            Box::new(MemLedger::new(cell_identity.clone(), mint_authority))
1132        }
1133    };
1134
1135    let verifier: Arc<dyn crate::services::ledger::GrantVerifier + Send + Sync> =
1136        Arc::new(StaticGrantVerifier::new());
1137    let sink: Arc<dyn crate::services::ledger::ReceiptSink + Send + Sync> =
1138        Arc::new(LoggingReceiptSink);
1139
1140    let service = LedgerService::spawn(lcfg, backend, signer, verifier, sink, cell_identity);
1141    Ok(Box::new(service))
1142}
1143
1144// ═══════════════════════════════════════════════════════════════════════════════
1145// Policy Service Factory
1146// ═══════════════════════════════════════════════════════════════════════════════
1147
1148/// Factory for PolicyService (Casbin policy management)
1149#[service_factory("policy", schema = "../../../hyprstream-rpc-std/schema/policy.capnp", metadata = hyprstream_rpc_std::policy_client::schema_metadata)]
1150fn create_policy_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
1151    info!("Creating PolicyService");
1152    register_service_key(ctx, "policy", &ctx.service_signing_key("policy"))?;
1153
1154    let policies_dir = ctx.models_dir().join(".registry").join("policies");
1155
1156    // Get shared Git2DB instance (initializes .registry as git repo if needed)
1157    let git2db = get_or_init_git2db(ctx.models_dir())?;
1158
1159    // Create policy manager (blocking since we're in sync context)
1160    let policy_manager = Arc::new(
1161        tokio::task::block_in_place(|| {
1162            let rt = tokio::runtime::Handle::current();
1163            rt.block_on(async {
1164                let pm = PolicyManager::new(&policies_dir).await?;
1165                // Idempotent migration: ensure required bootstrap rules are present.
1166                // These rules are in DEFAULT_POLICY_CSV for new installs; existing
1167                // deployments need them added once.
1168                let rules = pm.get_policy().await;
1169                let has_anon_tui = rules
1170                    .iter()
1171                    .any(|r| r.len() >= 3 && r[0] == "anonymous" && r[2] == "tui:*");
1172                if !has_anon_tui {
1173                    let _ = pm
1174                        .add_policy_with_domain("anonymous", "*", "tui:*", "*", "allow")
1175                        .await;
1176                    tracing::info!("policy migration: added 'anonymous' TUI access grant");
1177                }
1178                // Migration: persist service base rules to disk if not already there.
1179                // PolicyManager::new() already injected them into memory, but older
1180                // policy.csv files won't have them on disk. Save writes the full
1181                // enforcer state (including base rules) to disk.
1182                let has_service_policy = rules
1183                    .iter()
1184                    .any(|r| r.len() >= 2 && r[0] == "service:policy");
1185                if !has_anon_tui || !has_service_policy {
1186                    let _ = pm.save().await;
1187                    if !has_service_policy {
1188                        tracing::info!(
1189                            "policy migration: persisted service-to-service base rules to disk"
1190                        );
1191                    }
1192                }
1193                Ok::<_, anyhow::Error>(pm)
1194            })
1195        })
1196        .context("Failed to initialize policy manager")?,
1197    );
1198
1199    // Expose globally so other services (e.g. OAuthService) can write policy rules
1200    // for federated users without a ZMQ round-trip.
1201    crate::auth::set_global_policy_manager(Arc::clone(&policy_manager));
1202
1203    // Spawn file watcher for policy hot-reload
1204    let pm_clone = Arc::clone(&policy_manager);
1205    let policy_csv = policies_dir.join("policy.csv");
1206    tokio::task::block_in_place(|| {
1207        let rt = tokio::runtime::Handle::current();
1208        rt.spawn(async move {
1209            super::policy::watch_policy_file(pm_clone, policy_csv).await;
1210        });
1211    });
1212
1213    let config = load_config()?;
1214    // Bind issuance and revocation horizons by construction: the revocation
1215    // authority's retention bound derives from every configured issuance
1216    // maximum (plus one day of clock-skew margin), so an operator raising
1217    // token TTLs automatically raises the revocation horizon — no issuable
1218    // credential can outlive its revocability. The service-JWT renewal clamp
1219    // (policy.capnp RefreshServiceTokenRequest, hard 30 days) has no config
1220    // field and is included as a constant.
1221    const SERVICE_JWT_MAX_TTL_SECS: u32 = 2_592_000; // 30 days
1222    let revocation_max_ttl_secs = i64::from(
1223        config
1224            .token
1225            .max_ttl_seconds
1226            .max(config.oauth.token_ttl_seconds)
1227            .max(config.oauth.refresh_token_ttl_seconds)
1228            .max(SERVICE_JWT_MAX_TTL_SECS),
1229    ) + 86_400;
1230    let mut policy_service = PolicyService::new(
1231        policy_manager,
1232        Arc::new(ctx.signing_key().clone()),
1233        config.token.clone(),
1234        git2db,
1235        ctx.transport("policy", SocketKind::Rep),
1236    );
1237    policy_service = policy_service.with_revocation_max_ttl_secs(revocation_max_ttl_secs);
1238    if let Some(issuer) = ctx.oauth_issuer_url() {
1239        policy_service = policy_service.with_default_audience(issuer.to_owned());
1240    }
1241    // Use the same configured HTTP resource URLs consumed by the MCP/OAI
1242    // factories. A delegated token can select only these exact audiences.
1243    policy_service = policy_service
1244        .with_delegated_resource_audiences(config.mcp.resource_url(), config.oai.resource_url());
1245    policy_service = policy_service.with_jwt_key_source(ctx.cluster_key_source());
1246    let secrets_dir = crate::config::HyprConfig::resolve_secrets_dir()?;
1247
1248    // Federate identity and grants are stateless. Policy owns the current
1249    // tenant/grant decision; the shared proof-replay backend is installed by
1250    // the common RPC startup path. Do not require session rows or a profile
1251    // bootstrap for short-lived signed credentials.
1252    #[cfg(feature = "postgres")]
1253    {
1254        let host = crate::services::oauth::federate_source::FederateHost::new(
1255            ctx.oauth_issuer_url().unwrap_or(&config.oauth.issuer_url()),
1256        );
1257        if let (Some(enrollment), Ok(host)) = (
1258            crate::auth::service_enrollment::global_service_enrollment().cloned(),
1259            host,
1260        ) {
1261            // The complete manifest-validated bootstrap roster is the
1262            // deployed service authority. Compiled but unprovisioned factories
1263            // must not block otherwise-valid Policy startup.
1264            let required_services = enrollment
1265                .services
1266                .keys()
1267                .map(String::as_str)
1268                .collect::<Vec<_>>();
1269            let collision_inventory =
1270                crate::services::policy::inventory::complete_collision_inventory(
1271                    &config.oauth,
1272                    &secrets_dir,
1273                    ctx.signing_key(),
1274                    &required_services,
1275                )?;
1276            // W1.2: the optional second (TUI) client class from the
1277            // trusted-issuer config joins Policy's per-client grant/profile
1278            // table. A configured-but-invalid class disables the TUI entry
1279            // (fail closed) rather than widening the staging pin.
1280            let federate_tui_client = config
1281                .oauth
1282                .trusted_issuers
1283                .get(crate::services::oauth::federate_source::ISSUER)
1284                .and_then(|issuer| issuer.federate_tui_client.as_ref())
1285                .and_then(|class| {
1286                    match crate::services::oauth::federate_source::ClientClass::new(
1287                        &class.client_id,
1288                        &class.aud,
1289                        class.connector_id.as_deref(),
1290                    ) {
1291                        Ok(class) => Some(class.client_id().to_owned()),
1292                        Err(error) => {
1293                            tracing::warn!(
1294                                %error,
1295                                "Federate TUI client class disabled: invalid trusted-issuer configuration"
1296                            );
1297                            None
1298                        }
1299                    }
1300                });
1301            policy_service = policy_service.with_federate_runtime(
1302                enrollment,
1303                collision_inventory,
1304                host,
1305                federate_tui_client.as_deref(),
1306            );
1307            info!("Federate stateless Policy and request-use authorities installed");
1308        } else {
1309            tracing::warn!("Federate Policy runtime disabled: service enrollment or canonical HTTPS OAuth origin is unavailable");
1310        }
1311    }
1312
1313    // Wire ES256 + ML-DSA rotation stores into PolicyService for composite token issuance.
1314    // Uses global singletons so PolicyService shares the same store the rotation task updates.
1315    let es256_store =
1316        crate::auth::key_rotation::global_es256_key_store(&secrets_dir, &config.oauth);
1317    policy_service = policy_service.with_es256_key_store(es256_store);
1318    {
1319        let ml_dsa_store =
1320            crate::auth::key_rotation::global_ml_dsa_key_store(&secrets_dir, &config.oauth);
1321        let ed_store =
1322            crate::auth::key_rotation::global_ed25519_key_store(&secrets_dir, &config.oauth);
1323        let ca_key = Arc::new(hyprstream_rpc::node_identity::derive_purpose_key(
1324            ctx.signing_key(),
1325            "hyprstream-jwt-v1",
1326        ));
1327        tokio::task::block_in_place(|| {
1328            tokio::runtime::Handle::current().block_on(
1329                crate::auth::key_rotation::initialize_composite_key_set(
1330                    &secrets_dir,
1331                    &ed_store,
1332                    &ml_dsa_store,
1333                    ca_key,
1334                    config.oauth.drain_secs(),
1335                ),
1336            )
1337        })?;
1338        policy_service = policy_service.with_ml_dsa_key_store(ml_dsa_store);
1339    }
1340
1341    // The authority stores (credential revocation + session registry) are
1342    // published once per process by
1343    // `services::revocation::init_process_authority_stores` from
1344    // the main.rs startup block, before any factory runs — not here.
1345
1346    Ok(ctx.into_spawnable_quic(policy_service, config.policy.quic_port))
1347}
1348
1349// ═══════════════════════════════════════════════════════════════════════════════
1350// Registry Service Factory
1351// ═══════════════════════════════════════════════════════════════════════════════
1352
1353/// Factory for RegistryService (git2db model registry)
1354#[service_factory("registry", schema = "../../../hyprstream-rpc-std/schema/registry.capnp", metadata = hyprstream_rpc_std::registry_client::schema_metadata, depends_on = ["policy", "discovery"])]
1355fn create_registry_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
1356    info!("Creating RegistryService");
1357
1358    // RegistryService publishes clone-progress streams via StreamChannel::run_stream
1359    // (which fails loudly if no moq origin is registered in this process).
1360    // Initialize this process's local moq plane. Idempotent.
1361    init_local_moq_stream_plane("registry", ctx.iroh_required());
1362
1363    let config = load_config()?;
1364    let sk = ctx.service_signing_key("registry");
1365
1366    // Register this service's verifying key with PolicyService
1367    register_service_key(ctx, "registry", &sk)?;
1368
1369    // Create policy client for authorization checks
1370    let policy_vk = hyprstream_service::global_trust_store()
1371        .resolve_one("policy")
1372        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
1373    let policy_client =
1374        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
1375    #[cfg(feature = "postgres")]
1376    let federate_dispatch =
1377        federate_dispatch_for_service(policy_client.clone(), ctx.oauth_issuer_url())?;
1378
1379    // #910a — the registry service is the sole PDS-record writer AND the sole
1380    // holder of the `#atproto` private key: it opens the durable store
1381    // read-write and, on register/commit, signs the repo's commit ONCE and
1382    // persists the signed bytes. Reads (the discovery service) are keyless.
1383    // The record `repo` authority is the root `did:web` document that publishes
1384    // the `#atproto` commit-verification key. The `#atproto` signing key is the *active* key
1385    // from the shared `Es256SigningKeyStore` — the same P-256 key
1386    // `oauth::did_document` publishes as the `#atproto` verification method, so
1387    // the writer and the published key are one source of truth (classical —
1388    // atproto has no PQ variant). Failure handling is backend-dependent
1389    // (#1257): with an RDS binding resolved the store contract is
1390    // FATAL-on-unavailable — construction errors propagate and the registry
1391    // refuses to start (the resolver side is already fatal, so degrading the
1392    // writer to warn+disable would silently stall record/repo state across
1393    // every AZ). With no RDS binding (local/workstation) it stays
1394    // best-effort: any failure here disables PDS publish with a warning
1395    // rather than failing the registry. The key lives
1396    // only in the writer's memory — never in the record DB (#910a H1). Paths
1397    // fail closed rather than fall back to /tmp (H2).
1398    #[cfg(feature = "rocksdb")]
1399    let pds_rds_configured = config.rds.resolved_from_env()?.is_configured();
1400    #[cfg(feature = "rocksdb")]
1401    let pds_publisher = match (|| -> anyhow::Result<crate::services::discovery::PdsPublisher> {
1402        let store_dir = pds_store_dir(ctx)?;
1403        let secrets_dir = crate::config::HyprConfig::resolve_secrets_dir()?;
1404        // C4 (#1170, closes #1123): the publisher resolves the active
1405        // `#atproto` generation from the **sealed op-log head** at sign time,
1406        // not a key frozen here at construction. The head is written by the
1407        // OAuth rotation task to a dedicated shared state dir (NOT the
1408        // read-only credentials dir), and is signed by a dedicated head key
1409        // whose *public* verifying key the registry loads here — the registry
1410        // holds NO CA private key. Under `--ipc` a rotation by the OAuth
1411        // process is observed with no event-delivery mechanism. If the head is
1412        // absent (OAuth not booted, or the shared state dir not provisioned —
1413        // #808 under systemd), `active_generation()` returns `None` and the
1414        // publisher declines to sign: fail-closed, never a stale frozen key.
1415        let oplog_state_dir = crate::auth::resolve_oplog_state_dir(&secrets_dir)?;
1416        let generation_source: Arc<dyn crate::auth::ActiveGenerationSource> = Arc::new(
1417            crate::auth::SealedHeadEs256Source::new(&oplog_state_dir, &secrets_dir),
1418        );
1419        let es256_store =
1420            crate::auth::key_rotation::global_es256_key_store(&secrets_dir, &config.oauth);
1421        let acceptance_identity = ctx.service_signing_key("registry");
1422        anyhow::ensure!(
1423            hyprstream_discovery::deployment_registry_verifier()?
1424                .matches(&acceptance_identity.verifying_key()),
1425            "registry signing credential does not match authenticated deployment identity"
1426        );
1427        let audit_ed = hyprstream_rpc::node_identity::derive_purpose_key(
1428            &acceptance_identity,
1429            "hyprstream-at9p-audit-ed25519-v1",
1430        );
1431        let audit_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&audit_ed);
1432        let store = Arc::new(
1433            open_pds_record_store(ctx, false)?
1434                .with_at9p_acceptance_identity(acceptance_identity.verifying_key()),
1435        );
1436        let alarm_path = store_dir.join("at9p-duplicity.wal");
1437        let at9p_state = crate::services::discovery::At9pStateIngest::open(
1438            Arc::clone(&store),
1439            &alarm_path,
1440            acceptance_identity,
1441            audit_ed,
1442            audit_pq,
1443        )?;
1444        let node_did = hyprstream_rpc::did_key::ed25519_to_did_key(&ctx.verifying_key().to_bytes());
1445        Ok(
1446            crate::services::discovery::PdsPublisher::with_generation_source(
1447                store,
1448                node_did,
1449                generation_source,
1450            )
1451            .with_at9p_state_ingest(at9p_state)
1452            .with_es256_store(es256_store),
1453        )
1454    })() {
1455        Ok(publisher) => Some(publisher),
1456        Err(error) if pds_rds_configured => {
1457            return Err(error).context(
1458                "PDS publisher construction failed while an RDS records backend is \
1459                 configured — the store contract is FATAL-on-unavailable; refusing \
1460                 to start the registry with PDS publish disabled",
1461            );
1462        }
1463        Err(error) => {
1464            tracing::warn!("PDS publish disabled: {error}");
1465            None
1466        }
1467    };
1468
1469    // Create registry service with infrastructure (blocking since we're in sync context)
1470    let mut registry_service = tokio::task::block_in_place(|| {
1471        let rt = tokio::runtime::Handle::current();
1472        rt.block_on(async {
1473            let cas_pep = crate::mac::production_cas_pep(sk.clone(), &config.oauth, "cas-registry")
1474                .await
1475                .context("construct registry CAS MAC PEP")?;
1476            Ok::<_, anyhow::Error>(
1477                RegistryService::new(
1478                    ctx.models_dir(),
1479                    policy_client,
1480                    ctx.transport("registry", SocketKind::Rep),
1481                    sk.clone(),
1482                )
1483                .await?
1484                .with_cas_pep(cas_pep),
1485            )
1486        })
1487    })?;
1488    if let Some(issuer) = ctx.oauth_issuer_url() {
1489        registry_service = registry_service.with_expected_audience(issuer.to_owned());
1490    }
1491    if let Some(export) = config.server.ninep_canonical_read.clone() {
1492        let issuer = ctx
1493            .oauth_issuer_url()
1494            .ok_or_else(|| anyhow::anyhow!("9P issuer unavailable"))?;
1495        registry_service = registry_service.with_ninep_canonical_read(export, issuer.to_owned())?;
1496    }
1497    registry_service = registry_service.with_jwt_key_source(ctx.cluster_key_source());
1498    #[cfg(feature = "postgres")]
1499    if let Some(adapter) = federate_dispatch {
1500        registry_service = registry_service.with_federate_dispatch(adapter);
1501    }
1502    #[cfg(feature = "rocksdb")]
1503    if let Some(publisher) = pds_publisher {
1504        // A promotion publishes the former active key as a bounded drain slot;
1505        // no writer-local re-sign callback is needed, which keeps `--ipc`
1506        // rotation viable when OAuth and registry run in different processes.
1507        let publisher_arc = Arc::new(publisher);
1508        spawn_registry_at9p_renewal(&publisher_arc, &config)?;
1509        registry_service = registry_service.with_pds_publisher_arc(publisher_arc);
1510    }
1511
1512    Ok(ctx.into_spawnable_quic(registry_service, config.registry.quic_port))
1513}
1514
1515/// Spawn the in-daemon at9p service-identity renewal timer (design O-Ia: the
1516/// registry process renews every sibling identity itself, loading each
1517/// service's on-volume key from the shared credentials volume the deployment
1518/// mounts wholesale). Spawned only when PDS publisher construction succeeded
1519/// — that construction already established the deployment posture (registry
1520/// key matches the authenticated deployment verifier; the read-write store
1521/// and daemon-owned ingest are open).
1522///
1523/// The timer reuses the publisher's LIVE handles (never re-opens the store:
1524/// the RocksDB directory LOCK excludes the second read-write opener this
1525/// process already is) and holds the publisher by weak reference so it cannot
1526/// outlive the service (a strong handle would pin the directory LOCK and
1527/// block an in-process registry restart). It never re-authenticates the
1528/// deployment credential per tick — the credential/UCAN binds at process
1529/// starts — so the timer renews for the lifetime of this process, and
1530/// restarts still gate on the deployment credential.
1531#[cfg(feature = "rocksdb")]
1532fn validate_pds_head_refresh_interval(
1533    oauth: &crate::config::OAuthConfig,
1534    interval: std::time::Duration,
1535) -> anyhow::Result<()> {
1536    // Reject a nominal cadence that cannot cover even healthy rotation. This
1537    // is not a remaining-overlap guarantee: after downtime a lead may already
1538    // be due, and scheduler/I/O delays or failed ticks can exhaust overlap.
1539    // The immediate startup tick repairs retained heads without making an
1540    // expired/evicted verifier key valid again.
1541    let window = oauth
1542        .drain_secs()
1543        .min(oauth.active_secs().saturating_sub(oauth.lead_secs()));
1544    anyhow::ensure!(
1545        window > 0 && interval.as_secs() < u64::try_from(window).unwrap_or(0),
1546        "Registry maintenance interval must be shorter than both the ES256 drain \
1547         window and active-minus-lead promotion interval"
1548    );
1549    Ok(())
1550}
1551
1552#[cfg(feature = "rocksdb")]
1553fn spawn_registry_at9p_renewal(
1554    publisher: &Arc<crate::services::discovery::PdsPublisher>,
1555    config: &crate::config::HyprConfig,
1556) -> anyhow::Result<()> {
1557    // Resolve through the ALREADY-LOADED config so an explicit `--config`
1558    // `[secrets].path` stays authoritative — a bare resolve_secrets_dir()
1559    // here would reload default config and read the wrong credentials tree
1560    // on every tick (the CLI provisioning path uses the same _for form).
1561    let secrets_dir = crate::config::HyprConfig::resolve_secrets_dir_for(Some(config))?;
1562    // Cross-validated: the check interval must stay below half the TTL or a
1563    // schedule exists where an identity expires between due ticks (an expired
1564    // head can never be renewed). Refuses registry startup, fail-loud.
1565    let (check_interval, valid_for_seconds) = config.registry.at9p_renewal.validated()?;
1566    validate_pds_head_refresh_interval(&config.oauth, check_interval)?;
1567    let roster_export = config
1568        .registry
1569        .at9p_renewal
1570        .roster_export
1571        .clone()
1572        .or_else(crate::services::at9p_renewal::default_roster_export_path);
1573    if roster_export.is_none() {
1574        tracing::warn!(
1575            "at9p renewal: no roster export path resolved; the roster projection will not refresh"
1576        );
1577    }
1578    // Dropping the handle detaches the timer; it exits on its own when the
1579    // publisher drops (Weak) or the process runtime shuts down.
1580    std::mem::drop(crate::services::at9p_renewal::spawn_at9p_renewal_task(
1581        Arc::clone(publisher),
1582        secrets_dir,
1583        check_interval,
1584        valid_for_seconds,
1585        roster_export,
1586    ));
1587    Ok(())
1588}
1589
1590/// Fail-closed twin: without the `rocksdb` feature there is no
1591/// checkpoint-verifying PDS read to populate announcements from.
1592#[cfg(not(feature = "rocksdb"))]
1593pub fn with_checkpointed_native_announcements(
1594    _ctx: ServiceContext,
1595    _service_names: &[String],
1596) -> anyhow::Result<ServiceContext> {
1597    anyhow::bail!("checkpointed native announcements require the `rocksdb` feature")
1598}
1599
1600// ═══════════════════════════════════════════════════════════════════════════════
1601// Streams Service Factory
1602// ═══════════════════════════════════════════════════════════════════════════════
1603
1604/// Initialize this process's local moq stream plane (origin + UDS server).
1605///
1606/// Every service process that **publishes** moq streams (the central `streams`
1607/// service, and stream-publisher services such as `tui`/`notification`/`registry`/
1608/// `metrics`/`model`) needs its OWN moq plane in-process: the process-global
1609/// [`MoqStreamOrigin`] that `StreamChannel::publisher()` appends into, plus a
1610/// compatibility per-PID UDS server. Native required profiles initialize the
1611/// process plane without sockets; each publisher is served by its own Iroh origin.
1612///
1613/// In a multi-process (systemd one-process-per-service) deployment, only the
1614/// `streams` factory used to do this, so other publisher processes had a `None`
1615/// origin (nothing to publish to) and returned an empty `moq_uds_path` to the
1616/// client (→ client `ensure!` fails). This helper closes that gap.
1617///
1618/// Idempotent: if the process already has a moq origin (the `streams` factory
1619/// ran in this process, or this helper was already called), it returns early
1620/// without double-initializing the origin or double-serving the UDS. This lets
1621/// it compose with the `streams` factory and with multiple publisher factories
1622/// co-located in one process.
1623fn init_local_moq_stream_plane(service_name: &str, iroh_required: bool) {
1624    if iroh_required {
1625        hyprstream_rpc::moq_stream::require_native_iroh();
1626    }
1627    // Guard: a moq origin already exists in this process — nothing to do.
1628    if hyprstream_rpc::moq_stream::global_moq_origin().is_some() {
1629        return;
1630    }
1631
1632    let gate = |pubkey: &[u8; 32]| -> bool {
1633        use ed25519_dalek::VerifyingKey;
1634        let Ok(vk) = VerifyingKey::from_bytes(pubkey) else {
1635            return false;
1636        };
1637        hyprstream_service::global_trust_store().get(&vk).is_some()
1638    };
1639
1640    // Use DEFAULT_PREFIX ("local/streams") so the publisher's broadcast paths
1641    // path from DEFAULT_PREFIX; TUI/registry/metrics echo the origin's own
1642    // broadcast_path back to the client, so any prefix is self-consistent there).
1643    let moq_origin = hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone()
1644        .with_prefix(hyprstream_rpc::moq_stream::DEFAULT_PREFIX)
1645        .with_authorize_signer(gate)
1646        .build();
1647
1648    // Register the global BEFORE serving — downstream code that calls
1649    // StreamChannel::publisher() will see it immediately.
1650    if !hyprstream_rpc::moq_stream::init_global_moq_origin(moq_origin.clone()) {
1651        // Lost a race to another initializer in this process; that init owns the
1652        // UDS server too, so don't start a second one.
1653        return;
1654    }
1655
1656    // Native publishers are served by their service-owned Iroh handler.
1657    if hyprstream_rpc::moq_stream::native_iroh_required() {
1658        return;
1659    }
1660    let moq_uds_path = {
1661        let dir = std::env::temp_dir().join(format!("hyprstream-{}", std::process::id()));
1662        let _ = std::fs::create_dir_all(&dir);
1663        dir.join("moq.sock")
1664    };
1665    info!(
1666        service = service_name,
1667        path = %moq_uds_path.display(),
1668        "Initializing local moq stream plane",
1669    );
1670    hyprstream_rpc::moq_stream::serve_moq_uds_background(moq_origin, moq_uds_path);
1671}
1672
1673/// Factory for the moq stream origin (#138 N4 — ZMQ StreamService removed).
1674///
1675/// Compatibility serves a local UDS plane. Required native mode owns an
1676/// authenticated Iroh rendezvous origin and announces only its MoQ capability.
1677#[service_factory("streams")]
1678fn create_streams_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
1679    info!("Creating moq stream origin (ZMQ StreamService removed)");
1680
1681    init_local_moq_stream_plane("streams", ctx.iroh_required());
1682
1683    if ctx.iroh_required() {
1684        return Ok(Box::new(
1685            crate::services::stream_network::StreamsNetworkService::new(ctx)?,
1686        ));
1687    }
1688    Ok(Box::new(MoqStreamBarrierService::new()))
1689}
1690
1691/// Minimal `Spawnable` that holds the moq stream origin lifetime and satisfies
1692/// the service lifecycle contract. The origin itself is a process-global with
1693/// no dedicated thread; this service just waits for shutdown.
1694struct MoqStreamBarrierService;
1695
1696impl MoqStreamBarrierService {
1697    fn new() -> Self {
1698        Self
1699    }
1700}
1701
1702impl Spawnable for MoqStreamBarrierService {
1703    fn name(&self) -> &str {
1704        "streams"
1705    }
1706
1707    fn registrations(
1708        &self,
1709    ) -> Vec<(
1710        hyprstream_rpc::registry::SocketKind,
1711        hyprstream_rpc::transport::TransportConfig,
1712    )> {
1713        vec![]
1714    }
1715
1716    fn run(
1717        self: Box<Self>,
1718        shutdown: std::sync::Arc<tokio::sync::Notify>,
1719        on_ready: Option<tokio::sync::oneshot::Sender<()>>,
1720    ) -> hyprstream_rpc::error::Result<()> {
1721        if let Some(ready) = on_ready {
1722            let _ = ready.send(());
1723        }
1724        // systemd Type=notify: send READY=1 so the unit reaches `active` rather
1725        // than timing out (~45s) and restart-looping. These moq barrier services
1726        // don't go through the RPC serve path (serve.rs::signal_ready) that
1727        // normally notifies systemd, so they must signal readiness themselves —
1728        // the moq origin/event-bus is already initialized in the factory before
1729        // run() is called, so the service is genuinely ready here.
1730        let _ = hyprstream_rpc::notify::ready();
1731        let rt = tokio::runtime::Builder::new_current_thread()
1732            .enable_all()
1733            .build()
1734            .map_err(|e| hyprstream_rpc::error::RpcError::Other(e.to_string()))?;
1735        rt.block_on(shutdown.notified());
1736        Ok(())
1737    }
1738}
1739
1740// ═══════════════════════════════════════════════════════════════════════════════
1741// Model Service Factory
1742// ═══════════════════════════════════════════════════════════════════════════════
1743
1744/// Factory for ModelService (model lifecycle management)
1745#[service_factory("model", schema = "../../../hyprstream-rpc-std/schema/model.capnp", metadata = hyprstream_rpc_std::model_client::schema_metadata, depends_on = ["policy", "registry", "discovery", ])]
1746fn create_model_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
1747    info!("Creating ModelService");
1748
1749    // ModelService spawns InferenceService instances in-process, which publish
1750    // generation streams via StreamChannel::run_stream (fails loudly without a
1751    // moq origin). Initialize this process's local moq plane. Idempotent.
1752    init_local_moq_stream_plane("model", ctx.iroh_required());
1753
1754    use crate::services::{ModelService, ModelServiceConfig};
1755
1756    let config = load_config()?;
1757    let sk = ctx.service_signing_key("model");
1758
1759    // Register this service's verifying key with PolicyService
1760    register_service_key(ctx, "model", &sk)?;
1761
1762    // Create policy client for authorization checks
1763    let policy_vk = hyprstream_service::global_trust_store()
1764        .resolve_one("policy")
1765        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
1766    let policy_client =
1767        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
1768    #[cfg(feature = "postgres")]
1769    let federate_dispatch =
1770        federate_dispatch_for_service(policy_client.clone(), ctx.oauth_issuer_url())?;
1771
1772    // Create registry client
1773    let registry_client: RegistryClient = RegistryClient::from_provider(
1774        &hyprstream_discovery::ProductionRpcClientProvider,
1775        sk.clone(),
1776        service_token(&sk),
1777    )?;
1778
1779    #[allow(clippy::expect_used)]
1780    let mut model_service = tokio::task::block_in_place(|| {
1781        let rt = tokio::runtime::Builder::new_current_thread()
1782            .enable_all()
1783            .build()
1784            .expect("failed to create runtime for model factory");
1785        let local = tokio::task::LocalSet::new();
1786        local.block_on(&rt, {
1787            let mut model_config = ModelServiceConfig::default();
1788            model_config.inference_deployment =
1789                crate::runtime::inference_profile::InferenceDeploymentProfile::from_env()?;
1790            model_config.staging_model_pin = crate::services::StagingModelPin::from_env()?;
1791            ModelService::new(
1792                model_config,
1793                sk.clone(),
1794                policy_client,
1795                registry_client,
1796                ctx.transport("model", SocketKind::Rep),
1797                ctx.transport("policy", SocketKind::Rep),
1798            )
1799        })
1800    })?;
1801    if let Some(issuer) = ctx.oauth_issuer_url() {
1802        model_service = model_service.with_expected_audience(issuer.to_owned());
1803    }
1804    if let Some(issuer) = ctx.oauth_issuer_url() {
1805        let oai_resource = config.oai.resource_url();
1806        let mcp_resource = config.mcp.resource_url();
1807        anyhow::ensure!(
1808            oai_resource != mcp_resource && issuer != oai_resource && issuer != mcp_resource,
1809            "native, OAI and MCP resource audiences must be distinct"
1810        );
1811        model_service = model_service.with_delegated_resource_audiences(oai_resource, mcp_resource);
1812    }
1813    model_service = model_service.with_jwt_key_source(ctx.cluster_key_source());
1814    #[cfg(feature = "postgres")]
1815    if let Some(adapter) = federate_dispatch {
1816        model_service = model_service.with_federate_dispatch(adapter);
1817    }
1818
1819    // #431 — DiscoveryClient for federated at:// record resolution. The discovery
1820    // key is in the trust store (depends_on includes "discovery"). Best-effort:
1821    // if discovery isn't resolvable, ModelService simply has no federation client
1822    // and at:// refs fall through to local resolution.
1823    match hyprstream_rpc_std::discovery_client::DiscoveryClient::from_provider(
1824        &hyprstream_discovery::ProductionRpcClientProvider,
1825        sk.clone(),
1826        None,
1827    ) {
1828        Ok(dc) => {
1829            model_service = model_service.with_discovery_client(std::sync::Arc::new(dc));
1830        }
1831        Err(e) => {
1832            tracing::warn!("ModelService: failed to build DiscoveryClient for federation: {e}");
1833        }
1834    }
1835
1836    Ok(ctx.into_spawnable_quic(model_service, config.model.quic_port))
1837}
1838
1839// ═══════════════════════════════════════════════════════════════════════════════
1840// Standalone CPU Inference Service Factory (#1236/#1247)
1841// ═══════════════════════════════════════════════════════════════════════════════
1842
1843/// One model, one tenant, one CPU-only process. Metal starts two independent
1844/// copies with replica ordinals 0 and 1; no engine or KV-cache state is shared.
1845#[service_factory("inference", schema = "../../../hyprstream-rpc-std/schema/inference.capnp", metadata = hyprstream_rpc_std::inference_client::schema_metadata, depends_on = ["policy", "discovery"])]
1846fn create_inference_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
1847    info!("Creating standalone CPU InferenceService");
1848
1849    let config = load_config()?;
1850    config.inference.validate()?;
1851    config.inference.verify_materialized_oid()?;
1852
1853    let sk = ctx.service_signing_key("inference");
1854    register_service_key(ctx, "inference", &sk)?;
1855    let instance_name = config.inference.instance_name();
1856
1857    // The standalone image is a CPU fault domain even if a host happens to
1858    // expose CUDA. Explicitly clear every GPU selector at the service boundary.
1859    let runtime = crate::config::RuntimeConfig {
1860        use_gpu: false,
1861        gpu_device_id: None,
1862        devices: Vec::new(),
1863        gpu_layers: None,
1864        ..config.runtime.clone()
1865    };
1866    // Do not announce both replicas under the singleton Discovery key:
1867    // Discovery currently replaces, rather than pools, duplicate service
1868    // endpoints. Metal #1309 owns the canonical two-backend load-balancer.
1869    let quic = ctx
1870        .quic_shared()
1871        .map(|shared| shared.for_service(&instance_name, config.inference.quic_port.unwrap_or(0)))
1872        .ok_or_else(|| {
1873            anyhow::anyhow!("standalone inference requires [quic].enabled=true for browser MoQ")
1874        })?;
1875
1876    let mut service = crate::services::InferenceServiceConfig::new(
1877        &config.inference.model_path,
1878        runtime,
1879        sk.verifying_key(),
1880        sk.clone(),
1881        ctx.transport(&instance_name, SocketKind::Rep),
1882        ctx.transport("policy", SocketKind::Rep),
1883        None,
1884    )
1885    .with_instance_identity(
1886        instance_name.clone(),
1887        config.inference.tenant.clone(),
1888        sk.verifying_key(),
1889    )
1890    .with_quic_config(Some(quic))
1891    .with_quic_advertise_addr(config.inference.advertise_addr);
1892    if let Some(issuer) = ctx.oauth_issuer_url() {
1893        service = service.with_expected_audience(issuer.to_owned());
1894    }
1895    service = service.with_jwt_key_source(ctx.cluster_key_source());
1896
1897    info!(
1898        instance = %config.inference.instance_name(),
1899        model_ref = %config.inference.model_ref,
1900        model_oid = %config.inference.model_oid,
1901        tenant = %config.inference.tenant,
1902        "standalone CPU inference configured"
1903    );
1904    Ok(Box::new(service))
1905}
1906
1907// ═══════════════════════════════════════════════════════════════════════════════
1908// Worker Service Factory
1909// ═══════════════════════════════════════════════════════════════════════════════
1910
1911/// Factory for WorkerService (Kata container/sandbox management)
1912///
1913/// Note: This service requires worker configuration. If not configured,
1914/// the factory will use sensible defaults.
1915#[service_factory("worker", depends_on = ["policy", "discovery", "event"])]
1916fn create_worker_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
1917    info!("Creating WorkerService");
1918
1919    #[cfg(feature = "oci-image")]
1920    use hyprstream_workers::config::ImageConfig;
1921    use hyprstream_workers::config::PoolConfig;
1922    #[cfg(feature = "oci-image")]
1923    use hyprstream_workers::image::RafsStore;
1924    use hyprstream_workers::{resolve_backend, BackendCtx, SandboxBackend, WorkerService};
1925
1926    let config = load_config()?;
1927    let sk = ctx.service_signing_key("worker");
1928    let worker_quic_port = config.worker.as_ref().and_then(|w| w.quic_port);
1929    // Operator-selected backend name ("auto" or a registered backend); resolved
1930    // fail-closed against the inventory registry below.
1931    let backend_name: String = config
1932        .worker
1933        .as_ref()
1934        .map(|w| w.backend.clone())
1935        .unwrap_or_else(|| "auto".to_owned());
1936
1937    info!("WorkerService backend selection: {}", backend_name);
1938
1939    // Use default paths based on XDG directories
1940    let data_dir = dirs::data_local_dir()
1941        .unwrap_or_else(|| std::path::PathBuf::from("."))
1942        .join("hyprstream");
1943    let runtime_dir = dirs::runtime_dir()
1944        .unwrap_or_else(std::env::temp_dir)
1945        .join("hyprstream");
1946
1947    let kata_boot_path = std::env::var("KATA_BOOT_PATH")
1948        .map(std::path::PathBuf::from)
1949        .unwrap_or_else(|_| std::path::PathBuf::from("/opt/kata/share/kata-containers"));
1950
1951    let pool_config = PoolConfig {
1952        warm_pool_size: 0,
1953        runtime_dir: runtime_dir.join("sandboxes"),
1954        kernel_path: kata_boot_path.join("vmlinux.container"),
1955        vm_image: kata_boot_path.join("kata-containers.img"),
1956        cloud_init_dir: data_dir.join("cloud-init"),
1957        ..PoolConfig::default()
1958    };
1959
1960    // RAFS/nydus image store is built whenever the image filesystem service is
1961    // compiled in (`oci-image`), so both kata (virtio-fs) and nspawn (FUSE
1962    // tenant-VFS root, Model B #715) can compose a per-sandbox VFS from it.
1963    #[cfg(feature = "oci-image")]
1964    let image_config = ImageConfig {
1965        blobs_dir: data_dir.join("images/blobs"),
1966        bootstrap_dir: data_dir.join("images/bootstrap"),
1967        refs_dir: data_dir.join("images/refs"),
1968        cache_dir: data_dir.join("images/cache"),
1969        runtime_dir: runtime_dir.join("nydus"),
1970        ..ImageConfig::default()
1971    };
1972
1973    #[cfg(feature = "oci-image")]
1974    let rafs_store = Arc::new(RafsStore::new(image_config.clone())?);
1975
1976    let ninep_decider = tokio::task::block_in_place(|| {
1977        tokio::runtime::Handle::current().block_on(crate::mac::production_ninep_decider(
1978            sk.clone(),
1979            &config.oauth,
1980            "ninep-worker",
1981        ))
1982    })
1983    .context("construct worker 9P MAC PEP")?;
1984
1985    // The worker UDS/vsock carrier still has no verified attach credential.
1986    // Keep that boundary local: enrollment_ninep_reference_monitor installs an
1987    // AnonymousAuthenticator, so every 9P attach denies. It must not narrow
1988    // process-global RPC activation, which would turn authenticated Worker
1989    // RPCs into anonymous-floor requests before their handler runs.
1990    let ninep_monitor = Some(crate::mac::enrollment_ninep_reference_monitor(Arc::clone(
1991        &ninep_decider,
1992    )));
1993
1994    // Resolve + construct the backend fail-closed against the inventory registry
1995    // (config-driven by name; explicit requests are authoritative, missing
1996    // prerequisites error out rather than silently downgrading isolation; "auto"
1997    // picks the strongest available). Single seam — no scattered cfg, no
1998    // `_ => nspawn` fallback (#507 / #518).
1999    let backend_ctx = BackendCtx {
2000        pool_config: pool_config.clone(),
2001        ninep_decider,
2002        ninep_monitor,
2003        #[cfg(feature = "oci-image")]
2004        image_config,
2005        #[cfg(feature = "oci-image")]
2006        rafs_store: Arc::clone(&rafs_store),
2007    };
2008    let backend: Arc<dyn SandboxBackend> = resolve_backend(&backend_name, &backend_ctx)?;
2009
2010    // Register this service's verifying key with PolicyService
2011    register_service_key(ctx, "worker", &sk)?;
2012
2013    // Service includes infrastructure - directly Spawnable via blanket impl
2014    let mut worker_service = WorkerService::new(
2015        pool_config,
2016        backend,
2017        // `kata-vm = ["kata"]` is one-way: a `--features kata` build must still
2018        // wire rafs_store, so gate on either the canonical feature or its alias
2019        // rather than `kata-vm` alone (#518).
2020        #[cfg(any(feature = "kata", feature = "kata-vm"))]
2021        Some(rafs_store),
2022        #[cfg(not(any(feature = "kata", feature = "kata-vm")))]
2023        None,
2024        ctx.transport("worker", SocketKind::Rep),
2025        sk.clone(),
2026    )?;
2027
2028    // Wire up policy-backed authorization
2029    let policy_vk = hyprstream_service::global_trust_store()
2030        .resolve_one("policy")
2031        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
2032    let policy_client =
2033        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
2034    worker_service.set_authorize_fn(super::worker::build_authorize_fn(policy_client));
2035    if let Some(issuer) = ctx.oauth_issuer_url() {
2036        worker_service.set_expected_audience(issuer.to_owned());
2037    }
2038    worker_service.set_jwt_key_source(ctx.cluster_key_source());
2039
2040    Ok(ctx.into_spawnable_quic(worker_service, worker_quic_port))
2041}
2042
2043// ═══════════════════════════════════════════════════════════════════════════════
2044// Workflow Service Factory
2045// ═══════════════════════════════════════════════════════════════════════════════
2046
2047/// Factory for `WorkflowService` (#989, epic #1427).
2048///
2049/// The engine (`crates/hyprstream-workers/src/workflow/`) is fully implemented
2050/// but historically had no factory and was never started by the daemon. This
2051/// wires it into the service inventory: it resolves config, constructs the
2052/// service with a VFS namespace for the runner, attaches policy-backed
2053/// authorization, and returns it as a `Spawnable`.
2054///
2055/// **Default-off / opt-in:** the workflow config ships `enabled = false`, so
2056/// this factory bails unless an operator explicitly sets
2057/// `[worker.workflow] enabled = true`. First activation must not change daemon
2058/// behavior for everyone (amend-#989 §4).
2059///
2060/// Scope notes (amend-#989): only factory activation + namespace wiring land
2061/// here. `set_job_scheduler` is left unwired because `WorkerService` owns its
2062/// `SandboxPool` privately with no cross-factory handle today; jobs therefore
2063/// run in-proc (the documented `set_namespace` default) until Phase 1 adds pool
2064/// sharing. Event-bus subscription (`service.start()`) is #990's lifecycle
2065/// scope; the RPC surface (list/dispatch/getRun) works without it.
2066#[service_factory(
2067    "workflow",
2068    schema = "../../../hyprstream-rpc-std/schema/workflow.capnp",
2069    metadata = hyprstream_rpc_std::workflow_client::schema_metadata,
2070    depends_on = ["worker", "event", "policy", "registry", "discovery"]
2071)]
2072fn create_workflow_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
2073    use hyprstream_workers::WorkflowService;
2074
2075    info!("Creating WorkflowService");
2076
2077    let config = load_config()?;
2078    let wcfg = config
2079        .worker
2080        .as_ref()
2081        .map(|w| w.workflow.clone())
2082        .unwrap_or_default();
2083    if !wcfg.enabled {
2084        anyhow::bail!(
2085            "workflow service requested but [worker.workflow] enabled = false \
2086             (the engine is opt-in, #989; set `enabled = true` to activate)"
2087        );
2088    }
2089
2090    let sk = ctx.service_signing_key("workflow");
2091
2092    // Namespace the runner resolves actions/env/outputs through. Phase 0 ships
2093    // empty `/bin`, `/env`, `/out` skeletons; real action mounts and repo-scan
2094    // population land with #990/#992. The runner unmounts `/config` and
2095    // `/private` per-job for isolation regardless.
2096    let ns = Arc::new(build_workflow_namespace());
2097
2098    let mut workflow_service =
2099        WorkflowService::new(ctx.transport("workflow", SocketKind::Rep), sk.clone());
2100    workflow_service.set_namespace_with_config(ns, &wcfg);
2101
2102    // Policy-backed authorization — same seam as WorkerService. Without this the
2103    // dispatch handler fails closed on every request, so wire it before serving.
2104    register_service_key(ctx, "workflow", &sk)?;
2105    let policy_vk = hyprstream_service::global_trust_store()
2106        .resolve_one("policy")
2107        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
2108    let policy_client =
2109        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
2110    workflow_service.set_authorize_fn(crate::services::worker::build_authorize_fn(policy_client));
2111    if let Some(issuer) = ctx.oauth_issuer_url() {
2112        workflow_service.set_expected_audience(issuer.to_owned());
2113    }
2114    workflow_service.set_jwt_key_source(ctx.cluster_key_source());
2115
2116    // Inject a LIVE MCP relay authorizer so accept_delegated_bearer consults
2117    // current MCP-scope authorization at call time (not a frozen snapshot —
2118    // retired keys are rejected, new keys accepted after factory start). The
2119    // closure captures the global trust store and checks is_authorized for the
2120    // "mcp" scope — independently scoped, fail-closed (#989 review).
2121    let trust = hyprstream_service::global_trust_store();
2122    let has_mcp_keys = !trust.keys_for_scope("mcp").is_empty();
2123    if !has_mcp_keys {
2124        tracing::warn!(
2125            "workflow service started with no authorized MCP relay keys; \
2126             delegated-bearer MCP tool calls will be rejected until the mcp \
2127             service key(s) are registered (#989)"
2128        );
2129    }
2130    workflow_service.set_relay_authorizer(std::sync::Arc::new(move |pubkey: &[u8; 32]| {
2131        match hyprstream_rpc::prelude::VerifyingKey::from_bytes(pubkey) {
2132            Ok(vk) => trust.is_authorized(&vk, "mcp"),
2133            Err(_) => false,
2134        }
2135    }));
2136
2137    Ok(ctx.into_spawnable(workflow_service))
2138}
2139
2140/// Build the minimal Phase-0 workflow namespace: a synthetic root with empty
2141/// `/bin`, `/env`, `/out` directories for the runner to resolve through.
2142///
2143/// This is a skeleton — no action handlers, service mounts, or repo worktrees
2144/// are bound here yet. `#990` (git lifecycle) and `#992` (repo scan on clone)
2145/// populate the real content; the sandbox `JobScheduler` routing (#527) lands
2146/// once `WorkerService` exposes its `SandboxPool` cross-factory.
2147fn build_workflow_namespace() -> Namespace {
2148    use crate::services::fs::{SyntheticNode, SyntheticTree};
2149
2150    fn empty_dir() -> SyntheticNode {
2151        SyntheticNode::Dir {
2152            children: HashMap::new(),
2153        }
2154    }
2155
2156    let mut root = HashMap::new();
2157    root.insert("bin".to_owned(), empty_dir());
2158    root.insert("env".to_owned(), empty_dir());
2159    root.insert("out".to_owned(), empty_dir());
2160
2161    let tree = SyntheticTree::new(SyntheticNode::Dir { children: root });
2162    let mut ns = Namespace::new();
2163    // Mounting can only fail on a malformed prefix; "/" is well-formed.
2164    let _ = ns.mount("/", Arc::new(tree) as MountTarget);
2165    ns
2166}
2167
2168// ═══════════════════════════════════════════════════════════════════════════════
2169// OAI Service Factory
2170// ═══════════════════════════════════════════════════════════════════════════════
2171
2172/// Factory for OAIService (OpenAI-compatible HTTP API)
2173///
2174/// This service provides the HTTP API for inference requests.
2175/// It communicates with ModelService, PolicyService, and RegistryService over
2176/// the configured authenticated RPC resolver.
2177#[service_factory("oai", depends_on = ["policy", "model", "registry", "discovery"])]
2178fn create_oai_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
2179    info!("Creating OAIService");
2180
2181    use crate::server::state::ServerState;
2182    use crate::services::OAIService;
2183    use hyprstream_rpc_std::model_client::ModelClient;
2184
2185    // Load full config for OAI settings
2186    let config = load_config()?;
2187    let sk = ctx.service_signing_key("oai");
2188
2189    // Register this service's verifying key with PolicyService
2190    register_service_key(ctx, "oai", &sk)?;
2191
2192    // Create authenticated clients for Model and Policy services.
2193    let model_client = ModelClient::from_provider(
2194        &hyprstream_discovery::ProductionRpcClientProvider,
2195        sk.clone(),
2196        service_token(&sk),
2197    )?;
2198    let policy_vk = hyprstream_service::global_trust_store()
2199        .resolve_one("policy")
2200        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
2201    let policy_client =
2202        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
2203
2204    // Create registry client
2205    let registry_client: RegistryClient = RegistryClient::from_provider(
2206        &hyprstream_discovery::ProductionRpcClientProvider,
2207        sk.clone(),
2208        service_token(&sk),
2209    )?;
2210
2211    // Create server state (blocking since we're in sync context)
2212    let resource_url = config.oai.resource_url();
2213    let oauth_issuer_url = config.oauth.issuer_url();
2214    let server_state = tokio::task::block_in_place(|| {
2215        let rt = tokio::runtime::Handle::current();
2216        rt.block_on(async {
2217            let secrets_dir = crate::config::HyprConfig::resolve_secrets_dir()?;
2218            let ml_dsa_store =
2219                crate::auth::key_rotation::global_ml_dsa_key_store(&secrets_dir, &config.oauth);
2220            let signer = crate::mac::audit::cose::OwnedCoseAuditSigner::new(
2221                Arc::new(sk.clone()),
2222                ml_dsa_store.active_key().await,
2223                hyprstream_rpc::envelope::mandatory_envelope_policy(),
2224            );
2225            anyhow::ensure!(
2226                signer.can_sign(),
2227                "9P MAC PEP audit signer unavailable under mandatory Hybrid policy"
2228            );
2229            let audit_store = crate::mac::audit::WalAuditStore::open(
2230                secrets_dir.join("mac-audit").join("ninep"),
2231                signer,
2232            )
2233            .map_err(|error| anyhow::anyhow!("open 9P MAC audit store: {error}"))?;
2234            let resolver = crate::mac::GenesisGate::production().into_resolver();
2235            let ninep_decider: Arc<dyn hyprstream_9p::AccessDecider> = Arc::new(
2236                crate::mac::NinePAccessDecider::new(Arc::new(resolver), Arc::new(audit_store)),
2237            );
2238            let ninep_vfs_pep = if config.server.ninep_canonical_read.is_some() {
2239                Some(crate::mac::production_vfs_pep(sk.clone(), &config.oauth, "vfs-ninep").await?)
2240            } else {
2241                None
2242            };
2243
2244            ServerState::new(
2245                config.server.clone(),
2246                model_client,
2247                policy_client,
2248                registry_client,
2249                sk.clone(),
2250                ctx.jwt_verifying_key(),
2251                resource_url,
2252                oauth_issuer_url,
2253                &config.oauth.trusted_issuers,
2254                ninep_decider,
2255                ninep_vfs_pep,
2256            )
2257            .await
2258        })
2259    })
2260    .context("Failed to create server state")?;
2261
2262    let oai_service = OAIService::new(
2263        config.oai.clone(),
2264        config.tls.clone(),
2265        config.account.clone(),
2266        server_state,
2267        ctx.iroh_required(),
2268    );
2269
2270    Ok(Box::new(oai_service))
2271}
2272
2273// ═══════════════════════════════════════════════════════════════════════════════
2274// Xet Service Factory
2275// ═══════════════════════════════════════════════════════════════════════════════
2276
2277/// Factory for XetService (HuggingFace-XET CAS HTTP face, epic #654).
2278///
2279/// HTTP service that speaks the HF-XET CAS wire protocol so a standard
2280/// xet-enabled git repo can point its CAS endpoint at hyprstream. It dials the
2281/// `registry` service (reusing the authenticated `putBlob`/`getBlob` core) and
2282/// holds no standing CAS write authority of its own. Reads come from the shared
2283/// L1 CAS substrate (`crate::storage::CasSubstrate`, #812).
2284#[service_factory("xet", depends_on = ["policy", "registry", "discovery"])]
2285fn create_xet_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
2286    info!("Creating XetService");
2287
2288    use crate::server::state::ResourceAuthState;
2289    use crate::services::{XetService, XetState};
2290
2291    let config = load_config()?;
2292    let sk = ctx.service_signing_key("xet");
2293
2294    // Register this service's verifying key with PolicyService.
2295    register_service_key(ctx, "xet", &sk)?;
2296
2297    // Dial the registry — the authenticated write core the HTTP face translates to.
2298    let registry_client: RegistryClient = RegistryClient::from_provider(
2299        &hyprstream_discovery::ProductionRpcClientProvider,
2300        sk.clone(),
2301        service_token(&sk),
2302    )?;
2303
2304    // Reuse the same narrow authentication core as OAI without constructing an
2305    // inference-oriented ServerState. The policy client is used by federated
2306    // issuer admission and key resolution.
2307    let policy_vk = hyprstream_service::global_trust_store()
2308        .resolve_one("policy")
2309        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
2310    let policy_client =
2311        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
2312    let federation_resolver = Arc::new(
2313        crate::auth::FederationKeyResolver::new(&config.oauth.trusted_issuers)
2314            .with_policy_client(Arc::new(policy_client)),
2315    );
2316    let auth = ResourceAuthState::new(
2317        ctx.jwt_verifying_key(),
2318        config.xet.resource_url(),
2319        config.oauth.issuer_url(),
2320        federation_resolver,
2321    );
2322    let cas_pep = tokio::task::block_in_place(|| {
2323        tokio::runtime::Handle::current().block_on(crate::mac::production_cas_pep(
2324            sk.clone(),
2325            &config.oauth,
2326            "cas-xet",
2327        ))
2328    })
2329    .context("construct Xet CAS MAC PEP")?;
2330
2331    let state = XetState {
2332        // Reads share the same L1 CAS substrate the registry's getBlob uses (#812).
2333        store: crate::storage::CasSubstrate::from_env(),
2334        registry: Some(registry_client),
2335        auth,
2336        // T8: the production CAS hook resolves only contexts remembered from
2337        // verified Claims × VerifiedKeyMaterial.  The activation control keeps
2338        // that source at anonymous_floor until an operator widens it.
2339        cas_authorizer: Arc::new(crate::mac::MacCasAuthorizer::new(cas_pep)),
2340    };
2341
2342    let xet_service = XetService::new(
2343        config.xet.clone(),
2344        config.tls.clone(),
2345        config.account.clone(),
2346        state,
2347    );
2348
2349    Ok(Box::new(xet_service))
2350}
2351
2352// ═══════════════════════════════════════════════════════════════════════════════
2353// Flight Service Factory
2354// ═══════════════════════════════════════════════════════════════════════════════
2355
2356/// Factory for FlightService (Arrow Flight SQL server)
2357///
2358/// This service provides Flight SQL protocol for dataset queries.
2359/// It optionally uses RegistryClient for dataset lookup.
2360#[cfg(feature = "metrics")]
2361#[service_factory("flight", depends_on = ["policy", "registry", "discovery"])]
2362fn create_flight_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
2363    info!("Creating FlightService");
2364
2365    use crate::services::FlightService;
2366
2367    // Load full config for Flight settings
2368    let config = load_config()?;
2369    let sk = ctx.service_signing_key("flight");
2370
2371    // Register this service's verifying key with PolicyService
2372    register_service_key(ctx, "flight", &sk)?;
2373
2374    let policy_vk = hyprstream_service::global_trust_store()
2375        .resolve_one("policy")
2376        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
2377    let policy_client =
2378        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
2379    let federation_resolver = Arc::new(
2380        crate::auth::FederationKeyResolver::new(&config.oauth.trusted_issuers)
2381            .with_policy_client(Arc::new(policy_client.clone())),
2382    );
2383    let auth = crate::server::state::ResourceAuthState::new(
2384        ctx.jwt_verifying_key(),
2385        config.flight.resource_url(),
2386        config.oauth.issuer_url(),
2387        federation_resolver,
2388    );
2389    let authorizer = Arc::new(crate::services::flight::TenantFlightAuthorizer::new(
2390        auth,
2391        policy_client,
2392    ));
2393
2394    // Create registry client for dataset lookup (if default_dataset is configured)
2395    // Adapt the canonical RPC client to the metrics checkpoint interface.
2396    let registry_client: Option<Arc<dyn hyprstream_metrics::RegistryClient>> =
2397        if config.flight.default_dataset.is_some() {
2398            let registry_client: RegistryClient = RegistryClient::from_provider(
2399                &hyprstream_discovery::ProductionRpcClientProvider,
2400                sk.clone(),
2401                service_token(&sk),
2402            )?;
2403            Some(Arc::new(crate::services::registry::MetricsRegistryAdapter(
2404                registry_client,
2405            )))
2406        } else {
2407            None
2408        };
2409
2410    let flight_service = FlightService::new(
2411        config.flight.clone(),
2412        registry_client,
2413        ctx.transport("flight", SocketKind::Rep),
2414        ctx.verifying_key(),
2415        authorizer,
2416    );
2417
2418    Ok(Box::new(flight_service))
2419}
2420
2421// ═══════════════════════════════════════════════════════════════════════════════
2422// OAuth Service Factory
2423// ═══════════════════════════════════════════════════════════════════════════════
2424
2425/// Factory for OAuthService (OAuth 2.1 Authorization Server)
2426///
2427/// This service provides OAuth 2.1 authorization for MCP and OAI services.
2428/// It delegates token issuance to PolicyService over ZMQ.
2429#[service_factory("oauth", schema = "../../../hyprstream-rpc-std/schema/oauth.capnp", metadata = hyprstream_rpc_std::oauth_client::schema_metadata, depends_on = ["policy", "discovery"])]
2430fn create_oauth_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
2431    info!("Creating OAuthService");
2432
2433    use crate::services::OAuthService;
2434
2435    let config = load_config()?;
2436    let sk = ctx.service_signing_key("oauth");
2437
2438    // Register this service's verifying key with PolicyService
2439    register_service_key(ctx, "oauth", &sk)?;
2440
2441    let pds_root = ctx.deployment_data_dir()?.join("pds");
2442    // #698 Phase 2 (D3): one enrollment-intent journal per deployment, shared
2443    // by the hosted-mint and OIDC JIT hooks. Write-once; a conflicting wiring
2444    // fails startup closed rather than silently splitting intents across
2445    // journals. The reconciler reads this same directory.
2446    let journal_dir =
2447        pds_root.join(crate::services::oauth::identity_registration::ENROLLMENT_JOURNAL_DIR);
2448    match crate::mac::enrollment_journal::global_journal_dir() {
2449        None => crate::mac::enrollment_journal::set_global_journal_dir(journal_dir)?,
2450        Some(existing) if existing == journal_dir => {}
2451        Some(existing) => anyhow::bail!(
2452            "enrollment journal directory already wired to {} — refusing {}",
2453            existing.display(),
2454            journal_dir.display()
2455        ),
2456    }
2457    let identity_registration_api =
2458        crate::services::oauth::identity_registration::production_identity_registration_api(
2459            &config.oauth,
2460            &config.account,
2461            &config.quic,
2462            sk.clone(),
2463            pds_root.clone(),
2464        )
2465        .context("compose production identity registration API")?;
2466
2467    // Pass signing key instead of a pre-created PolicyClient.
2468    // OAuthService runs in its own tokio runtime (separate thread), so the
2469    // PolicyClient must be created inside that runtime for ZMQ async I/O to work.
2470
2471    // The registry factory and `ServerState::new` validate the canonical 9P
2472    // export where they consume it; a dedicated oauth-only `--ipc` process
2473    // runs neither, so this wiring validates too — an invalid export must
2474    // fail oauth startup closed rather than seed HolderGate rows from it at
2475    // signup.
2476    if let Some(export) = config.server.ninep_canonical_read.as_ref() {
2477        export.validate()?;
2478    }
2479    let oauth_service = OAuthService::new(
2480        config.oauth.clone(),
2481        config.tls.clone(),
2482        config.account.clone(),
2483        sk,
2484        ctx.transport("oauth", SocketKind::Rep),
2485        ctx.transport("policy", SocketKind::Rep),
2486        ctx.transport("discovery", SocketKind::Rep),
2487        ctx.verifying_key(),
2488        ctx.jwt_verifying_key(),
2489    )
2490    .with_quic_config(config.quic.clone())
2491    .with_identity_registration_api(identity_registration_api)
2492    .with_pds_root(pds_root)
2493    .with_ninep_canonical_read(config.server.ninep_canonical_read.clone())
2494    .with_dedicated_process(ctx.is_dedicated_process_for("oauth"));
2495
2496    Ok(Box::new(oauth_service))
2497}
2498
2499// ═══════════════════════════════════════════════════════════════════════════════
2500// MCP Service Factory
2501// ═══════════════════════════════════════════════════════════════════════════════
2502
2503async fn verify_mcp_http_token(
2504    auth: &crate::server::state::ResourceAuthState,
2505    token: &str,
2506) -> std::result::Result<crate::auth::jwt::Claims, &'static str> {
2507    crate::server::middleware::verify_resource_token_claims(auth, token).await
2508}
2509
2510fn validated_mcp_subject(
2511    claims: &crate::auth::jwt::Claims,
2512    local_issuers: &[&str],
2513) -> Option<String> {
2514    let subject = claims.subject(local_issuers);
2515    crate::auth::valid_resource_subject(
2516        &subject,
2517        &claims.iss,
2518        local_issuers,
2519        claims.tenant.as_deref(),
2520    )
2521    .then(|| subject.name().map(str::to_owned))
2522    .flatten()
2523}
2524
2525/// Factory for McpService (Model Context Protocol)
2526///
2527/// This service provides an MCP-compliant interface for AI coding assistants
2528/// (Claude Code, Cursor, etc.) to interact with hyprstream via:
2529/// - ZMQ control plane (for internal service communication)
2530/// - HTTP/SSE (for external MCP clients)
2531///
2532/// Note: The HTTP/SSE server is spawned as a background task in the factory.
2533#[service_factory("mcp", schema = "../../../hyprstream-rpc-std/schema/mcp.capnp", metadata = hyprstream_rpc_std::mcp_client::schema_metadata, depends_on = ["policy", "discovery"])]
2534fn create_mcp_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
2535    info!("Creating McpService");
2536
2537    // Load full config for MCP settings
2538    let config = load_config()?;
2539
2540    // Create McpConfig for the service
2541    let _oauth_issuer = ctx.oauth_issuer_url().map(str::to_owned);
2542    let federation_key_source = ctx.federation_key_source();
2543    let sk = ctx.service_signing_key("mcp");
2544
2545    // Register this service's verifying key with PolicyService
2546    register_service_key(ctx, "mcp", &sk)?;
2547
2548    let policy_vk = hyprstream_service::global_trust_store()
2549        .resolve_one("policy")
2550        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
2551
2552    let mcp_config = McpConfig {
2553        verifying_key: ctx.verifying_key(),
2554        signing_key: sk.clone(),
2555        transport: ctx.transport("mcp", SocketKind::Rep),
2556        policy_transport: ctx.transport("policy", SocketKind::Rep),
2557        ctx: None, // ServiceContext not yet available as Arc — handlers use signing_key directly
2558        policy_verifying_key: policy_vk,
2559        expected_audience: Some(config.mcp.resource_url()),
2560        jwt_key_source: Some(ctx.cluster_key_source()),
2561    };
2562
2563    // Clone config for HTTP/SSE server before consuming it for ZMQ service
2564    let mcp_config_clone = mcp_config.clone();
2565
2566    // Create the service (includes ZMQ infrastructure)
2567    let mcp_service = McpService::new(mcp_config)?;
2568
2569    // Spawn rmcp HTTP/SSE server as background task
2570    let mcp_host = config.mcp.host.clone();
2571    let http_port = config.mcp.http_port;
2572    let mcp_cors_config = config.mcp.cors.clone();
2573    let mcp_tls_config = config.tls.clone();
2574    let mcp_tls_cert = config.mcp.tls_cert.clone();
2575    let mcp_tls_key = config.mcp.tls_key.clone();
2576    // RFC 9728 protected-resource metadata values come from THIS fail-closed
2577    // load — reloading the config inside the spawned server task and falling
2578    // back to defaults on error would silently discard a configured [rds]
2579    // backend (and serve wrong issuer metadata) instead of failing closed.
2580    let mcp_resource_url = config.mcp.resource_url();
2581    let mcp_oauth_issuer = config.oauth.issuer_url();
2582    // Use the shared FederationKeySource from ServiceContext if available,
2583    // otherwise fall back to a locally-constructed resolver from config.
2584    // The fallback path wires its own PolicyClient so the unified
2585    // federation:register trust gate stays in effect — never downgrade
2586    // security posture just because the shared resolver wasn't provided.
2587    let mcp_federation_resolver: std::sync::Arc<dyn hyprstream_rpc::auth::FederationKeySource> =
2588        if let Some(fed) = federation_key_source {
2589            fed
2590        } else {
2591            let fallback_policy_client = std::sync::Arc::new(policy_client_for_deployment(
2592                ctx,
2593                ctx.service_signing_key("mcp"),
2594                policy_vk,
2595                service_token(&sk),
2596            )?);
2597            std::sync::Arc::new(
2598                crate::auth::FederationKeyResolver::new(&config.oauth.trusted_issuers)
2599                    .with_policy_client(fallback_policy_client),
2600            )
2601        };
2602    let mcp_resource_auth = crate::server::state::ResourceAuthState::new(
2603        ctx.jwt_verifying_key(),
2604        mcp_resource_url.clone(),
2605        mcp_oauth_issuer.clone(),
2606        mcp_federation_resolver,
2607    );
2608    tokio::task::block_in_place(|| {
2609        let rt = tokio::runtime::Handle::current();
2610        rt.spawn(async move {
2611            let router = build_mcp_http_router(
2612                mcp_config_clone,
2613                mcp_resource_url,
2614                mcp_oauth_issuer,
2615                mcp_resource_auth,
2616                mcp_cors_config,
2617            );
2618
2619            let addr: std::net::SocketAddr = format!("{}:{}", mcp_host, http_port)
2620                .parse()
2621                .unwrap_or_else(|_| ([0, 0, 0, 0], http_port).into());
2622
2623            // Resolve TLS configuration for MCP HTTP server.
2624            // If the user explicitly configured cert/key paths and TLS fails,
2625            // refuse to start (don't silently degrade to HTTP).
2626            let has_explicit_tls = mcp_tls_cert.is_some() || mcp_tls_key.is_some()
2627                || mcp_tls_config.cert_path.is_some() || mcp_tls_config.key_path.is_some();
2628
2629            let rustls_config = match crate::server::tls::resolve_rustls_config(
2630                &mcp_tls_config,
2631                &config.account,
2632                mcp_tls_cert.as_ref(),
2633                mcp_tls_key.as_ref(),
2634            ).await {
2635                Ok(cfg) => cfg,
2636                Err(e) => {
2637                    if has_explicit_tls {
2638                        tracing::error!(
2639                            "MCP TLS config error with explicit cert/key paths: {} — refusing to start without TLS", e
2640                        );
2641                        return;
2642                    }
2643                    tracing::warn!("MCP TLS config error (self-signed): {} — falling back to HTTP", e);
2644                    None
2645                }
2646            };
2647
2648            let scheme = if rustls_config.is_some() { "https" } else { "http" };
2649            tracing::info!("MCP HTTP/SSE server listening on {scheme}://{addr}");
2650
2651            match rustls_config {
2652                Some(tls) => {
2653                    // MCP HTTP is fire-and-forget (no Arc<Notify> shutdown signal),
2654                    // so no Handle is wired for graceful shutdown. The process exit
2655                    // will terminate this task. OAI/OAuth use serve_app() instead.
2656                    let serve_result = if let Some(acme_acceptor) = tls.acme_acceptor {
2657                        axum_server::bind(addr)
2658                            .acceptor(acme_acceptor)
2659                            .serve(router.into_make_service())
2660                            .await
2661                    } else {
2662                        axum_server::bind_rustls(addr, tls.rustls)
2663                            .serve(router.into_make_service())
2664                            .await
2665                    };
2666                    if let Err(e) = serve_result {
2667                        tracing::error!("MCP HTTPS server error: {}", e);
2668                    }
2669                }
2670                None => {
2671                    let listener = match tokio::net::TcpListener::bind(addr).await {
2672                        Ok(l) => l,
2673                        Err(e) => {
2674                            tracing::error!("Failed to bind MCP HTTP/SSE on {}: {}", addr, e);
2675                            return;
2676                        }
2677                    };
2678                    if let Err(e) = axum::serve(listener, router).await {
2679                        tracing::error!("MCP HTTP/SSE server error: {}", e);
2680                    }
2681                }
2682            }
2683        });
2684    });
2685    info!(
2686        "McpService created (HTTP/SSE on {}:{})",
2687        config.mcp.host, http_port
2688    );
2689
2690    Ok(ctx.into_spawnable_quic(mcp_service, config.mcp.quic_port))
2691}
2692
2693/// Build the production MCP Streamable HTTP router. Kept separate from socket
2694/// binding so protocol and authentication behavior can be exercised in-process.
2695fn build_mcp_http_router(
2696    mcp_config: McpConfig,
2697    mcp_resource_url: String,
2698    mcp_oauth_issuer: String,
2699    mcp_resource_auth: crate::server::state::ResourceAuthState,
2700    mcp_cors_config: crate::server::state::CorsConfig,
2701) -> axum::Router {
2702    use rmcp::transport::streamable_http_server::{
2703        StreamableHttpServerConfig, StreamableHttpService,
2704    };
2705
2706    use rmcp::transport::streamable_http_server::session::local::LocalSessionManager;
2707
2708    let session_mgr = std::sync::Arc::new(LocalSessionManager::default());
2709    let service: StreamableHttpService<McpService, LocalSessionManager> =
2710        StreamableHttpService::new(
2711            move || {
2712                McpService::new(mcp_config.clone())
2713                    .map_err(|e| std::io::Error::other(e.to_string()))
2714            },
2715            session_mgr,
2716            StreamableHttpServerConfig::default(),
2717        );
2718    // Add protected resource metadata (RFC 9728) for OAuth discovery
2719    // (values captured from the factory's fail-closed config load).
2720    let www_authenticate = format!(
2721        "Bearer resource_metadata=\"{}/.well-known/oauth-protected-resource\"",
2722        mcp_resource_url
2723    );
2724    let router = axum::Router::new()
2725        .route(
2726            "/.well-known/oauth-protected-resource",
2727            axum::routing::get({
2728                let mcp_resource_url = mcp_resource_url.clone();
2729                let mcp_oauth_issuer = mcp_oauth_issuer.clone();
2730                move || async move {
2731                    let mut meta = crate::services::oauth::protected_resource_metadata(
2732                        &mcp_resource_url,
2733                        &mcp_oauth_issuer,
2734                    );
2735                    meta.resource_name = Some("HyprStream MCP Server".to_owned());
2736                    meta.scopes_supported = Some(vec![
2737                        "read:model:*".into(),
2738                        "infer:model:*".into(),
2739                        "write:model:*".into(),
2740                        // Advertise the registry read scope: a spec-compliant native
2741                        // OAuth client requests exactly the advertised scopes, and
2742                        // the signed scope ceiling
2743                        // (policy.rs delegated_resource_scope_allows) denies every
2744                        // registry 9P read leaf without it. Advertisement is not a
2745                        // grant: tenant policy rows and MAC clearance still gate
2746                        // every call, and write-tier stays unmappable for delegated
2747                        // user tokens.
2748                        "read:registry:*".into(),
2749                    ]);
2750                    axum::Json(meta)
2751                }
2752            }),
2753        )
2754        .nest_service("/mcp", service)
2755        .layer(axum::middleware::from_fn({
2756            let mcp_resource_url = mcp_resource_url.clone();
2757            let mcp_oauth_issuer = mcp_oauth_issuer.clone();
2758            let mcp_resource_auth = mcp_resource_auth.clone();
2759            // The credential-revocation store is global; no per-service
2760            // capture needed.
2761            // DPoP JTI replay cache (separate from OAI server's, RFC 9449).
2762            // 1,000 sustained DPoP proofs/s for the admitted 180s
2763            // maximum residency (60s future iat skew + 120s), plus
2764            // 20% headroom. Fixed digest keys plan 216,000 entries at
2765            // about 26.4 MiB before allocator slack; this barrier
2766            // never evicts live JTIs.
2767            let mcp_dpop_jti_seen: std::sync::Arc<hyprstream_util::TtlCache<
2768                crate::services::oauth::replay_key::ReplayKey,
2769                (),
2770            >> =
2771                std::sync::Arc::new(hyprstream_util::TtlCache::new(216_000, 64));
2772            move |mut req: axum::extract::Request, next: axum::middleware::Next| {
2773                let www_authenticate = www_authenticate.clone();
2774                let mcp_resource_url = mcp_resource_url.clone();
2775                let mcp_oauth_issuer = mcp_oauth_issuer.clone();
2776                let mcp_resource_auth = mcp_resource_auth.clone();
2777                let dpop_jti_seen = mcp_dpop_jti_seen.clone();
2778                async move {
2779                    use axum::http::{header, StatusCode};
2780                    use axum::response::IntoResponse;
2781                    use subtle::ConstantTimeEq as _;
2782                    let method = req.method().clone();
2783                    let uri = req.uri().clone();
2784                    // Allow OAuth discovery endpoint without auth
2785                    if req.uri().path().starts_with("/.well-known/") {
2786                        tracing::debug!(%method, %uri, "MCP discovery request (no auth required)");
2787                        return next.run(req).await;
2788                    }
2789                    let has_auth_header = req.headers().contains_key(header::AUTHORIZATION);
2790                    let auth_value = req.headers()
2791                        .get(header::AUTHORIZATION)
2792                        .and_then(|v| v.to_str().ok())
2793                        .map(str::to_owned);
2794                    // Accept both Bearer (RFC 6750) and DPoP (RFC 9449) schemes
2795                    let (scheme, t) = match auth_value.as_deref().and_then(|h| {
2796                        if h.len() > 7 && h[..7].eq_ignore_ascii_case("bearer ") {
2797                            Some(("bearer", h[7..].trim().to_owned()))
2798                        } else if h.len() > 5 && h[..5].eq_ignore_ascii_case("dpop ") {
2799                            Some(("dpop", h[5..].trim().to_owned()))
2800                        } else {
2801                            None
2802                        }
2803                    }) {
2804                        Some(pair) => pair,
2805                        None => {
2806                            tracing::info!(%method, %uri, has_auth_header, "MCP auth MISSING token");
2807                            let mut res = (StatusCode::UNAUTHORIZED, "Authentication required").into_response();
2808                            if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2809                                res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2810                            }
2811                            return res;
2812                        }
2813                    };
2814                    let result = verify_mcp_http_token(&mcp_resource_auth, &t).await;
2815                    let mut claims = match result {
2816                        Ok(c) => c,
2817                        Err(e) => {
2818                            tracing::warn!(%method, %uri, error = %e, "MCP auth REJECTED");
2819                            let mut res = (StatusCode::UNAUTHORIZED, "Invalid or expired token").into_response();
2820                            if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2821                                res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2822                            }
2823                            return res;
2824                        }
2825                    };
2826                    let atproto_issuer =
2827                        crate::services::oauth::state::canonical_issuer_origin(
2828                            &mcp_oauth_issuer,
2829                        )
2830                        .unwrap_or_else(|| mcp_oauth_issuer.clone());
2831                    let local_issuers =
2832                        [mcp_oauth_issuer.as_str(), atproto_issuer.as_str()];
2833                    claims.strip_federated_tenant(&local_issuers);
2834                    let Some(subject_name) = validated_mcp_subject(&claims, &local_issuers) else {
2835                        tracing::warn!(%method, %uri, "MCP auth rejected: invalid subject");
2836                        return (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2837                    };
2838                    // Credential revocation check — fail-closed on
2839                    // store absence: a token with a jti that cannot
2840                    // be checked for revocation is rejected. Local
2841                    // tokens MUST carry a jti (profile REQUIRED
2842                    // claim); a local token without one is rejected.
2843                    let token_is_local =
2844                        hyprstream_rpc::auth::is_local_iss(&claims.iss, &local_issuers);
2845                    if token_is_local && claims.jti.is_none() {
2846                        tracing::warn!(%method, %uri, iss = %claims.iss, "MCP: local token without jti rejected");
2847                        let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2848                        if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2849                            res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2850                        }
2851                        return res;
2852                    }
2853                    if let Some(ref jti) = claims.jti {
2854                        let cred_id = hyprstream_rpc::auth::CredentialId::jwt(&claims.iss, jti);
2855                        let revoked_or_unavailable = match hyprstream_rpc::auth::global_credential_revocation_store() {
2856                            Some(bl) => bl.is_revoked(&cred_id).await,
2857                            None => {
2858                                tracing::warn!(%method, %uri, %jti, "MCP: no revocation store configured — rejecting token with jti");
2859                                true // fail-closed
2860                            }
2861                        };
2862                        if revoked_or_unavailable {
2863                            tracing::warn!(%method, %uri, %jti, sub = %claims.sub, "MCP: revoked token presented");
2864                            let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2865                            if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2866                                res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2867                            }
2868                            return res;
2869                        }
2870                    }
2871                    // Session check (v16 §3.3): a local token carrying
2872                    // a session ID is rejected when the session is
2873                    // revoked, unknown, or cannot be checked.
2874                    if token_is_local {
2875                        let session_key = match claims.session_key() {
2876                            Ok(key) => key,
2877                            Err(e) => {
2878                                tracing::warn!(%method, %uri, error = %e, "MCP: malformed session claims rejected");
2879                                let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2880                                if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2881                                    res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2882                                }
2883                                return res;
2884                            }
2885                        };
2886                        if let Some(session_key) = session_key {
2887                            let session_inactive = match hyprstream_rpc::auth::global_session_registry() {
2888                                Some(registry) => registry.is_revoked(&session_key).await,
2889                                None => {
2890                                    tracing::warn!(%method, %uri, "MCP: no session registry configured — rejecting token with session id");
2891                                    true // fail-closed
2892                                }
2893                            };
2894                            if session_inactive {
2895                                tracing::warn!(%method, %uri, sub = %claims.sub, "MCP: token with revoked/unknown session presented");
2896                                let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2897                                if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2898                                    res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2899                                }
2900                                return res;
2901                            }
2902                        }
2903                    }
2904                    // RFC 9068 §2.2.1 (v16 credential profile): an
2905                    // `at+jwt` access token presented to MCP MUST carry
2906                    // a non-empty `client_id`. Positively typed on the
2907                    // JOSE `typ`; `wit+jwt`/other types are exempt.
2908                    if hyprstream_rpc::auth::parse_protected_header(&t)
2909                        .ok()
2910                        .is_some_and(|h| {
2911                            hyprstream_rpc::auth::is_rfc9068_access_token_type(&h.typ)
2912                        })
2913                        && claims
2914                            .client_id
2915                            .as_deref()
2916                            .is_none_or(|c| c.trim().is_empty())
2917                    {
2918                        tracing::warn!(%method, %uri, sub = %claims.sub, "MCP: at+jwt without client_id rejected");
2919                        let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2920                        if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2921                            res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2922                        }
2923                        return res;
2924                    }
2925                    // DPoP binding enforcement (RFC 9449 §7):
2926                    // cnf.jkt tokens MUST be presented with DPoP scheme + proof header.
2927                    if let Some(expected_jkt) = claims.cnf_jkt() {
2928                        if scheme != "dpop" {
2929                            tracing::warn!(%method, %uri, sub = %claims.sub, "MCP: DPoP-bound token presented with Bearer scheme");
2930                            let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2931                            if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2932                                res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2933                            }
2934                            return res;
2935                        }
2936                        let dpop_proof = match req.headers().get("DPoP").and_then(|v| v.to_str().ok()) {
2937                            Some(p) => p.to_owned(),
2938                            None => {
2939                                tracing::debug!(%method, %uri, "MCP: DPoP-bound token missing DPoP proof header");
2940                                let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2941                                if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2942                                    res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2943                                }
2944                                return res;
2945                            }
2946                        };
2947                        let method_str = method.as_str().to_owned();
2948                        let path = uri.path().to_owned();
2949                        let htu = format!("{}{}", mcp_resource_url.trim_end_matches('/'), path);
2950                        let proof = match crate::services::oauth::dpop::verify_dpop_proof(
2951                            &dpop_proof,
2952                            &method_str,
2953                            &htu,
2954                            Some(&t),
2955                        ) {
2956                            Ok(p) => p,
2957                            Err(e) => {
2958                                tracing::debug!(%method, %uri, error = %e, "MCP: DPoP proof verification failed");
2959                                let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2960                                if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2961                                    res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2962                                }
2963                                return res;
2964                            }
2965                        };
2966                        // Replay prevention: atomic check-and-record on the shared TtlCache.
2967                        {
2968                            let now = chrono::Utc::now().timestamp();
2969                            let Some(ttl_secs) = proof
2970                                .iat
2971                                .checked_add(120)
2972                                .and_then(|deadline| deadline.checked_sub(now))
2973                                .filter(|remaining| *remaining > 0 && *remaining <= 180)
2974                                .and_then(|remaining| u64::try_from(remaining).ok())
2975                            else {
2976                                let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
2977                                if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
2978                                    res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
2979                                }
2980                                return res;
2981                            };
2982                            let result = dpop_jti_seen.insert_if_absent_no_evict(
2983                                crate::services::oauth::replay_key::dpop_jti(&proof.jti),
2984                                (),
2985                                std::time::Duration::from_secs(ttl_secs),
2986                            );
2987                            if result != hyprstream_util::InsertIfAbsentNoEvictResult::Inserted {
2988                                crate::services::oauth::replay_metrics::record_rejection(
2989                                    crate::services::oauth::replay_metrics::DPOP,
2990                                    result,
2991                                );
2992                                if crate::services::oauth::replay_metrics::should_warn_full(
2993                                    crate::services::oauth::replay_metrics::DPOP,
2994                                    result,
2995                                ) {
2996                                    tracing::warn!(%method, %uri, "MCP: DPoP replay barrier is full; refusing fresh proof");
2997                                } else if result
2998                                    == hyprstream_util::InsertIfAbsentNoEvictResult::Duplicate
2999                                {
3000                                    tracing::debug!(%method, %uri, "MCP: DPoP proof replayed");
3001                                }
3002                                let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
3003                                if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
3004                                    res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
3005                                }
3006                                return res;
3007                            }
3008                        }
3009                        // cnf.jkt must match proof key thumbprint (constant-time)
3010                        if expected_jkt.as_bytes().ct_eq(proof.jkt.as_bytes()).unwrap_u8() == 0 {
3011                            tracing::warn!(%method, %uri, sub = %claims.sub, "MCP: cnf.jkt mismatch");
3012                            let mut res = (StatusCode::UNAUTHORIZED, "Authentication failed").into_response();
3013                            if let Ok(val) = header::HeaderValue::from_str(&www_authenticate) {
3014                                res.headers_mut().insert(header::WWW_AUTHENTICATE, val);
3015                            }
3016                            return res;
3017                        }
3018                    }
3019                    tracing::debug!(%method, %uri, sub = %claims.sub, "MCP auth OK");
3020                    // Insert AuthenticatedUser so MCP handlers see validated identity
3021                    let authenticated = crate::server::middleware::AuthenticatedUser {
3022                        user: subject_name,
3023                        verified_tenant: claims.tenant.clone(),
3024                        token: Some(t.clone()),
3025                        exp: Some(claims.exp),
3026                    };
3027                    if authenticated.authorization_domain().is_err() {
3028                        tracing::warn!(%method, %uri, sub = %claims.sub, "MCP auth rejected: no valid hosted-account tenant binding");
3029                        return (StatusCode::FORBIDDEN, "Verified hosted-account tenant binding required").into_response();
3030                    }
3031                    req.extensions_mut().insert(authenticated);
3032                    next.run(req).await
3033                }
3034            }
3035        }));
3036
3037    // CORS must be outermost layer (added last) so OPTIONS preflights
3038    // are handled before auth middleware rejects them.
3039    if mcp_cors_config.enabled {
3040        router.layer(crate::server::middleware::cors_layer(&mcp_cors_config))
3041    } else {
3042        router
3043    }
3044}
3045
3046// ═══════════════════════════════════════════════════════════════════════════════
3047// TUI Service Factory
3048// ═══════════════════════════════════════════════════════════════════════════════
3049
3050/// Factory for TuiService (terminal multiplexer display server)
3051///
3052/// This service provides a terminal multiplexer with session persistence,
3053/// multi-pane layouts, and remote access via ZMQ RPC and WebTransport.
3054#[service_factory("tui", schema = "../../../hyprstream-rpc-std/schema/tui.capnp", depends_on = ["policy", "discovery"])]
3055fn create_tui_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
3056    info!("Creating TuiService");
3057
3058    use crate::tui::{service::TuiService, TuiState};
3059
3060    // TUI publishes terminal frames (stdin/stdout) over moq via
3061    // StreamChannel::publisher(), and returns its per-PID moq UDS path to the
3062    // client. In a per-process deployment this process has no moq plane unless
3063    // we initialize one here. Idempotent — no-op if already set.
3064    init_local_moq_stream_plane("tui", ctx.iroh_required());
3065
3066    let config = load_config()?;
3067    let tui_config = &config.tui;
3068    let sk = ctx.service_signing_key("tui");
3069
3070    // Register this service's verifying key with PolicyService
3071    register_service_key(ctx, "tui", &sk)?;
3072    let service_token = service_token(&sk)
3073        .filter(|token| !token.trim().is_empty())
3074        .ok_or_else(|| anyhow::anyhow!("TUI service credential absent after registration"))?;
3075
3076    let state = Arc::new(RwLock::new(TuiState::new(
3077        80,
3078        24,
3079        tui_config.scrollback_lines,
3080    )));
3081
3082    let policy_vk = hyprstream_service::global_trust_store()
3083        .resolve_one("policy")
3084        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
3085    let policy_client =
3086        policy_client_for_deployment(ctx, sk.clone(), policy_vk, Some(service_token.clone()))?;
3087
3088    // Build the direct-VFS PEP before exposing the namespace. Failure to open
3089    // its signed WAL aborts construction; there is no unarmed fallback.
3090    let vfs_pep = tokio::task::block_in_place(|| {
3091        tokio::runtime::Handle::current().block_on(crate::mac::production_vfs_pep(
3092            sk.clone(),
3093            &config.oauth,
3094            "vfs-tui",
3095        ))
3096    })
3097    .context("construct TUI VFS MAC PEP")?;
3098    let (vfs_ns, vfs_subject) = crate::tui::vfs::build_chat_vfs_namespace(&sk, vfs_pep)?;
3099
3100    let mut tui_service = TuiService::new(state, ctx.transport("tui", SocketKind::Rep), sk.clone())
3101        .with_policy_client(policy_client)
3102        .with_vfs(vfs_ns, vfs_subject);
3103
3104    if let Some(issuer) = ctx.oauth_issuer_url() {
3105        tui_service = tui_service.with_expected_audience(issuer.to_owned());
3106    }
3107    tui_service = tui_service.with_jwt_key_source(ctx.cluster_key_source());
3108
3109    // W1.3: the configured Federate TUI client class for the consumer gate.
3110    // A configured-but-invalid class leaves the gate fail-closed (never
3111    // widening), mirroring the oauth/policy factory wiring.
3112    let federate_tui_client = config
3113        .oauth
3114        .trusted_issuers
3115        .get(crate::services::oauth::federate_source::ISSUER)
3116        .and_then(|issuer| issuer.federate_tui_client.as_ref())
3117        .and_then(|class| {
3118            match crate::services::oauth::federate_source::ClientClass::new(
3119                &class.client_id,
3120                &class.aud,
3121                class.connector_id.as_deref(),
3122            ) {
3123                Ok(class) => Some(class.client_id().to_owned()),
3124                Err(error) => {
3125                    tracing::warn!(
3126                        %error,
3127                        "Federate TUI consumer gate disabled: invalid trusted-issuer configuration"
3128                    );
3129                    None
3130                }
3131            }
3132        });
3133    tui_service = tui_service.with_federate_tui_client(federate_tui_client);
3134
3135    Ok(ctx.into_spawnable_quic(tui_service, tui_config.quic_port))
3136}
3137
3138/// Open the PDS record store read-only, selecting the backend from config.
3139///
3140/// For the **Postgres** backend (#1257): no bootstrap dance is needed — the
3141/// schema migration runs on `connect`, and there is no single-writer lock to
3142/// race. Just calls [`open_pds_record_store`] with `readonly = true`.
3143///
3144/// For the **RocksDB** backend: bootstraps an empty database at `dir` first if
3145/// nothing has been published yet (the registry service creates the DB files;
3146/// on a fresh install the discovery service may start before any model has
3147/// been registered). When this path creates a fresh empty store it also
3148/// writes the durable first-boot RocksDB key, so the QUIC startup gate
3149/// recognizes it as a genuine first boot rather than data loss.
3150///
3151/// Known limitation: if the registry and discovery services start
3152/// concurrently on a brand-new install, both may race to bootstrap the same
3153/// directory; one loses the RocksDB lock and its factory call fails, which
3154/// the service manager will retry.
3155#[cfg(feature = "rocksdb")]
3156fn open_pds_store_readonly(
3157    ctx: &ServiceContext,
3158) -> anyhow::Result<crate::services::discovery::PdsRecordStore> {
3159    let rds = load_config()?.rds.resolved_from_env()?;
3160    if rds.is_configured() {
3161        // Postgres: no bootstrap dance needed.
3162        return open_pds_record_store(ctx, true);
3163    }
3164    let dir = pds_store_dir(ctx)?;
3165    match crate::services::discovery::PdsRecordStore::open_readonly(&dir) {
3166        Ok(store) => Ok(store),
3167        Err(orig) => {
3168            // Bootstrap the DB files by briefly opening read-write, then retry
3169            // read-only. If bootstrap itself fails (e.g. the writer holds the
3170            // lock, or the path is corrupt), surface BOTH errors so the real
3171            // cause is visible rather than masked by the retry (#910a, fable M3).
3172            let store =
3173                crate::services::discovery::PdsRecordStore::open(&dir).with_context(|| {
3174                    format!("read-only open failed ({orig}); bootstrap open also failed")
3175                })?;
3176            // This bootstrap created a fresh empty store — record first-boot
3177            // lifecycle evidence as a durable RocksDB key so the QUIC gate
3178            // doesn't mistake it for data loss on the next classification.
3179            // Propagate the error: a failed marker write must not appear
3180            // successful.
3181            store.mark_first_boot()?;
3182            drop(store);
3183            crate::services::discovery::PdsRecordStore::open_readonly(&dir)
3184        }
3185    }
3186}
3187
3188// ═══════════════════════════════════════════════════════════════════════════════
3189// Discovery Service Factory
3190// ═══════════════════════════════════════════════════════════════════════════════
3191
3192/// Factory for DiscoveryService (endpoint registry over ZMQ RPC)
3193///
3194/// This service exposes the EndpointRegistry so remote clients can discover
3195/// registered services, their endpoints, socket kinds, and schemas.
3196#[service_factory("discovery", schema = "../../../hyprstream-rpc-std/schema/discovery.capnp", metadata = hyprstream_rpc_std::discovery_client::schema_metadata, depends_on = ["policy"])]
3197#[cfg_attr(not(feature = "rocksdb"), allow(unreachable_code, unused_variables))]
3198fn create_discovery_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
3199    info!("Creating DiscoveryService");
3200    // The checkpoint-verifying record resolver (#431/#910a) reads the
3201    // RocksDB-backed PDS store; without the `rocksdb` feature the factory
3202    // fails closed instead of serving resolver-less discovery.
3203    #[cfg(not(feature = "rocksdb"))]
3204    anyhow::bail!("discovery service requires the `rocksdb` feature (PDS record resolver)");
3205
3206    let config = load_config()?;
3207    let sk = ctx.service_signing_key("discovery");
3208
3209    // Register this service's verifying key with PolicyService
3210    register_service_key(ctx, "discovery", &sk)?;
3211
3212    // Create policy-based authorization provider
3213    let policy_vk = hyprstream_service::global_trust_store()
3214        .resolve_one("policy")
3215        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
3216    let policy_client =
3217        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
3218    let auth_provider = crate::services::discovery::PolicyAuthProvider::new(policy_client);
3219
3220    // #431 — record resolver backing getRecord/getRepo, over the durable
3221    // RocksDB-backed PDS record store (#910a). The registry service is the
3222    // sole writer (it opens the same directory read-write, signs each commit
3223    // ONCE and persists it — see `create_registry_service`); this factory
3224    // opens the directory read-only, matching `RocksDbUserStore::open_readonly`.
3225    // The resolver holds **no signing key** at all: reads are keyless (rebuild
3226    // the deterministic MST, load the writer's already-signed commit, serve a
3227    // proof). This is the #910a security fix — a read path that re-signed a
3228    // commit on every `getRecord` (and so needed the private key) was the root
3229    // of the key-exposure problem; atproto never re-signs on read.
3230    // In-process factories share the stable node root. In IPC mode the
3231    // registry process signs with its stable service credential, whose public
3232    // key is anchored in the global service trust store.
3233    #[cfg(feature = "rocksdb")]
3234    let at9p_acceptance_identity = hyprstream_discovery::deployment_registry_verifier()?;
3235    #[cfg(feature = "rocksdb")]
3236    let pds_store = std::sync::Arc::new(
3237        open_pds_store_readonly(ctx)
3238            .context("failed to open PDS record store (read-only)")?
3239            .with_at9p_deployment_verifier(at9p_acceptance_identity),
3240    );
3241    // #918 — the local repo subject is the root did:web authority whose
3242    // document is fed by this ES256 store. The resolver uses the same bounded
3243    // publication snapshot, including drain, before placement ingest.
3244    #[cfg(feature = "rocksdb")]
3245    let secrets_dir = crate::config::HyprConfig::resolve_secrets_dir()?;
3246    #[cfg(feature = "rocksdb")]
3247    let es256_store =
3248        crate::auth::key_rotation::global_es256_key_store(&secrets_dir, &config.oauth);
3249    #[cfg(feature = "rocksdb")]
3250    let issuer = ctx
3251        .oauth_issuer_url()
3252        .map(str::to_owned)
3253        .unwrap_or_else(|| config.oauth.issuer_url());
3254    #[cfg(feature = "rocksdb")]
3255    let node_did = crate::services::oauth::state::atproto_service_did_for_origin(&issuer)
3256        .context("OAuth issuer has no supported service DID")?;
3257    #[cfg(feature = "rocksdb")]
3258    let record_resolver = std::sync::Arc::new(
3259        crate::services::discovery::PdsRecordResolver::new(pds_store)
3260            .with_es256_rotation(es256_store, node_did),
3261    );
3262
3263    let mut discovery_service = DiscoveryService::new(
3264        Arc::new(sk),
3265        ctx.jwt_verifying_key(),
3266        ctx.transport("discovery", SocketKind::Rep),
3267    )
3268    .with_state({
3269        // Shared backends retain their own driver runtime after this temporary
3270        // bootstrap runtime exits; service state owns its shutdown lifetime.
3271        let state_config = config.discovery.state.clone();
3272        std::thread::spawn(
3273            move || -> anyhow::Result<hyprstream_discovery::DiscoveryState> {
3274                let runtime = tokio::runtime::Builder::new_current_thread()
3275                    .enable_all()
3276                    .build()
3277                    .context("build Discovery state bootstrap runtime")?;
3278                runtime.block_on(hyprstream_discovery::DiscoveryState::connect(&state_config))
3279            },
3280        )
3281        .join()
3282        .map_err(|_| anyhow::anyhow!("Discovery state bootstrap thread panicked"))??
3283    })
3284    .with_auth_provider(Box::new(auth_provider));
3285    #[cfg(feature = "rocksdb")]
3286    {
3287        discovery_service = discovery_service
3288            .with_record_resolver(std::sync::Arc::clone(&record_resolver)
3289                as std::sync::Arc<dyn hyprstream_discovery::RecordResolver>);
3290        discovery_service.attach_process_accepted_state_source()?;
3291    }
3292    if let Some(issuer) = ctx.oauth_issuer_url() {
3293        discovery_service = discovery_service.with_oauth_issuer(issuer.to_owned());
3294        // Use the issuer URL as the audience for discovery tokens
3295        discovery_service = discovery_service.with_expected_audience(issuer.to_owned());
3296    }
3297    discovery_service = discovery_service.with_jwt_key_source(ctx.cluster_key_source());
3298
3299    // Pre-compute TLS endorsement if QUIC is enabled with a TLS cert.
3300    // Uses the root verifying key — TLS endorsement is a node-level trust assertion,
3301    // not specific to any per-service key. Clients verify against the pinned root pubkey.
3302    if let Some(quic) = ctx.quic_shared() {
3303        let ed25519_pubkey = ctx.verifying_key().to_bytes();
3304        let domain = &quic.server_name;
3305        match compute_tls_endorsement(&quic.key_der, &ed25519_pubkey, domain) {
3306            Ok(endorsement) => {
3307                if !endorsement.is_empty() {
3308                    info!(
3309                        "TLS endorsement computed for domain '{}' ({} bytes)",
3310                        domain,
3311                        endorsement.len()
3312                    );
3313                    discovery_service =
3314                        discovery_service.with_tls_endorsement(endorsement, domain.clone());
3315                }
3316            }
3317            Err(e) => {
3318                // Non-fatal: TLS endorsement is optional additive trust
3319                tracing::warn!("Failed to compute TLS endorsement for '{}': {}", domain, e);
3320            }
3321        }
3322    }
3323    // TODO: DiscoveryService federation key source support
3324    // (federation_key_source not yet implemented on DiscoveryService)
3325
3326    let publisher = if ctx.iroh_required() {
3327        Some(discovery_self_publisher(
3328            discovery_service.self_announcer()?,
3329        ))
3330    } else {
3331        None
3332    };
3333    Ok(ctx.into_spawnable_quic_with_publisher(
3334        discovery_service,
3335        config.discovery.quic_port,
3336        publisher,
3337    ))
3338}
3339
3340fn discovery_self_publisher(
3341    owner: hyprstream_discovery::DiscoverySelfAnnouncer,
3342) -> hyprstream_service::NativeAnnouncementPublisher {
3343    let owner = Arc::new(owner);
3344    Arc::new(move |mut request| {
3345        let owner = owner.clone();
3346        let (first_tx, first_rx) = std::sync::mpsc::sync_channel(1);
3347        std::thread::spawn(move || {
3348            let runtime = match tokio::runtime::Builder::new_current_thread()
3349                .enable_all()
3350                .build()
3351            {
3352                Ok(runtime) => runtime,
3353                Err(error) => {
3354                    let _ = first_tx.send(Err(error.to_string()));
3355                    return;
3356                }
3357            };
3358            runtime.block_on(async move {
3359                let mut first_tx = Some(first_tx);
3360                let mut delay = std::time::Duration::from_secs(5);
3361                let cancellation = request.cancellation.clone();
3362                loop {
3363                    let result = tokio::select! {
3364                        biased;
3365                        _ = cancellation.cancelled() => Err(anyhow::anyhow!("Discovery self publication cancelled")),
3366                        result = async {
3367                            let announcement = current_native_announcement(&mut request)?;
3368                            owner.publish(&announcement).await
3369                        } => result,
3370                    };
3371                    if let Some(tx) = first_tx.take() {
3372                        let _ = tx.send(result.as_ref().map(|_| ()).map_err(ToString::to_string));
3373                        if result.is_err() { return; }
3374                    }
3375                    if cancellation.is_cancelled() { return; }
3376                    let wait = match result {
3377                        Ok(()) => { delay = std::time::Duration::from_secs(5); std::time::Duration::from_secs(25) }
3378                        Err(error) => {
3379                            tracing::warn!("Discovery self publication refresh failed: {error}");
3380                            let wait = delay;
3381                            delay = delay.saturating_mul(2).min(std::time::Duration::from_secs(25));
3382                            wait
3383                        }
3384                    };
3385                    tokio::select! {
3386                        biased;
3387                        _ = cancellation.cancelled() => return,
3388                        _ = tokio::time::sleep(wait) => {}
3389                    }
3390                }
3391            });
3392        });
3393        first_rx
3394            .recv()
3395            .map_err(|_| anyhow::anyhow!("Discovery self publication ended before readiness"))?
3396            .map_err(anyhow::Error::msg)
3397    })
3398}
3399
3400// ═══════════════════════════════════════════════════════════════════════════════
3401// Notification Service Factory
3402// ═══════════════════════════════════════════════════════════════════════════════
3403
3404// ═══════════════════════════════════════════════════════════════════════════════
3405// Metrics Service Factory
3406// ═══════════════════════════════════════════════════════════════════════════════
3407
3408/// Factory for MetricsService (DuckDB-backed time-series ingest + DataFusion query)
3409#[cfg(feature = "metrics")]
3410#[service_factory("metrics", schema = "../../../hyprstream-rpc-std/schema/metrics.capnp", metadata = hyprstream_rpc_std::metrics_client::schema_metadata, depends_on = ["policy", "discovery"])]
3411fn create_metrics_service(ctx: &ServiceContext) -> anyhow::Result<Box<dyn Spawnable>> {
3412    info!("Creating MetricsService");
3413
3414    // MetricsService publishes query-result streams via StreamChannel::run_stream
3415    // (fails loudly without a moq origin). Initialize this process's local moq
3416    // plane. Idempotent.
3417    init_local_moq_stream_plane("metrics", ctx.iroh_required());
3418
3419    use crate::services::MetricsService;
3420    use hyprstream_metrics::query::QueryOrchestrator;
3421    use hyprstream_metrics::storage::duckdb::DuckDbBackend;
3422
3423    let config = load_config()?;
3424    let mc = &config.metrics;
3425
3426    let backend = Arc::new(
3427        DuckDbBackend::new(mc.db_path.clone(), Default::default(), None)
3428            .map_err(|e| anyhow::anyhow!("DuckDbBackend init: {e}"))?,
3429    );
3430
3431    let orchestrator = Arc::new(
3432        tokio::task::block_in_place(|| {
3433            tokio::runtime::Handle::current().block_on(async {
3434                hyprstream_metrics::create_schema_v2_tables(backend.as_ref())
3435                    .await
3436                    .map_err(|e| anyhow::anyhow!("metrics schema v2 init: {e}"))?;
3437                QueryOrchestrator::new(backend as Arc<dyn hyprstream_metrics::StorageBackend>)
3438                    .await
3439                    .map_err(|e| anyhow::anyhow!("QueryOrchestrator init: {e}"))
3440            })
3441        })
3442        .map_err(|e| anyhow::anyhow!("metrics service init: {e}"))?,
3443    );
3444
3445    let sk = ctx.service_signing_key("metrics");
3446
3447    // Register this service's verifying key with PolicyService
3448    register_service_key(ctx, "metrics", &sk)?;
3449
3450    let policy_vk = hyprstream_service::global_trust_store()
3451        .resolve_one("policy")
3452        .ok_or_else(|| anyhow::anyhow!("trust store has no policy key"))?;
3453    let policy_client =
3454        policy_client_for_deployment(ctx, sk.clone(), policy_vk, service_token(&sk))?;
3455
3456    let mut metrics_service = MetricsService::new(
3457        orchestrator,
3458        ctx.transport("metrics", SocketKind::Rep),
3459        sk,
3460        policy_client,
3461    );
3462    if let Some(issuer) = ctx.oauth_issuer_url() {
3463        metrics_service = metrics_service.with_expected_audience(issuer.to_owned());
3464    }
3465    metrics_service = metrics_service.with_jwt_key_source(ctx.cluster_key_source());
3466
3467    Ok(ctx.into_spawnable_quic(metrics_service, mc.quic_port))
3468}
3469
3470// ═══════════════════════════════════════════════════════════════════════════════
3471// TLS Endorsement Computation
3472// ═══════════════════════════════════════════════════════════════════════════════
3473
3474/// Domain separator for TLS endorsement messages.
3475const TLS_ENDORSEMENT_V1: &[u8] = b"TLS_ENDORSEMENT_V1";
3476
3477/// Compute a TLS endorsement signature.
3478///
3479/// Signs `TLS_ENDORSEMENT_V1 || ed25519_pubkey || domain` with the TLS private key.
3480/// Handles ECDSA P-256, RSA, and Ed25519 key types (auto-detected from PKCS8 DER).
3481///
3482/// Returns the raw signature bytes, or an empty vec if the key type is unsupported.
3483fn compute_tls_endorsement(
3484    tls_key_der: &[u8],
3485    ed25519_pubkey: &[u8; 32],
3486    domain: &str,
3487) -> anyhow::Result<Vec<u8>> {
3488    // Build message: TLS_ENDORSEMENT_V1 || ed25519_pubkey (32) || domain
3489    let mut message = Vec::with_capacity(TLS_ENDORSEMENT_V1.len() + 32 + domain.len());
3490    message.extend_from_slice(TLS_ENDORSEMENT_V1);
3491    message.extend_from_slice(ed25519_pubkey);
3492    message.extend_from_slice(domain.as_bytes());
3493
3494    let rng = ring::rand::SystemRandom::new();
3495
3496    // Try Ed25519 first (most modern, smallest signature)
3497    if let Ok(key_pair) = ring::signature::Ed25519KeyPair::from_pkcs8(tls_key_der) {
3498        return Ok(key_pair.sign(&message).as_ref().to_vec());
3499    }
3500
3501    // Try ECDSA P-256 SHA-256
3502    if let Ok(key_pair) = ring::signature::EcdsaKeyPair::from_pkcs8(
3503        &ring::signature::ECDSA_P256_SHA256_FIXED_SIGNING,
3504        tls_key_der,
3505        &rng,
3506    ) {
3507        let signature = key_pair.sign(&rng, &message)?;
3508        return Ok(signature.as_ref().to_vec());
3509    }
3510
3511    // Try ECDSA P-384 SHA-384
3512    if let Ok(key_pair) = ring::signature::EcdsaKeyPair::from_pkcs8(
3513        &ring::signature::ECDSA_P384_SHA384_FIXED_SIGNING,
3514        tls_key_der,
3515        &rng,
3516    ) {
3517        let signature = key_pair.sign(&rng, &message)?;
3518        return Ok(signature.as_ref().to_vec());
3519    }
3520
3521    // Try RSA (PKCS1v15 + SHA-256, then PSS + SHA-256)
3522    if let Ok(key_pair) = ring::signature::RsaKeyPair::from_pkcs8(tls_key_der) {
3523        let mut signature = vec![0u8; key_pair.public().modulus_len()];
3524        let padding = &ring::signature::RSA_PKCS1_SHA256;
3525        key_pair.sign(padding, &rng, &message, &mut signature)?;
3526        return Ok(signature);
3527    }
3528
3529    anyhow::bail!("unsupported TLS key type in PKCS8 DER")
3530}
3531
3532#[cfg(test)]
3533#[allow(clippy::unwrap_used, clippy::expect_used)]
3534mod tests {
3535    use super::*;
3536
3537    #[cfg(feature = "rocksdb")]
3538    #[test]
3539    fn pds_head_refresh_interval_refuses_a_verifier_gap() {
3540        use std::time::Duration;
3541        let mut oauth = crate::config::OAuthConfig::default();
3542        assert!(validate_pds_head_refresh_interval(&oauth, Duration::from_secs(6 * 3600)).is_ok());
3543        oauth.jwt_key_active_secs = Some(1000);
3544        oauth.jwt_key_lead_secs = Some(100);
3545        oauth.jwt_key_drain_secs = Some(120);
3546        assert!(validate_pds_head_refresh_interval(&oauth, Duration::from_secs(60)).is_ok());
3547        assert!(validate_pds_head_refresh_interval(&oauth, Duration::from_secs(120)).is_err());
3548        oauth.jwt_key_drain_secs = Some(2000);
3549        assert!(validate_pds_head_refresh_interval(&oauth, Duration::from_secs(900)).is_err());
3550        oauth.jwt_key_lead_secs = Some(1000);
3551        assert!(validate_pds_head_refresh_interval(&oauth, Duration::from_secs(60)).is_err());
3552    }
3553
3554    fn mcp_jsonrpc_events(body: &str) -> Vec<serde_json::Value> {
3555        body.lines()
3556            .filter_map(|line| line.strip_prefix("data: ").or(Some(line)))
3557            .filter_map(|line| serde_json::from_str(line).ok())
3558            .collect()
3559    }
3560
3561    #[tokio::test]
3562    async fn mcp_http_router_runs_authenticated_streamable_http_protocol() {
3563        use hyprstream_rpc::auth::{
3564            global_credential_revocation_store, set_global_credential_revocation_store,
3565            CompositeKeyPair, CompositeKeySet, CompositePairRole, CompositePairState,
3566            InMemoryCredentialRevocationStore,
3567        };
3568
3569        let signing_key = SigningKey::from_bytes(&[0x61; 32]);
3570        let ed = SigningKey::from_bytes(&[0x62; 32]);
3571        let (pq, pq_vk) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
3572        let pair = CompositeKeyPair::verifying(
3573            crate::auth::jwt::composite_kid(&pq_vk, &ed.verifying_key()),
3574            pq_vk,
3575            ed.verifying_key(),
3576            CompositePairRole::OAuth,
3577            CompositePairState::Active,
3578            0,
3579            i64::MAX,
3580        );
3581        let issuer = "https://oauth.mcp-loopback.example.test";
3582        let resource = "https://mcp.mcp-loopback.example.test/mcp";
3583        let key_set = Arc::new(CompositeKeySet::default());
3584        key_set
3585            .publish(1, "mcp-loopback-fixture".to_owned(), vec![pair])
3586            .unwrap();
3587        let mut auth = crate::server::state::ResourceAuthState::new(
3588            signing_key.verifying_key(),
3589            resource.to_owned(),
3590            issuer.to_owned(),
3591            Arc::new(crate::auth::FederationKeyResolver::new(&Default::default())),
3592        );
3593        auth.composite_key_set = key_set;
3594        if global_credential_revocation_store().is_none() {
3595            let _ = set_global_credential_revocation_store(Arc::new(
3596                InMemoryCredentialRevocationStore::new(),
3597            ));
3598        }
3599
3600        let now = chrono::Utc::now().timestamp();
3601        let claims = crate::auth::jwt::Claims::new(
3602            "did:web:agent.publisher.mcp-loopback.example.test".to_owned(),
3603            now,
3604            now + 300,
3605        )
3606        .with_issuer(issuer.to_owned())
3607        .with_audience(Some(resource.to_owned()))
3608        .with_client_id("mcp-loopback-test-client")
3609        .with_tenant("publisher.mcp-loopback.example.test".to_owned())
3610        .with_jti();
3611        let token = crate::auth::jwt::encode_composite_ml_dsa_65_ed25519(&claims, &pq, &ed);
3612
3613        let router = build_mcp_http_router(
3614            McpConfig {
3615                verifying_key: signing_key.verifying_key(),
3616                signing_key,
3617                transport: TransportConfig::ipc("/tmp/hyprstream-mcp-loopback.sock"),
3618                policy_transport: TransportConfig::ipc("/tmp/hyprstream-policy-loopback.sock"),
3619                ctx: None,
3620                policy_verifying_key: SigningKey::from_bytes(&[0x63; 32]).verifying_key(),
3621                expected_audience: Some(resource.to_owned()),
3622                jwt_key_source: None,
3623            },
3624            resource.to_owned(),
3625            issuer.to_owned(),
3626            auth,
3627            crate::server::state::CorsConfig {
3628                enabled: false,
3629                ..Default::default()
3630            },
3631        );
3632
3633        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
3634            .await
3635            .expect("bind loopback listener");
3636        let addr = listener.local_addr().expect("read loopback address");
3637        let server = tokio::spawn(async move {
3638            axum::serve(listener, router)
3639                .await
3640                .expect("serve MCP fixture");
3641        });
3642        let endpoint = format!("http://{addr}/mcp");
3643        let client = reqwest::Client::new();
3644        let initialize = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"loopback-acceptance","version":"0.1"}}}"#;
3645
3646        let missing_auth = client
3647            .post(&endpoint)
3648            .header("Content-Type", "application/json")
3649            .header("Accept", "application/json, text/event-stream")
3650            .body(initialize)
3651            .send()
3652            .await
3653            .expect("send unauthenticated initialize");
3654        assert_eq!(missing_auth.status(), reqwest::StatusCode::UNAUTHORIZED);
3655        assert!(missing_auth
3656            .headers()
3657            .get(reqwest::header::WWW_AUTHENTICATE)
3658            .and_then(|value| value.to_str().ok())
3659            .is_some_and(|value| value.starts_with("Bearer resource_metadata=")));
3660
3661        let invalid_auth = client
3662            .post(&endpoint)
3663            .header("Authorization", "Bearer malformed-fixture-token")
3664            .header("Content-Type", "application/json")
3665            .header("Accept", "application/json, text/event-stream")
3666            .body(initialize)
3667            .send()
3668            .await
3669            .expect("send malformed bearer initialize");
3670        assert_eq!(invalid_auth.status(), reqwest::StatusCode::UNAUTHORIZED);
3671
3672        let initialized = client
3673            .post(&endpoint)
3674            .header("Authorization", format!("Bearer {token}"))
3675            .header("Content-Type", "application/json")
3676            .header("Accept", "application/json, text/event-stream")
3677            .body(initialize)
3678            .send()
3679            .await
3680            .expect("send authenticated initialize");
3681        assert_eq!(initialized.status(), reqwest::StatusCode::OK);
3682        let session_id = initialized
3683            .headers()
3684            .get("mcp-session-id")
3685            .expect("initialize returns MCP session")
3686            .to_str()
3687            .expect("session id is text")
3688            .to_owned();
3689        let init_body = initialized.text().await.expect("read initialize body");
3690        let init_response = mcp_jsonrpc_events(&init_body)
3691            .into_iter()
3692            .find(|event| event.get("id") == Some(&serde_json::json!(1)))
3693            .expect("initialize returns a JSON-RPC result");
3694        assert_eq!(init_response["result"]["serverInfo"]["name"], "hyprstream");
3695
3696        let initialized_notification = client
3697            .post(&endpoint)
3698            .header("Authorization", format!("Bearer {token}"))
3699            .header("MCP-Protocol-Version", "2025-11-25")
3700            .header("Mcp-Session-Id", &session_id)
3701            .header("Content-Type", "application/json")
3702            .header("Accept", "application/json, text/event-stream")
3703            .body(r#"{"jsonrpc":"2.0","method":"notifications/initialized"}"#)
3704            .send()
3705            .await
3706            .expect("send initialized notification");
3707        assert_eq!(
3708            initialized_notification.status(),
3709            reqwest::StatusCode::ACCEPTED
3710        );
3711
3712        let tools = client
3713            .post(&endpoint)
3714            .header("Authorization", format!("Bearer {token}"))
3715            .header("MCP-Protocol-Version", "2025-11-25")
3716            .header("Mcp-Session-Id", &session_id)
3717            .header("Content-Type", "application/json")
3718            .header("Accept", "application/json, text/event-stream")
3719            .body(r#"{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}"#)
3720            .send()
3721            .await
3722            .expect("send tools/list");
3723        assert_eq!(tools.status(), reqwest::StatusCode::OK);
3724        let tools_body = tools.text().await.expect("read tools/list body");
3725        let tools_response = mcp_jsonrpc_events(&tools_body)
3726            .into_iter()
3727            .find(|event| event.get("id") == Some(&serde_json::json!(2)))
3728            .expect("tools/list returns a JSON-RPC result");
3729        let refresh = tools_response["result"]["tools"]
3730            .as_array()
3731            .expect("tools/list result includes tools")
3732            .iter()
3733            .find(|tool| tool["title"] == "mcp.refresh_tools")
3734            .expect("refresh tool is advertised");
3735        let refresh_id = refresh["name"]
3736            .as_str()
3737            .expect("refresh tool has UUID name");
3738        let call_body = serde_json::json!({
3739            "jsonrpc": "2.0",
3740            "id": 3,
3741            "method": "tools/call",
3742            "params": { "name": refresh_id, "arguments": {} }
3743        });
3744        let call = client
3745            .post(&endpoint)
3746            .header("Authorization", format!("Bearer {token}"))
3747            .header("MCP-Protocol-Version", "2025-11-25")
3748            .header("Mcp-Session-Id", &session_id)
3749            .header("Content-Type", "application/json")
3750            .header("Accept", "application/json, text/event-stream")
3751            .json(&call_body)
3752            .send()
3753            .await
3754            .expect("send authenticated refresh tools call");
3755        assert_eq!(call.status(), reqwest::StatusCode::OK);
3756        let call_body = call.text().await.expect("read tool call body");
3757        let call_response = mcp_jsonrpc_events(&call_body)
3758            .into_iter()
3759            .find(|event| event.get("id") == Some(&serde_json::json!(3)))
3760            .expect("tools/call returns a JSON-RPC result");
3761        assert_eq!(call_response["result"]["isError"], false);
3762        assert!(call_response["result"]["content"][0]["text"]
3763            .as_str()
3764            .is_some_and(|text| text.contains("Refreshed tool registry")));
3765
3766        server.abort();
3767    }
3768
3769    #[tokio::test]
3770    async fn mcp_http_verifies_published_composite_access_token_without_ed_downgrade() {
3771        use hyprstream_rpc::auth::{
3772            global_credential_revocation_store, set_global_credential_revocation_store,
3773            CompositeKeyPair, CompositeKeySet, CompositePairRole, CompositePairState,
3774            InMemoryCredentialRevocationStore,
3775        };
3776
3777        let ca = SigningKey::from_bytes(&[0x41; 32]);
3778        let ed = SigningKey::from_bytes(&[0x42; 32]);
3779        let (pq, pq_vk) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
3780        let (other_pq, _) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
3781        let pair = CompositeKeyPair::verifying(
3782            crate::auth::jwt::composite_kid(&pq_vk, &ed.verifying_key()),
3783            pq_vk,
3784            ed.verifying_key(),
3785            CompositePairRole::OAuth,
3786            CompositePairState::Active,
3787            0,
3788            i64::MAX,
3789        );
3790        let published = Arc::new(CompositeKeySet::default());
3791        published
3792            .publish(1, "mcp-http-test".to_owned(), vec![pair])
3793            .unwrap();
3794        let issuer = "https://oauth.example.test";
3795        let resource = "https://mcp.example.test/mcp";
3796        let mut auth = crate::server::state::ResourceAuthState::new(
3797            ca.verifying_key(),
3798            resource.to_owned(),
3799            issuer.to_owned(),
3800            Arc::new(crate::auth::FederationKeyResolver::new(&Default::default())),
3801        );
3802        auth.composite_key_set = published;
3803        if global_credential_revocation_store().is_none() {
3804            let _ = set_global_credential_revocation_store(Arc::new(
3805                InMemoryCredentialRevocationStore::new(),
3806            ));
3807        }
3808        let now = chrono::Utc::now().timestamp();
3809        let claims = crate::auth::jwt::Claims::new("alice".to_owned(), now, now + 300)
3810            .with_issuer(issuer.to_owned())
3811            .with_audience(Some(resource.to_owned()))
3812            .with_client_id("mcp-client")
3813            .with_tenant("hosted-tenant".to_owned())
3814            .with_jti();
3815        let encode = |claims: &crate::auth::jwt::Claims, pq| {
3816            crate::auth::jwt::encode_composite_ml_dsa_65_ed25519(claims, pq, &ed)
3817        };
3818        let token = encode(&claims, &pq);
3819        let verified = verify_mcp_http_token(&auth, &token).await.unwrap();
3820        assert_eq!(verified.sub, "alice");
3821        assert_eq!(verified.aud.as_deref(), Some(resource));
3822
3823        // Exercise the same verified claims and subject gate used by the MCP
3824        // HTTP layer. A hosted DID reaches the later tenant/Policy gate only
3825        // when the signed tenant is its exact account zone.
3826        let did = "did:web:canary-agent.publisher.staging.lab.hyprstream.com";
3827        let tenant = "publisher.staging.lab.hyprstream.com";
3828        let hosted = crate::auth::jwt::Claims::new(did.to_owned(), now, now + 300)
3829            .with_issuer(issuer.to_owned())
3830            .with_audience(Some(resource.to_owned()))
3831            .with_client_id("mcp-client")
3832            .with_tenant(tenant.to_owned())
3833            .with_jti();
3834        let local_issuers = [issuer];
3835        let checked = verify_mcp_http_token(&auth, &encode(&hosted, &pq))
3836            .await
3837            .unwrap();
3838        assert_eq!(
3839            validated_mcp_subject(&checked, &local_issuers).as_deref(),
3840            Some(did)
3841        );
3842        let identity = crate::server::middleware::AuthenticatedUser {
3843            user: did.to_owned(),
3844            verified_tenant: checked.tenant,
3845            token: None,
3846            exp: Some(checked.exp),
3847        };
3848        assert_eq!(identity.authorization_domain().unwrap(), tenant);
3849        for invalid in [
3850            hosted
3851                .clone()
3852                .with_tenant("foreign.staging.lab.hyprstream.com".to_owned()),
3853            hosted.clone().with_issuer(String::new()),
3854            crate::auth::jwt::Claims::new(
3855                "did:web:canary-agent.publisher.staging.lab.hyprstream.com:users".to_owned(),
3856                now,
3857                now + 300,
3858            )
3859            .with_issuer(issuer.to_owned())
3860            .with_audience(Some(resource.to_owned()))
3861            .with_client_id("mcp-client")
3862            .with_tenant(tenant.to_owned()),
3863        ] {
3864            assert!(validated_mcp_subject(&invalid, &local_issuers).is_none());
3865        }
3866        use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
3867        let (signing_input, signature) = token.rsplit_once('.').unwrap();
3868        let mut signature = URL_SAFE_NO_PAD.decode(signature).unwrap();
3869        signature[0] ^= 1;
3870        let tampered = format!("{signing_input}.{}", URL_SAFE_NO_PAD.encode(signature));
3871        assert!(verify_mcp_http_token(&auth, &tampered).await.is_err());
3872        assert!(
3873            verify_mcp_http_token(&auth, &crate::auth::jwt::encode(&claims, &ca))
3874                .await
3875                .is_err()
3876        );
3877        assert!(verify_mcp_http_token(
3878            &auth,
3879            &encode(&claims.clone().with_cnf_jwk(&[0x43; 32]), &pq)
3880        )
3881        .await
3882        .is_err());
3883        assert!(verify_mcp_http_token(&auth, &encode(&claims, &other_pq))
3884            .await
3885            .is_err());
3886        assert!(verify_mcp_http_token(
3887            &auth,
3888            &encode(
3889                &claims
3890                    .clone()
3891                    .with_audience(Some("https://other.example.test/mcp".to_owned())),
3892                &pq
3893            )
3894        )
3895        .await
3896        .is_err());
3897        assert!(verify_mcp_http_token(
3898            &auth,
3899            &encode(
3900                &claims
3901                    .clone()
3902                    .with_issuer("https://other.example.test".to_owned()),
3903                &pq
3904            )
3905        )
3906        .await
3907        .is_err());
3908        assert!(
3909            verify_mcp_http_token(&auth, &encode(&claims.clone().with_client_id(""), &pq))
3910                .await
3911                .is_err()
3912        );
3913    }
3914
3915    /// Rootless Quadlets run each service in a distinct process, so the
3916    /// process-local endpoint registry cannot resolve Policy for Discovery or
3917    /// its downstream peers. The typed IPC transport remains lazy: creating a
3918    /// client for the shared socket must not require a co-located registration.
3919    #[test]
3920    fn policy_client_accepts_unregistered_ipc_transport() {
3921        let signing_key = SigningKey::from_bytes(&[0x63; 32]);
3922        let transport =
3923            hyprstream_rpc::transport::TransportConfig::ipc("/run/hyprstream/policy.sock");
3924
3925        let client = policy_client_for_transport(
3926            &transport,
3927            signing_key.clone(),
3928            signing_key.verifying_key(),
3929            None,
3930        );
3931        assert!(
3932            client.is_ok(),
3933            "IPC policy client must be lazy at first boot"
3934        );
3935    }
3936
3937    #[test]
3938    fn checkpointed_service_selection_uses_canonical_id_and_current_signer() {
3939        use hyprstream_pds::at9p::{
3940            CapsuleBody, HybridKeyPair, ServiceEndpoint, ServiceEntry, ServiceType, Transport,
3941        };
3942        use hyprstream_pds::at9p_duplicity::AcceptedAt9pState;
3943        use hyprstream_pds::at9p_gate::verify_genesis_capsule;
3944        use hyprstream_pds::at9p_sign::sign_capsule;
3945        use hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes;
3946
3947        let signer = ed25519_dalek::SigningKey::from_bytes(&[0x71; 32]);
3948        let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signer);
3949        let key = HybridKeyPair::new(
3950            signer.verifying_key().to_bytes().to_vec(),
3951            ml_dsa_sk_to_vk_bytes(&pq),
3952        )
3953        .unwrap();
3954        let endpoint = ServiceEndpoint::new(
3955            Transport::Iroh,
3956            format!("iroh://{}", hex::encode([0x72; 32])),
3957        )
3958        .unwrap();
3959        let service = ServiceEntry::new("#model", ServiceType::NinePExport, endpoint).unwrap();
3960        let body = CapsuleBody::new(vec![key], vec![service]).unwrap();
3961        let genesis = sign_capsule(body, &signer, &pq).unwrap();
3962        let verified =
3963            verify_genesis_capsule(&genesis.cid512().unwrap(), &genesis.to_dag_cbor().unwrap())
3964                .unwrap();
3965        let state = AcceptedAt9pState::from_verified_genesis(&verified).unwrap();
3966        let key = signer.verifying_key().to_bytes();
3967        assert!(accepted_state_matches_service(&state, "model", &key));
3968        assert!(!accepted_state_matches_service(&state, "#model", &key));
3969        assert!(!accepted_state_matches_service(&state, "registry", &key));
3970        assert!(!accepted_state_matches_service(
3971            &state,
3972            "model",
3973            &[0x73; 32]
3974        ));
3975        // Selection does not weaken the separate bounded-successor gate.
3976        let error = hyprstream_service::NativeServiceAnnouncement::from_accepted_state(
3977            "model", &signer, &state,
3978        )
3979        .err()
3980        .expect("genesis alone cannot authorize a production announcement");
3981        assert!(error.to_string().contains("bounded production expiry"));
3982    }
3983
3984    /// Compatibility checkpoints may legitimately predate a `#discovery`
3985    /// identity (the compatibility discovery factory installs its
3986    /// self-publisher only under `iroh_required`), so the checkpoint
3987    /// announcement loop must exempt Discovery outside the required profile
3988    /// while required mode keeps demanding the checkpoint.
3989    #[test]
3990    fn checkpoint_announcements_keep_discovery_optional_outside_required_profile() {
3991        assert!(!checkpoint_announces_service(false, "discovery"));
3992        assert!(checkpoint_announces_service(false, "model"));
3993        assert!(checkpoint_announces_service(false, "policy"));
3994        // Required mode stays strict: the owned publication path refuses to
3995        // publish without the checkpoint bundle, so Discovery stays gated.
3996        assert!(checkpoint_announces_service(true, "discovery"));
3997        assert!(checkpoint_announces_service(true, "model"));
3998
3999        // The exemption must live inside the loop itself, not beside it: the
4000        // loop routes every service through the compatibility decision.
4001        let source = include_str!("factories.rs");
4002        let start = source
4003            .find("pub fn with_checkpointed_native_announcements(")
4004            .expect("checkpoint announcement loop");
4005        let rest = &source[start..];
4006        let end = rest.find("\npub fn ").unwrap_or(rest.len());
4007        let body = &rest[..end];
4008        assert!(
4009            body.contains("checkpoint_announces_service(iroh_required, name)"),
4010            "the checkpoint loop must route services through the compatibility decision"
4011        );
4012        assert!(
4013            body.contains("hyprstream_discovery::native_network_required()"),
4014            "required native bootstrap must gate Discovery proof installation before QUIC setup"
4015        );
4016    }
4017
4018    #[test]
4019    fn native_deployment_chain_has_no_local_policy_bootstrap() {
4020        let source = include_str!("factories.rs");
4021        for function in [
4022            "fn register_service_key(",
4023            "fn create_registry_service(",
4024            "fn create_model_service(",
4025            "fn create_oai_service(",
4026        ] {
4027            let start = source.find(function).expect("production factory function");
4028            let rest = &source[start..];
4029            let end = rest.find("\nfn ").unwrap_or(rest.len());
4030            let body = &rest[..end];
4031            assert!(
4032                !body.contains("for_local_bootstrap"),
4033                "{function} must not construct a local Policy client in the deployed chain"
4034            );
4035        }
4036        assert!(source.contains(
4037            "PolicyClient::from_provider(&hyprstream_discovery::ProductionRpcClientProvider"
4038        ));
4039    }
4040
4041    /// Slice one top-level item out of a source file: from `marker` to the
4042    /// first column-0 item/comment/attribute line that follows it.
4043    /// (Owned return, no lifetime parameters: a `'` token desyncs the
4044    /// loopback-burndown test-region skipper, which is not a full lexer.)
4045    fn top_level_body(source: &str, marker: &str) -> String {
4046        let start = source.find(marker).expect("item exists");
4047        let rest = &source[start + marker.len()..];
4048        let end = [
4049            "\nfn ",
4050            "\npub fn",
4051            "\npub(crate) fn",
4052            "\npub enum",
4053            "\nenum ",
4054            "\n///",
4055            "\n//",
4056            "\n#[",
4057        ]
4058        .iter()
4059        .filter_map(|needle| rest.find(needle))
4060        .min()
4061        .unwrap_or(rest.len());
4062        rest[..end].to_owned()
4063    }
4064
4065    /// #1257 followup: in RDS posture no production caller may bypass the
4066    /// configured records backend with a direct local RocksDB open, and no
4067    /// RDS-configured failure may degrade to warn+disable. These source-level
4068    /// contract checks pin the exact seams; the functional fail-closed
4069    /// behavior is pinned in `cli::deployment_bootstrap` tests.
4070    #[test]
4071    fn rds_mode_has_no_silent_local_fallback_in_pds_callers() {
4072        // Production code only: include_str! would otherwise let these
4073        // pattern assertions match this very test module.
4074        let source = include_str!("factories.rs");
4075        let source = &source[..source.find("\nmod tests").expect("tests module")];
4076
4077        // QUIC startup gate: resolves the binding and opens the configured
4078        // backend in RDS posture before ever touching the local directory.
4079        let gate = top_level_body(source, "pub fn classify_pds_store_for_quic(");
4080        assert!(
4081            gate.contains("resolved_from_env()?"),
4082            "the QUIC gate must resolve the records-role binding"
4083        );
4084        assert!(
4085            gate.contains("open_pds_record_store(ctx, true)"),
4086            "the QUIC gate must open the configured backend in RDS posture"
4087        );
4088
4089        // Announcement refresh: routes through the backend-selecting helper;
4090        // no direct local open remains in the function.
4091        let announcement = top_level_body(source, "pub fn current_native_announcement(");
4092        assert!(
4093            announcement.contains("open_pds_record_store_at("),
4094            "announcement refresh must open the configured backend"
4095        );
4096        assert!(
4097            !announcement.contains("PdsRecordStore::open_readonly("),
4098            "announcement refresh must not bypass backend selection with a direct local open"
4099        );
4100
4101        // Publisher construction: warn+disable survives only for the local
4102        // posture; an RDS-configured failure returns instead of degrading.
4103        let registry = top_level_body(source, "fn create_registry_service(");
4104        assert!(
4105            registry.contains("Err(error) if pds_rds_configured =>"),
4106            "publisher construction must fail closed when RDS is configured"
4107        );
4108        assert!(
4109            registry.contains("tracing::warn!(\"PDS publish disabled"),
4110            "local posture keeps the documented best-effort warn+disable"
4111        );
4112
4113        // The selecting helper itself: RDS-configured without the feature is
4114        // a hard error, and every configured open goes through open_postgres.
4115        let helper = top_level_body(source, "fn open_pds_record_store_at(");
4116        assert!(helper.contains("resolved_from_env()?"));
4117        assert!(helper.contains("PdsRecordStore::open_postgres("));
4118        assert!(helper.contains("not built with the `pds-postgres` feature"));
4119
4120        // Config loading is fail-closed everywhere in this module: no
4121        // HyprConfig load may silently discard a configured [rds] section.
4122        assert!(
4123            !source.contains("Config::load().unwrap_or_default()"),
4124            "config load must fail closed, never fall back to defaults"
4125        );
4126    }
4127
4128    #[test]
4129    fn worker_ninep_denial_is_local_not_global_rpc_floor_only() {
4130        let source = include_str!("factories.rs");
4131        let production = &source[..source.find("\nmod tests").expect("tests module")];
4132        let worker = top_level_body(production, "fn create_worker_service(");
4133        assert!(worker.contains("enrollment_ninep_reference_monitor"));
4134        assert!(
4135            !worker.contains("block_identity_widening_for_unverified_attach_transport"),
4136            "an unauthenticated 9P attach must not narrow ordinary RPC dispatch"
4137        );
4138        let standalone_worker = include_str!("../bin/main.rs");
4139        assert!(standalone_worker.contains("enrollment_ninep_reference_monitor"));
4140        assert!(
4141            !standalone_worker.contains("block_identity_widening_for_unverified_attach_transport"),
4142            "the standalone Worker constructor must preserve identity-aware RPC activation"
4143        );
4144    }
4145
4146    /// The registry factory and `ServerState::new` each validate the canonical
4147    /// 9P export where they consume it; a dedicated oauth-only `--ipc` process
4148    /// runs neither, so the oauth factory must validate the export itself —
4149    /// otherwise signup seeding would turn a never-validated export (e.g.
4150    /// repo_name "*") into broad HolderGate rows.
4151    #[test]
4152    fn oauth_factory_validates_ninep_canonical_read_export() {
4153        let source = include_str!("factories.rs");
4154        let production = &source[..source.find("\nmod tests").expect("tests module")];
4155        let oauth = top_level_body(production, "fn create_oauth_service(");
4156        assert!(
4157            oauth.contains("ninep_canonical_read.as_ref()"),
4158            "the oauth factory must inspect the export it wires into OAuthState"
4159        );
4160        assert!(
4161            oauth.contains("export.validate()"),
4162            "the oauth factory must fail closed on an invalid ninep_canonical_read export"
4163        );
4164    }
4165
4166    /// #989: the workflow factory is in the service inventory (the daemon can
4167    /// discover and start it). This is the compile-time/registration half of the
4168    /// acceptance criterion "WorkflowService appears in service status/discovery"
4169    /// without spinning a full daemon.
4170    #[test]
4171    fn workflow_factory_registered() {
4172        let factory = hyprstream_service::get_factory("workflow")
4173            .expect("workflow factory must be registered (#989)");
4174        assert_eq!(factory.name, "workflow");
4175        // amend-#989 §1 dependency set — P0 prerequisite wiring.
4176        assert_eq!(
4177            factory.depends_on,
4178            &["worker", "event", "policy", "registry", "discovery"]
4179        );
4180        // Schema + metadata must be wired so PolicyService can discover scopes
4181        // and MCP can surface methods.
4182        assert!(
4183            factory.schema.is_some(),
4184            "workflow factory must embed its capnp schema"
4185        );
4186        assert!(
4187            factory.metadata.is_some(),
4188            "workflow factory must expose schema_metadata for scope discovery"
4189        );
4190    }
4191
4192    #[test]
4193    fn required_startup_uses_discovery_before_policy_from_real_inventory() {
4194        use hyprstream_service::service::ordering::startup_stages_for_profile;
4195        let roster = ["discovery", "policy", "registry", "model"];
4196        for required in [false, true] {
4197            let stages = startup_stages_for_profile(&roster, required);
4198            let stage = |name: &str| {
4199                stages
4200                    .iter()
4201                    .position(|items| items.iter().any(|item| item == name))
4202                    .expect("registered service")
4203            };
4204            assert_eq!(stage("discovery") < stage("policy"), required);
4205            assert!(stage("registry") > stage("discovery"));
4206            assert!(stage("model") > stage("policy"));
4207        }
4208    }
4209
4210    /// The Phase-0 namespace skeleton mounts `/bin`, `/env`, `/out` for the
4211    /// runner (amend-#989 §2). Smoke-check the three directories exist as direct
4212    /// children of the synthetic root before the namespace is handed off.
4213    #[test]
4214    fn workflow_namespace_phase0_skeleton() {
4215        use crate::services::fs::{SyntheticNode, SyntheticTree};
4216
4217        // Rebuild the same tree shape build_workflow_namespace produces, proving
4218        // the node shape compiles and the three Phase-0 dirs are present.
4219        let mut root = std::collections::HashMap::new();
4220        root.insert(
4221            "bin".to_owned(),
4222            SyntheticNode::Dir {
4223                children: std::collections::HashMap::new(),
4224            },
4225        );
4226        root.insert(
4227            "env".to_owned(),
4228            SyntheticNode::Dir {
4229                children: std::collections::HashMap::new(),
4230            },
4231        );
4232        root.insert(
4233            "out".to_owned(),
4234            SyntheticNode::Dir {
4235                children: std::collections::HashMap::new(),
4236            },
4237        );
4238        let _tree = SyntheticTree::new(SyntheticNode::Dir { children: root });
4239
4240        // build_workflow_namespace itself must not panic and must expose a root mount.
4241        let ns = build_workflow_namespace();
4242        assert!(
4243            ns.mount_prefixes().iter().any(|p| p == &"/"),
4244            "Phase-0 workflow namespace must mount a root prefix"
4245        );
4246    }
4247
4248    /// amend-#989 §4: default-off / opt-in. The engine must stay dormant unless
4249    /// an operator explicitly enables it.
4250    #[test]
4251    fn workflow_config_default_off() {
4252        let cfg = hyprstream_workers::config::WorkflowConfig::default();
4253        assert!(
4254            !cfg.enabled,
4255            "WorkflowConfig must default to enabled = false (#989)"
4256        );
4257    }
4258
4259    /// Helper: generate an ECDSA P-256 key pair and return (pkcs8_der, public_key_der)
4260    fn generate_ecdsa_p256_pair() -> (Vec<u8>, Vec<u8>) {
4261        let key_pair = rcgen::KeyPair::generate_for(&rcgen::PKCS_ECDSA_P256_SHA256).unwrap();
4262        let pkcs8 = key_pair.serialize_der();
4263        let pub_der = key_pair.public_key_der();
4264        (pkcs8, pub_der.clone())
4265    }
4266
4267    fn build_endorsement_message(ed25519_pubkey: &[u8; 32], domain: &str) -> Vec<u8> {
4268        let mut msg = Vec::with_capacity(TLS_ENDORSEMENT_V1.len() + 32 + domain.len());
4269        msg.extend_from_slice(TLS_ENDORSEMENT_V1);
4270        msg.extend_from_slice(ed25519_pubkey);
4271        msg.extend_from_slice(domain.as_bytes());
4272        msg
4273    }
4274
4275    /// #441 fail-closed: with no JWT in the trust store and none on disk,
4276    /// registration MUST error (naming the real cause) rather than silently
4277    /// skip — a service that can't register its key must not serve signed
4278    /// responses.
4279    #[test]
4280    fn resolve_registration_jwt_fails_closed_when_missing() {
4281        let dir = tempfile::tempdir().unwrap();
4282        let err = resolve_registration_jwt(
4283            "model",
4284            dir.path(),
4285            crate::auth::identity_store::SecretsProfile::SharedDirectory,
4286            None,
4287        )
4288        .expect_err("missing JWT must fail closed, not skip");
4289        let msg = err.to_string();
4290        assert!(msg.contains("model"), "error names the service: {msg}");
4291        assert!(
4292            msg.contains("cannot register its signing key"),
4293            "error names the real cause: {msg}",
4294        );
4295    }
4296
4297    #[test]
4298    fn required_policy_loads_provisioned_jwt_without_self_rpc() -> anyhow::Result<()> {
4299        const CHILD: &str = "HYPRSTREAM_POLICY_REGISTRATION_CHILD";
4300        if std::env::var_os(CHILD).is_none() {
4301            let status = std::process::Command::new(std::env::current_exe()?)
4302                .args(["--exact", "services::factories::tests::required_policy_loads_provisioned_jwt_without_self_rpc", "--nocapture"])
4303                .env(CHILD, "1").status()?;
4304            anyhow::ensure!(status.success(), "isolated Policy credential test failed");
4305            return Ok(());
4306        }
4307        let dir = tempfile::tempdir()?;
4308        std::env::set_var("HYPRSTREAM__SECRETS__PATH", dir.path());
4309        std::env::remove_var(crate::auth::identity_store::SECRETS_PROFILE_ENV);
4310        let signer = SigningKey::from_bytes(&[0x69; 32]);
4311        let ctx = ServiceContext::new(
4312            signer.clone(),
4313            signer.verifying_key(),
4314            true,
4315            dir.path().to_owned(),
4316        )
4317        .with_quic(hyprstream_service::QuicSharedConfig {
4318            cert_chain: Vec::new(),
4319            certificate_hashes: Vec::new(),
4320            key_der: zeroize::Zeroizing::new(Vec::new()),
4321            base_ip: std::net::Ipv4Addr::LOCALHOST.into(),
4322            server_name: "policy.test".to_owned(),
4323            oauth_issuer_url: None,
4324            jwt_verifying_key: None,
4325            iroh_enabled: true,
4326            iroh_required: true,
4327            moq_relay: None,
4328            native_announcement_publisher: None,
4329            moq_relay_server_identity: None,
4330            moq_admission: None,
4331            moq_ingress_authorizer: None,
4332            moq_admission_proof: None,
4333        });
4334        assert!(register_service_key(&ctx, "policy", &signer).is_err());
4335        let now = chrono::Utc::now().timestamp();
4336        let claims =
4337            hyprstream_rpc::auth::Claims::new("service:policy".to_owned(), now, now + 3600)
4338                .with_cnf_jwk(signer.verifying_key().as_bytes());
4339        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &signer);
4340        crate::auth::identity_store::write_service_jwt(dir.path(), "policy", &jwt)?;
4341        // No runtime, resolver or RPC endpoint exists in this child.
4342        register_service_key(&ctx, "policy", &signer)?;
4343        assert_eq!(
4344            hyprstream_service::global_trust_store()
4345                .get(&signer.verifying_key())
4346                .and_then(|attestation| attestation.jwt),
4347            Some(jwt)
4348        );
4349        Ok(())
4350    }
4351
4352    /// Compatibility-profile Policy must also load and seed its provisioned
4353    /// service JWT: nothing else ever seeds a credential onto the flat root/CA
4354    /// key (#759), and the announcement bearer plus internal policy→discovery
4355    /// calls read it from the trust store. Missing JWT fails closed.
4356    #[test]
4357    fn compat_policy_loads_provisioned_jwt_without_self_rpc() -> anyhow::Result<()> {
4358        const CHILD: &str = "HYPRSTREAM_POLICY_COMPAT_REGISTRATION_CHILD";
4359        if std::env::var_os(CHILD).is_none() {
4360            let status = std::process::Command::new(std::env::current_exe()?)
4361                .args(["--exact", "services::factories::tests::compat_policy_loads_provisioned_jwt_without_self_rpc", "--nocapture"])
4362                .env(CHILD, "1").status()?;
4363            anyhow::ensure!(
4364                status.success(),
4365                "isolated compat Policy credential test failed"
4366            );
4367            return Ok(());
4368        }
4369        let dir = tempfile::tempdir()?;
4370        std::env::set_var("HYPRSTREAM__SECRETS__PATH", dir.path());
4371        std::env::remove_var(crate::auth::identity_store::SECRETS_PROFILE_ENV);
4372        let signer = SigningKey::from_bytes(&[0x6c; 32]);
4373        // No with_quic: the compatibility profile (iroh_required == false).
4374        let ctx = ServiceContext::new(
4375            signer.clone(),
4376            signer.verifying_key(),
4377            true,
4378            dir.path().to_owned(),
4379        );
4380        assert!(
4381            register_service_key(&ctx, "policy", &signer).is_err(),
4382            "missing provisioned JWT must fail closed in the compatibility profile too"
4383        );
4384        let now = chrono::Utc::now().timestamp();
4385        let claims =
4386            hyprstream_rpc::auth::Claims::new("service:policy".to_owned(), now, now + 3600)
4387                .with_cnf_jwk(signer.verifying_key().as_bytes());
4388        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &signer);
4389        crate::auth::identity_store::write_service_jwt(dir.path(), "policy", &jwt)?;
4390        // No runtime, resolver or RPC endpoint exists in this child.
4391        register_service_key(&ctx, "policy", &signer)?;
4392        assert_eq!(
4393            hyprstream_service::global_trust_store()
4394                .get(&signer.verifying_key())
4395                .and_then(|attestation| attestation.jwt),
4396            Some(jwt)
4397        );
4398        Ok(())
4399    }
4400
4401    /// A required-native Policy process carries its loaded custom secrets path
4402    /// into the factory through the trust store. The factory's config-free
4403    /// fallback must not need the XDG default path to exist.
4404    #[test]
4405    fn required_policy_uses_loaded_custom_jwt_before_factory_fallback() -> anyhow::Result<()> {
4406        const CHILD: &str = "HYPRSTREAM_POLICY_CUSTOM_JWT_CHILD";
4407        if std::env::var_os(CHILD).is_none() {
4408            let status = std::process::Command::new(std::env::current_exe()?)
4409                .args(["--exact", "services::factories::tests::required_policy_uses_loaded_custom_jwt_before_factory_fallback", "--nocapture"])
4410                .env(CHILD, "1")
4411                .status()?;
4412            anyhow::ensure!(status.success(), "isolated custom Policy JWT test failed");
4413            return Ok(());
4414        }
4415        let root = tempfile::tempdir()?;
4416        let custom = root.path().join("custom-credentials");
4417        let xdg = root.path().join("xdg-config");
4418        std::env::set_var("XDG_CONFIG_HOME", &xdg);
4419        std::env::remove_var("HYPRSTREAM__SECRETS__PATH");
4420        std::env::remove_var(crate::auth::identity_store::SECRETS_PROFILE_ENV);
4421
4422        let signer = SigningKey::from_bytes(&[0x6a; 32]);
4423        let ctx = ServiceContext::new(signer.clone(), signer.verifying_key(), true, custom.clone())
4424            .with_quic(hyprstream_service::QuicSharedConfig {
4425                cert_chain: Vec::new(),
4426                certificate_hashes: Vec::new(),
4427                key_der: zeroize::Zeroizing::new(Vec::new()),
4428                base_ip: std::net::Ipv4Addr::LOCALHOST.into(),
4429                server_name: "policy.test".to_owned(),
4430                oauth_issuer_url: None,
4431                jwt_verifying_key: None,
4432                iroh_enabled: true,
4433                iroh_required: true,
4434                moq_relay: None,
4435                native_announcement_publisher: None,
4436                moq_relay_server_identity: None,
4437                moq_admission: None,
4438                moq_ingress_authorizer: None,
4439                moq_admission_proof: None,
4440            });
4441        // With the default XDG directory absent, registration fails before
4442        // startup carries the loaded custom credential into the trust store.
4443        assert!(register_service_key(&ctx, "policy", &signer).is_err());
4444
4445        let now = chrono::Utc::now().timestamp();
4446        let claims =
4447            hyprstream_rpc::auth::Claims::new("service:policy".to_owned(), now, now + 3600)
4448                .with_cnf_jwk(signer.verifying_key().as_bytes());
4449        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &signer);
4450        crate::auth::identity_store::write_service_jwt(&custom, "policy", &jwt)?;
4451        let mut loaded_config = HyprConfig::default();
4452        loaded_config.secrets.path = Some(custom.clone());
4453        let loaded_secrets = HyprConfig::resolve_secrets_dir_for(Some(&loaded_config))?;
4454        crate::auth::identity_store::seed_service_jwt_into_trust_store(
4455            "policy",
4456            &signer,
4457            &loaded_secrets,
4458            crate::auth::identity_store::SecretsProfile::SharedDirectory,
4459        );
4460        register_service_key(&ctx, "policy", &signer)?;
4461        assert_eq!(
4462            hyprstream_service::global_trust_store()
4463                .get(&signer.verifying_key())
4464                .and_then(|attestation| attestation.jwt),
4465            Some(jwt),
4466        );
4467        Ok(())
4468    }
4469
4470    /// The loaded custom `[secrets].path` must reach the factory itself, not
4471    /// only the startup trust-store bridge: with the service JWT on disk
4472    /// solely at the config-selected directory (no env override, no manual
4473    /// seeding), `register_service_key` resolves it there. This is the exact
4474    /// selection expression that feeds `spawn_jwt_renewal_task` — the hourly
4475    /// renewal reads and persists at this directory — so a config-free
4476    /// fallback here makes custom-path deployments renew against the wrong
4477    /// directory and silently skip.
4478    #[test]
4479    fn register_service_key_resolves_custom_config_secrets_dir() -> anyhow::Result<()> {
4480        const CHILD: &str = "HYPRSTREAM_POLICY_CTX_SECRETS_DIR_CHILD";
4481        if std::env::var_os(CHILD).is_none() {
4482            let status = std::process::Command::new(std::env::current_exe()?)
4483                .args(["--exact", "services::factories::tests::register_service_key_resolves_custom_config_secrets_dir", "--nocapture"])
4484                .env(CHILD, "1")
4485                .status()?;
4486            anyhow::ensure!(
4487                status.success(),
4488                "isolated ctx secrets-dir registration test failed"
4489            );
4490            return Ok(());
4491        }
4492        let root = tempfile::tempdir()?;
4493        let custom = root.path().join("custom-credentials");
4494        let xdg = root.path().join("xdg-config");
4495        std::env::set_var("XDG_CONFIG_HOME", &xdg);
4496        std::env::remove_var("HYPRSTREAM__SECRETS__PATH");
4497        std::env::remove_var(crate::auth::identity_store::SECRETS_PROFILE_ENV);
4498
4499        let signer = SigningKey::from_bytes(&[0x6b; 32]);
4500        let ctx = ServiceContext::new(signer.clone(), signer.verifying_key(), true, custom.clone())
4501            .with_quic(hyprstream_service::QuicSharedConfig {
4502                cert_chain: Vec::new(),
4503                certificate_hashes: Vec::new(),
4504                key_der: zeroize::Zeroizing::new(Vec::new()),
4505                base_ip: std::net::Ipv4Addr::LOCALHOST.into(),
4506                server_name: "policy.test".to_owned(),
4507                oauth_issuer_url: None,
4508                jwt_verifying_key: None,
4509                iroh_enabled: true,
4510                iroh_required: true,
4511                moq_relay: None,
4512                native_announcement_publisher: None,
4513                moq_relay_server_identity: None,
4514                moq_admission: None,
4515                moq_ingress_authorizer: None,
4516                moq_admission_proof: None,
4517            })
4518            .with_secrets_dir(custom.clone());
4519        // No trust-store bridge: the JWT exists only at the config-selected
4520        // directory. Registration must resolve it from the context-carried
4521        // directory — the same binding handed to the renewal task.
4522        let now = chrono::Utc::now().timestamp();
4523        let claims =
4524            hyprstream_rpc::auth::Claims::new("service:policy".to_owned(), now, now + 3600)
4525                .with_cnf_jwk(signer.verifying_key().as_bytes());
4526        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &signer);
4527        crate::auth::identity_store::write_service_jwt(&custom, "policy", &jwt)?;
4528        register_service_key(&ctx, "policy", &signer)?;
4529        assert_eq!(
4530            hyprstream_service::global_trust_store()
4531                .get(&signer.verifying_key())
4532                .and_then(|attestation| attestation.jwt),
4533            Some(jwt),
4534        );
4535        Ok(())
4536    }
4537
4538    /// A JWT already present in the trust store is used directly (no disk read).
4539    #[test]
4540    fn resolve_registration_jwt_prefers_trust_store() {
4541        let dir = tempfile::tempdir().unwrap();
4542        let jwt = resolve_registration_jwt(
4543            "model",
4544            dir.path(),
4545            crate::auth::identity_store::SecretsProfile::SharedDirectory,
4546            Some("trust.jwt.token".to_owned()),
4547        )
4548        .unwrap();
4549        assert_eq!(jwt, "trust.jwt.token");
4550    }
4551
4552    /// When not in the trust store, the authoritative on-disk JWT is loaded.
4553    #[test]
4554    fn resolve_registration_jwt_falls_back_to_disk() {
4555        let dir = tempfile::tempdir().unwrap();
4556        crate::auth::identity_store::write_service_jwt(dir.path(), "model", "disk.jwt.token")
4557            .unwrap();
4558        let jwt = resolve_registration_jwt(
4559            "model",
4560            dir.path(),
4561            crate::auth::identity_store::SecretsProfile::SharedDirectory,
4562            None,
4563        )
4564        .unwrap();
4565        assert_eq!(jwt, "disk.jwt.token");
4566    }
4567
4568    #[test]
4569    fn test_tls_endorsement_with_ecdsa_p256() {
4570        let (pkcs8, _pub_der) = generate_ecdsa_p256_pair();
4571        let ed25519_pubkey = [0xAB_u8; 32];
4572
4573        let endorsement = compute_tls_endorsement(&pkcs8, &ed25519_pubkey, "example.com").unwrap();
4574        assert!(!endorsement.is_empty());
4575        // ECDSA P-256 fixed-length signature is 64 bytes
4576        assert_eq!(endorsement.len(), 64);
4577    }
4578
4579    #[test]
4580    fn test_tls_endorsement_wrong_domain_differs() {
4581        let (pkcs8, _) = generate_ecdsa_p256_pair();
4582        let ed25519_pubkey = [0xAB_u8; 32];
4583
4584        let endorsement_a =
4585            compute_tls_endorsement(&pkcs8, &ed25519_pubkey, "example.com").unwrap();
4586        let endorsement_b = compute_tls_endorsement(&pkcs8, &ed25519_pubkey, "evil.com").unwrap();
4587
4588        // ECDSA signatures are randomized so they'll differ anyway, but the important
4589        // thing is that the message content changes — verified by the factory logic.
4590        // Just confirm both succeed.
4591        assert!(!endorsement_a.is_empty());
4592        assert!(!endorsement_b.is_empty());
4593    }
4594
4595    #[test]
4596    fn test_tls_endorsement_message_format() {
4597        let ed25519_pubkey = [0x42_u8; 32];
4598        let msg = build_endorsement_message(&ed25519_pubkey, "test.local");
4599
4600        let expected_len = TLS_ENDORSEMENT_V1.len() + 32 + "test.local".len();
4601        assert_eq!(msg.len(), expected_len);
4602
4603        // Starts with domain separator
4604        assert_eq!(&msg[..TLS_ENDORSEMENT_V1.len()], TLS_ENDORSEMENT_V1);
4605        // Followed by pubkey
4606        assert_eq!(
4607            &msg[TLS_ENDORSEMENT_V1.len()..TLS_ENDORSEMENT_V1.len() + 32],
4608            &[0x42_u8; 32]
4609        );
4610        // Followed by domain
4611        assert_eq!(&msg[TLS_ENDORSEMENT_V1.len() + 32..], b"test.local");
4612    }
4613
4614    #[test]
4615    fn test_tls_endorsement_invalid_key() {
4616        let ed25519_pubkey = [0xAB_u8; 32];
4617        let result = compute_tls_endorsement(&[0xFF; 32], &ed25519_pubkey, "example.com");
4618        assert!(result.is_err());
4619    }
4620
4621    #[test]
4622    fn required_stream_initializer_never_serves_uds() -> anyhow::Result<()> {
4623        const CHILD: &str = "HYPRSTREAM_NATIVE_STREAM_INIT_TEST";
4624        if std::env::var_os(CHILD).is_none() {
4625            let status = std::process::Command::new(std::env::current_exe()?)
4626                .args([
4627                    "--exact",
4628                    "services::factories::tests::required_stream_initializer_never_serves_uds",
4629                    "--nocapture",
4630                ])
4631                .env(CHILD, "1")
4632                .status()?;
4633            anyhow::ensure!(
4634                status.success(),
4635                "isolated native stream initializer failed"
4636            );
4637            return Ok(());
4638        }
4639        init_local_moq_stream_plane("model", true);
4640        init_local_moq_stream_plane("registry", false);
4641        assert!(hyprstream_rpc::moq_stream::global_moq_origin().is_some());
4642        assert!(hyprstream_rpc::moq_stream::global_moq_uds_path().is_none());
4643        assert!(hyprstream_rpc::moq_stream::native_iroh_required());
4644        Ok(())
4645    }
4646
4647    /// `init_local_moq_stream_plane` sets both process-global moq state
4648    /// (`global_moq_origin` + `global_moq_uds_path`) and is idempotent: a second
4649    /// call is a no-op and must not panic (composes with the streams factory and
4650    /// multiple co-located publisher factories).
4651    ///
4652    /// Uses process-global `OnceLock`s, so it runs in a dedicated single-test
4653    /// binary (`#[cfg(test)]` integration is impractical for OnceLock); the
4654    /// assertions tolerate a plane already initialized by an earlier test in the
4655    /// same process — the contract under test is "set after call" + "idempotent".
4656    #[tokio::test]
4657    async fn init_local_moq_stream_plane_sets_globals_and_is_idempotent() {
4658        use hyprstream_rpc::moq_stream::{global_moq_origin, global_moq_uds_path};
4659
4660        // First call (or pre-set by another test) → plane is initialized.
4661        init_local_moq_stream_plane("test", false);
4662        assert!(
4663            global_moq_origin().is_some(),
4664            "origin must be set after init_local_moq_stream_plane",
4665        );
4666        let uds = global_moq_uds_path();
4667        assert!(
4668            uds.is_some(),
4669            "uds path must be set after init_local_moq_stream_plane",
4670        );
4671        let path_after_first = uds.map(std::path::Path::to_path_buf);
4672
4673        // Second call must be a no-op (idempotent) — no panic, no change.
4674        init_local_moq_stream_plane("test", false);
4675        assert!(global_moq_origin().is_some());
4676        assert_eq!(
4677            global_moq_uds_path().map(std::path::Path::to_path_buf),
4678            path_after_first,
4679            "second call must not change the served UDS path",
4680        );
4681    }
4682}