Skip to main content

hyprstream_core/cli/
deployment_bootstrap.rs

1//! Offline service identity admission before network services start.
2
3use anyhow::{ensure, Context, Result};
4use chrono::{DateTime, Duration, SecondsFormat, Utc};
5use ed25519_dalek::SigningKey;
6use hyprstream_pds::at9p::{
7    CapsuleBody, HybridKeyPair, ServiceEndpoint, ServiceEntry, ServiceType, Transport,
8};
9use hyprstream_pds::at9p_duplicity::AcceptedAt9pState;
10use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
11use hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes;
12use std::io::Write as _;
13use std::path::Path;
14use std::sync::Arc;
15
16use crate::auth::identity_store::{load_existing_service_signing_key, SecretsProfile};
17use crate::config::HyprConfig;
18use crate::services::discovery::{At9pStateIngest, PdsRecordStore};
19
20/// Machine-readable manifest schema emitted by `--roster-export`.
21const VERIFIED_ROSTER_SCHEMA: &str = "hyprstream/verified-service-roster@1";
22
23pub use crate::config::{MAX_SERVICE_IDENTITY_TTL_SECS, MIN_SERVICE_IDENTITY_TTL_SECS};
24pub use hyprstream_discovery::DeploymentStoreInit;
25
26/// Open the deployment checkpoint store on the CONFIGURED backend (#1257).
27///
28/// With an RDS binding resolved (`[rds]` TOML, the role-scoped env vars, or
29/// the shared credentials directory), the store is the shared RDS Postgres
30/// instance — the bootstrap flow must write/read the same accepted states the
31/// RDS-backed services and the QUIC checkpoint gate read; opening the local
32/// RocksDB instead would provision states no RDS-backed reader ever sees.
33/// With no binding resolved, `directory` is the local RocksDB path.
34///
35/// The at9p duplicity alarm WAL stays host-local under `directory` in both
36/// modes (it is a per-host tamper journal, not shared state). In RDS posture
37/// the local directory may hold only that WAL — never a record store.
38fn open_checkpoint_store(
39    config: &HyprConfig,
40    directory: &Path,
41    readonly: bool,
42) -> Result<PdsRecordStore> {
43    let rds = config.rds.resolved_from_env()?;
44    if rds.is_configured() {
45        #[cfg(feature = "pds-postgres")]
46        return PdsRecordStore::open_postgres(&rds, readonly)
47            .context("open RDS-backed deployment checkpoint store");
48        #[cfg(not(feature = "pds-postgres"))]
49        anyhow::bail!(
50            "RDS is configured ([rds] section or HYPRSTREAM_RECORDS_URL_FILE) but this \
51             binary was not built with the `pds-postgres` feature — refusing to silently \
52             use the local checkpoint store. Rebuild with --features pds-postgres."
53        );
54    }
55    if readonly {
56        PdsRecordStore::open_readonly(directory)
57    } else {
58        PdsRecordStore::open(directory)
59    }
60}
61
62/// Explicitly create the empty deployment checkpoint store for a newly
63/// provisioned deployment, or reuse a healthy retained one, on the
64/// configured backend (#1257).
65///
66/// Local backend: delegates to
67/// [`hyprstream_discovery::initialize_deployment_checkpoint_store`] (creates
68/// the RocksDB store and writes the first-boot marker when the store is
69/// safely absent; validates and reuses a healthy retained store; fails
70/// closed on invalid retained state).
71///
72/// RDS backend, normal mode: proves the shared store holds no accepted
73/// states before writing the first-boot marker. A shared store that already
74/// holds accepted states is healthy retained state: the verified readback
75/// of those states is the validation, and the store is reused
76/// ([`DeploymentStoreInit::Reused`]) rather than refused — a reboot of an
77/// initialized deployment is not an initialization error. Fresh-genesis mode
78/// is separate and explicit: it clears only the configured records KV and
79/// writes that marker atomically. The registry deletes the marker in the
80/// same transaction as its first accepted-state commit.
81pub fn init_checkpoint_store(config: &HyprConfig) -> Result<DeploymentStoreInit> {
82    init_checkpoint_store_mode(config, false)
83}
84
85/// Fresh-genesis variant used only by the protected staging bootstrap when its
86/// saved-plan manifest is bound to the approved pre-genesis RDS snapshot.
87/// Unlike normal initialization, this deliberately clears the records KV.
88pub fn init_checkpoint_store_for_fresh_genesis(config: &HyprConfig) -> Result<DeploymentStoreInit> {
89    init_checkpoint_store_mode(config, true)
90}
91
92fn init_checkpoint_store_mode(
93    config: &HyprConfig,
94    fresh_genesis: bool,
95) -> Result<DeploymentStoreInit> {
96    let rds = config.rds.resolved_from_env()?;
97    ensure!(
98        !fresh_genesis || rds.is_configured(),
99        "fresh-genesis reset requires the configured RDS records backend"
100    );
101    if rds.is_configured() {
102        #[cfg(feature = "pds-postgres")]
103        {
104            let verifier = hyprstream_discovery::authenticate_local_deployment_registry()?;
105            let directory = hyprstream_service::deployment_data_dir()?.join("pds-store");
106            let store = open_checkpoint_store(config, &directory, false)?
107                .with_at9p_deployment_verifier(verifier);
108            if fresh_genesis {
109                store.reset_for_fresh_genesis()?;
110                return Ok(DeploymentStoreInit::Created);
111            }
112            // A shared store already holding accepted states is healthy
113            // retained state: `accepted_at9p_states` is a verified readback,
114            // so reaching this point with a non-empty result IS the
115            // validation. Reuse it; a readback failure fails closed above.
116            if !store.accepted_at9p_states()?.is_empty() {
117                return Ok(DeploymentStoreInit::Reused);
118            }
119            store.mark_first_boot()?;
120            return Ok(DeploymentStoreInit::Created);
121        }
122        #[cfg(not(feature = "pds-postgres"))]
123        anyhow::bail!(
124            "RDS is configured ([rds] section or HYPRSTREAM_RECORDS_URL_FILE) but this \
125             binary was not built with the `pds-postgres` feature — refusing to silently \
126             initialize the local checkpoint store. Rebuild with --features pds-postgres."
127        );
128    }
129    hyprstream_discovery::initialize_deployment_checkpoint_store()
130}
131
132/// One public roster member projected from the checkpoint-verified accepted
133/// state read back from the store after admission. Contains no secret
134/// material: only the admitted DID, epoch, bounded validity, and the
135/// accepted-state head digest binding the entry to that exact state.
136#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
137pub struct VerifiedServiceRosterEntry {
138    pub service: String,
139    pub did: String,
140    pub epoch: u64,
141    pub expires_at: String,
142    pub accepted_head_digest: String,
143}
144
145/// Public verified roster manifest. This document is not a trust root: it is a
146/// convenience projection for tooling, and the authenticated checkpoint store
147/// remains the authoritative source the values were verified against.
148#[derive(serde::Serialize)]
149struct VerifiedServiceRosterManifest {
150    schema: &'static str,
151    generated_at: String,
152    services: Vec<VerifiedServiceRosterEntry>,
153}
154
155/// Provision or renew a complete local service roster. The database writer
156/// lock excludes a running registry; callers must order this before services.
157///
158/// `roster_export` is opt-in: when absent, behavior is unchanged. When given,
159/// a JSON manifest of only the verified readback public fields is written
160/// atomically (temporary file + rename); any member failure or unsafe output
161/// path fails the whole command without leaving partial or new output.
162pub fn provision_services(
163    config: &HyprConfig,
164    services: &[String],
165    valid_for_seconds: i64,
166    roster_export: Option<&Path>,
167) -> Result<()> {
168    ensure!(
169        (MIN_SERVICE_IDENTITY_TTL_SECS..=MAX_SERVICE_IDENTITY_TTL_SECS)
170            .contains(&valid_for_seconds),
171        "service identity lifetime must be {}..{} seconds",
172        MIN_SERVICE_IDENTITY_TTL_SECS,
173        MAX_SERVICE_IDENTITY_TTL_SECS
174    );
175    validate_service_roster(services)?;
176    let verifier = hyprstream_discovery::authenticate_local_deployment_registry()?;
177    let secrets = provisioning_secrets_dir(config)?;
178    let acceptance =
179        load_existing_service_signing_key(&secrets, "registry", SecretsProfile::SharedDirectory)?;
180    ensure!(
181        verifier.matches(&acceptance.verifying_key()),
182        "registry signing key does not match authenticated deployment credential"
183    );
184    // Load the entire roster before writing any state; missing keys are errors,
185    // never a reason to silently generate a replacement service identity.
186    let keys = services
187        .iter()
188        .map(|name| {
189            load_existing_service_signing_key(&secrets, name, SecretsProfile::SharedDirectory)
190                .map(|key| (name, key))
191        })
192        .collect::<Result<Vec<_>>>()?;
193    let directory = hyprstream_service::deployment_data_dir()?.join("pds-store");
194    let probe = open_checkpoint_store(config, &directory, true)?
195        .with_at9p_deployment_verifier(verifier.clone());
196    ensure!(
197        probe.first_boot_pending()? || !probe.accepted_at9p_states()?.is_empty(),
198        "empty unmarked checkpoint store requires explicit initialization or recovery"
199    );
200    drop(probe);
201    let store = Arc::new(
202        open_checkpoint_store(config, &directory, false)?
203            .with_at9p_deployment_verifier(verifier.clone()),
204    );
205    let audit = hyprstream_rpc::node_identity::derive_purpose_key(
206        &acceptance,
207        "hyprstream-at9p-audit-ed25519-v1",
208    );
209    let audit_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&audit);
210    let ingest = At9pStateIngest::open(
211        Arc::clone(&store),
212        &directory.join("at9p-duplicity.wal"),
213        acceptance,
214        audit,
215        audit_pq,
216    )?;
217    let now = Utc::now();
218    let mut admitted = Vec::new();
219    for (name, key) in &keys {
220        admitted.push((
221            name.as_str(),
222            key,
223            provision_one(&store, &ingest, name, key, now, valid_for_seconds)?,
224        ));
225    }
226    // Verify every roster member, not just the first DID that clears the
227    // first-boot marker. No service starts while this writer is held. The
228    // manifest is built only from these verified readback states, never from
229    // the provisioned inputs.
230    let now_text = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
231    let entries = build_verified_roster_entries(&store, &admitted, &now_text)?;
232    if let Some(path) = roster_export {
233        write_verified_roster_manifest(path, entries)?;
234        tracing::info!(path = %path.display(), "verified service roster manifest exported");
235    }
236    Ok(())
237}
238
239/// Read the current accepted roster without acquiring a writer or admitting
240/// state. Returns a complete public JSON document only after every member has
241/// passed the existing deployment/checkpoint/key-binding/freshness validators.
242/// The caller emits these buffered bytes; no output or store is written here.
243pub fn inspect_services(config: &HyprConfig, services: &[String]) -> Result<Vec<u8>> {
244    validate_service_roster(services)?;
245    let verifier = hyprstream_discovery::authenticate_local_deployment_registry()?;
246    let secrets = provisioning_secrets_dir(config)?;
247    let keys = load_roster_keys(&secrets, services)?;
248    let directory = hyprstream_service::deployment_data_dir()?.join("pds-store");
249    let store =
250        open_checkpoint_store(config, &directory, true)?.with_at9p_deployment_verifier(verifier);
251    inspect_verified_roster(
252        &store,
253        &keys,
254        &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
255    )
256}
257
258fn validate_service_roster(services: &[String]) -> Result<()> {
259    ensure!(!services.is_empty(), "at least one service is required");
260    let mut unique = std::collections::HashSet::new();
261    for name in services {
262        ensure!(unique.insert(name), "duplicate service in bootstrap roster");
263        ensure!(
264            hyprstream_service::get_factory(name).is_some(),
265            "unknown service in bootstrap roster: {name}"
266        );
267    }
268    Ok(())
269}
270
271fn load_roster_keys(secrets: &Path, services: &[String]) -> Result<Vec<(String, SigningKey)>> {
272    services
273        .iter()
274        .map(|name| {
275            load_existing_service_signing_key(secrets, name, SecretsProfile::SharedDirectory)
276                .map(|key| (name.clone(), key))
277        })
278        .collect()
279}
280
281fn inspect_verified_roster(
282    store: &PdsRecordStore,
283    keys: &[(String, SigningKey)],
284    now_text: &str,
285) -> Result<Vec<u8>> {
286    // Enumerate through the authenticated checkpoint path, never deserialize
287    // raw DB bytes or use an editable prior export as authority. Missing or
288    // ambiguous service IDs cannot silently become first boot or a new DID.
289    let states = store.accepted_at9p_states()?;
290    let mut admitted = Vec::with_capacity(keys.len());
291    for (name, key) in keys {
292        let service_id = format!("#{name}");
293        let mut matching = states.iter().filter(|state| {
294            state
295                .current
296                .services
297                .iter()
298                .any(|service| service.id == service_id)
299        });
300        let state = matching
301            .next()
302            .with_context(|| format!("no accepted identity for service {name}"))?;
303        ensure!(
304            matching.next().is_none(),
305            "multiple accepted identities match service {name}"
306        );
307        admitted.push((name.as_str(), key, state.clone()));
308    }
309    let entries = build_verified_roster_entries(store, &admitted, now_text)?;
310    let mut bytes = serialize_verified_roster_manifest(entries)?;
311    bytes.push(b'\n');
312    Ok(bytes)
313}
314
315fn serialize_verified_roster_manifest(
316    services: Vec<VerifiedServiceRosterEntry>,
317) -> Result<Vec<u8>> {
318    ensure!(
319        !services.is_empty(),
320        "refusing to export an empty verified service roster"
321    );
322    let manifest = VerifiedServiceRosterManifest {
323        schema: VERIFIED_ROSTER_SCHEMA,
324        generated_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
325        services,
326    };
327    Ok(serde_json::to_vec_pretty(&manifest)?)
328}
329
330/// Re-read and re-verify every admitted member from the checkpoint store,
331/// projecting only public fields. Any missing, mismatched, expired, or
332/// unbound member fails the whole roster.
333pub(crate) fn build_verified_roster_entries(
334    store: &PdsRecordStore,
335    admitted: &[(&str, &SigningKey, AcceptedAt9pState)],
336    now_text: &str,
337) -> Result<Vec<VerifiedServiceRosterEntry>> {
338    let mut entries = Vec::with_capacity(admitted.len());
339    for (name, key, state) in admitted {
340        let verified = store
341            .accepted_at9p_state(&state.did, Some(now_text))?
342            .context("provisioned service state disappeared")?;
343        hyprstream_service::NativeServiceAnnouncement::from_accepted_state(name, key, &verified)?;
344        tracing::info!(service = name, did = %verified.did, epoch = verified.epoch, "checkpoint-accepted service identity ready");
345        let expires_at = verified
346            .expires_at
347            .clone()
348            .context("verified accepted state lacks bounded expiry")?;
349        entries.push(VerifiedServiceRosterEntry {
350            service: (*name).to_owned(),
351            did: verified.did,
352            epoch: verified.epoch,
353            expires_at,
354            accepted_head_digest: hex::encode(verified.head_digest),
355        });
356    }
357    Ok(entries)
358}
359
360/// Atomically write the verified roster manifest: serialize fully, write to a
361/// sibling temporary file (created exclusively), sync, then rename. A failure
362/// at any step never creates, truncates, or replaces the target path. Only a
363/// temporary file this invocation successfully created is ever removed; if
364/// exclusive creation fails because the predictable sibling already exists,
365/// that preexisting file belongs to someone else and is left untouched.
366pub(crate) fn write_verified_roster_manifest(
367    path: &Path,
368    services: Vec<VerifiedServiceRosterEntry>,
369) -> Result<()> {
370    ensure!(
371        !services.is_empty(),
372        "refusing to export an empty verified service roster"
373    );
374    ensure!(
375        !path.is_dir(),
376        "roster export path is a directory: {}",
377        path.display()
378    );
379    let file_name = path
380        .file_name()
381        .context("roster export path has no file name")?;
382    let parent = path
383        .parent()
384        .filter(|parent| !parent.as_os_str().is_empty())
385        .unwrap_or_else(|| Path::new("."));
386    ensure!(
387        parent.is_dir(),
388        "roster export parent directory does not exist: {}",
389        parent.display()
390    );
391    let temp = parent.join(format!(
392        ".{}.tmp-{}",
393        file_name.to_string_lossy(),
394        std::process::id()
395    ));
396    let bytes = serialize_verified_roster_manifest(services)?;
397    // Ownership boundary: until the exclusive create succeeds, `temp` is not
398    // ours — a collision means a leftover from an interrupted earlier process
399    // (possibly with a reused PID) or another entry in the caller's directory,
400    // and must survive this failed invocation.
401    let file = std::fs::OpenOptions::new()
402        .write(true)
403        .create_new(true)
404        .open(&temp)
405        .with_context(|| format!("create roster export temporary {}", temp.display()))?;
406    publish_owned_roster_temp(file, &temp, path, &bytes)
407}
408
409/// Publish through a temporary file this invocation created exclusively. On
410/// any failure before the rename lands, only that owned file is removed;
411/// nothing else in the directory is touched.
412fn publish_owned_roster_temp(
413    mut file: std::fs::File,
414    temp: &Path,
415    path: &Path,
416    bytes: &[u8],
417) -> Result<()> {
418    let result = (|| -> Result<()> {
419        file.write_all(bytes)?;
420        file.write_all(b"\n")?;
421        file.sync_all()?;
422        drop(file);
423        std::fs::rename(temp, path)
424            .with_context(|| format!("publish roster export {}", path.display()))?;
425        Ok(())
426    })();
427    if result.is_err() {
428        let _ = std::fs::remove_file(temp);
429    }
430    result
431}
432
433/// Re-export the verified roster projection from the LIVE daemon store: the
434/// same readback-verified entries and atomic writer the boot command uses, so
435/// host-side tooling that reads the roster (the metal renewal monitor, which
436/// flags `roster_stale` when `generated_at` ages past 24 h) observes fresh
437/// expiries after in-daemon renewals. All-or-nothing like the boot export:
438/// any member that fails verification fails the whole snapshot, never a
439/// partial manifest that would hide a sick member from the monitor.
440///
441/// Unlike the boot writer, the periodic path gives its temporary file an
442/// ATTEMPT-UNIQUE name: the boot writer's predictable `.tmp-<pid>` sibling is
443/// deliberately preserved on collision (ownership boundary for a one-shot
444/// command), but a daemon crash between create and rename can leave that
445/// predictable file behind, and a container daemon restarting with the SAME
446/// pid would then fail every periodic export on the preserved leftover — a
447/// permanently stale roster despite healthy renewals. An attempt-unique name
448/// cannot collide with any prior attempt's leftover; a crash leaks one small
449/// stale temp file, never a permanent failure.
450pub(crate) fn export_verified_roster_snapshot(
451    store: &PdsRecordStore,
452    admitted: &[(String, SigningKey, AcceptedAt9pState)],
453    path: &Path,
454) -> Result<()> {
455    let now_text = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
456    let borrowed = admitted
457        .iter()
458        .map(|(name, key, state)| (name.as_str(), key, state.clone()))
459        .collect::<Vec<_>>();
460    let entries = build_verified_roster_entries(store, &borrowed, &now_text)?;
461    ensure!(
462        !path.is_dir(),
463        "roster export path is a directory: {}",
464        path.display()
465    );
466    let file_name = path
467        .file_name()
468        .context("roster export path has no file name")?;
469    let parent = path
470        .parent()
471        .filter(|parent| !parent.as_os_str().is_empty())
472        .unwrap_or_else(|| Path::new("."));
473    ensure!(
474        parent.is_dir(),
475        "roster export parent directory does not exist: {}",
476        parent.display()
477    );
478    let bytes = serialize_verified_roster_manifest(entries)?;
479    let temp = parent.join(format!(
480        ".{}.tmp-{}-{}",
481        file_name.to_string_lossy(),
482        std::process::id(),
483        Utc::now().timestamp_nanos_opt().unwrap_or_default()
484    ));
485    let file = std::fs::OpenOptions::new()
486        .write(true)
487        .create_new(true)
488        .open(&temp)
489        .with_context(|| format!("create roster export temporary {}", temp.display()))?;
490    publish_owned_roster_temp(file, &temp, path, &bytes)
491}
492
493/// The command is dispatched after `main` has loaded and validated the
494/// operator-selected configuration. Reusing that exact value keeps an explicit
495/// `--config` `[secrets].path` authoritative rather than reloading defaults.
496fn provisioning_secrets_dir(config: &HyprConfig) -> Result<std::path::PathBuf> {
497    HyprConfig::resolve_secrets_dir_for(Some(config))
498}
499
500/// Provision or renew one service identity against ALREADY-OPEN store/ingest
501/// handles. This is the reusable mint core shared by the offline boot command
502/// and the in-daemon renewal timer (`services::at9p_renewal`).
503///
504/// Contract for in-daemon callers: pass the daemon's LIVE `PdsRecordStore` and
505/// `At9pStateIngest` handles — never re-open the store (the RocksDB directory
506/// LOCK excludes the second read-write opener the daemon itself is) and never
507/// re-authenticate the deployment registry per tick (the credential/UCAN
508/// check happens exactly once at process start; reintroducing it per tick
509/// would cap renewal at the delegation's TTL). Every refusal invariant is
510/// preserved verbatim: key mismatch refuses (never substitutes), terminal
511/// identity refuses, a provisioned key absent from `next_key_commitments`
512/// refuses, the same key is carried forward, the successor is `epoch+1`, and
513/// `now` is the caller's (daemon) clock with no skew tolerance.
514pub(crate) fn provision_one(
515    store: &PdsRecordStore,
516    ingest: &At9pStateIngest,
517    service_name: &str,
518    signer: &SigningKey,
519    now: DateTime<Utc>,
520    valid_for_seconds: i64,
521) -> Result<AcceptedAt9pState> {
522    let service_id = format!("#{service_name}");
523    let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(signer);
524    let hybrid = HybridKeyPair::new(
525        signer.verifying_key().to_bytes().to_vec(),
526        ml_dsa_sk_to_vk_bytes(&pq),
527    )?;
528    let commitment = hybrid.commitment_digest();
529    let carrier =
530        hyprstream_rpc::node_identity::derive_purpose_key(signer, "hyprstream-iroh-transport-v1");
531    let mut endpoint = ServiceEndpoint::new(
532        Transport::Iroh,
533        format!("iroh://{}", hex::encode(carrier.verifying_key().to_bytes())),
534    )?;
535    endpoint.request_kem = Some(
536        hyprstream_rpc::node_identity::derive_mesh_kem_recipient(signer)?
537            .public()
538            .encode(),
539    );
540    let service = ServiceEntry::new(&service_id, ServiceType::NinePExport, endpoint)?;
541    let states = store.accepted_at9p_states()?;
542    let mut matching = states.into_iter().filter(|state| {
543        state
544            .current
545            .services
546            .iter()
547            .any(|entry| entry.id == service_id)
548    });
549    let existing = matching.next();
550    ensure!(
551        matching.next().is_none(),
552        "multiple accepted identities match service {service_name}"
553    );
554    let state = match existing {
555        Some(state) => {
556            ensure!(
557                state.current.subject_keys.contains(&hybrid),
558                "accepted service hybrid key does not match the provisioned signer"
559            );
560            state
561        }
562        None => {
563            let mut body = CapsuleBody::new(vec![hybrid.clone()], vec![service.clone()])?;
564            body.next_key_commitments = vec![commitment];
565            let genesis = sign_capsule(body, signer, &pq)?;
566            let bytes = genesis.to_dag_cbor()?;
567            let did = format!("did:at9p:{}", genesis.cid512()?);
568            ingest.ingest_genesis(&did, &bytes)?
569        }
570    };
571    let renew_after = now + Duration::seconds(valid_for_seconds / 2);
572    if state.epoch > 0
573        && state.current.services.iter().any(|entry| entry == &service)
574        && state
575            .expires_at
576            .as_deref()
577            .map(DateTime::parse_from_rfc3339)
578            .transpose()?
579            .is_some_and(|expiry| expiry >= renew_after)
580    {
581        return Ok(state);
582    }
583    ensure!(
584        !state.terminal,
585        "service {service_name} identity has no authorized successor"
586    );
587    ensure!(
588        state.current.next_key_commitments.contains(&commitment),
589        "provisioned service key is not committed for the next update"
590    );
591    // Preserve every other subject key, service entry and future commitment.
592    let mut body = state.current.clone();
593    let entry = body
594        .services
595        .iter_mut()
596        .find(|entry| entry.id == service_id)
597        .context("accepted service entry disappeared")?;
598    *entry = service;
599    let expires =
600        (now + Duration::seconds(valid_for_seconds)).to_rfc3339_opts(SecondsFormat::Secs, true);
601    let update = sign_update_record(
602        state.subject_cid512.clone(),
603        state
604            .epoch
605            .checked_add(1)
606            .context("service identity epoch exhausted")?,
607        state.head_digest,
608        body,
609        expires,
610        signer,
611        &pq,
612    )?;
613    ingest.ingest_successor(
614        &state.did,
615        &update.to_dag_cbor()?,
616        &now.to_rfc3339_opts(SecondsFormat::Secs, true),
617    )
618}
619
620#[cfg(test)]
621mod tests {
622    use super::*;
623
624    // Inventory contents and names (including directories) without relying on
625    // atime, which a read may legitimately update at the filesystem layer.
626    fn filesystem_contents(
627        root: &Path,
628    ) -> Result<std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>> {
629        fn visit(
630            root: &Path,
631            dir: &Path,
632            out: &mut std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>,
633        ) -> Result<()> {
634            for entry in std::fs::read_dir(dir)? {
635                let entry = entry?;
636                let path = entry.path();
637                let relative = path.strip_prefix(root)?.to_path_buf();
638                if entry.file_type()?.is_dir() {
639                    out.insert(relative, None);
640                    visit(root, &path, out)?;
641                } else {
642                    out.insert(relative, Some(std::fs::read(path)?));
643                }
644            }
645            Ok(())
646        }
647        let mut out = std::collections::BTreeMap::new();
648        visit(root, root, &mut out)?;
649        Ok(out)
650    }
651
652    #[test]
653    fn readonly_roster_reads_verified_state_without_changing_store_or_keys() -> Result<()> {
654        let (dir, store, ingest, model) = fixture()?;
655        let event = SigningKey::from_bytes(&[0x63; 32]);
656        let now = Utc::now();
657        let accepted_model = provision_one(&store, &ingest, "model", &model, now, 86400)?;
658        let accepted_event = provision_one(&store, &ingest, "event", &event, now, 86400)?;
659        let credentials = dir.path().join("retained-credentials");
660        crate::auth::identity_store::write_secret(
661            &credentials.join("model"),
662            "signing-key",
663            &model.to_bytes(),
664        )?;
665        crate::auth::identity_store::write_secret(
666            &credentials.join("event"),
667            "signing-key",
668            &event.to_bytes(),
669        )?;
670        drop(ingest);
671        drop(store);
672        let before = filesystem_contents(dir.path())?;
673        let keys = load_roster_keys(&credentials, &["model".into(), "event".into()])?;
674        let readonly = PdsRecordStore::open_readonly(dir.path())?
675            .with_at9p_acceptance_identity(SigningKey::from_bytes(&[0x61; 32]).verifying_key());
676        let bytes = inspect_verified_roster(
677            &readonly,
678            &keys,
679            &now.to_rfc3339_opts(SecondsFormat::Secs, true),
680        )?;
681        let document: serde_json::Value = serde_json::from_slice(&bytes)?;
682        assert_eq!(document["schema"], VERIFIED_ROSTER_SCHEMA);
683        assert_eq!(
684            document["services"].as_array().context("services")?.len(),
685            2
686        );
687        for (index, state) in [accepted_model, accepted_event].iter().enumerate() {
688            assert_eq!(document["services"][index]["did"], state.did);
689            assert_eq!(document["services"][index]["epoch"], state.epoch);
690            assert_eq!(
691                document["services"][index]["accepted_head_digest"],
692                hex::encode(state.head_digest)
693            );
694            assert_eq!(
695                document["services"][index]["expires_at"],
696                state.expires_at.as_deref().context("expiry")?
697            );
698        }
699        assert!(!String::from_utf8(bytes)?.contains(&hex::encode(model.to_bytes())));
700        assert_eq!(filesystem_contents(dir.path())?, before);
701        Ok(())
702    }
703
704    #[test]
705    fn readonly_roster_failure_preserves_state_and_never_initializes_missing_input() -> Result<()> {
706        let missing_root = tempfile::tempdir()?;
707        let before = filesystem_contents(missing_root.path())?;
708        assert!(PdsRecordStore::open_readonly(&missing_root.path().join("missing-store")).is_err());
709        assert!(load_roster_keys(missing_root.path(), &["model".into()]).is_err());
710        assert_eq!(filesystem_contents(missing_root.path())?, before);
711        let empty = missing_root.path().join("empty-store");
712        std::fs::create_dir(&empty)?;
713        let before = filesystem_contents(missing_root.path())?;
714        assert!(PdsRecordStore::open_readonly(&empty).is_err());
715        assert_eq!(filesystem_contents(missing_root.path())?, before);
716        std::fs::write(empty.join("CURRENT"), b"invalid current manifest")?;
717        let before = filesystem_contents(missing_root.path())?;
718        assert!(PdsRecordStore::open_readonly(&empty).is_err());
719        assert_eq!(filesystem_contents(missing_root.path())?, before);
720
721        let (dir, store, ingest, key) = fixture()?;
722        let now = Utc::now();
723        provision_one(&store, &ingest, "model", &key, now, 86400)?;
724        drop(ingest);
725        drop(store);
726        let before = filesystem_contents(dir.path())?;
727        let readonly = PdsRecordStore::open_readonly(dir.path())?
728            .with_at9p_acceptance_identity(SigningKey::from_bytes(&[0x61; 32]).verifying_key());
729        // A successful first member cannot produce a partial JSON result when
730        // a later requested service is missing or its retained key mismatches.
731        assert!(inspect_verified_roster(
732            &readonly,
733            &[("model".into(), key.clone()), ("event".into(), key.clone())],
734            &now.to_rfc3339_opts(SecondsFormat::Secs, true)
735        )
736        .is_err());
737        assert!(inspect_verified_roster(
738            &readonly,
739            &[("model".into(), SigningKey::from_bytes(&[0x77; 32]))],
740            &now.to_rfc3339_opts(SecondsFormat::Secs, true)
741        )
742        .is_err());
743        assert!(inspect_verified_roster(
744            &readonly,
745            &[("model".into(), key.clone())],
746            &(now + Duration::hours(25)).to_rfc3339_opts(SecondsFormat::Secs, true)
747        )
748        .is_err());
749        let wrong_verifier = PdsRecordStore::open_readonly(dir.path())?
750            .with_at9p_acceptance_identity(SigningKey::from_bytes(&[0x78; 32]).verifying_key());
751        assert!(inspect_verified_roster(
752            &wrong_verifier,
753            &[("model".into(), key)],
754            &now.to_rfc3339_opts(SecondsFormat::Secs, true)
755        )
756        .is_err());
757        assert_eq!(filesystem_contents(dir.path())?, before);
758        Ok(())
759    }
760
761    #[test]
762    fn readonly_roster_rejects_corrupt_checkpoint_and_preserves_failure_evidence() -> Result<()> {
763        let (dir, store, ingest, key) = fixture()?;
764        let now = Utc::now();
765        let accepted = provision_one(&store, &ingest, "model", &key, now, 86400)?;
766        drop(ingest);
767        drop(store);
768        // Tamper only in fixture setup, then close the writer before capturing
769        // the failed reader's complete on-disk before/after evidence.
770        let db = rocksdb::DB::open(&rocksdb::Options::default(), dir.path())?;
771        db.put(
772            format!("at9p-checkpoint\0{}", accepted.subject_cid512),
773            b"invalid checkpoint",
774        )?;
775        drop(db);
776        let before = filesystem_contents(dir.path())?;
777        let readonly = PdsRecordStore::open_readonly(dir.path())?
778            .with_at9p_acceptance_identity(SigningKey::from_bytes(&[0x61; 32]).verifying_key());
779        assert!(inspect_verified_roster(
780            &readonly,
781            &[("model".into(), key)],
782            &now.to_rfc3339_opts(SecondsFormat::Secs, true)
783        )
784        .is_err());
785        assert_eq!(filesystem_contents(dir.path())?, before);
786        Ok(())
787    }
788
789    #[test]
790    fn readonly_roster_rejects_empty_first_boot_store_without_consuming_marker() -> Result<()> {
791        let dir = tempfile::tempdir()?;
792        let store = PdsRecordStore::open(dir.path())?;
793        store.mark_first_boot()?;
794        drop(store);
795        let before = filesystem_contents(dir.path())?;
796        let readonly = PdsRecordStore::open_readonly(dir.path())?
797            .with_at9p_acceptance_identity(SigningKey::from_bytes(&[0x61; 32]).verifying_key());
798        assert!(inspect_verified_roster(
799            &readonly,
800            &[("model".into(), SigningKey::from_bytes(&[0x62; 32]))],
801            &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true)
802        )
803        .is_err());
804        assert!(readonly.first_boot_pending()?);
805        assert_eq!(filesystem_contents(dir.path())?, before);
806        assert!(validate_service_roster(&[]).is_err());
807        assert!(validate_service_roster(&["model".into(), "model".into()]).is_err());
808        assert!(validate_service_roster(&["unknown-service".into()]).is_err());
809        Ok(())
810    }
811
812    #[test]
813    fn deployment_bootstrap_rejects_invalid_roster_and_lifetime_before_credentials() {
814        let config = HyprConfig::default();
815        assert!(provision_services(&config, &["model".to_owned()], 599, None).is_err());
816        assert!(provision_services(
817            &config,
818            &["model".to_owned()],
819            MAX_SERVICE_IDENTITY_TTL_SECS + 1,
820            None
821        )
822        .is_err());
823        assert!(provision_services(
824            &config,
825            &["model".to_owned()],
826            MAX_SERVICE_IDENTITY_TTL_SECS,
827            None
828        )
829        .is_err());
830        assert!(provision_services(&config, &[], 86400, None).is_err());
831        assert!(provision_services(
832            &config,
833            &["model".to_owned(), "model".to_owned()],
834            86400,
835            None
836        )
837        .is_err());
838        assert!(
839            provision_services(&config, &["../not-a-service".to_owned()], 86400, None).is_err()
840        );
841    }
842
843    #[test]
844    fn deployment_bootstrap_uses_loaded_config_for_secrets_dir() -> Result<()> {
845        let mut config = HyprConfig::default();
846        let custom_secrets = tempfile::tempdir()?.path().join("custom-secrets");
847        config.secrets.path = Some(custom_secrets.clone());
848        assert_eq!(provisioning_secrets_dir(&config)?, custom_secrets);
849        Ok(())
850    }
851
852    /// A config with an explicit (TOML-level) records-role binding: URL and CA
853    /// files rendered under `dir`, so resolution never consults process env.
854    fn rds_bound_config(dir: &Path, url: &str) -> Result<HyprConfig> {
855        std::fs::write(dir.join("records-url"), format!("{url}\n"))?;
856        std::fs::write(dir.join("rds-ca.pem"), b"test-ca-pem")?;
857        let mut config = HyprConfig::default();
858        config.rds = crate::config::RdsConfig {
859            url_file: Some(dir.join("records-url")),
860            root_cert_file: Some(dir.join("rds-ca.pem")),
861            cell_id: "test-cell".to_owned(),
862        };
863        Ok(config)
864    }
865
866    /// Slice one top-level item out of this file: from `marker` to the first
867    /// column-0 item/comment/attribute line that follows it. (Owned return,
868    /// no lifetime parameters: a `'` token desyncs the loopback-burndown
869    /// test-region skipper, which is not a full lexer.)
870    fn top_level_body(source: &str, marker: &str) -> String {
871        let start = source
872            .find(marker)
873            .unwrap_or_else(|| panic!("item exists: {marker}"));
874        let rest = &source[start + marker.len()..];
875        let end = ["\nfn ", "\npub fn", "\n///", "\n//", "\n#["]
876            .iter()
877            .filter_map(|needle| rest.find(needle))
878            .min()
879            .unwrap_or(rest.len());
880        rest[..end].to_owned()
881    }
882
883    #[test]
884    fn bootstrap_paths_route_through_the_backend_selecting_open() {
885        let source = include_str!("deployment_bootstrap.rs");
886        for marker in ["pub fn provision_services(", "pub fn inspect_services("] {
887            let body = top_level_body(source, marker);
888            assert!(
889                body.contains("open_checkpoint_store("),
890                "{marker} must open the configured backend"
891            );
892            assert!(
893                !body.contains("PdsRecordStore::open_readonly(")
894                    && !body.contains("PdsRecordStore::open("),
895                "{marker} must not open the local store directly"
896            );
897        }
898        let init = top_level_body(source, "fn init_checkpoint_store_mode(");
899        assert!(init.contains("resolved_from_env()?"));
900        assert!(init.contains("open_checkpoint_store("));
901        assert!(init.contains("mark_first_boot"));
902        assert!(init.contains("reset_for_fresh_genesis"));
903        assert!(init.contains("!fresh_genesis || rds.is_configured()"));
904    }
905
906    #[cfg(feature = "pds-postgres")]
907    #[test]
908    fn fresh_genesis_reset_refuses_local_store() -> Result<()> {
909        let dir = tempfile::tempdir()?;
910        let store_dir = dir.path().join("pds-store");
911        let store = PdsRecordStore::open(&store_dir)?;
912        assert!(store.reset_for_fresh_genesis().is_err());
913        assert!(!store.first_boot_pending()?);
914        Ok(())
915    }
916
917    /// RDS-configured bootstrap paths must fail closed and never silently
918    /// open the local store — even when a valid local store exists. The
919    /// failure here is deterministic (the contract rejects a loopback URL
920    /// before any network I/O), and it must come from the RDS contract path,
921    /// proving the RDS branch was taken.
922    #[cfg(feature = "pds-postgres")]
923    #[test]
924    fn rds_configured_bootstrap_store_fails_closed_and_never_touches_local() -> Result<()> {
925        let dir = tempfile::tempdir()?;
926        let store_dir = dir.path().join("pds-store");
927        // A valid local store exists: a silent local fallback would succeed.
928        drop(PdsRecordStore::open(&store_dir)?);
929        let creds = tempfile::tempdir()?;
930        let config = rds_bound_config(
931            creds.path(),
932            "postgresql://127.0.0.1/records?sslmode=verify-full",
933        )?;
934        for readonly in [true, false] {
935            let err = open_checkpoint_store(&config, &store_dir, readonly)
936                .err()
937                .context("RDS-configured open must fail closed, not fall back to local")?;
938            assert!(
939                format!("{err:?}").contains("loopback"),
940                "the failure must come from the RDS contract path: {err}"
941            );
942        }
943        assert!(init_checkpoint_store(&config).is_err());
944        Ok(())
945    }
946
947    /// Without the `pds-postgres` feature an RDS-configured bootstrap path is
948    /// a hard error naming the missing feature — never a silent local store.
949    #[cfg(not(feature = "pds-postgres"))]
950    #[test]
951    fn rds_configured_without_feature_bails_on_bootstrap_paths() -> Result<()> {
952        let dir = tempfile::tempdir()?;
953        let store_dir = dir.path().join("pds-store");
954        drop(PdsRecordStore::open(&store_dir)?);
955        let creds = tempfile::tempdir()?;
956        let config = rds_bound_config(
957            creds.path(),
958            "postgresql://db.internal.example/records?sslmode=verify-full",
959        )?;
960        for readonly in [true, false] {
961            let err = open_checkpoint_store(&config, &store_dir, readonly)
962                .err()
963                .context("RDS-configured open must fail closed without the feature")?;
964            assert!(
965                err.to_string().contains("pds-postgres"),
966                "the failure must name the missing feature: {err}"
967            );
968        }
969        assert!(init_checkpoint_store(&config).is_err());
970        Ok(())
971    }
972
973    fn fixture() -> Result<(
974        tempfile::TempDir,
975        Arc<PdsRecordStore>,
976        At9pStateIngest,
977        SigningKey,
978    )> {
979        let dir = tempfile::tempdir()?;
980        let acceptance = SigningKey::from_bytes(&[0x61; 32]);
981        let store = Arc::new(
982            PdsRecordStore::open(dir.path())?
983                .with_at9p_acceptance_identity(acceptance.verifying_key()),
984        );
985        let audit = hyprstream_rpc::node_identity::derive_purpose_key(
986            &acceptance,
987            "hyprstream-at9p-audit-ed25519-v1",
988        );
989        let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&audit);
990        let ingest = At9pStateIngest::open(
991            Arc::clone(&store),
992            &dir.path().join("audit.wal"),
993            acceptance,
994            audit,
995            pq,
996        )?;
997        Ok((dir, store, ingest, SigningKey::from_bytes(&[0x62; 32])))
998    }
999
1000    #[test]
1001    fn deployment_bootstrap_admits_bounded_successor_and_reuses_identity() -> Result<()> {
1002        let (_dir, store, ingest, key) = fixture()?;
1003        let now = Utc::now();
1004        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1005        assert_eq!(first.epoch, 1);
1006        assert!(!first.terminal);
1007        assert_eq!(
1008            first.current.services[0].endpoint.request_kem.as_deref(),
1009            Some(
1010                hyprstream_rpc::node_identity::derive_mesh_kem_recipient(&key)?
1011                    .public()
1012                    .encode()
1013                    .as_slice()
1014            )
1015        );
1016        hyprstream_service::NativeServiceAnnouncement::from_accepted_state("model", &key, &first)?;
1017        let retry = provision_one(
1018            &store,
1019            &ingest,
1020            "model",
1021            &key,
1022            now + Duration::minutes(1),
1023            86400,
1024        )?;
1025        assert_eq!(retry.did, first.did);
1026        assert_eq!(retry.head_digest, first.head_digest);
1027        let renewed = provision_one(
1028            &store,
1029            &ingest,
1030            "model",
1031            &key,
1032            now + Duration::hours(13),
1033            86400,
1034        )?;
1035        assert_eq!(renewed.did, first.did);
1036        assert_eq!(renewed.epoch, 2);
1037        assert_ne!(renewed.head_digest, first.head_digest);
1038        assert_eq!(store.accepted_at9p_states()?.len(), 1);
1039        Ok(())
1040    }
1041
1042    #[test]
1043    fn deployment_bootstrap_resumes_after_genesis_only_partial_progress() -> Result<()> {
1044        let (_dir, store, ingest, key) = fixture()?;
1045        let now = Utc::now();
1046        // Deliberately invalid internal update lifetime simulates interruption
1047        // after durable genesis; the public command rejects this input earlier.
1048        assert!(provision_one(&store, &ingest, "model", &key, now, -1).is_err());
1049        let states = store.accepted_at9p_states()?;
1050        assert_eq!(states.len(), 1);
1051        assert_eq!(states[0].epoch, 0);
1052        let resumed = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1053        assert_eq!(resumed.did, states[0].did);
1054        assert_eq!(resumed.epoch, 1);
1055        Ok(())
1056    }
1057
1058    #[test]
1059    fn deployment_bootstrap_rejects_wrong_key_without_duplicate_genesis() -> Result<()> {
1060        let (_dir, store, ingest, key) = fixture()?;
1061        let now = Utc::now();
1062        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1063        let wrong_key = SigningKey::from_bytes(&[0x65; 32]);
1064        let error = provision_one(&store, &ingest, "model", &wrong_key, now, 86400)
1065            .err()
1066            .context("an enrolled service cannot be treated as absent after a key change")?;
1067        assert!(error.to_string().contains("hybrid key does not match"));
1068        let states = store.accepted_at9p_states()?;
1069        assert_eq!(states.len(), 1);
1070        assert_eq!(states[0].did, first.did);
1071        assert_eq!(states[0].epoch, first.epoch);
1072        assert_eq!(states[0].head_digest, first.head_digest);
1073        Ok(())
1074    }
1075
1076    #[test]
1077    fn deployment_bootstrap_rejects_multiple_service_identities_across_keys() -> Result<()> {
1078        let (_dir, store, ingest, key) = fixture()?;
1079        let now = Utc::now();
1080        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1081        // Simulate a pre-existing duplicate, admitted through the real signed
1082        // genesis/checkpoint path, with the same service ID but a different key.
1083        let other = SigningKey::from_bytes(&[0x66; 32]);
1084        let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&other);
1085        let pair = HybridKeyPair::new(
1086            other.verifying_key().to_bytes().to_vec(),
1087            ml_dsa_sk_to_vk_bytes(&pq),
1088        )?;
1089        let body = CapsuleBody::new(vec![pair], first.current.services.clone())?;
1090        let capsule = sign_capsule(body, &other, &pq)?;
1091        let duplicate = ingest.ingest_genesis(
1092            &format!("did:at9p:{}", capsule.cid512()?),
1093            &capsule.to_dag_cbor()?,
1094        )?;
1095        let error = provision_one(&store, &ingest, "model", &key, now, 86400)
1096            .err()
1097            .context("matching a key cannot hide a second service authority")?;
1098        assert!(error.to_string().contains("multiple accepted identities"));
1099        let states = store.accepted_at9p_states()?;
1100        assert_eq!(states.len(), 2);
1101        assert!(states
1102            .iter()
1103            .any(|state| state.head_digest == first.head_digest));
1104        assert!(states
1105            .iter()
1106            .any(|state| state.head_digest == duplicate.head_digest));
1107        Ok(())
1108    }
1109
1110    #[test]
1111    fn deployment_bootstrap_checkpoint_survives_reopen_and_covers_roster() -> Result<()> {
1112        let (dir, store, ingest, key) = fixture()?;
1113        let now = Utc::now();
1114        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1115        let second_key = SigningKey::from_bytes(&[0x63; 32]);
1116        let second = provision_one(&store, &ingest, "event", &second_key, now, 86400)?;
1117        assert_ne!(first.did, second.did);
1118        drop(ingest);
1119        drop(store);
1120        let reopened = PdsRecordStore::open_readonly(dir.path())?
1121            .with_at9p_acceptance_identity(SigningKey::from_bytes(&[0x61; 32]).verifying_key());
1122        let states = reopened.accepted_at9p_states()?;
1123        assert_eq!(states.len(), 2);
1124        assert!(states.iter().all(|state| state.epoch == 1));
1125        let wrong = PdsRecordStore::open_readonly(dir.path())?
1126            .with_at9p_acceptance_identity(SigningKey::from_bytes(&[0x64; 32]).verifying_key());
1127        assert!(wrong.accepted_at9p_states().is_err());
1128        Ok(())
1129    }
1130
1131    #[test]
1132    fn deployment_bootstrap_never_generates_missing_service_keys() -> Result<()> {
1133        let dir = tempfile::tempdir()?;
1134        assert!(load_existing_service_signing_key(
1135            dir.path(),
1136            "model",
1137            SecretsProfile::SharedDirectory
1138        )
1139        .is_err());
1140        assert!(load_existing_service_signing_key(
1141            dir.path(),
1142            "policy",
1143            SecretsProfile::SharedDirectory
1144        )
1145        .is_err());
1146        assert!(!dir.path().join("model").exists());
1147        assert!(!dir.path().join("signing-key").exists());
1148        let model_dir = dir.path().join("model");
1149        crate::auth::identity_store::write_secret(&model_dir, "signing-key", &[0x65; 32])?;
1150        let loaded = load_existing_service_signing_key(
1151            dir.path(),
1152            "model",
1153            SecretsProfile::SharedDirectory,
1154        )?;
1155        assert_eq!(
1156            loaded.verifying_key(),
1157            SigningKey::from_bytes(&[0x65; 32]).verifying_key()
1158        );
1159        assert!(load_existing_service_signing_key(
1160            dir.path(),
1161            "../model",
1162            SecretsProfile::SharedDirectory
1163        )
1164        .is_err());
1165        Ok(())
1166    }
1167
1168    #[test]
1169    fn deployment_bootstrap_rejects_existing_hybrid_mismatch_without_replacing_did() -> Result<()> {
1170        let (_dir, store, ingest, key) = fixture()?;
1171        let other_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(
1172            &SigningKey::from_bytes(&[0x66; 32]),
1173        );
1174        let hybrid = HybridKeyPair::new(
1175            key.verifying_key().to_bytes().to_vec(),
1176            ml_dsa_sk_to_vk_bytes(&other_pq),
1177        )?;
1178        let endpoint = ServiceEndpoint::new(
1179            Transport::Iroh,
1180            format!("iroh://{}", hex::encode([0x67; 32])),
1181        )?;
1182        let service = ServiceEntry::new("#model", ServiceType::NinePExport, endpoint)?;
1183        let body = CapsuleBody::new(vec![hybrid], vec![service])?;
1184        let genesis = sign_capsule(body, &key, &other_pq)?;
1185        let did = format!("did:at9p:{}", genesis.cid512()?);
1186        ingest.ingest_genesis(&did, &genesis.to_dag_cbor()?)?;
1187        assert!(provision_one(&store, &ingest, "model", &key, Utc::now(), 86400).is_err());
1188        let states = store.accepted_at9p_states()?;
1189        assert_eq!(states.len(), 1);
1190        assert_eq!(states[0].did, did);
1191        assert_eq!(states[0].epoch, 0);
1192        Ok(())
1193    }
1194
1195    #[test]
1196    fn deployment_bootstrap_roster_export_matches_verified_readback_not_inputs() -> Result<()> {
1197        let (_dir, store, ingest, key) = fixture()?;
1198        let now = Utc::now();
1199        let now_text = now.to_rfc3339_opts(SecondsFormat::Secs, true);
1200        let event_key = SigningKey::from_bytes(&[0x63; 32]);
1201        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1202        let event = provision_one(&store, &ingest, "event", &event_key, now, 86400)?;
1203        let admitted = [
1204            ("model", &key, first.clone()),
1205            ("event", &event_key, event.clone()),
1206        ];
1207        let entries = build_verified_roster_entries(&store, &admitted, &now_text)?;
1208        assert_eq!(entries.len(), 2);
1209        // Every exported field equals an independent store readback of the
1210        // accepted state, not the values handed to the provisioner.
1211        let model_readback = store
1212            .accepted_at9p_state(&first.did, Some(&now_text))?
1213            .context("model readback")?;
1214        assert_eq!(entries[0].service, "model");
1215        assert_eq!(entries[0].did, model_readback.did);
1216        assert_eq!(entries[0].epoch, model_readback.epoch);
1217        assert_eq!(
1218            entries[0].expires_at,
1219            model_readback.expires_at.clone().context("model expiry")?
1220        );
1221        assert_eq!(
1222            entries[0].accepted_head_digest,
1223            hex::encode(model_readback.head_digest)
1224        );
1225        assert_eq!(entries[1].service, "event");
1226        assert_eq!(entries[1].did, event.did);
1227        assert_ne!(entries[0].did, entries[1].did);
1228        // Provenance: after a real signed renewal advances the store, the
1229        // rebuilt export tracks the new verified state even though the stale
1230        // in-memory `first` value is still passed in as the admitted input.
1231        let renewed = provision_one(
1232            &store,
1233            &ingest,
1234            "model",
1235            &key,
1236            now + Duration::hours(13),
1237            86400,
1238        )?;
1239        assert_eq!(renewed.epoch, first.epoch + 1);
1240        let admitted = [("model", &key, first.clone()), ("event", &event_key, event)];
1241        let entries = build_verified_roster_entries(&store, &admitted, &now_text)?;
1242        assert_eq!(entries[0].did, renewed.did);
1243        assert_eq!(entries[0].epoch, renewed.epoch);
1244        assert_eq!(
1245            entries[0].accepted_head_digest,
1246            hex::encode(renewed.head_digest)
1247        );
1248        assert_eq!(
1249            entries[0].expires_at,
1250            renewed.expires_at.clone().context("renewed expiry")?
1251        );
1252        Ok(())
1253    }
1254
1255    #[test]
1256    fn deployment_bootstrap_roster_export_is_atomic_and_never_partial() -> Result<()> {
1257        let (dir, store, ingest, key) = fixture()?;
1258        let now = Utc::now();
1259        let now_text = now.to_rfc3339_opts(SecondsFormat::Secs, true);
1260        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1261        let out_dir = tempfile::tempdir()?;
1262        let target = out_dir.path().join("roster.json");
1263        // A member whose key is not the accepted key fails the whole export;
1264        // the target and any temporary sibling must not exist afterwards.
1265        let wrong_key = SigningKey::from_bytes(&[0x65; 32]);
1266        let admitted = [("model", &wrong_key, first.clone())];
1267        let error = build_verified_roster_entries(&store, &admitted, &now_text)
1268            .err()
1269            .context("unaccepted key must fail roster verification")?;
1270        assert!(error.to_string().contains("accepted current hybrid"));
1271        let write_result = build_verified_roster_entries(&store, &admitted, &now_text)
1272            .and_then(|entries| write_verified_roster_manifest(&target, entries));
1273        assert!(write_result.is_err());
1274        assert!(!target.exists());
1275        assert!(std::fs::read_dir(out_dir.path())?.next().is_none());
1276        // A failed renewal-time export leaves a previously published manifest
1277        // byte-identical instead of clobbering it with partial content.
1278        let good = [("model", &key, first.clone())];
1279        let entries = build_verified_roster_entries(&store, &good, &now_text)?;
1280        write_verified_roster_manifest(&target, entries)?;
1281        let published = std::fs::read(&target)?;
1282        let failed = build_verified_roster_entries(&store, &admitted, &now_text)
1283            .and_then(|entries| write_verified_roster_manifest(&target, entries));
1284        assert!(failed.is_err());
1285        assert_eq!(std::fs::read(&target)?, published);
1286        // Unsafe output targets fail without touching the store directory.
1287        let entries = build_verified_roster_entries(&store, &good, &now_text)?;
1288        assert!(write_verified_roster_manifest(dir.path(), entries.clone()).is_err());
1289        assert!(write_verified_roster_manifest(
1290            &out_dir.path().join("missing-parent").join("roster.json"),
1291            entries,
1292        )
1293        .is_err());
1294        assert!(!out_dir.path().join("missing-parent").exists());
1295        Ok(())
1296    }
1297
1298    #[test]
1299    fn deployment_bootstrap_roster_export_rejects_expired_and_unbounded_state() -> Result<()> {
1300        let (_dir, store, ingest, key) = fixture()?;
1301        let now = Utc::now();
1302        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1303        // Readback at a time beyond the bounded validity fails the export.
1304        let later = (now + Duration::days(2)).to_rfc3339_opts(SecondsFormat::Secs, true);
1305        let admitted = [("model", &key, first)];
1306        let error = build_verified_roster_entries(&store, &admitted, &later)
1307            .err()
1308            .context("expired accepted state must fail roster export")?;
1309        assert!(error.to_string().contains("expired"));
1310        // A genesis-only accepted state has no bounded validity and is refused.
1311        let genesis = SigningKey::from_bytes(&[0x67; 32]);
1312        let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&genesis);
1313        let pair = HybridKeyPair::new(
1314            genesis.verifying_key().to_bytes().to_vec(),
1315            ml_dsa_sk_to_vk_bytes(&pq),
1316        )?;
1317        let endpoint = ServiceEndpoint::new(
1318            Transport::Iroh,
1319            format!("iroh://{}", hex::encode([0x68; 32])),
1320        )?;
1321        let service = ServiceEntry::new("#event", ServiceType::NinePExport, endpoint)?;
1322        let body = CapsuleBody::new(vec![pair], vec![service])?;
1323        let capsule = sign_capsule(body, &genesis, &pq)?;
1324        let genesis_did = format!("did:at9p:{}", capsule.cid512()?);
1325        let genesis_state = ingest.ingest_genesis(&genesis_did, &capsule.to_dag_cbor()?)?;
1326        assert_eq!(genesis_state.epoch, 0);
1327        assert!(genesis_state.expires_at.is_none());
1328        let now_text = now.to_rfc3339_opts(SecondsFormat::Secs, true);
1329        let admitted = [("event", &genesis, genesis_state)];
1330        let error = build_verified_roster_entries(&store, &admitted, &now_text)
1331            .err()
1332            .context("genesis-only state must fail roster export")?;
1333        assert!(error.to_string().contains("no bounded production expiry"));
1334        Ok(())
1335    }
1336
1337    #[test]
1338    fn deployment_bootstrap_roster_manifest_is_public_json_only() -> Result<()> {
1339        let (_dir, store, ingest, key) = fixture()?;
1340        let now = Utc::now();
1341        let now_text = now.to_rfc3339_opts(SecondsFormat::Secs, true);
1342        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1343        let admitted = [("model", &key, first.clone())];
1344        let entries = build_verified_roster_entries(&store, &admitted, &now_text)?;
1345        let out_dir = tempfile::tempdir()?;
1346        let target = out_dir.path().join("roster.json");
1347        write_verified_roster_manifest(&target, entries)?;
1348        let text = std::fs::read_to_string(&target)?;
1349        let parsed: serde_json::Value = serde_json::from_str(&text)?;
1350        assert_eq!(parsed["schema"], VERIFIED_ROSTER_SCHEMA);
1351        let services = parsed["services"].as_array().context("services array")?;
1352        assert_eq!(services.len(), 1);
1353        assert_eq!(services[0]["service"], "model");
1354        assert_eq!(services[0]["did"], first.did);
1355        assert_eq!(services[0]["epoch"], serde_json::Value::from(first.epoch));
1356        assert_eq!(
1357            services[0]["expires_at"].as_str().context("expires_at")?,
1358            first.expires_at.as_deref().context("first expiry")?
1359        );
1360        assert_eq!(
1361            services[0]["accepted_head_digest"],
1362            hex::encode(first.head_digest)
1363        );
1364        // The manifest must never leak private key material: neither the raw
1365        // signing seed nor the derived ML-DSA secret appears in any form.
1366        assert!(!text.contains(&hex::encode([0x62; 32])));
1367        assert!(!text.contains("signing-key"));
1368        Ok(())
1369    }
1370
1371    /// The predictable sibling temp path of the current process, exactly as
1372    /// `write_verified_roster_manifest` computes it.
1373    fn roster_temp_sibling(target: &Path) -> Result<std::path::PathBuf> {
1374        let file_name = target
1375            .file_name()
1376            .context("roster target has no file name")?;
1377        Ok(target.with_file_name(format!(
1378            ".{}.tmp-{}",
1379            file_name.to_string_lossy(),
1380            std::process::id()
1381        )))
1382    }
1383
1384    #[test]
1385    fn deployment_bootstrap_roster_export_collision_preserves_foreign_sibling_and_target(
1386    ) -> Result<()> {
1387        let out_dir = tempfile::tempdir()?;
1388        let target = out_dir.path().join("roster.json");
1389        // A leftover from an interrupted earlier process (possibly a reused
1390        // PID) occupies the predictable temp path, and an older published
1391        // manifest occupies the target. The failed invocation must not delete
1392        // or modify either: it never owned the sibling.
1393        let foreign = roster_temp_sibling(&target)?;
1394        std::fs::write(&foreign, b"interrupted earlier process")?;
1395        std::fs::write(&target, b"previous good manifest")?;
1396        let error = write_verified_roster_manifest(
1397            &target,
1398            vec![VerifiedServiceRosterEntry {
1399                service: "model".to_owned(),
1400                did: "did:at9p:collision".to_owned(),
1401                epoch: 1,
1402                expires_at: "2099-01-01T00:00:00Z".to_owned(),
1403                accepted_head_digest: hex::encode([0x42; 64]),
1404            }],
1405        )
1406        .err()
1407        .context("temp collision must fail the export")?;
1408        assert!(error.to_string().contains("create roster export temporary"));
1409        assert_eq!(std::fs::read(&foreign)?, b"interrupted earlier process");
1410        assert_eq!(std::fs::read(&target)?, b"previous good manifest");
1411        Ok(())
1412    }
1413
1414    #[test]
1415    fn deployment_bootstrap_roster_export_failure_after_create_removes_only_owned_temp(
1416    ) -> Result<()> {
1417        let out_dir = tempfile::tempdir()?;
1418        let target = out_dir.path().join("roster.json");
1419        let temp = roster_temp_sibling(&target)?;
1420        // Simulate ownership exactly as the writer acquires it, then fail the
1421        // publication step: the rename destination's parent does not exist.
1422        let owned = std::fs::OpenOptions::new()
1423            .write(true)
1424            .create_new(true)
1425            .open(&temp)?;
1426        let missing_target = out_dir.path().join("gone").join("roster.json");
1427        let result = publish_owned_roster_temp(owned, &temp, &missing_target, b"{}");
1428        assert!(result.is_err());
1429        assert!(!temp.exists(), "owned temp must be removed after failure");
1430        // An unrelated preexisting sibling entry is never cleanup scope.
1431        let foreign = out_dir.path().join(".roster.json.tmp-foreign");
1432        std::fs::write(&foreign, b"not ours")?;
1433        let owned = std::fs::OpenOptions::new()
1434            .write(true)
1435            .create_new(true)
1436            .open(&temp)?;
1437        assert!(publish_owned_roster_temp(owned, &temp, &missing_target, b"{}").is_err());
1438        assert!(!temp.exists());
1439        assert_eq!(std::fs::read(&foreign)?, b"not ours");
1440        // Success control: the owned path publishes and removes the temp.
1441        let owned = std::fs::OpenOptions::new()
1442            .write(true)
1443            .create_new(true)
1444            .open(&temp)?;
1445        publish_owned_roster_temp(owned, &temp, &target, b"{\"ok\":true}")?;
1446        assert!(!temp.exists());
1447        assert_eq!(std::fs::read(&target)?, b"{\"ok\":true}\n");
1448        Ok(())
1449    }
1450
1451    #[test]
1452    fn deployment_bootstrap_roster_export_success_control_publishes_atomically() -> Result<()> {
1453        let (_dir, store, ingest, key) = fixture()?;
1454        let now = Utc::now();
1455        let now_text = now.to_rfc3339_opts(SecondsFormat::Secs, true);
1456        let first = provision_one(&store, &ingest, "model", &key, now, 86400)?;
1457        let admitted = [("model", &key, first.clone())];
1458        let entries = build_verified_roster_entries(&store, &admitted, &now_text)?;
1459        let out_dir = tempfile::tempdir()?;
1460        let target = out_dir.path().join("roster.json");
1461        write_verified_roster_manifest(&target, entries)?;
1462        // Successful publication leaves exactly the target: no temp sibling.
1463        let names = std::fs::read_dir(out_dir.path())?
1464            .map(|entry| entry.map(|entry| entry.file_name()))
1465            .collect::<std::io::Result<Vec<_>>>()?;
1466        assert_eq!(names, [std::ffi::OsString::from("roster.json")]);
1467        let parsed: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&target)?)?;
1468        assert_eq!(parsed["services"][0]["did"], first.did);
1469        Ok(())
1470    }
1471}