Skip to main content

hyprstream_core/auth/
key_rotation.rs

1//! JWT signing key rotation — multi-slot drain/active/lead lifecycle.
2//!
3//! Keys live in three ordered slots:
4//!   lead   — pre-published (nbf in the future); clients see it in JWKS but no tokens use it yet
5//!   active — current issuance key
6//!   drain  — old active, still valid for token verification until its exp passes
7//!
8//! The background task checks every 6 h:
9//!   1. lead.nbf <= now  → promote lead → active, old active → drain
10//!   2. drain exp + drain_days * 86400 < now → remove drain
11//!   3. lead is None and active.exp - now < lead_days * 86400 → generate new lead, persist
12
13use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
14use ed25519_dalek::SigningKey;
15use serde::{Deserialize, Serialize};
16use sha2::{Digest as _, Sha256};
17use std::path::{Path, PathBuf};
18use std::sync::atomic::{AtomicBool, Ordering};
19use std::sync::Arc;
20use tokio::sync::RwLock;
21use tokio::time::MissedTickBehavior;
22use tracing::{error, info, warn};
23
24use crate::config::OAuthConfig;
25
26/// Global shared ML-DSA-65 verifying keys for PQ-hybrid JWT verification.
27///
28/// Populated at startup from the ML-DSA rotation store's current slots.
29/// Updated by the rotation task after each promotion. Services read from this
30/// via their `JwtKeySource` implementation.
31// Cross-crate `Arc<std::sync::RwLock<..>>` contract with `JwtKeySource` and the
32// service factory; intentionally `std::sync::RwLock`, not `parking_lot`.
33#[allow(clippy::disallowed_types)]
34static ML_DSA_VERIFYING_KEYS: std::sync::OnceLock<
35    std::sync::Arc<std::sync::RwLock<Vec<hyprstream_rpc::crypto::pq::MlDsaVerifyingKey>>>,
36> = std::sync::OnceLock::new();
37
38/// Get or initialize the global ML-DSA verifying keys Arc.
39#[allow(clippy::disallowed_types)]
40pub fn global_ml_dsa_verifying_keys(
41) -> std::sync::Arc<std::sync::RwLock<Vec<hyprstream_rpc::crypto::pq::MlDsaVerifyingKey>>> {
42    ML_DSA_VERIFYING_KEYS
43        .get_or_init(|| std::sync::Arc::new(std::sync::RwLock::new(Vec::new())))
44        .clone()
45}
46
47/// Live, shared handle to the node's published Ed25519 rotation-slot verifying
48/// keys (drain/active/lead). This is the SAME Ed25519 key set the `/oauth/jwks`
49/// endpoint publishes — the keys that sign rotation-issued at+JWTs (WIT, S6
50/// grant / token-exchange, and any issuance path using the active slot). It is
51/// public-key material only; no signing keys are ever exposed through it.
52///
53/// Consumers (e.g. the shared HTTP validator `verify_token_claims`) hold a clone
54/// and verify locally-issued tokens against these keys in addition to the CA key.
55// Cross-service `Arc<std::sync::RwLock<..>>` contract mirroring the ML-DSA
56// verifying-key list above; intentionally `std::sync::RwLock`, not `parking_lot`.
57#[allow(clippy::disallowed_types)]
58pub type PublishedEd25519Keys = std::sync::Arc<std::sync::RwLock<Vec<ed25519_dalek::VerifyingKey>>>;
59
60/// Global shared Ed25519 published verifying keys (rotation slots).
61///
62/// Populated at OAuth service init from the signing-key store's current slots
63/// and refreshed by the rotation task after each promotion/eviction. The CA key
64/// is NOT included here — callers already hold it separately (`verifying_key`).
65static ED25519_VERIFYING_KEYS: std::sync::OnceLock<PublishedEd25519Keys> =
66    std::sync::OnceLock::new();
67
68/// Get or initialize the global published-Ed25519 verifying-key handle.
69///
70/// Returns a live, shared `Arc` — mutations by the rotation task are visible to
71/// every holder. Starts empty until the OAuth service populates it.
72#[allow(clippy::disallowed_types)]
73pub fn global_ed25519_verifying_keys() -> PublishedEd25519Keys {
74    ED25519_VERIFYING_KEYS
75        .get_or_init(|| std::sync::Arc::new(std::sync::RwLock::new(Vec::new())))
76        .clone()
77}
78
79/// Refresh the global published-Ed25519 verifying keys from a signing-key store
80/// snapshot (all current drain/active/lead slots).
81///
82/// Called at OAuth init and after every rotation so the shared HTTP validator
83/// keeps accepting tokens signed by any currently-published slot.
84pub async fn refresh_ed25519_verifying_keys(store: &SigningKeyStore) {
85    let vks: Vec<ed25519_dalek::VerifyingKey> = store
86        .all_slots_snapshot()
87        .await
88        .iter()
89        .map(|slot| slot.key.verifying_key())
90        .collect();
91    let shared = global_ed25519_verifying_keys();
92    let _ = shared.write().map(|mut guard| *guard = vks);
93}
94
95/// Refresh the published ML-DSA component-key snapshot.
96pub async fn refresh_ml_dsa_verifying_keys(store: &Arc<MlDsaSigningKeyStore>) {
97    let vks: Vec<hyprstream_rpc::crypto::pq::MlDsaVerifyingKey> = store
98        .all_slots_snapshot()
99        .await
100        .iter()
101        .map(ml_dsa_rotation::MlDsaKeySlot::verifying_key)
102        .collect();
103    let shared = global_ml_dsa_verifying_keys();
104    let _ = shared.write().map(|mut guard| *guard = vks);
105}
106
107#[derive(Clone, Serialize, Deserialize)]
108struct CompositeLedger {
109    version: u64,
110    #[serde(default)]
111    component_digest: String,
112    pairs: Vec<CompositeLedgerPair>,
113}
114
115#[derive(Clone, Serialize, Deserialize)]
116struct CompositeCommit {
117    version: u64,
118    component_digest: String,
119}
120
121#[derive(Clone, Serialize, Deserialize)]
122struct CompositeAcknowledgement {
123    version: u64,
124    component_digest: String,
125}
126
127#[derive(Clone, Serialize, Deserialize)]
128struct CompositeLedgerPair {
129    kid: String,
130    ml_dsa_public: String,
131    ed25519_public: String,
132    role: String,
133    state: String,
134    not_before: i64,
135    expires_at: i64,
136}
137
138fn composite_ledger_path(secrets_dir: &Path) -> PathBuf {
139    secrets_dir.join("jwt-composite-pairs.json")
140}
141
142fn composite_committed_path(dir: &Path) -> PathBuf {
143    dir.join("jwt-composite-pairs.committed")
144}
145
146/// An existing composite commit marker means this store already has signing
147/// authority. Missing/unreadable rotation slots must not be mistaken for a
148/// first boot: generating replacement Ed/PQ keys would mutate the component
149/// state while the committed ledger still names the previous pair. Treat any
150/// marker entry (including a malformed or unreadable one) as committed so the
151/// normal restore path fails closed without creating keys.
152fn has_committed_composite_marker(dir: &Path) -> bool {
153    !matches!(
154        std::fs::symlink_metadata(composite_committed_path(dir)),
155        Err(error) if error.kind() == std::io::ErrorKind::NotFound
156    )
157}
158fn composite_committed_ledger_path(dir: &Path, commit: &CompositeCommit) -> PathBuf {
159    dir.join(format!(
160        "jwt-composite-pairs.committed-{}-{}.json",
161        commit.version, commit.component_digest
162    ))
163}
164fn composite_ledger_lock_path(dir: &Path) -> PathBuf {
165    dir.join("jwt-composite-pairs.lock")
166}
167fn composite_writer_lock_path(dir: &Path) -> PathBuf {
168    dir.join("jwt-composite-pairs.writer.lock")
169}
170fn composite_subscribers_dir(dir: &Path) -> PathBuf {
171    dir.join("jwt-composite-subscribers")
172}
173
174static COMPOSITE_SUBSCRIPTION_STARTED: AtomicBool = AtomicBool::new(false);
175static COMPOSITE_PUBLISHING: AtomicBool = AtomicBool::new(false);
176static COMPOSITE_CA_SIGNING_KEY: std::sync::OnceLock<parking_lot::RwLock<Option<Arc<SigningKey>>>> =
177    std::sync::OnceLock::new();
178fn composite_ca_signing_key() -> &'static parking_lot::RwLock<Option<Arc<SigningKey>>> {
179    COMPOSITE_CA_SIGNING_KEY.get_or_init(|| parking_lot::RwLock::new(None))
180}
181fn configure_composite_authority(dir: &Path) {
182    hyprstream_rpc::auth::global_composite_key_set().configure_authority(
183        composite_ledger_path(dir),
184        composite_committed_path(dir),
185        dir.join("jwt-composite-pairs.committed"),
186        composite_ledger_lock_path(dir),
187    );
188}
189
190/// Load the immutable ledger selected by the commit marker. The fallback is
191/// only for ledgers written before immutable committed snapshots were added,
192/// and is safe only when the mutable ledger still exactly matches the marker.
193fn read_committed_composite_ledger(
194    dir: &Path,
195) -> anyhow::Result<(CompositeCommit, CompositeLedger)> {
196    let commit: CompositeCommit =
197        serde_json::from_slice(&std::fs::read(composite_committed_path(dir))?)?;
198    let ledger = read_composite_ledger_selected_by_commit(dir, &commit)?;
199    Ok((commit, ledger))
200}
201
202fn read_composite_ledger_selected_by_commit(
203    dir: &Path,
204    commit: &CompositeCommit,
205) -> anyhow::Result<CompositeLedger> {
206    let immutable = composite_committed_ledger_path(dir, commit);
207    let ledger: CompositeLedger = match std::fs::read(&immutable) {
208        Ok(bytes) => serde_json::from_slice(&bytes)?,
209        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
210            match std::fs::symlink_metadata(&immutable) {
211                Err(metadata_error) if metadata_error.kind() == std::io::ErrorKind::NotFound => {
212                    serde_json::from_slice(&std::fs::read(composite_ledger_path(dir))?)?
213                }
214                Ok(_) => return Err(error.into()),
215                Err(metadata_error) => return Err(metadata_error.into()),
216            }
217        }
218        Err(error) => return Err(error.into()),
219    };
220    anyhow::ensure!(
221        ledger.version == commit.version
222            && !ledger.component_digest.is_empty()
223            && ledger.component_digest == commit.component_digest,
224        "committed composite ledger does not match its commit marker"
225    );
226    Ok(ledger)
227}
228
229/// Load the marker-selected authority while the caller holds the exclusive
230/// ledger lock. Only an absent marker means first bootstrap. A legacy marker
231/// whose matching generation still lives in the mutable ledger is migrated to
232/// an immutable snapshot before the lock can be released and a publisher can
233/// replace the mutable file.
234fn load_or_migrate_committed_composite_ledger(
235    dir: &Path,
236) -> anyhow::Result<Option<CompositeLedger>> {
237    let marker_path = composite_committed_path(dir);
238    let marker_bytes = match std::fs::read(&marker_path) {
239        Ok(bytes) => bytes,
240        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
241            match std::fs::symlink_metadata(&marker_path) {
242                Err(metadata_error) if metadata_error.kind() == std::io::ErrorKind::NotFound => {
243                    return Ok(None);
244                }
245                Ok(_) => return Err(error.into()),
246                Err(metadata_error) => return Err(metadata_error.into()),
247            }
248        }
249        Err(error) => return Err(error.into()),
250    };
251    let commit: CompositeCommit = serde_json::from_slice(&marker_bytes)?;
252    let immutable = composite_committed_ledger_path(dir, &commit);
253    let immutable_missing = match std::fs::symlink_metadata(&immutable) {
254        Ok(_) => false,
255        Err(error) if error.kind() == std::io::ErrorKind::NotFound => true,
256        Err(error) => return Err(error.into()),
257    };
258    let ledger = read_composite_ledger_selected_by_commit(dir, &commit)?;
259    let name = immutable
260        .file_name()
261        .and_then(|name| name.to_str())
262        .ok_or_else(|| anyhow::anyhow!("invalid committed composite ledger path"))?;
263    if immutable_missing {
264        super::identity_store::write_secret_durable(dir, name, &serde_json::to_vec(&ledger)?)?;
265    } else {
266        super::identity_store::sync_secret_durable(dir, name)?;
267    }
268    // An earlier attempt can leave either rename visible after its directory
269    // sync failed. Under the exclusive ledger lock, establish both barriers
270    // before restoring authority or replacing mutable staging. Existence alone
271    // does not establish durability, including on a retry after uncertain I/O.
272    super::identity_store::sync_secret_durable(dir, "jwt-composite-pairs.committed")?;
273    Ok(Some(ledger))
274}
275
276fn digest_component_slot(
277    digest: &mut Sha256,
278    family: &[u8],
279    name: &[u8],
280    public: Option<&[u8]>,
281    validity: Option<(i64, i64)>,
282) {
283    digest.update((family.len() as u64).to_be_bytes());
284    digest.update(family);
285    digest.update((name.len() as u64).to_be_bytes());
286    digest.update(name);
287    match (public, validity) {
288        (Some(public), Some((nbf, exp))) => {
289            digest.update([1]);
290            digest.update((public.len() as u64).to_be_bytes());
291            digest.update(public);
292            digest.update(nbf.to_be_bytes());
293            digest.update(exp.to_be_bytes());
294        }
295        _ => digest.update([0]),
296    }
297}
298
299fn component_state_digest_from_slots(
300    ed: &KeySlots,
301    pq: &ml_dsa_rotation::MlDsaKeySlots,
302    ca_key: ed25519_dalek::VerifyingKey,
303) -> String {
304    let mut digest = Sha256::new();
305    digest.update(b"hyprstream-composite-component-state-v1");
306    let ed_drain = ed.drain.as_ref().map(KeySlot::verifying_key_bytes);
307    let ed_active = ed.active.as_ref().map(KeySlot::verifying_key_bytes);
308    let ed_lead = ed.lead.as_ref().map(KeySlot::verifying_key_bytes);
309    digest_component_slot(
310        &mut digest,
311        b"ed25519",
312        b"drain",
313        ed_drain.as_ref().map(<[u8; 32]>::as_slice),
314        ed.drain.as_ref().map(|slot| (slot.nbf, slot.exp)),
315    );
316    digest_component_slot(
317        &mut digest,
318        b"ed25519",
319        b"active",
320        ed_active.as_ref().map(<[u8; 32]>::as_slice),
321        ed.active.as_ref().map(|slot| (slot.nbf, slot.exp)),
322    );
323    digest_component_slot(
324        &mut digest,
325        b"ed25519",
326        b"lead",
327        ed_lead.as_ref().map(<[u8; 32]>::as_slice),
328        ed.lead.as_ref().map(|slot| (slot.nbf, slot.exp)),
329    );
330    for (name, slot) in [
331        (b"drain".as_slice(), pq.drain.as_ref()),
332        (b"active".as_slice(), pq.active.as_ref()),
333        (b"lead".as_slice(), pq.lead.as_ref()),
334    ] {
335        let public =
336            slot.map(|slot| hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&slot.verifying_key()));
337        digest_component_slot(
338            &mut digest,
339            b"ml-dsa-65",
340            name,
341            public.as_deref(),
342            slot.map(|slot| (slot.nbf, slot.exp)),
343        );
344    }
345    digest_component_slot(
346        &mut digest,
347        b"ed25519",
348        b"policy-ca",
349        Some(&ca_key.to_bytes()),
350        Some((i64::MIN, i64::MAX)),
351    );
352    URL_SAFE_NO_PAD.encode(digest.finalize())
353}
354
355/// Cheaply detect a component-key change before deciding that a rejected
356/// proposal is still stale. Slot persistence uses atomic file replacement, so
357/// identity plus modification time detects a new key without repeatedly
358/// deserializing private ML-DSA material on the verifier hot path.
359fn fingerprint_component_slot_files(
360    secrets_dir: &Path,
361    fingerprint: &mut Sha256,
362) -> anyhow::Result<()> {
363    use std::os::unix::fs::MetadataExt;
364    let state_dir = rotation_state_dir(secrets_dir);
365    for dir in [secrets_dir, state_dir.as_path()] {
366        for family in ["jwt-signing-key", "ml-dsa-signing-key"] {
367            for slot in ["drain", "active", "lead"] {
368                for suffix in ["", ".meta"] {
369                    let path = dir.join(format!("{family}.{slot}{suffix}"));
370                    match std::fs::metadata(&path) {
371                        Ok(metadata) => {
372                            fingerprint.update([1]);
373                            fingerprint.update(metadata.dev().to_be_bytes());
374                            fingerprint.update(metadata.ino().to_be_bytes());
375                            fingerprint.update(metadata.len().to_be_bytes());
376                            fingerprint.update(metadata.mtime().to_be_bytes());
377                            fingerprint.update(metadata.mtime_nsec().to_be_bytes());
378                        }
379                        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
380                            fingerprint.update([0]);
381                        }
382                        Err(error) => return Err(error.into()),
383                    }
384                }
385            }
386        }
387    }
388    Ok(())
389}
390
391async fn component_state_digest(
392    ed_store: &SigningKeyStore,
393    pq_store: &MlDsaSigningKeyStore,
394    ca_key: ed25519_dalek::VerifyingKey,
395) -> String {
396    let ed = ed_store.0.read().await;
397    let pq = pq_store.0.read().await;
398    component_state_digest_from_slots(&ed, &pq, ca_key)
399}
400
401/// Restore the public exact-pair ledger in verifier-only service processes.
402pub async fn restore_composite_verifying_key_set(
403    secrets_dir: &Path,
404    _ed_store: &SigningKeyStore,
405    _ml_dsa_store: &MlDsaSigningKeyStore,
406    _ca_key: ed25519_dalek::VerifyingKey,
407) -> anyhow::Result<()> {
408    use hyprstream_rpc::auth::{CompositeKeyPair, CompositePairRole, CompositePairState};
409
410    configure_composite_authority(secrets_dir);
411    #[cfg(not(test))]
412    start_composite_authority_subscription(secrets_dir.to_path_buf(), _ca_key)?;
413    use nix::fcntl::{flock, FlockArg};
414    use std::os::fd::AsRawFd;
415    let lock = std::fs::OpenOptions::new()
416        .create(true)
417        .truncate(false)
418        .read(true)
419        .write(true)
420        .open(composite_ledger_lock_path(secrets_dir))?;
421    flock(lock.as_raw_fd(), FlockArg::LockShared)?;
422    let (_, ledger) = read_committed_composite_ledger(secrets_dir)?;
423    let now = chrono::Utc::now().timestamp();
424    let mut pairs = Vec::new();
425    for record in ledger.pairs {
426        if record.expires_at <= now {
427            continue;
428        }
429        let pq_bytes = URL_SAFE_NO_PAD.decode(&record.ml_dsa_public)?;
430        let pq = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&pq_bytes)?;
431        let ed_bytes: [u8; 32] = URL_SAFE_NO_PAD
432            .decode(&record.ed25519_public)?
433            .try_into()
434            .map_err(|_| anyhow::anyhow!("invalid committed Ed25519 key length"))?;
435        let ed = ed25519_dalek::VerifyingKey::from_bytes(&ed_bytes)?;
436        pairs.push(CompositeKeyPair::verifying(
437            record.kid,
438            pq,
439            ed,
440            if record.role == "policy" {
441                CompositePairRole::Policy
442            } else {
443                CompositePairRole::OAuth
444            },
445            if record.state == "active" {
446                CompositePairState::Active
447            } else {
448                CompositePairState::Drain
449            },
450            record.not_before,
451            record.expires_at,
452        ));
453    }
454    let key_set = hyprstream_rpc::auth::global_composite_key_set();
455    if ledger.version > key_set.snapshot().version() {
456        key_set.publish(ledger.version, ledger.component_digest, pairs)?;
457    }
458    Ok(())
459}
460
461fn start_composite_authority_subscription(
462    secrets_dir: PathBuf,
463    ca_key: ed25519_dalek::VerifyingKey,
464) -> anyhow::Result<()> {
465    if COMPOSITE_SUBSCRIPTION_STARTED.swap(true, Ordering::AcqRel) {
466        return Ok(());
467    }
468    configure_composite_authority(&secrets_dir);
469    let subscribers = composite_subscribers_dir(&secrets_dir);
470    std::fs::create_dir_all(&subscribers)?;
471    let name = format!("{}-{}", std::process::id(), uuid::Uuid::new_v4());
472    super::identity_store::write_secret(&subscribers, &name, b"0")?;
473    std::thread::Builder::new()
474        .name("composite-authority".into())
475        .spawn(move || {
476            use nix::fcntl::{flock, FlockArg};
477            use std::os::fd::AsRawFd;
478            let mut last_error: Option<String> = None;
479            let mut rejected_pending: Option<([u8; 32], std::time::Instant)> = None;
480            loop {
481                if COMPOSITE_PUBLISHING.load(Ordering::Acquire) {
482                    std::thread::sleep(std::time::Duration::from_millis(10));
483                    continue;
484                }
485                let load = (|| -> anyhow::Result<()> {
486                    let lock = std::fs::OpenOptions::new()
487                        .create(true)
488                        .truncate(false)
489                        .read(true)
490                        .write(true)
491                        .open(composite_ledger_lock_path(&secrets_dir))?;
492                    flock(lock.as_raw_fd(), FlockArg::LockShared)?;
493                    let pending_bytes = std::fs::read(composite_ledger_path(&secrets_dir))?;
494                    let marker_bytes = std::fs::read(composite_committed_path(&secrets_dir))?;
495                    let mut fingerprint = Sha256::new();
496                    fingerprint.update(&pending_bytes);
497                    fingerprint.update(&marker_bytes);
498                    fingerprint_component_slot_files(&secrets_dir, &mut fingerprint)?;
499                    let fingerprint: [u8; 32] = fingerprint.finalize().into();
500                    if rejected_pending.as_ref().is_some_and(|(previous, at)| {
501                        *previous == fingerprint
502                            && at.elapsed() < std::time::Duration::from_secs(60)
503                    }) {
504                        // Neither the proposal nor its commit marker changed.
505                        // Retain the already-published authority without
506                        // repeatedly deserializing private signing keys.
507                        return Ok(());
508                    }
509                    let pending: CompositeLedger = serde_json::from_slice(&pending_bytes)?;
510                    let ed = load_or_init_key_store(&secrets_dir, &OAuthConfig::default());
511                    let pq = load_or_init_ml_dsa_key_store(&secrets_dir, &OAuthConfig::default());
512                    // Acknowledgement means only that the proposed generation is
513                    // completely loadable. It must not become live authority until
514                    // the matching commit marker is durable.
515                    let local_digest = {
516                        let ed_slots = ed.0.blocking_read();
517                        let pq_slots = pq.0.blocking_read();
518                        component_state_digest_from_slots(&ed_slots, &pq_slots, ca_key)
519                    };
520                    if pending.component_digest != local_digest {
521                        rejected_pending = Some((fingerprint, std::time::Instant::now()));
522                        anyhow::bail!(
523                            "pending composite ledger does not match local component authority"
524                        );
525                    }
526                    rejected_pending = None;
527                    let _staged = ledger_pairs_from_local_keys(&pending, &ed, &pq, ca_key, true)?;
528                    super::identity_store::write_secret(
529                        &subscribers,
530                        &name,
531                        &serde_json::to_vec(&CompositeAcknowledgement {
532                            version: pending.version,
533                            component_digest: pending.component_digest,
534                        })?,
535                    )?;
536                    let (_, committed) = read_committed_composite_ledger(&secrets_dir)?;
537                    let key_set = hyprstream_rpc::auth::global_composite_key_set();
538                    if committed.version > key_set.snapshot().version()
539                        || (committed.version == key_set.snapshot().version()
540                            && committed.component_digest != key_set.snapshot().component_digest())
541                    {
542                        key_set.publish(
543                            committed.version,
544                            committed.component_digest.clone(),
545                            ledger_pairs_from_local_keys(&committed, &ed, &pq, ca_key, false)?,
546                        )?;
547                    }
548                    Ok(())
549                })();
550                match load {
551                    Ok(()) => {
552                        last_error = None;
553                        std::thread::sleep(std::time::Duration::from_millis(25));
554                    }
555                    Err(error) => {
556                        let message = format!("{error:#}");
557                        if last_error.as_deref() != Some(&message) {
558                            tracing::warn!("composite authority reload failed closed: {message}");
559                            last_error = Some(message);
560                        }
561                        // A stale or interrupted pending generation cannot become
562                        // live authority. Retrying its expensive key-store load
563                        // every 25 ms can exhaust a small verifier host before a
564                        // publisher has a chance to replace the pending ledger.
565                        std::thread::sleep(std::time::Duration::from_secs(1));
566                    }
567                }
568            }
569        })?;
570    Ok(())
571}
572
573fn ledger_pairs_from_local_keys(
574    ledger: &CompositeLedger,
575    ed_store: &SigningKeyStore,
576    pq_store: &MlDsaSigningKeyStore,
577    ca_key: ed25519_dalek::VerifyingKey,
578    require_local_pairs: bool,
579) -> anyhow::Result<Vec<hyprstream_rpc::auth::CompositeKeyPair>> {
580    use hyprstream_rpc::auth::{CompositeKeyPair, CompositePairRole, CompositePairState};
581    let ed_slots = ed_store.0.blocking_read();
582    let pq_slots = pq_store.0.blocking_read();
583    let ca_signing = composite_ca_signing_key().read().clone();
584    let now = chrono::Utc::now().timestamp();
585    ledger
586        .pairs
587        .iter()
588        .filter(|r| r.expires_at > now)
589        .map(|r| {
590            let pq_bytes = URL_SAFE_NO_PAD.decode(&r.ml_dsa_public)?;
591            let pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&pq_bytes)?;
592            let ed_bytes: [u8; 32] = URL_SAFE_NO_PAD
593                .decode(&r.ed25519_public)?
594                .try_into()
595                .map_err(|_| anyhow::anyhow!("invalid Ed25519 key length"))?;
596            let ed_vk = ed25519_dalek::VerifyingKey::from_bytes(&ed_bytes)?;
597            let role = if r.role == "policy" {
598                CompositePairRole::Policy
599            } else {
600                CompositePairRole::OAuth
601            };
602            let state = if r.state == "active" {
603                CompositePairState::Active
604            } else {
605                CompositePairState::Drain
606            };
607            let pq_signing = pq_slots
608                .all()
609                .into_iter()
610                .find(|s| {
611                    hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&s.verifying_key()) == pq_bytes
612                })
613                .map(|s| Arc::clone(&s.key));
614            let ed_signing = if ca_key == ed_vk {
615                ca_signing.clone()
616            } else {
617                ed_slots
618                    .all()
619                    .into_iter()
620                    .find(|s| s.key.verifying_key() == ed_vk)
621                    .map(|s| Arc::clone(&s.key))
622            };
623            if require_local_pairs {
624                anyhow::ensure!(
625                    pq_signing.is_some(),
626                    "composite PQ key is absent from the pending component generation"
627                );
628                anyhow::ensure!(
629                    ca_key == ed_vk || ed_signing.is_some(),
630                    "composite Ed25519 key is absent from the pending component generation"
631                );
632            }
633            Ok(match (pq_signing, ed_signing) {
634                (Some(pq), Some(ed)) => CompositeKeyPair::signing(
635                    r.kid.clone(),
636                    pq,
637                    ed,
638                    role,
639                    state,
640                    r.not_before,
641                    r.expires_at,
642                ),
643                _ => CompositeKeyPair::verifying(
644                    r.kid.clone(),
645                    pq_vk,
646                    ed_vk,
647                    role,
648                    state,
649                    r.not_before,
650                    r.expires_at,
651                ),
652            })
653        })
654        .collect()
655}
656
657/// Restore exact persisted associations and atomically publish the complete
658/// signing/verifying/JWKS authority. Component stores are used only to resolve
659/// identities recorded in the ledger; they are never cross-paired.
660pub async fn initialize_composite_key_set(
661    secrets_dir: &Path,
662    ed_store: &SigningKeyStore,
663    ml_dsa_store: &MlDsaSigningKeyStore,
664    ca_key: Arc<SigningKey>,
665    drain_secs: i64,
666) -> anyhow::Result<()> {
667    publish_composite_key_set(
668        secrets_dir,
669        ed_store,
670        ml_dsa_store,
671        ca_key,
672        drain_secs,
673        true,
674        None,
675    )
676    .await
677}
678
679/// Publish the post-rotation lifecycle as one atomic exact-pair snapshot.
680pub async fn refresh_composite_key_set(
681    secrets_dir: &Path,
682    ed_store: &SigningKeyStore,
683    ml_dsa_store: &MlDsaSigningKeyStore,
684    ca_key: Arc<SigningKey>,
685    drain_secs: i64,
686    expected_component_digest: &str,
687) -> anyhow::Result<()> {
688    publish_composite_key_set(
689        secrets_dir,
690        ed_store,
691        ml_dsa_store,
692        ca_key,
693        drain_secs,
694        false,
695        Some(expected_component_digest.to_owned()),
696    )
697    .await
698}
699
700async fn publish_composite_key_set(
701    secrets_dir: &Path,
702    ed_store: &SigningKeyStore,
703    ml_dsa_store: &MlDsaSigningKeyStore,
704    ca_key: Arc<SigningKey>,
705    drain_secs: i64,
706    restore: bool,
707    expected_component_digest: Option<String>,
708) -> anyhow::Result<()> {
709    use hyprstream_rpc::auth::{CompositeKeyPair, CompositePairRole, CompositePairState};
710    use nix::fcntl::{flock, FlockArg};
711    use std::os::fd::AsRawFd;
712    struct Guard;
713    impl Drop for Guard {
714        fn drop(&mut self) {
715            COMPOSITE_PUBLISHING.store(false, Ordering::Release);
716        }
717    }
718    configure_composite_authority(secrets_dir);
719    *composite_ca_signing_key().write() = Some(Arc::clone(&ca_key));
720    let writer = std::fs::OpenOptions::new()
721        .create(true)
722        .truncate(false)
723        .read(true)
724        .write(true)
725        .open(composite_writer_lock_path(secrets_dir))?;
726    flock(writer.as_raw_fd(), FlockArg::LockExclusive)?;
727    COMPOSITE_PUBLISHING.store(true, Ordering::Release);
728    let _guard = Guard;
729
730    let now = chrono::Utc::now().timestamp();
731    let ed_component_slots = ed_store.0.read().await.clone();
732    let pq_component_slots = ml_dsa_store.0.read().await.clone();
733    let ed_slots: Vec<_> = ed_component_slots.all().into_iter().cloned().collect();
734    let pq_slots: Vec<_> = pq_component_slots.all().into_iter().cloned().collect();
735    let proposed_component_digest = component_state_digest_from_slots(
736        &ed_component_slots,
737        &pq_component_slots,
738        ca_key.verifying_key(),
739    );
740    let key_set = hyprstream_rpc::auth::global_composite_key_set();
741    let current = key_set.snapshot();
742    let ledger_lock = std::fs::OpenOptions::new()
743        .create(true)
744        .truncate(false)
745        .read(true)
746        .write(true)
747        .open(composite_ledger_lock_path(secrets_dir))?;
748    flock(ledger_lock.as_raw_fd(), FlockArg::LockExclusive)?;
749    let persisted = load_or_migrate_committed_composite_ledger(secrets_dir)?;
750    let restoring_committed = restore && persisted.is_some();
751    let persisted_version = persisted.as_ref().map_or(0, |ledger| ledger.version);
752    let persisted_digest = persisted
753        .as_ref()
754        .map(|ledger| ledger.component_digest.clone());
755    if persisted.is_none() {
756        // First bootstrap: every required active signing component must be
757        // loadable before any pending ledger, immutable snapshot, or commit
758        // marker is created. Publishing from an incomplete local signer set
759        // (a failed first-boot persistence, for example) would commit a
760        // generation with missing OAuth/Policy active pairs — and the commit
761        // marker then suppresses first-boot key generation on every restart,
762        // so a transient failure would require manual state recovery. With an
763        // existing commit marker this branch is skipped and the committed
764        // fail-closed behavior is unchanged.
765        let ed_active = ed_component_slots.active.is_some();
766        let pq_active = pq_component_slots.active.is_some();
767        anyhow::ensure!(
768            ed_active && pq_active,
769            "refusing to initialize composite authority from an incomplete \
770             local signer set (ed25519_active={} ml_dsa_active={}): resolve \
771             first-boot persistence and retry",
772            ed_active,
773            pq_active
774        );
775    }
776    match (&persisted, expected_component_digest.as_deref()) {
777        (Some(ledger), Some(expected)) => anyhow::ensure!(
778            !ledger.component_digest.is_empty() && ledger.component_digest == expected,
779            "stale composite component authority: expected {expected}, authoritative digest is {}",
780            ledger.component_digest
781        ),
782        (Some(ledger), None) => anyhow::ensure!(
783            restore && !ledger.component_digest.is_empty(),
784            "persisted composite authority is not restorable"
785        ),
786        (None, Some(_)) => {
787            anyhow::bail!("stale composite component authority: expected generation has no ledger")
788        }
789        (None, None) => anyhow::ensure!(restore, "composite authority is not initialized"),
790    }
791    let mut pairs = Vec::new();
792
793    for record in persisted.into_iter().flat_map(|ledger| ledger.pairs) {
794        if record.expires_at <= now {
795            continue;
796        }
797        let pq_bytes = URL_SAFE_NO_PAD.decode(&record.ml_dsa_public)?;
798        let pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&pq_bytes)?;
799        let pq_signing = pq_slots.iter().find(|slot| {
800            URL_SAFE_NO_PAD.encode(hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
801                &slot.verifying_key(),
802            )) == record.ml_dsa_public
803        });
804        let ed_bytes: [u8; 32] = URL_SAFE_NO_PAD
805            .decode(&record.ed25519_public)?
806            .try_into()
807            .map_err(|_| anyhow::anyhow!("invalid persisted Ed25519 key length"))?;
808        let ed_vk = ed25519_dalek::VerifyingKey::from_bytes(&ed_bytes)?;
809        let policy_ca_ed_matched =
810            URL_SAFE_NO_PAD.encode(ca_key.verifying_key().to_bytes()) == record.ed25519_public;
811        let ed_signing = if policy_ca_ed_matched {
812            Some(Arc::clone(&ca_key))
813        } else {
814            ed_slots
815                .iter()
816                .find(|slot| {
817                    URL_SAFE_NO_PAD.encode(slot.key.verifying_key().to_bytes())
818                        == record.ed25519_public
819                })
820                .map(|slot| Arc::clone(&slot.key))
821        };
822        let role = if record.role == "policy" {
823            CompositePairRole::Policy
824        } else {
825            CompositePairRole::OAuth
826        };
827        let state = if restoring_committed && record.state == "active" {
828            CompositePairState::Active
829        } else {
830            CompositePairState::Drain
831        };
832        // A refresh carries forward only committed pairs whose private
833        // components remain in the local drain/active/lead authority. Restore
834        // may retain verification-only drains from the immutable ledger.
835        if !restoring_committed && (pq_signing.is_none() || ed_signing.is_none()) {
836            continue;
837        }
838        if restoring_committed && state == CompositePairState::Active {
839            anyhow::ensure!(
840                pq_signing.is_some() && ed_signing.is_some(),
841                "committed active composite signing pair is unavailable after restart: \
842                 kid={:?} role={:?} ed_component_available={} pq_component_available={} \
843                 policy_ca_ed_matched={}",
844                record.kid,
845                record.role,
846                ed_signing.is_some(),
847                pq_signing.is_some(),
848                policy_ca_ed_matched,
849            );
850        }
851        pairs.push(match (pq_signing, ed_signing) {
852            (Some(pq), Some(ed)) => CompositeKeyPair::signing(
853                record.kid,
854                Arc::clone(&pq.key),
855                ed,
856                role,
857                state,
858                record.not_before,
859                record.expires_at,
860            ),
861            _ => CompositeKeyPair::verifying(
862                record.kid,
863                pq_vk,
864                ed_vk,
865                role,
866                state,
867                record.not_before,
868                record.expires_at,
869            ),
870        });
871    }
872
873    if restoring_committed {
874        anyhow::ensure!(
875            [CompositePairRole::OAuth, CompositePairRole::Policy]
876                .into_iter()
877                .all(|role| pairs.iter().any(|pair| {
878                    pair.role() == role
879                        && pair.state() == CompositePairState::Active
880                        && pair.signing_keys().is_some()
881                })),
882            "committed active composite signing authority is incomplete"
883        );
884        let digest = persisted_digest.ok_or_else(|| {
885            anyhow::anyhow!("committed composite digest disappeared during restore")
886        })?;
887        if persisted_version > current.version() {
888            key_set.publish(persisted_version, digest, pairs)?;
889        } else {
890            // Standalone OAuth first installs this generation as a verifier in
891            // main. Restoring its local signer must attach the exact private
892            // handles without inventing a new ledger generation. Release our
893            // exclusive ledger lock before the attachment rechecks the commit
894            // under its own shared lock; the writer lock remains held.
895            drop(ledger_lock);
896            key_set.attach_committed_signing_keys(persisted_version, &digest, pairs)?;
897        }
898        #[cfg(not(test))]
899        start_composite_authority_subscription(secrets_dir.to_path_buf(), ca_key.verifying_key())?;
900        return Ok(());
901    }
902
903    let active_pq_key = ml_dsa_store.active_key().await;
904    let active_pq = active_pq_key.as_ref().and_then(|active| {
905        let active_vk = ml_dsa::Keypair::verifying_key(&**active);
906        pq_slots
907            .iter()
908            .find(|slot| {
909                hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&active_vk)
910                    == hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&slot.verifying_key())
911            })
912            .cloned()
913    });
914    let active_ed = ed_store.active_key().await;
915    if let Some(pq) = active_pq {
916        if let Some(ed) = active_ed {
917            upsert_active_pair(
918                &mut pairs,
919                Arc::clone(&pq.key),
920                ed,
921                CompositePairRole::OAuth,
922                pq.nbf,
923                pq.exp + drain_secs,
924            );
925        }
926        upsert_active_pair(
927            &mut pairs,
928            pq.key,
929            Arc::clone(&ca_key),
930            CompositePairRole::Policy,
931            pq.nbf,
932            pq.exp + drain_secs,
933        );
934    }
935
936    let version = current.version().max(persisted_version).saturating_add(1);
937    let ledger = CompositeLedger {
938        version,
939        component_digest: proposed_component_digest.clone(),
940        pairs: pairs.iter().map(ledger_record).collect(),
941    };
942    super::identity_store::write_secret(
943        secrets_dir,
944        "jwt-composite-pairs.json",
945        &serde_json::to_vec(&ledger)?,
946    )?;
947    #[cfg(test)]
948    if let Some(marker) = std::env::var_os("HYPRSTREAM_COMPOSITE_STAGE_MARKER") {
949        super::identity_store::write_secret(
950            Path::new(&marker).parent().ok_or_else(|| {
951                anyhow::anyhow!("composite stage marker must have a parent directory")
952            })?,
953            Path::new(&marker)
954                .file_name()
955                .and_then(|name| name.to_str())
956                .ok_or_else(|| anyhow::anyhow!("invalid composite stage marker"))?,
957            b"1",
958        )?;
959    }
960    #[cfg(test)]
961    if std::env::var_os("HYPRSTREAM_COMPOSITE_CRASH_AFTER_STAGE").is_some() {
962        std::process::exit(86);
963    }
964    drop(ledger_lock);
965    wait_for_composite_acknowledgements(secrets_dir, version, &proposed_component_digest)?;
966    let ledger_lock = std::fs::OpenOptions::new()
967        .create(true)
968        .truncate(false)
969        .read(true)
970        .write(true)
971        .open(composite_ledger_lock_path(secrets_dir))?;
972    flock(ledger_lock.as_raw_fd(), FlockArg::LockExclusive)?;
973    let pending: CompositeLedger =
974        serde_json::from_slice(&std::fs::read(composite_ledger_path(secrets_dir))?)?;
975    anyhow::ensure!(
976        pending.version == version && pending.component_digest == proposed_component_digest,
977        "composite publication changed before commit"
978    );
979    persist_committed_composite_ledger(secrets_dir, &pending)?;
980    if version > key_set.snapshot().version() {
981        key_set.publish(version, proposed_component_digest, pairs)?;
982    }
983    #[cfg(not(test))]
984    start_composite_authority_subscription(secrets_dir.to_path_buf(), ca_key.verifying_key())?;
985    Ok(())
986}
987
988fn persist_committed_composite_ledger(dir: &Path, ledger: &CompositeLedger) -> anyhow::Result<()> {
989    let commit = CompositeCommit {
990        version: ledger.version,
991        component_digest: ledger.component_digest.clone(),
992    };
993    // Keep every committed generation immutable. A restart racing a later
994    // pending publication can therefore restore the marker-selected snapshot
995    // instead of either installing pending authority or losing the old one.
996    let committed_name = composite_committed_ledger_path(dir, &commit)
997        .file_name()
998        .and_then(|name| name.to_str())
999        .ok_or_else(|| anyhow::anyhow!("invalid committed composite ledger path"))?
1000        .to_owned();
1001    // The immutable file's data and rename must be durable before the marker
1002    // can select it. Then make the marker's own data and rename durable before
1003    // reporting publication success or installing the in-memory authority.
1004    super::identity_store::write_secret_durable(
1005        dir,
1006        &committed_name,
1007        &serde_json::to_vec(ledger)?,
1008    )?;
1009    super::identity_store::write_secret_durable(
1010        dir,
1011        "jwt-composite-pairs.committed",
1012        &serde_json::to_vec(&commit)?,
1013    )?;
1014    Ok(())
1015}
1016
1017fn wait_for_composite_acknowledgements(
1018    dir: &Path,
1019    version: u64,
1020    component_digest: &str,
1021) -> anyhow::Result<()> {
1022    let subscribers = composite_subscribers_dir(dir);
1023    std::fs::create_dir_all(&subscribers)?;
1024    let own = format!("{}-", std::process::id());
1025    let required: Vec<PathBuf> = std::fs::read_dir(&subscribers)?
1026        .filter_map(Result::ok)
1027        .map(|e| e.path())
1028        .filter(|path| {
1029            let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
1030                return false;
1031            };
1032            if name.starts_with(&own) {
1033                return false;
1034            }
1035            let pid = name
1036                .split_once('-')
1037                .and_then(|(p, _)| p.parse::<u32>().ok());
1038            if pid.is_some_and(|p| !composite_subscriber_alive(p)) {
1039                let _ = std::fs::remove_file(path);
1040                return false;
1041            }
1042            true
1043        })
1044        .collect();
1045    let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
1046    loop {
1047        if required.iter().all(|p| {
1048            std::fs::read(p)
1049                .ok()
1050                .and_then(|bytes| serde_json::from_slice::<CompositeAcknowledgement>(&bytes).ok())
1051                .is_some_and(|ack| {
1052                    ack.version == version && ack.component_digest == component_digest
1053                })
1054        }) {
1055            return Ok(());
1056        }
1057        anyhow::ensure!(
1058            std::time::Instant::now() < deadline,
1059            "composite generation {version} was not acknowledged by all live service processes"
1060        );
1061        std::thread::sleep(std::time::Duration::from_millis(10));
1062    }
1063}
1064
1065fn composite_subscriber_alive(pid: u32) -> bool {
1066    use nix::errno::Errno;
1067    use nix::sys::signal::kill;
1068    use nix::unistd::Pid;
1069    !matches!(kill(Pid::from_raw(pid as i32), None), Err(Errno::ESRCH))
1070}
1071
1072fn upsert_active_pair(
1073    pairs: &mut Vec<hyprstream_rpc::auth::CompositeKeyPair>,
1074    ml_dsa: Arc<hyprstream_rpc::crypto::pq::MlDsaSigningKey>,
1075    ed25519: Arc<SigningKey>,
1076    role: hyprstream_rpc::auth::CompositePairRole,
1077    not_before: i64,
1078    expires_at: i64,
1079) {
1080    let ml_vk = ml_dsa::Keypair::verifying_key(&*ml_dsa).clone();
1081    let kid = crate::auth::jwt::composite_kid(&ml_vk, &ed25519.verifying_key());
1082    pairs.retain(|pair| {
1083        pair.kid() != kid
1084            && !(pair.role() == role
1085                && pair.state() == hyprstream_rpc::auth::CompositePairState::Active)
1086    });
1087    pairs.push(hyprstream_rpc::auth::CompositeKeyPair::signing(
1088        kid,
1089        ml_dsa,
1090        ed25519,
1091        role,
1092        hyprstream_rpc::auth::CompositePairState::Active,
1093        not_before,
1094        expires_at,
1095    ));
1096}
1097
1098fn ledger_record(pair: &hyprstream_rpc::auth::CompositeKeyPair) -> CompositeLedgerPair {
1099    CompositeLedgerPair {
1100        kid: pair.kid().to_owned(),
1101        ml_dsa_public: URL_SAFE_NO_PAD
1102            .encode(hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(pair.ml_dsa())),
1103        ed25519_public: URL_SAFE_NO_PAD.encode(pair.ed25519().to_bytes()),
1104        role: if pair.role() == hyprstream_rpc::auth::CompositePairRole::Policy {
1105            "policy"
1106        } else {
1107            "oauth"
1108        }
1109        .to_owned(),
1110        state: if pair.state() == hyprstream_rpc::auth::CompositePairState::Active {
1111            "active"
1112        } else {
1113            "drain"
1114        }
1115        .to_owned(),
1116        not_before: pair.not_before(),
1117        expires_at: pair.expires_at(),
1118    }
1119}
1120
1121/// Global ML-DSA signing key store singleton.
1122///
1123/// Ensures all services (PolicyService, OAuthService, rotation task) share
1124/// the same store instance — rotation applies universally.
1125static ML_DSA_SIGNING_STORE: std::sync::OnceLock<Arc<MlDsaSigningKeyStore>> =
1126    std::sync::OnceLock::new();
1127static ED25519_SIGNING_STORE: std::sync::OnceLock<Arc<SigningKeyStore>> =
1128    std::sync::OnceLock::new();
1129
1130/// Get or initialize the process-wide Ed25519 rotation store.
1131pub fn global_ed25519_key_store(secrets_dir: &Path, config: &OAuthConfig) -> Arc<SigningKeyStore> {
1132    ED25519_SIGNING_STORE
1133        .get_or_init(|| Arc::new(load_or_init_key_store(secrets_dir, config)))
1134        .clone()
1135}
1136
1137/// Get or initialize the global ML-DSA signing key store.
1138///
1139/// First call initializes from disk; subsequent calls return the same Arc.
1140pub fn global_ml_dsa_key_store(
1141    secrets_dir: &Path,
1142    config: &OAuthConfig,
1143) -> Arc<MlDsaSigningKeyStore> {
1144    ML_DSA_SIGNING_STORE
1145        .get_or_init(|| Arc::new(load_or_init_ml_dsa_key_store(secrets_dir, config)))
1146        .clone()
1147}
1148
1149/// Global ES256 signing key store singleton.
1150static ES256_SIGNING_STORE: std::sync::OnceLock<Arc<Es256SigningKeyStore>> =
1151    std::sync::OnceLock::new();
1152
1153/// Get or initialize the global ES256 signing key store.
1154pub fn global_es256_key_store(
1155    secrets_dir: &Path,
1156    config: &OAuthConfig,
1157) -> Arc<Es256SigningKeyStore> {
1158    let store = ES256_SIGNING_STORE
1159        .get_or_init(|| Arc::new(load_or_init_es256_key_store(secrets_dir, config)))
1160        .clone();
1161    store.bind_secrets_dir(rotation_state_dir(secrets_dir));
1162    store
1163}
1164
1165// ── Key slot ────────────────────────────────────────────────────────────────
1166
1167#[derive(Clone)]
1168pub struct KeySlot {
1169    pub key: Arc<SigningKey>,
1170    /// Unix timestamp at which this key may be used for issuance (not_before).
1171    pub nbf: i64,
1172    /// Unix timestamp at which this key expires.
1173    pub exp: i64,
1174}
1175
1176impl KeySlot {
1177    pub fn new(key: SigningKey, nbf: i64, exp: i64) -> Self {
1178        Self {
1179            key: Arc::new(key),
1180            nbf,
1181            exp,
1182        }
1183    }
1184
1185    pub fn verifying_key_bytes(&self) -> [u8; 32] {
1186        self.key.verifying_key().to_bytes()
1187    }
1188
1189    pub fn kid(&self) -> String {
1190        crate::services::oauth::jwks::compute_kid(&self.verifying_key_bytes())
1191    }
1192}
1193
1194// ── Slot container ───────────────────────────────────────────────────────────
1195
1196#[derive(Clone, Default)]
1197pub struct KeySlots {
1198    pub drain: Option<KeySlot>,
1199    pub active: Option<KeySlot>,
1200    pub lead: Option<KeySlot>,
1201}
1202
1203impl KeySlots {
1204    /// All non-None slots (drain, active, lead) in that order.
1205    pub fn all(&self) -> Vec<&KeySlot> {
1206        [&self.drain, &self.active, &self.lead]
1207            .into_iter()
1208            .flatten()
1209            .collect()
1210    }
1211}
1212
1213// ── SigningKeyStore ──────────────────────────────────────────────────────────
1214
1215#[derive(Clone)]
1216pub struct SigningKeyStore(pub Arc<RwLock<KeySlots>>);
1217
1218impl SigningKeyStore {
1219    pub fn new(slots: KeySlots) -> Self {
1220        Self(Arc::new(RwLock::new(slots)))
1221    }
1222
1223    pub async fn active_key(&self) -> Option<Arc<SigningKey>> {
1224        let slots = self.0.read().await;
1225        slots.active.as_ref().map(|s| Arc::clone(&s.key))
1226    }
1227
1228    pub async fn active_verifying_key_bytes(&self) -> Option<[u8; 32]> {
1229        let slots = self.0.read().await;
1230        slots.active.as_ref().map(KeySlot::verifying_key_bytes)
1231    }
1232
1233    pub async fn all_slots_snapshot(&self) -> Vec<KeySlot> {
1234        let slots = self.0.read().await;
1235        slots.all().into_iter().cloned().collect()
1236    }
1237
1238    /// Snapshot the named drain/active/lead roles under one read guard.
1239    pub async fn slots_snapshot(&self) -> KeySlots {
1240        self.0.read().await.clone()
1241    }
1242
1243    /// Find a verifying key by kid across all slots (for token verification).
1244    pub async fn verifying_key_for_kid(&self, kid: &str) -> Option<ed25519_dalek::VerifyingKey> {
1245        let slots = self.0.read().await;
1246        for slot in slots.all() {
1247            if slot.kid() == kid {
1248                return Some(slot.key.verifying_key());
1249            }
1250        }
1251        None
1252    }
1253}
1254
1255// ── Persistence ─────────────────────────────────────────────────────────────
1256
1257/// Resolve the directory JWT rotation *state* is persisted to.
1258///
1259/// Rotation slots are runtime-mutated state, not provisioned credentials, so
1260/// they need a writable home (#803). When `secrets_dir` is writable (bare
1261/// metal, dev, non-systemd) the slots live there, as before. When it is
1262/// read-only — the systemd `$CREDENTIALS_DIRECTORY` ramfs — slots fall back to
1263/// the unit's `$STATE_DIRECTORY` (`StateDirectory=`), or to the XDG state dir
1264/// when not running under systemd. Loading always checks the state dir first
1265/// and falls back to `secrets_dir`, so credentials provisioned into the
1266/// credstore still bootstrap the first boot.
1267pub fn rotation_state_dir(secrets_dir: &Path) -> PathBuf {
1268    if super::identity_store::is_writable(secrets_dir) {
1269        return secrets_dir.to_path_buf();
1270    }
1271    // systemd: `StateDirectory=` sets $STATE_DIRECTORY to a writable,
1272    // service-private dir (colon-separated list when multiple are declared;
1273    // hyprstream units declare exactly one).
1274    let systemd_state = std::env::var("STATE_DIRECTORY").ok().and_then(|v| {
1275        let first = v.split(':').next().unwrap_or_default();
1276        if first.is_empty() {
1277            None
1278        } else {
1279            Some(PathBuf::from(first))
1280        }
1281    });
1282    let base = match systemd_state {
1283        // Namespace by instance to match the socket/config isolation.
1284        Some(dir) => match std::env::var("HYPRSTREAM_INSTANCE") {
1285            Ok(inst) if !inst.is_empty() => Some(dir.join("instances").join(inst)),
1286            _ => Some(dir),
1287        },
1288        // Not systemd: XDG state home (already instance-namespaced via the
1289        // StoragePaths prefix).
1290        None => crate::storage::paths::StoragePaths::new()
1291            .and_then(|s| s.state_dir())
1292            .ok(),
1293    };
1294    match base {
1295        Some(dir) => dir.join("credentials"),
1296        None => {
1297            error!(
1298                "secrets dir '{}' is read-only and no writable state directory \
1299                 could be resolved; JWT rotation state will NOT survive restart \
1300                 (all issued tokens are invalidated on every restart)",
1301                secrets_dir.display()
1302            );
1303            secrets_dir.to_path_buf()
1304        }
1305    }
1306}
1307
1308#[derive(Serialize, Deserialize)]
1309struct SlotMeta {
1310    nbf: i64,
1311    exp: i64,
1312}
1313
1314fn slot_paths(secrets_dir: &Path, name: &str) -> (PathBuf, PathBuf) {
1315    (
1316        secrets_dir.join(format!("jwt-signing-key.{name}")),
1317        secrets_dir.join(format!("jwt-signing-key.{name}.meta")),
1318    )
1319}
1320
1321fn load_slot(secrets_dir: &Path, name: &str) -> Option<KeySlot> {
1322    let (key_path, meta_path) = slot_paths(secrets_dir, name);
1323    let seed = std::fs::read(&key_path).ok()?;
1324    if seed.len() != 32 {
1325        warn!(
1326            "JWT key slot '{name}': unexpected seed length {}",
1327            seed.len()
1328        );
1329        return None;
1330    }
1331    let meta_bytes = std::fs::read(&meta_path).ok()?;
1332    let meta: SlotMeta = serde_json::from_slice(&meta_bytes).ok()?;
1333    let mut seed_arr = [0u8; 32];
1334    seed_arr.copy_from_slice(&seed);
1335    Some(KeySlot::new(
1336        SigningKey::from_bytes(&seed_arr),
1337        meta.nbf,
1338        meta.exp,
1339    ))
1340}
1341
1342fn persist_slot(secrets_dir: &Path, name: &str, slot: &KeySlot) -> anyhow::Result<()> {
1343    // Atomic write + 0600 perms (#179) — replaces bare std::fs::write which
1344    // used 0644 (world-readable) and was non-atomic (torn file on crash).
1345    super::identity_store::write_secret(
1346        secrets_dir,
1347        &format!("jwt-signing-key.{name}"),
1348        &slot.key.to_bytes(),
1349    )?;
1350    let meta = SlotMeta {
1351        nbf: slot.nbf,
1352        exp: slot.exp,
1353    };
1354    super::identity_store::write_secret(
1355        secrets_dir,
1356        &format!("jwt-signing-key.{name}.meta"),
1357        &serde_json::to_vec(&meta)?,
1358    )?;
1359    Ok(())
1360}
1361
1362fn delete_slot(secrets_dir: &Path, name: &str) {
1363    let (key_path, meta_path) = slot_paths(secrets_dir, name);
1364    let _ = std::fs::remove_file(&key_path);
1365    let _ = std::fs::remove_file(&meta_path);
1366}
1367
1368fn generate_slot(nbf: i64, exp: i64) -> KeySlot {
1369    let key = SigningKey::generate(&mut rand::rngs::OsRng);
1370    KeySlot::new(key, nbf, exp)
1371}
1372
1373// ── Load or initialize slots at startup ────────────────────────────────────
1374
1375/// Serialize component reads/creation with other starters and slot mutations.
1376/// A slot is two files, so atomic replacement of each file alone does not make
1377/// first boot atomic. Every contender must re-read under this lock and adopt
1378/// the durable winner, never publish a process-local losing key.
1379///
1380/// This is deliberately NOT the composite cycle/ledger lock: subscribers load
1381/// slots while holding the ledger lock. Rotators release this short-lived lock
1382/// before publication/acknowledgements, avoiding a ledger -> cycle inversion.
1383fn lock_signing_slots(state_dir: &Path, nonblocking: bool) -> std::io::Result<std::fs::File> {
1384    use nix::fcntl::{flock, FlockArg};
1385    use std::os::fd::AsRawFd;
1386    use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
1387
1388    std::fs::DirBuilder::new()
1389        .recursive(true)
1390        .mode(0o700)
1391        .create(state_dir)?;
1392    let file = std::fs::OpenOptions::new()
1393        .create(true)
1394        .truncate(false)
1395        .read(true)
1396        .write(true)
1397        .mode(0o600)
1398        .open(state_dir.join("signing-slots.lock"))?;
1399    flock(
1400        file.as_raw_fd(),
1401        if nonblocking {
1402            FlockArg::LockExclusiveNonblock
1403        } else {
1404            FlockArg::LockExclusive
1405        },
1406    )?;
1407    Ok(file)
1408}
1409
1410/// Load all three JWT key slots from `secrets_dir`.
1411///
1412/// If no active slot exists, generate one immediately (first boot) — but only
1413/// activate it when it persists; a persistence failure leaves the store
1414/// without an active key rather than returning a process-only signer.
1415/// Slot files: `jwt-signing-key.{active,drain,lead}` + `.meta` JSON.
1416pub fn load_or_init_key_store(secrets_dir: &Path, config: &OAuthConfig) -> SigningKeyStore {
1417    let now = chrono::Utc::now().timestamp();
1418    let active_secs = config.active_secs();
1419    let lead_secs = config.lead_secs();
1420
1421    // Slots are state: read/write them via the writable state dir (#803),
1422    // falling back to the (possibly read-only, provisioned) secrets dir.
1423    let state_dir = rotation_state_dir(secrets_dir);
1424    let _slot_lock = match lock_signing_slots(&state_dir, false) {
1425        Ok(lock) => lock,
1426        Err(error) => {
1427            error!("Could not lock JWT signing slots: {error}; refusing initialization");
1428            return SigningKeyStore::new(KeySlots::default());
1429        }
1430    };
1431    let has_committed_authority = has_committed_composite_marker(secrets_dir);
1432    let drain = load_slot(&state_dir, "drain").or_else(|| load_slot(secrets_dir, "drain"));
1433    let mut active = load_slot(&state_dir, "active").or_else(|| load_slot(secrets_dir, "active"));
1434    let lead = load_slot(&state_dir, "lead").or_else(|| load_slot(secrets_dir, "lead"));
1435
1436    if active.is_none() {
1437        if has_committed_authority {
1438            error!(
1439                "Committed composite authority exists but no active JWT signing slot is \
1440                 available; refusing implicit key generation"
1441            );
1442        } else {
1443            info!("No active JWT signing key found — generating on first boot");
1444            let slot = generate_slot(now, now + active_secs);
1445            // Disk-before-memory: the generated key becomes active only after
1446            // its slot persisted. An unpersisted key is process-only — it can
1447            // never be recovered after restart — so on failure the store stays
1448            // without an active key instead of serving one.
1449            match persist_slot(&state_dir, "active", &slot) {
1450                Ok(()) => {
1451                    info!("Active JWT key generated (kid={})", slot.kid());
1452                    active = load_slot(&state_dir, "active");
1453                }
1454                Err(e) => error!(
1455                    "Could not persist active JWT key to '{}': {e}. Refusing to \
1456                     activate the unpersisted key; no active signer is available \
1457                     until persistence succeeds on a restart.",
1458                    state_dir.display()
1459                ),
1460            }
1461        }
1462    }
1463
1464    // If we have active but no lead and active is close to expiry, generate lead now.
1465    let should_gen_lead = !has_committed_authority
1466        && lead.is_none()
1467        && active.as_ref().is_some_and(|a| a.exp - now < lead_secs);
1468    if should_gen_lead {
1469        let lead_nbf = active.as_ref().map(|a| a.exp).unwrap_or(now) - lead_secs;
1470        let lead_exp = lead_nbf + active_secs;
1471        let slot = generate_slot(lead_nbf, lead_exp);
1472        if let Err(e) = persist_slot(&state_dir, "lead", &slot) {
1473            error!(
1474                "Could not persist lead JWT key to '{}': {e}. Rotation state will \
1475                 not survive restart.",
1476                state_dir.display()
1477            );
1478        } else {
1479            info!("Lead JWT key pre-generated at startup (kid={})", slot.kid());
1480        }
1481    }
1482
1483    // Re-load lead if we just generated it
1484    let lead = if should_gen_lead {
1485        load_slot(&state_dir, "lead")
1486    } else {
1487        lead
1488    };
1489
1490    SigningKeyStore::new(KeySlots {
1491        drain,
1492        active,
1493        lead,
1494    })
1495}
1496
1497/// Load the root-DID identity rotation set, seeding the first active slot from
1498/// the already-provisioned node identity key.
1499///
1500/// `secrets_dir` is a dedicated subdirectory, so these durable slot files never
1501/// collide with the JWT rotation set. Seeding from `initial_key` makes the first
1502/// deployment a compatibility-preserving transition rather than a flag day.
1503pub fn load_or_init_root_identity_key_store(
1504    secrets_dir: &Path,
1505    config: &OAuthConfig,
1506    initial_key: SigningKey,
1507) -> SigningKeyStore {
1508    let now = chrono::Utc::now().timestamp();
1509    let active_secs = config.active_secs();
1510    let lead_secs = config.lead_secs();
1511
1512    let drain = load_slot(secrets_dir, "drain");
1513    let mut active = load_slot(secrets_dir, "active");
1514    let lead = load_slot(secrets_dir, "lead");
1515
1516    // Documented follow-up (outside this repair): activation below is not
1517    // gated on persistence success, unlike `load_or_init_key_store`. Recovery
1518    // differs — the seed is the provisioned node identity key, reloadable
1519    // from its own store — but the rotation slot can stay absent on disk
1520    // until the next successful persist.
1521    if active.is_none() {
1522        let slot = KeySlot::new(initial_key, now, now + active_secs);
1523        if let Err(error) = persist_slot(secrets_dir, "active", &slot) {
1524            warn!("Could not persist root-DID active identity key: {error}");
1525        } else {
1526            info!("Seeded root-DID active identity key (kid={})", slot.kid());
1527        }
1528        active = Some(slot);
1529    }
1530
1531    let should_generate_lead = lead.is_none()
1532        && active
1533            .as_ref()
1534            .is_some_and(|slot| slot.exp - now < lead_secs);
1535    if should_generate_lead {
1536        let lead_nbf = active.as_ref().map_or(now, |slot| slot.exp - lead_secs);
1537        let slot = generate_slot(lead_nbf, lead_nbf + active_secs);
1538        if let Err(error) = persist_slot(secrets_dir, "lead", &slot) {
1539            warn!("Could not persist root-DID lead identity key: {error}");
1540        }
1541    }
1542    let lead = if should_generate_lead {
1543        load_slot(secrets_dir, "lead")
1544    } else {
1545        lead
1546    };
1547
1548    SigningKeyStore::new(KeySlots {
1549        drain,
1550        active,
1551        lead,
1552    })
1553}
1554
1555// ── Rotation logic ──────────────────────────────────────────────────────────
1556
1557/// Returns `true` when every signer slot this cycle touched finished fully
1558/// restored (or promoted). `false` means a promotion rollback failed: the
1559/// durable state is only partially restored, the old active signer stays
1560/// recoverable in its drain-slot copy, and callers must not treat the cycle
1561/// as a clean success. A later tick retries the queued lead.
1562pub async fn rotate_jwt_keys(
1563    config: &OAuthConfig,
1564    secrets_dir: &Path,
1565    store: &SigningKeyStore,
1566    now: i64,
1567) -> bool {
1568    let state_dir = rotation_state_dir(secrets_dir);
1569    let _slot_lock = match lock_signing_slots(&state_dir, true) {
1570        Ok(lock) => lock,
1571        Err(error) => {
1572            warn!("Could not lock JWT signing slots; deferring rotation: {error}");
1573            return false;
1574        }
1575    };
1576    rotate_jwt_keys_in_state_dir(config, &state_dir, store, now).await
1577}
1578
1579/// Rotation against an explicit state directory. Private test seam: passing a
1580/// state dir that cannot hold files injects deterministic persistence
1581/// failures without relying on permission bits.
1582async fn rotate_jwt_keys_in_state_dir(
1583    config: &OAuthConfig,
1584    state_dir: &Path,
1585    store: &SigningKeyStore,
1586    now: i64,
1587) -> bool {
1588    let mut slots = store.0.write().await;
1589
1590    let active_secs = config.active_secs();
1591    let lead_secs = config.lead_secs();
1592    let drain_secs = config.drain_secs();
1593
1594    // 1. Promote lead → active if lead.nbf has passed.
1595    //
1596    // Disk-before-memory: every persistence operation required to retain the
1597    // old active signer (the durable drain write) and to durably install the
1598    // new active signer must succeed before any in-memory slot changes. On a
1599    // failure the previous active stays active in memory, the lead stays
1600    // queued for the next tick, and no promotion is reported. The gate is
1601    // successful completion of `persist_slot` (temp-file + rename); that is
1602    // not proven power-loss durability, and key bytes and metadata remain two
1603    // separate files (documented residuals of this repair).
1604    if let Some(new_lead) = slots.lead.as_ref().filter(|l| l.nbf <= now).cloned() {
1605        let old_active = slots.active.clone();
1606        let old_drain = slots.drain.clone();
1607
1608        info!("Promoting lead JWT key (kid={}) to active", new_lead.kid());
1609
1610        // Old active → drain, persisted first so a process that reloads
1611        // between the writes can still verify the previous generation. The
1612        // previous drain is not evicted up front: the atomic write replaces
1613        // it, and it is restored if that write fails.
1614        if let Some(ref old) = old_active {
1615            if let Err(e) = persist_slot(state_dir, "drain", old) {
1616                // A failed drain write can already have replaced the seed
1617                // before its metadata write failed, so restoring (or
1618                // removing) the previous drain is part of the rollback and
1619                // its outcome must be confirmed before the cycle may claim
1620                // fully restored state.
1621                let drain_restored = if let Some(ref prev) = old_drain {
1622                    persist_slot(state_dir, "drain", prev)
1623                } else {
1624                    delete_slot(state_dir, "drain");
1625                    // Error-preserving removal proof: only a definite
1626                    // NotFound for both files counts as removed; any other
1627                    // metadata error leaves restoration unconfirmed.
1628                    let (drain_key, drain_meta) = slot_paths(state_dir, "drain");
1629                    let removed = [&drain_key, &drain_meta].iter().all(|path| {
1630                        matches!(
1631                            std::fs::symlink_metadata(path),
1632                            Err(error) if error.kind() == std::io::ErrorKind::NotFound
1633                        )
1634                    });
1635                    if !removed {
1636                        Err(anyhow::anyhow!(
1637                            "partial replacement drain slot still present after removal"
1638                        ))
1639                    } else {
1640                        Ok(())
1641                    }
1642                };
1643                if let Err(drain_error) = drain_restored {
1644                    error!(
1645                        "Could not restore previous drain JWT key to '{}': \
1646                         {drain_error}. The durable drain slot is not fully \
1647                         restored; retaining the old active in memory and \
1648                         retrying next tick.",
1649                        state_dir.display()
1650                    );
1651                    return false;
1652                }
1653                error!(
1654                    "Could not persist drain JWT key to '{}': {e}. Retaining the \
1655                     old active in memory; promotion will be retried next tick.",
1656                    state_dir.display()
1657                );
1658                return true;
1659            }
1660        }
1661
1662        if let Err(e) = persist_slot(state_dir, "active", &new_lead) {
1663            // Roll back the active slot first, and only after that restore is
1664            // confirmed touch the drain slot: the drain copy written above is
1665            // the only durable copy of the old active signer, so a failing
1666            // rollback must leave it in place for restart recovery instead of
1667            // deleting or overwriting it with the previous drain.
1668            let restored = match old_active {
1669                Some(ref old) => persist_slot(state_dir, "active", old),
1670                None => {
1671                    delete_slot(state_dir, "active");
1672                    Ok(())
1673                }
1674            };
1675            if let Err(restore_error) = restored {
1676                error!(
1677                    "Could not restore active JWT key to '{}': {restore_error}. The \
1678                     failed promotion left the old active signer only in its drain \
1679                     slot; that recoverable copy is kept untouched and promotion \
1680                     will be retried next tick.",
1681                    state_dir.display()
1682                );
1683                return false;
1684            }
1685            if old_active.is_some() {
1686                if let Some(ref prev) = old_drain {
1687                    if let Err(drain_error) = persist_slot(state_dir, "drain", prev) {
1688                        error!(
1689                            "Could not restore previous drain JWT key to '{}': \
1690                             {drain_error}. The old active signer is restored in the \
1691                             active slot, but the durable drain slot may hold a \
1692                             partial write; promotion will be retried next tick.",
1693                            state_dir.display()
1694                        );
1695                        return false;
1696                    }
1697                } else {
1698                    delete_slot(state_dir, "drain");
1699                }
1700            }
1701            error!(
1702                "Could not persist active JWT key to '{}': {e}. Retaining the old \
1703                 active in memory; promotion will be retried next tick.",
1704                state_dir.display()
1705            );
1706            return true;
1707        }
1708
1709        // All persistence succeeded — swap the in-memory slots and clear the
1710        // persisted lead slot.
1711        if let Some(ref prev) = old_drain {
1712            info!(
1713                "Evicting previous drain JWT key (kid={}) during promotion",
1714                prev.kid()
1715            );
1716        }
1717        delete_slot(state_dir, "lead");
1718        if let Some(old) = old_active {
1719            slots.drain = Some(old);
1720        }
1721        slots.active = Some(new_lead);
1722        slots.lead = None;
1723    }
1724
1725    // 2. Remove drain if drain window has closed.
1726    if slots
1727        .drain
1728        .as_ref()
1729        .is_some_and(|d| now >= d.exp + drain_secs)
1730    {
1731        let kid = slots.drain.as_ref().map(KeySlot::kid).unwrap_or_default();
1732        info!("Removing expired drain JWT key (kid={kid})");
1733        delete_slot(state_dir, "drain");
1734        slots.drain = None;
1735    }
1736
1737    // 3. Generate lead if active is approaching expiry and lead is absent.
1738    if slots.lead.is_none() {
1739        if let Some(active) = &slots.active {
1740            if active.exp - now < lead_secs {
1741                let lead_nbf = active.exp - lead_secs;
1742                let lead_exp = lead_nbf + active_secs;
1743                let new_lead = generate_slot(lead_nbf, lead_exp);
1744                info!(
1745                    "Generated new lead JWT key (kid={}, nbf={})",
1746                    new_lead.kid(),
1747                    new_lead.nbf
1748                );
1749                if let Err(e) = persist_slot(state_dir, "lead", &new_lead) {
1750                    error!(
1751                        "Could not persist lead JWT key to '{}': {e}. Rotation state \
1752                         will not survive restart.",
1753                        state_dir.display()
1754                    );
1755                }
1756                slots.lead = Some(new_lead);
1757            }
1758        }
1759    }
1760    true
1761}
1762
1763/// Rotate only the root-DID Ed25519 identity set.
1764///
1765/// The same proven lead → active → bounded-drain state machine is reused, but
1766/// the dedicated directory keeps its authority and persistence separate from
1767/// JWT signing keys.
1768pub fn spawn_root_identity_rotation_task(
1769    config: Arc<OAuthConfig>,
1770    secrets_dir: PathBuf,
1771    store: Arc<SigningKeyStore>,
1772) {
1773    tokio::task::spawn_local(async move {
1774        let mut interval = tokio::time::interval(
1775            config
1776                .rotation_check_interval()
1777                .min(std::time::Duration::from_secs(60)),
1778        );
1779        interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
1780        interval.tick().await;
1781        loop {
1782            interval.tick().await;
1783            rotate_jwt_keys(
1784                &config,
1785                &secrets_dir,
1786                &store,
1787                chrono::Utc::now().timestamp(),
1788            )
1789            .await;
1790        }
1791    });
1792}
1793
1794// ── Background task ─────────────────────────────────────────────────────────
1795
1796/// Additional stores to rotate alongside the primary Ed25519 store.
1797pub struct RotationStores {
1798    pub es256: Option<Arc<Es256SigningKeyStore>>,
1799    pub ml_dsa: Option<Arc<MlDsaSigningKeyStore>>,
1800    pub composite_ca_key: Arc<SigningKey>,
1801}
1802
1803/// The recovery transaction recorded when a rotation cycle defers composite
1804/// publication: `committed_digest` names the marker-selected committed
1805/// generation the cycle deferred against, and `states` contains only
1806/// component digests computed from this process's in-memory stores. A fresh
1807/// reload from mutable slot files is never certified as part of the failed
1808/// cycle. A later tick may anchor publication only when its tick-start memory
1809/// digest is in `states`, and the committed digest is still checked against
1810/// the ledger. After restart, a matching reload may continue; a different
1811/// reload remains untrusted and cannot advance the ledger. This prefers
1812/// explicit repair over guessing which writer produced divergent disk state.
1813#[derive(Clone, Serialize, Deserialize)]
1814struct DeferredPredecessorRecord {
1815    committed_digest: String,
1816    states: Vec<String>,
1817}
1818
1819static DEFERRED_PREDECESSOR_FALLBACK: parking_lot::RwLock<Option<DeferredPredecessorRecord>> =
1820    parking_lot::RwLock::new(None);
1821
1822fn deferred_predecessor_path(secrets_dir: &Path) -> PathBuf {
1823    rotation_state_dir(secrets_dir).join("jwt-composite-deferred-predecessor")
1824}
1825
1826/// Cross-process serialization for one rotation cycle: component slot
1827/// mutation, committed-predecessor validation, recovery-state capture and
1828/// record lifecycle all happen while this lock is held, so no competing
1829/// writer can interleave a component advance between them. The composite
1830/// publisher's own ledger flock is acquired later, inside publication, so
1831/// the lock order is cycle lock -> publisher lock and never the reverse.
1832fn cycle_lock_path(secrets_dir: &Path) -> PathBuf {
1833    rotation_state_dir(secrets_dir).join("jwt-composite-rotation-cycle.lock")
1834}
1835
1836/// Read the recovery record. The in-process register is authoritative when
1837/// present (it is set whenever the durable write could not be persisted);
1838/// otherwise the durable file copy is parsed. A missing, unreadable, or
1839/// malformed record reads as `None` — never as supersession.
1840fn read_deferred_predecessor(secrets_dir: &Path) -> Option<DeferredPredecessorRecord> {
1841    if let Some(record) = DEFERRED_PREDECESSOR_FALLBACK.read().clone() {
1842        return Some(record);
1843    }
1844    let value = std::fs::read_to_string(deferred_predecessor_path(secrets_dir)).ok()?;
1845    serde_json::from_str::<DeferredPredecessorRecord>(value.trim()).ok()
1846}
1847
1848/// Persist the durable copy of the record; returns whether it was written.
1849/// The in-process register retains the record until it is cleared or
1850/// successfully advanced, so a failed durable write never loses the recovery
1851/// transaction on its own.
1852fn write_deferred_predecessor(secrets_dir: &Path, record: &DeferredPredecessorRecord) -> bool {
1853    let state_dir = rotation_state_dir(secrets_dir);
1854    match serde_json::to_vec(record) {
1855        Ok(bytes) => super::identity_store::write_secret(
1856            &state_dir,
1857            "jwt-composite-deferred-predecessor",
1858            &bytes,
1859        )
1860        .is_ok(),
1861        Err(error) => {
1862            warn!("could not serialize the deferred predecessor record: {error}");
1863            false
1864        }
1865    }
1866}
1867
1868fn clear_deferred_predecessor(secrets_dir: &Path) {
1869    *DEFERRED_PREDECESSOR_FALLBACK.write() = None;
1870    let _ = std::fs::remove_file(deferred_predecessor_path(secrets_dir));
1871}
1872
1873/// One background rotation tick: rotate every algorithm store, refresh the
1874/// published verifier snapshots, and — only when every rotation finished
1875/// fully restored — republish the composite authority. A cycle that ended
1876/// not fully restored (a failed promotion rollback) must not publish: the
1877/// durable slots diverge from what memory would propose, so a staged pending
1878/// ledger could never be acknowledged and would only strand a divergent
1879/// proposal next to the retained prior authority.
1880///
1881/// When a cycle defers, the cycle records a recovery transaction: the
1882/// committed component digest it deferred against (validated to match the
1883/// marker-selected committed ledger) plus the certified component states it
1884/// left behind (post-rotation memory and the durable disk state). A later
1885/// all-restored tick anchors publication on the recorded committed digest
1886/// only when its own tick-start memory digest is one of the certified
1887/// states — proving its stores continue exactly that recorded transition;
1888/// any other record is ignored, and stores that match nothing keep the
1889/// pre-existing stale-authority rejection. Every anchored cycle advances
1890/// the certified states to its own post-rotation outcome, so recovery
1891/// survives chained deferrals, failed recovery publications, and restarts.
1892/// The record is durable (survives restart), cleared on successful
1893/// publication, retained on validation I/O errors, and discarded only after
1894/// a verified supersession, followed by one retry anchored on current
1895/// memory. Recovery therefore survives chained deferrals and failed
1896/// recovery publications (each certified cycle re-certifies its own
1897/// outcome); a restart recovers while the on-disk state still digests to a
1898/// certified state and fail-closes (retaining prior authority) after a
1899/// foreign writer mutates the shared slot files. Returns `true` when every
1900/// rotation finished fully restored; a lock-contended tick skips without
1901/// rotating and also returns `true` (the competing cycle owns the
1902/// transition; this tick changes nothing).
1903async fn run_rotation_cycle(
1904    config: &OAuthConfig,
1905    secrets_dir: &Path,
1906    store: &SigningKeyStore,
1907    extra: &RotationStores,
1908    now: i64,
1909) -> bool {
1910    // Serialize the whole cycle against competing writers: slot mutation,
1911    // predecessor validation, recovery-state capture and record lifecycle
1912    // must not interleave with another process's cycle, or a captured state
1913    // could certify a transition it did not produce. A contending writer
1914    // skips its tick entirely and retries on the next interval. The
1915    // publisher's ledger flock is acquired later, inside publication, so the
1916    // lock order is cycle lock -> publisher lock and never the reverse.
1917    let cycle_lock = std::fs::OpenOptions::new()
1918        .create(true)
1919        .truncate(false)
1920        .read(true)
1921        .write(true)
1922        .open(cycle_lock_path(secrets_dir))
1923        .map_err(|error| {
1924            warn!("could not open the rotation cycle lock: {error}");
1925        });
1926    let cycle_lock = match cycle_lock {
1927        Ok(file) => file,
1928        Err(()) => return true,
1929    };
1930    use nix::fcntl::{flock, FlockArg};
1931    use std::os::fd::AsRawFd as _;
1932    if let Err(error) = flock(cycle_lock.as_raw_fd(), FlockArg::LockExclusiveNonblock) {
1933        warn!(
1934            "another rotation cycle holds the cycle lock; skipping this tick \
1935             ({error})"
1936        );
1937        return true;
1938    }
1939
1940    let pre_rotation_digest = if let Some(ref ml_dsa) = extra.ml_dsa {
1941        Some(component_state_digest(store, ml_dsa, extra.composite_ca_key.verifying_key()).await)
1942    } else {
1943        None
1944    };
1945    let mut all_restored = rotate_jwt_keys(config, secrets_dir, store, now).await;
1946    if !all_restored {
1947        warn!(
1948            "Ed25519 rotation ended not fully restored: a failed promotion \
1949             rollback left the signer slots only partially restored; the old \
1950             active signer remains active in memory and publication is \
1951             deferred until a retry fully restores it"
1952        );
1953    }
1954    // Keep the shared HTTP validator's published Ed25519 key set current after
1955    // every rotation (drain evicted / lead promoted → active). Memory is
1956    // unchanged on a not-restored cycle, so the previously published verifier
1957    // state is retained there by construction.
1958    refresh_ed25519_verifying_keys(store).await;
1959    if let Some(ref es256) = extra.es256 {
1960        rotate_es256_keys(config, secrets_dir, es256, now).await;
1961        // C4 (#1170): re-seal the op-log head after the ES256
1962        // promotion so cross-process readers (the registry's
1963        // `PdsPublisher` under `--ipc`) observe the new active
1964        // `#atproto` generation. Dedicated head-signing key + shared
1965        // state dir; async because the ES256 store is a tokio RwLock
1966        // (F1). C2 (#1168) seam — see `auth::op_log`.
1967        if let Err(error) = super::op_log::advance_sealed_head(secrets_dir, es256).await {
1968            error!(
1969                "failed to re-seal op-log head after ES256 rotation; \
1970                 cross-process readers will retain the prior generation: {error}"
1971            );
1972        }
1973    }
1974    if let (Some(ref ml_dsa), Some(pre_rotation)) = (&extra.ml_dsa, pre_rotation_digest.as_deref())
1975    {
1976        let pq_restored = rotate_ml_dsa_keys(config, secrets_dir, ml_dsa, now).await;
1977        all_restored &= pq_restored;
1978        if !pq_restored {
1979            warn!(
1980                "ML-DSA rotation ended not fully restored: a failed promotion \
1981                 rollback left the signer slots only partially restored; the old \
1982                 active signer remains active in memory and publication is \
1983                 deferred until a retry fully restores it"
1984            );
1985        }
1986        refresh_ml_dsa_verifying_keys(ml_dsa).await;
1987        let post_memory =
1988            component_state_digest(store, ml_dsa, extra.composite_ca_key.verifying_key()).await;
1989        // The recovery record applies only when this tick's component memory
1990        // is one of the certified states of the recorded deferred transition;
1991        // a record describing some other transition is ignored (never used,
1992        // never treated as supersession).
1993        let retained = read_deferred_predecessor(secrets_dir);
1994        let certified = retained
1995            .as_ref()
1996            .map(|record| record.states.iter().any(|state| state == pre_rotation))
1997            .unwrap_or(false);
1998        let anchor = match retained.as_ref() {
1999            Some(record) if certified => {
2000                info!(
2001                    "anchoring deferred composite publication on the retained \
2002                     committed predecessor"
2003                );
2004                Some(record.committed_digest.clone())
2005            }
2006            _ => None,
2007        };
2008        // Post-rotation component state: the cycle's in-memory outcome and the
2009        // durable slot state (what a restarted process would reload). Anchored
2010        // cycles certify this outcome as the transition's next state.
2011        if all_restored {
2012            let expected = anchor.as_deref().unwrap_or(pre_rotation);
2013            match refresh_composite_key_set(
2014                secrets_dir,
2015                store,
2016                ml_dsa,
2017                Arc::clone(&extra.composite_ca_key),
2018                config.drain_secs(),
2019                expected,
2020            )
2021            .await
2022            {
2023                Ok(()) => {
2024                    if retained.is_some() {
2025                        clear_deferred_predecessor(secrets_dir);
2026                    }
2027                }
2028                Err(error) => {
2029                    // Distinguish verified supersession from unavailable
2030                    // authority: only a successfully read ledger whose digest
2031                    // differs proves the recorded recovery transaction was
2032                    // superseded by a competing publication. A read failure
2033                    // retains the record and the prior authority. An anchored
2034                    // (certified) cycle that failed to publish advances the
2035                    // certified states to this tick's post-rotation outcome,
2036                    // so the next tick can still anchor and publish.
2037                    let superseded = retained.as_ref().map(|record| {
2038                        read_committed_composite_ledger(secrets_dir)
2039                            .map(|(_, ledger)| ledger.component_digest != record.committed_digest)
2040                            .unwrap_or(false)
2041                    });
2042                    match (retained.as_ref(), superseded) {
2043                        (Some(_record), Some(true)) => {
2044                            // Verified supersession: the recorded committed
2045                            // predecessor no longer matches the ledger.
2046                            clear_deferred_predecessor(secrets_dir);
2047                            warn!(
2048                                "deferred composite predecessor superseded by a \
2049                                 competing publication; retrying once anchored on \
2050                                 current component memory"
2051                            );
2052                            if let Err(retry_error) = refresh_composite_key_set(
2053                                secrets_dir,
2054                                store,
2055                                ml_dsa,
2056                                Arc::clone(&extra.composite_ca_key),
2057                                config.drain_secs(),
2058                                pre_rotation,
2059                            )
2060                            .await
2061                            {
2062                                warn!(
2063                                    "composite key-set publication failed; retaining \
2064                                     prior authority: {retry_error}"
2065                                );
2066                            }
2067                        }
2068                        (Some(record), _) if certified => {
2069                            // Anchored (certified) cycle whose publication
2070                            // failed: advance the certified states to this
2071                            // tick's own post-rotation memory outcome, keeping
2072                            // the committed predecessor, so the next tick can
2073                            // still anchor and publish the recovery. Only the
2074                            // memory outcome is certified here: the durable
2075                            // slot files this tick did not rewrite cannot be
2076                            // proven to continue the transition (another
2077                            // writer may have replaced them), so the disk
2078                            // digest is never added by an advance.
2079                            let advanced = DeferredPredecessorRecord {
2080                                committed_digest: record.committed_digest.clone(),
2081                                states: vec![post_memory.clone()],
2082                            };
2083                            if write_deferred_predecessor(secrets_dir, &advanced) {
2084                                *DEFERRED_PREDECESSOR_FALLBACK.write() = None;
2085                            } else {
2086                                *DEFERRED_PREDECESSOR_FALLBACK.write() = Some(advanced);
2087                            }
2088                            warn!(
2089                                "composite key-set publication failed; the certified \
2090                                 recovery state advanced with this tick's rotations \
2091                                 and publication will be retried"
2092                            );
2093                        }
2094                        _ => {
2095                            // An uncertified record must never acquire this
2096                            // tick's outcome: the stale-authority rejection
2097                            // stands, the retained record is left untouched
2098                            // for its own writer, and the prior authority is
2099                            // kept.
2100                            warn!(
2101                                "composite key-set publication failed; retaining prior \
2102                                 authority: {error}"
2103                            );
2104                        }
2105                    }
2106                }
2107            }
2108        } else {
2109            // Seed only from the state computed in this process. The
2110            // pre-rotation digest must still match the committed ledger; an
2111            // already-drifted store keeps the stale-authority rejection. Do
2112            // not reload and certify mutable slot files: another writer may
2113            // have changed them before this record was created. After a
2114            // restart, a disk digest not in this record cannot authorize a
2115            // retry; explicit repair is safer than guessing.
2116            //
2117            // A chained certified deferral (record present, memory still
2118            // certified) advances the certified states to this tick's own
2119            // memory outcome only when a family durably advanced; when the
2120            // failure left the transition unchanged, the record is kept
2121            // as-is. Durable slot files a tick did not rewrite cannot be
2122            // proven to continue the transition (another writer may have
2123            // replaced them between cycles), so their digest is never
2124            // certified by an advance — restart recovery after such an
2125            // advance fail-closes instead of authorizing a foreign state.
2126            // A failed record write retains the record in memory, and the
2127            // durable copy is retried on later ticks.
2128            match retained.as_ref() {
2129                None => {
2130                    if let Ok((_, ledger)) = read_committed_composite_ledger(secrets_dir) {
2131                        if ledger.component_digest == pre_rotation {
2132                            let record = DeferredPredecessorRecord {
2133                                committed_digest: pre_rotation.to_owned(),
2134                                states: vec![post_memory.clone()],
2135                            };
2136                            if write_deferred_predecessor(secrets_dir, &record) {
2137                                info!(
2138                                    "composite publication deferred; committed predecessor \
2139                                     and certified deferred states recorded for recovery"
2140                                );
2141                            } else {
2142                                *DEFERRED_PREDECESSOR_FALLBACK.write() = Some(record);
2143                                warn!(
2144                                    "could not persist the deferred predecessor record; \
2145                                     retaining it in memory for this process"
2146                                );
2147                            }
2148                        }
2149                    }
2150                }
2151                Some(record) if certified => {
2152                    let states = if post_memory == *pre_rotation {
2153                        // Transition unchanged (the same failure repeated):
2154                        // the recorded states already describe it.
2155                        record.states.clone()
2156                    } else {
2157                        // A family durably advanced in this cycle: certify
2158                        // only the memory outcome.
2159                        vec![post_memory.clone()]
2160                    };
2161                    let advanced = DeferredPredecessorRecord {
2162                        committed_digest: record.committed_digest.clone(),
2163                        states,
2164                    };
2165                    if write_deferred_predecessor(secrets_dir, &advanced) {
2166                        *DEFERRED_PREDECESSOR_FALLBACK.write() = None;
2167                    } else {
2168                        *DEFERRED_PREDECESSOR_FALLBACK.write() = Some(advanced);
2169                    }
2170                }
2171                Some(_) => {
2172                    // Uncertified record: it is not this tick's transition, so
2173                    // it is left untouched. The durable copy of the in-process
2174                    // register is refreshed (the register holds solely
2175                    // certified-at-write-time records); a durable file copy is
2176                    // left as-is.
2177                    if !deferred_predecessor_path(secrets_dir).exists() {
2178                        if let Some(record) = DEFERRED_PREDECESSOR_FALLBACK.read().clone() {
2179                            write_deferred_predecessor(secrets_dir, &record);
2180                        }
2181                    }
2182                }
2183            }
2184            warn!(
2185                "rotation cycle ended not fully restored; deferring composite \
2186                 key-set publication so the cycle cannot publish as successfully \
2187                 restored"
2188            );
2189        }
2190    }
2191    all_restored
2192}
2193
2194pub fn spawn_rotation_task(
2195    config: Arc<OAuthConfig>,
2196    secrets_dir: PathBuf,
2197    store: Arc<SigningKeyStore>,
2198    extra: RotationStores,
2199) {
2200    tokio::task::spawn_local(async move {
2201        // Bounded overlap is an expiry contract, not merely six-hour
2202        // housekeeping. Do not leave an expired method served for a full
2203        // default rotation interval.
2204        let mut interval = tokio::time::interval(
2205            config
2206                .rotation_check_interval()
2207                .min(std::time::Duration::from_secs(60)),
2208        );
2209        interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
2210        // Skip the first tick (fires immediately on creation)
2211        interval.tick().await;
2212        loop {
2213            interval.tick().await;
2214            let now = chrono::Utc::now().timestamp();
2215            run_rotation_cycle(&config, &secrets_dir, &store, &extra, now).await;
2216        }
2217    });
2218}
2219
2220// ════════════════════════════════════════════════════════════════════════════════
2221// ES256 (P-256) Key Rotation Store
2222// ════════════════════════════════════════════════════════════════════════════════
2223
2224use p256::ecdsa::SigningKey as Es256SigningKey;
2225
2226#[derive(Clone)]
2227pub struct Es256KeySlot {
2228    pub key: Arc<Es256SigningKey>,
2229    pub nbf: i64,
2230    pub exp: i64,
2231}
2232
2233impl Es256KeySlot {
2234    pub fn new(key: Es256SigningKey, nbf: i64, exp: i64) -> Self {
2235        Self {
2236            key: Arc::new(key),
2237            nbf,
2238            exp,
2239        }
2240    }
2241
2242    pub fn kid(&self) -> String {
2243        crate::auth::jwt::es256_kid(&self.key)
2244    }
2245}
2246
2247#[derive(Default, Clone)]
2248pub struct Es256KeySlots {
2249    pub drain: Option<Es256KeySlot>,
2250    pub active: Option<Es256KeySlot>,
2251    pub lead: Option<Es256KeySlot>,
2252}
2253
2254impl Es256KeySlots {
2255    pub fn all(&self) -> Vec<&Es256KeySlot> {
2256        [&self.drain, &self.active, &self.lead]
2257            .into_iter()
2258            .flatten()
2259            .collect()
2260    }
2261}
2262
2263type Es256PromotionHook = dyn Fn(Arc<Es256SigningKey>) -> anyhow::Result<()> + Send + Sync;
2264
2265#[derive(Clone)]
2266pub struct Es256SigningKeyStore(
2267    pub Arc<parking_lot::RwLock<Es256KeySlots>>,
2268    Arc<parking_lot::RwLock<Option<Arc<Es256PromotionHook>>>>,
2269    Arc<parking_lot::RwLock<Option<PathBuf>>>,
2270);
2271
2272impl Es256SigningKeyStore {
2273    pub fn new(slots: Es256KeySlots) -> Self {
2274        Self(
2275            Arc::new(parking_lot::RwLock::new(slots)),
2276            Arc::new(parking_lot::RwLock::new(None)),
2277            Arc::new(parking_lot::RwLock::new(None)),
2278        )
2279    }
2280
2281    /// Bind this process-local cache to the durable slot directory.  Each
2282    /// process refreshes before it signs or resolves, so `--ipc` workers see a
2283    /// promotion made by the OAuth process rather than retaining a stale key.
2284    pub fn bind_secrets_dir(&self, secrets_dir: PathBuf) {
2285        *self.2.write() = Some(secrets_dir);
2286    }
2287
2288    pub fn refresh_from_disk(&self) {
2289        let Some(secrets_dir) = self.2.read().clone() else {
2290            return;
2291        };
2292        let active = load_es256_slot(&secrets_dir, "active");
2293        // Never replace a usable in-memory authority with a partially written
2294        // or unavailable filesystem snapshot.
2295        let Some(active) = active else {
2296            warn!("ES256: durable active slot unavailable; retaining cached authority");
2297            return;
2298        };
2299        *self.0.write() = Es256KeySlots {
2300            drain: load_es256_slot(&secrets_dir, "drain"),
2301            active: Some(active),
2302            lead: load_es256_slot(&secrets_dir, "lead"),
2303        };
2304    }
2305
2306    /// Install the in-process action run while a lead → active transition is
2307    /// still hidden behind the store write guard. The publisher stores a weak
2308    /// self-reference in this hook, so the shared key store does not keep the
2309    /// service alive. Cross-`--ipc` promotion delivery remains tracked by
2310    /// #1123.
2311    pub(crate) fn set_promotion_hook(&self, hook: Arc<Es256PromotionHook>) {
2312        *self.1.write() = Some(hook);
2313    }
2314
2315    fn notify_promotion(&self, key: Arc<Es256SigningKey>) -> anyhow::Result<()> {
2316        let hook = self.1.read().clone();
2317        if let Some(hook) = hook {
2318            hook(key)?;
2319        }
2320        Ok(())
2321    }
2322
2323    /// The current active ES256 signing key — the single `#atproto` key the
2324    /// DID document publishes and the repo head is signed with (#918
2325    /// re-sign-on-rotation). Synchronous (parking_lot lock) so the publisher
2326    /// can resolve the LIVE key at sign time without an async runtime.
2327    pub fn active_key(&self) -> Option<Arc<Es256SigningKey>> {
2328        self.refresh_from_disk();
2329        self.0.read().active.as_ref().map(|s| Arc::clone(&s.key))
2330    }
2331
2332    /// Take active, drain and lead from one read guard so a DID document never
2333    /// mixes two rotation generations.
2334    pub fn slots_snapshot(&self) -> Es256KeySlots {
2335        self.refresh_from_disk();
2336        self.0.read().clone()
2337    }
2338
2339    /// The active slot (key + `nbf`/`exp` bounds), if present.
2340    pub fn active_slot(&self) -> Option<Es256KeySlot> {
2341        self.0.read().active.clone()
2342    }
2343
2344    /// Snapshot the bounded verification-only drain slot, if any.
2345    pub fn drain_slot(&self) -> Option<Es256KeySlot> {
2346        self.0.read().drain.clone()
2347    }
2348
2349    /// Snapshot the bounded verification-only lead slot, if any. New commits
2350    /// are always signed by [`Es256SigningKeyStore::active_key`].
2351    pub fn lead_slot(&self) -> Option<Es256KeySlot> {
2352        self.0.read().lead.clone()
2353    }
2354
2355    pub fn all_slots_snapshot(&self) -> Vec<Es256KeySlot> {
2356        self.0.read().all().into_iter().cloned().collect()
2357    }
2358}
2359
2360/// Resolve an ES256 signing key by kid from the persisted slot files in
2361/// `secrets_dir` (scans drain/active/lead). Used by the sealed op-log head
2362/// reader ([`super::op_log`]) to materialize the active generation's signing
2363/// key once the head has authenticated *which* kid is active. Reading from
2364/// disk — not the in-memory `OnceLock` — is what makes a cross-process reader
2365/// observe a slot promoted by another process.
2366pub(crate) fn es256_signing_key_for_kid(secrets_dir: &Path, kid: &str) -> Option<Es256SigningKey> {
2367    let state_dir = rotation_state_dir(secrets_dir);
2368    for name in ["active", "drain", "lead"] {
2369        if let Some(slot) =
2370            load_es256_slot(&state_dir, name).or_else(|| load_es256_slot(secrets_dir, name))
2371        {
2372            if slot.kid() == kid {
2373                return Some((*slot.key).clone());
2374            }
2375        }
2376    }
2377    None
2378}
2379
2380// ── ES256 persistence ──────────────────────────────────────────────────────
2381
2382fn es256_slot_paths(secrets_dir: &Path, name: &str) -> (PathBuf, PathBuf) {
2383    (
2384        secrets_dir.join(format!("es256-signing-key.{name}")),
2385        secrets_dir.join(format!("es256-signing-key.{name}.meta")),
2386    )
2387}
2388
2389fn load_es256_slot(secrets_dir: &Path, name: &str) -> Option<Es256KeySlot> {
2390    let (key_path, meta_path) = es256_slot_paths(secrets_dir, name);
2391    let seed = std::fs::read(&key_path).ok()?;
2392    if seed.len() != 32 {
2393        warn!(
2394            "ES256 key slot '{name}': unexpected seed length {}",
2395            seed.len()
2396        );
2397        return None;
2398    }
2399    let meta_bytes = std::fs::read(&meta_path).ok()?;
2400    let meta: SlotMeta = serde_json::from_slice(&meta_bytes).ok()?;
2401    let key = Es256SigningKey::from_bytes(seed.as_slice().into()).ok()?;
2402    Some(Es256KeySlot::new(key, meta.nbf, meta.exp))
2403}
2404
2405pub(crate) fn persist_es256_slot(
2406    secrets_dir: &Path,
2407    name: &str,
2408    slot: &Es256KeySlot,
2409) -> anyhow::Result<()> {
2410    // Atomic write + 0600 perms (#179).
2411    super::identity_store::write_secret(
2412        secrets_dir,
2413        &format!("es256-signing-key.{name}"),
2414        &slot.key.to_bytes(),
2415    )?;
2416    let meta = SlotMeta {
2417        nbf: slot.nbf,
2418        exp: slot.exp,
2419    };
2420    super::identity_store::write_secret(
2421        secrets_dir,
2422        &format!("es256-signing-key.{name}.meta"),
2423        &serde_json::to_vec(&meta)?,
2424    )?;
2425    Ok(())
2426}
2427
2428fn delete_es256_slot(secrets_dir: &Path, name: &str) {
2429    let (key_path, meta_path) = es256_slot_paths(secrets_dir, name);
2430    let _ = std::fs::remove_file(&key_path);
2431    let _ = std::fs::remove_file(&meta_path);
2432}
2433
2434fn generate_es256_slot(nbf: i64, exp: i64) -> Es256KeySlot {
2435    let key = Es256SigningKey::random(&mut rand::rngs::OsRng);
2436    Es256KeySlot::new(key, nbf, exp)
2437}
2438
2439pub fn load_or_init_es256_key_store(
2440    secrets_dir: &Path,
2441    config: &OAuthConfig,
2442) -> Es256SigningKeyStore {
2443    match super::op_log::resolve_oplog_state_dir(secrets_dir) {
2444        Ok(state_dir) => {
2445            load_or_init_es256_key_store_with_oplog_state_dir(secrets_dir, config, &state_dir)
2446        }
2447        Err(error) => {
2448            error!(
2449                "Could not resolve durable ES256 operation-log authority: {error}; refusing implicit key generation"
2450            );
2451            load_or_init_es256_key_store_with_authority(secrets_dir, config, || true)
2452        }
2453    }
2454}
2455
2456fn load_or_init_es256_key_store_with_oplog_state_dir(
2457    secrets_dir: &Path,
2458    config: &OAuthConfig,
2459    oplog_state_dir: &Path,
2460) -> Es256SigningKeyStore {
2461    load_or_init_es256_key_store_with_authority(secrets_dir, config, || {
2462        match has_es256_oplog_authority(oplog_state_dir) {
2463            Ok(present) => present,
2464            Err(error) => {
2465                error!(
2466                    "Could not inspect durable ES256 operation-log authority: {error}; refusing implicit key generation"
2467                );
2468                true
2469            }
2470        }
2471    })
2472}
2473
2474fn has_es256_oplog_authority(state_dir: &Path) -> std::io::Result<bool> {
2475    use std::io::ErrorKind;
2476
2477    for name in [
2478        super::op_log::SEALED_HEAD_FILENAME,
2479        super::op_log::SEALED_HEAD_MAX_SEQ_FILENAME,
2480        super::op_log::HEAD_VERIFYING_KEY_FILENAME,
2481    ] {
2482        match std::fs::symlink_metadata(state_dir.join(name)) {
2483            Ok(_) => return Ok(true),
2484            Err(error) if error.kind() == ErrorKind::NotFound => {}
2485            Err(error) => return Err(error),
2486        }
2487    }
2488    Ok(false)
2489}
2490
2491fn load_or_init_es256_key_store_with_authority(
2492    secrets_dir: &Path,
2493    config: &OAuthConfig,
2494    has_durable_authority: impl FnOnce() -> bool,
2495) -> Es256SigningKeyStore {
2496    let now = chrono::Utc::now().timestamp();
2497    let active_secs = config.active_secs();
2498    let lead_secs = config.lead_secs();
2499
2500    let state_dir = rotation_state_dir(secrets_dir);
2501    let _slot_lock = match lock_signing_slots(&state_dir, false) {
2502        Ok(lock) => lock,
2503        Err(error) => {
2504            error!("Could not lock ES256 signing slots: {error}; refusing initialization");
2505            return Es256SigningKeyStore::new(Es256KeySlots::default());
2506        }
2507    };
2508    let has_durable_authority = has_durable_authority();
2509    let mut drain =
2510        load_es256_slot(&state_dir, "drain").or_else(|| load_es256_slot(secrets_dir, "drain"));
2511    let mut active =
2512        load_es256_slot(&state_dir, "active").or_else(|| load_es256_slot(secrets_dir, "active"));
2513    let mut lead =
2514        load_es256_slot(&state_dir, "lead").or_else(|| load_es256_slot(secrets_dir, "lead"));
2515
2516    // Do not serve an already-expired bounded method after restart. This is
2517    // the same exclusive boundary the verifier and runtime cleanup use.
2518    if drain.as_ref().is_some_and(|slot| now >= slot.exp) {
2519        delete_es256_slot(&state_dir, "drain");
2520        drain = None;
2521    }
2522    if lead.as_ref().is_some_and(|slot| now >= slot.exp) {
2523        delete_es256_slot(&state_dir, "lead");
2524        lead = None;
2525    }
2526
2527    if active.is_none() {
2528        if has_durable_authority {
2529            error!(
2530                "Durable ES256 operation-log authority exists but no active signing slot is \
2531                 available; refusing implicit key generation"
2532            );
2533        } else {
2534            info!("No active ES256 signing key found — generating on first boot");
2535            let slot = generate_es256_slot(now, now + active_secs);
2536            match persist_es256_slot(&state_dir, "active", &slot) {
2537                Ok(()) => {
2538                    info!("Active ES256 key generated (kid={})", slot.kid());
2539                    active = load_es256_slot(&state_dir, "active");
2540                }
2541                Err(error) => error!(
2542                    "Could not persist active ES256 key to '{}': {error}. Refusing \
2543                     to activate the unpersisted key; no active signer is available \
2544                     until persistence succeeds on a restart.",
2545                    state_dir.display()
2546                ),
2547            }
2548        }
2549    }
2550
2551    let should_gen_lead =
2552        lead.is_none() && active.as_ref().is_some_and(|a| a.exp - now < lead_secs);
2553    if should_gen_lead {
2554        let lead_nbf = active.as_ref().map(|a| a.exp).unwrap_or(now) - lead_secs;
2555        let lead_exp = lead_nbf + active_secs;
2556        let slot = generate_es256_slot(lead_nbf, lead_exp);
2557        if let Err(e) = persist_es256_slot(&state_dir, "lead", &slot) {
2558            error!(
2559                "Could not persist lead ES256 key to '{}': {e}. Rotation state will \
2560                 not survive restart.",
2561                state_dir.display()
2562            );
2563        }
2564    }
2565
2566    let lead = if should_gen_lead {
2567        load_es256_slot(&state_dir, "lead")
2568    } else {
2569        lead
2570    };
2571    Es256SigningKeyStore::new(Es256KeySlots {
2572        drain,
2573        active,
2574        lead,
2575    })
2576}
2577
2578/// Returns `true` if the active key was promoted (lead → active). An installed
2579/// in-process promotion hook is invoked before the candidate becomes visible.
2580///
2581/// Promotion is recoverable with respect to the live key: the store write
2582/// guard prevents readers from observing the candidate key until persistence
2583/// and the persisted repo-head re-sign both succeed. Deployments without an
2584/// in-process publisher may omit the hook and rely on the bounded overlap
2585/// slots used by `--ipc` workers.
2586pub async fn rotate_es256_keys(
2587    config: &OAuthConfig,
2588    secrets_dir: &Path,
2589    store: &Es256SigningKeyStore,
2590    now: i64,
2591) -> bool {
2592    let state_dir = rotation_state_dir(secrets_dir);
2593    let _slot_lock = match lock_signing_slots(&state_dir, true) {
2594        Ok(lock) => lock,
2595        Err(error) => {
2596            warn!("Could not lock ES256 signing slots; deferring rotation: {error}");
2597            return false;
2598        }
2599    };
2600    rotate_es256_keys_in_state_dir(config, &state_dir, store, now).await
2601}
2602
2603async fn rotate_es256_keys_in_state_dir(
2604    config: &OAuthConfig,
2605    state_dir: &Path,
2606    store: &Es256SigningKeyStore,
2607    now: i64,
2608) -> bool {
2609    let mut slots = store.0.write();
2610    let active_secs = config.active_secs();
2611    let lead_secs = config.lead_secs();
2612    let drain_secs = config.drain_secs();
2613    let mut promoted = false;
2614
2615    // Phase 1: promote lead → active if lead.nbf <= now
2616    if let Some(new_active) = slots.lead.as_ref().filter(|lead| lead.nbf <= now).cloned() {
2617        let old_active = slots.active.clone();
2618        let old_drain = slots.drain.clone();
2619        let new_drain = old_active.as_ref().map(|old_active| {
2620            // The old active key becomes verification-only for one bounded
2621            // drain interval. Store the actual publication expiry in the slot
2622            // itself so the DID document and cleanup use one boundary.
2623            Es256KeySlot::new(
2624                old_active.key.as_ref().clone(),
2625                old_active.nbf,
2626                now.saturating_add(drain_secs),
2627            )
2628        });
2629
2630        // Persist drain before active. A process that reloads between the two
2631        // writes can therefore still verify K-signed commits.
2632        if let Some(ref drain) = new_drain {
2633            if let Err(error) = persist_es256_slot(state_dir, "drain", drain) {
2634                if let Some(ref old_drain) = old_drain {
2635                    let _ = persist_es256_slot(state_dir, "drain", old_drain);
2636                } else {
2637                    delete_es256_slot(state_dir, "drain");
2638                }
2639                error!(
2640                    "ES256: failed to persist drain slot to '{}'; retaining old active: {error}",
2641                    state_dir.display()
2642                );
2643                return false;
2644            }
2645        }
2646
2647        if let Err(error) = persist_es256_slot(state_dir, "active", &new_active) {
2648            if let Some(ref old_active) = old_active {
2649                let _ = persist_es256_slot(state_dir, "active", old_active);
2650            } else {
2651                delete_es256_slot(state_dir, "active");
2652            }
2653            if let Some(ref old_drain) = old_drain {
2654                let _ = persist_es256_slot(state_dir, "drain", old_drain);
2655            } else {
2656                delete_es256_slot(state_dir, "drain");
2657            }
2658            error!(
2659                "ES256: failed to persist promoted active to '{}'; retaining old active: {error}",
2660                state_dir.display()
2661            );
2662            return false;
2663        }
2664
2665        if let Err(error) = store.notify_promotion(Arc::clone(&new_active.key)) {
2666            // A hook may have committed the candidate head before returning an
2667            // error. Re-sign with the old key defensively, then restore the
2668            // durable slot generation so the queued lead can be retried.
2669            if let Some(ref old_active) = old_active {
2670                if let Err(rollback_error) = store.notify_promotion(Arc::clone(&old_active.key)) {
2671                    warn!(
2672                        "ES256: failed to restore PDS head after promotion failure: {rollback_error}"
2673                    );
2674                }
2675                if let Err(rollback_error) = persist_es256_slot(state_dir, "active", old_active) {
2676                    warn!(
2677                        "ES256: failed to restore persisted active after promotion failure: {rollback_error}"
2678                    );
2679                }
2680            } else {
2681                delete_es256_slot(state_dir, "active");
2682            }
2683            if let Some(ref old_drain) = old_drain {
2684                if let Err(rollback_error) = persist_es256_slot(state_dir, "drain", old_drain) {
2685                    warn!(
2686                        "ES256: failed to restore persisted drain after promotion failure: {rollback_error}"
2687                    );
2688                }
2689            } else {
2690                delete_es256_slot(state_dir, "drain");
2691            }
2692            warn!(
2693                "ES256: failed to re-sign PDS head; retaining old active and retrying next tick: {error}"
2694            );
2695            return false;
2696        }
2697
2698        delete_es256_slot(state_dir, "lead");
2699        slots.drain = new_drain.or(old_drain);
2700        slots.active = Some(new_active);
2701        slots.lead = None;
2702        promoted = true;
2703        info!("ES256: promoted lead → active");
2704    }
2705
2706    // Phase 2: remove expired drain
2707    if let Some(ref drain) = slots.drain {
2708        if now >= drain.exp {
2709            delete_es256_slot(state_dir, "drain");
2710            slots.drain = None;
2711            info!("ES256: removed expired drain slot");
2712        }
2713    }
2714
2715    // A lead is publication-only and must not survive its own bounded window
2716    // if an administrator supplied inconsistent slot files.
2717    if slots.lead.as_ref().is_some_and(|lead| now >= lead.exp) {
2718        delete_es256_slot(state_dir, "lead");
2719        slots.lead = None;
2720        info!("ES256: removed expired lead slot");
2721    }
2722
2723    // Phase 3: generate lead if active is near expiry
2724    if let Some(ref active) = slots.active {
2725        if slots.lead.is_none() && active.exp - now < lead_secs {
2726            let lead_nbf = active.exp - lead_secs;
2727            let lead_exp = lead_nbf + active_secs;
2728            let new_lead = generate_es256_slot(lead_nbf, lead_exp);
2729            if let Err(e) = persist_es256_slot(state_dir, "lead", &new_lead) {
2730                error!(
2731                    "ES256: failed to persist new lead to '{}': {e}. Rotation state \
2732                     will not survive restart.",
2733                    state_dir.display()
2734                );
2735            } else {
2736                info!("ES256: generated new lead key (kid={})", new_lead.kid());
2737            }
2738            slots.lead = Some(new_lead);
2739        }
2740    }
2741    promoted
2742}
2743
2744// ════════════════════════════════════════════════════════════════════════════════
2745// ML-DSA-65 Key Rotation Store (always compiled; runtime CryptoPolicy)
2746// ════════════════════════════════════════════════════════════════════════════════
2747
2748mod ml_dsa_rotation {
2749    use super::*;
2750    use hyprstream_rpc::crypto::pq::{
2751        ml_dsa_generate_keypair, ml_dsa_sk_from_seed, ml_dsa_sk_to_seed, MlDsaSigningKey,
2752    };
2753
2754    #[derive(Clone)]
2755    pub struct MlDsaKeySlot {
2756        pub key: Arc<MlDsaSigningKey>,
2757        pub nbf: i64,
2758        pub exp: i64,
2759    }
2760
2761    impl MlDsaKeySlot {
2762        pub fn new(key: MlDsaSigningKey, nbf: i64, exp: i64) -> Self {
2763            Self {
2764                key: Arc::new(key),
2765                nbf,
2766                exp,
2767            }
2768        }
2769
2770        pub fn verifying_key(&self) -> hyprstream_rpc::crypto::pq::MlDsaVerifyingKey {
2771            ml_dsa::Keypair::verifying_key(&*self.key).clone()
2772        }
2773    }
2774
2775    #[derive(Clone, Default)]
2776    pub struct MlDsaKeySlots {
2777        pub drain: Option<MlDsaKeySlot>,
2778        pub active: Option<MlDsaKeySlot>,
2779        pub lead: Option<MlDsaKeySlot>,
2780    }
2781
2782    impl MlDsaKeySlots {
2783        pub fn all(&self) -> Vec<&MlDsaKeySlot> {
2784            [&self.drain, &self.active, &self.lead]
2785                .into_iter()
2786                .flatten()
2787                .collect()
2788        }
2789    }
2790
2791    #[derive(Clone)]
2792    pub struct MlDsaSigningKeyStore(pub Arc<RwLock<MlDsaKeySlots>>);
2793
2794    impl MlDsaSigningKeyStore {
2795        pub fn new(slots: MlDsaKeySlots) -> Self {
2796            Self(Arc::new(RwLock::new(slots)))
2797        }
2798
2799        pub async fn active_key(&self) -> Option<Arc<MlDsaSigningKey>> {
2800            self.0
2801                .read()
2802                .await
2803                .active
2804                .as_ref()
2805                .map(|s| Arc::clone(&s.key))
2806        }
2807
2808        pub async fn all_slots_snapshot(&self) -> Vec<MlDsaKeySlot> {
2809            self.0.read().await.all().into_iter().cloned().collect()
2810        }
2811    }
2812
2813    // ── ML-DSA persistence ─────────────────────────────────────────────────
2814
2815    fn ml_dsa_slot_paths(secrets_dir: &Path, name: &str) -> (PathBuf, PathBuf) {
2816        (
2817            secrets_dir.join(format!("ml-dsa-signing-key.{name}")),
2818            secrets_dir.join(format!("ml-dsa-signing-key.{name}.meta")),
2819        )
2820    }
2821
2822    pub(super) fn load_ml_dsa_slot(secrets_dir: &Path, name: &str) -> Option<MlDsaKeySlot> {
2823        let (key_path, meta_path) = ml_dsa_slot_paths(secrets_dir, name);
2824        let seed_bytes = std::fs::read(&key_path).ok()?;
2825        if seed_bytes.len() != 32 {
2826            warn!(
2827                "ML-DSA key slot '{name}': unexpected seed length {}",
2828                seed_bytes.len()
2829            );
2830            return None;
2831        }
2832        let meta_bytes = std::fs::read(&meta_path).ok()?;
2833        let meta: SlotMeta = serde_json::from_slice(&meta_bytes).ok()?;
2834        let mut seed = [0u8; 32];
2835        seed.copy_from_slice(&seed_bytes);
2836        let key = ml_dsa_sk_from_seed(&seed);
2837        Some(MlDsaKeySlot::new(key, meta.nbf, meta.exp))
2838    }
2839
2840    pub(super) fn persist_ml_dsa_slot(
2841        secrets_dir: &Path,
2842        name: &str,
2843        slot: &MlDsaKeySlot,
2844    ) -> anyhow::Result<()> {
2845        // Atomic write + 0600 perms (#179).
2846        let seed = ml_dsa_sk_to_seed(&slot.key);
2847        super::super::identity_store::write_secret(
2848            secrets_dir,
2849            &format!("ml-dsa-signing-key.{name}"),
2850            &seed,
2851        )?;
2852        let meta = SlotMeta {
2853            nbf: slot.nbf,
2854            exp: slot.exp,
2855        };
2856        super::super::identity_store::write_secret(
2857            secrets_dir,
2858            &format!("ml-dsa-signing-key.{name}.meta"),
2859            &serde_json::to_vec(&meta)?,
2860        )?;
2861        Ok(())
2862    }
2863
2864    fn delete_ml_dsa_slot(secrets_dir: &Path, name: &str) {
2865        let (key_path, meta_path) = ml_dsa_slot_paths(secrets_dir, name);
2866        let _ = std::fs::remove_file(&key_path);
2867        let _ = std::fs::remove_file(&meta_path);
2868    }
2869
2870    pub(super) fn generate_ml_dsa_slot(nbf: i64, exp: i64) -> MlDsaKeySlot {
2871        let (key, _vk) = ml_dsa_generate_keypair();
2872        MlDsaKeySlot::new(key, nbf, exp)
2873    }
2874
2875    pub fn load_or_init_ml_dsa_key_store(
2876        secrets_dir: &Path,
2877        config: &OAuthConfig,
2878    ) -> MlDsaSigningKeyStore {
2879        let now = chrono::Utc::now().timestamp();
2880        let active_secs = config.active_secs();
2881        let lead_secs = config.lead_secs();
2882
2883        let state_dir = rotation_state_dir(secrets_dir);
2884        let _slot_lock = match lock_signing_slots(&state_dir, false) {
2885            Ok(lock) => lock,
2886            Err(error) => {
2887                error!("Could not lock ML-DSA signing slots: {error}; refusing initialization");
2888                return MlDsaSigningKeyStore::new(MlDsaKeySlots::default());
2889            }
2890        };
2891        let has_committed_authority = has_committed_composite_marker(secrets_dir);
2892        let drain = load_ml_dsa_slot(&state_dir, "drain")
2893            .or_else(|| load_ml_dsa_slot(secrets_dir, "drain"));
2894        let mut active = load_ml_dsa_slot(&state_dir, "active")
2895            .or_else(|| load_ml_dsa_slot(secrets_dir, "active"));
2896        let lead =
2897            load_ml_dsa_slot(&state_dir, "lead").or_else(|| load_ml_dsa_slot(secrets_dir, "lead"));
2898
2899        if active.is_none() {
2900            if has_committed_authority {
2901                error!(
2902                    "Committed composite authority exists but no active ML-DSA signing slot is \
2903                     available; refusing implicit key generation"
2904                );
2905            } else {
2906                info!("No active ML-DSA-65 signing key found — generating on first boot");
2907                let slot = generate_ml_dsa_slot(now, now + active_secs);
2908                // Disk-before-memory, matching `load_or_init_key_store`: the
2909                // generated key activates only after its slot persisted.
2910                match persist_ml_dsa_slot(&state_dir, "active", &slot) {
2911                    Ok(()) => {
2912                        info!("Active ML-DSA-65 key generated");
2913                        active = load_ml_dsa_slot(&state_dir, "active");
2914                    }
2915                    Err(e) => error!(
2916                        "Could not persist active ML-DSA key to '{}': {e}. Refusing \
2917                         to activate the unpersisted key; no active signer is \
2918                         available until persistence succeeds on a restart.",
2919                        state_dir.display()
2920                    ),
2921                }
2922            }
2923        }
2924
2925        let should_gen_lead = !has_committed_authority
2926            && lead.is_none()
2927            && active.as_ref().is_some_and(|a| a.exp - now < lead_secs);
2928        if should_gen_lead {
2929            let lead_nbf = active.as_ref().map(|a| a.exp).unwrap_or(now) - lead_secs;
2930            let lead_exp = lead_nbf + active_secs;
2931            let slot = generate_ml_dsa_slot(lead_nbf, lead_exp);
2932            if let Err(e) = persist_ml_dsa_slot(&state_dir, "lead", &slot) {
2933                error!(
2934                    "Could not persist lead ML-DSA key to '{}': {e}. Rotation state \
2935                     will not survive restart.",
2936                    state_dir.display()
2937                );
2938            }
2939        }
2940
2941        let lead = if should_gen_lead {
2942            load_ml_dsa_slot(&state_dir, "lead")
2943        } else {
2944            lead
2945        };
2946        MlDsaSigningKeyStore::new(MlDsaKeySlots {
2947            drain,
2948            active,
2949            lead,
2950        })
2951    }
2952
2953    /// Returns `true` when every signer slot this cycle touched finished fully
2954    /// restored (or promoted). `false` means a promotion rollback failed: the
2955    /// durable state is only partially restored, the old active signer stays
2956    /// recoverable in its drain-slot copy, and callers must not treat the
2957    /// cycle as a clean success. A later tick retries the queued lead.
2958    pub async fn rotate_ml_dsa_keys(
2959        config: &OAuthConfig,
2960        secrets_dir: &Path,
2961        store: &MlDsaSigningKeyStore,
2962        now: i64,
2963    ) -> bool {
2964        let state_dir = rotation_state_dir(secrets_dir);
2965        let _slot_lock = match lock_signing_slots(&state_dir, true) {
2966            Ok(lock) => lock,
2967            Err(error) => {
2968                warn!("Could not lock ML-DSA signing slots; deferring rotation: {error}");
2969                return false;
2970            }
2971        };
2972        rotate_ml_dsa_keys_in_state_dir(config, &state_dir, store, now).await
2973    }
2974
2975    /// Rotation against an explicit state directory. Private test seam: a
2976    /// state dir that cannot hold files injects deterministic persistence
2977    /// failures without relying on permission bits.
2978    pub(super) async fn rotate_ml_dsa_keys_in_state_dir(
2979        config: &OAuthConfig,
2980        state_dir: &Path,
2981        store: &MlDsaSigningKeyStore,
2982        now: i64,
2983    ) -> bool {
2984        let mut slots = store.0.write().await;
2985        let active_secs = config.active_secs();
2986        let lead_secs = config.lead_secs();
2987        let drain_secs = config.drain_secs();
2988
2989        // Phase 1: promote lead → active. Same disk-before-memory contract as
2990        // the Ed25519 rotation: the durable drain write (retaining the old
2991        // active) and the durable active write (installing the new signer)
2992        // must both succeed before any in-memory slot changes. On a failure
2993        // the old active stays active in memory, the lead stays queued, the
2994        // pre-promotion durable state is restored best-effort, and no
2995        // promotion is reported. The gate is successful completion of
2996        // `persist_ml_dsa_slot`; not proven power-loss durability (residual).
2997        if let Some(new_active) = slots.lead.as_ref().filter(|lead| lead.nbf <= now).cloned() {
2998            let old_active = slots.active.clone();
2999            let old_drain = slots.drain.clone();
3000
3001            info!("ML-DSA: promoting lead to active");
3002
3003            if let Some(ref old) = old_active {
3004                if let Err(e) = persist_ml_dsa_slot(state_dir, "drain", old) {
3005                    // Mirror of the Ed25519 rollback: the restore (or removal)
3006                    // of the previous drain is part of the rollback and its
3007                    // outcome must be confirmed.
3008                    let drain_restored = if let Some(ref prev) = old_drain {
3009                        persist_ml_dsa_slot(state_dir, "drain", prev)
3010                    } else {
3011                        delete_ml_dsa_slot(state_dir, "drain");
3012                        // Error-preserving removal proof, mirroring Ed25519.
3013                        let (drain_key, drain_meta) = ml_dsa_slot_paths(state_dir, "drain");
3014                        let removed = [&drain_key, &drain_meta].iter().all(|path| {
3015                            matches!(
3016                                std::fs::symlink_metadata(path),
3017                                Err(error) if error.kind() == std::io::ErrorKind::NotFound
3018                            )
3019                        });
3020                        if !removed {
3021                            Err(anyhow::anyhow!(
3022                                "partial replacement drain slot still present after removal"
3023                            ))
3024                        } else {
3025                            Ok(())
3026                        }
3027                    };
3028                    if let Err(drain_error) = drain_restored {
3029                        error!(
3030                            "ML-DSA: could not restore previous drain slot to '{}': \
3031                             {drain_error}. The durable drain slot is not fully \
3032                             restored; retaining the old active in memory and \
3033                             retrying next tick.",
3034                            state_dir.display()
3035                        );
3036                        return false;
3037                    }
3038                    error!(
3039                        "ML-DSA: failed to persist drain slot to '{}': {e}. Retaining \
3040                         the old active in memory; promotion will be retried next \
3041                         tick.",
3042                        state_dir.display()
3043                    );
3044                    return true;
3045                }
3046            }
3047
3048            if let Err(e) = persist_ml_dsa_slot(state_dir, "active", &new_active) {
3049                // Roll back the active slot first, and only after that restore
3050                // is confirmed touch the drain slot: the drain copy written
3051                // above is the only durable copy of the old active signer, so
3052                // a failing rollback must leave it in place for restart
3053                // recovery instead of deleting or overwriting it with the
3054                // previous drain.
3055                let restored = match old_active {
3056                    Some(ref old) => persist_ml_dsa_slot(state_dir, "active", old),
3057                    None => {
3058                        delete_ml_dsa_slot(state_dir, "active");
3059                        Ok(())
3060                    }
3061                };
3062                if let Err(restore_error) = restored {
3063                    error!(
3064                        "ML-DSA: could not restore active key to '{}': {restore_error}. \
3065                         The failed promotion left the old active signer only in its \
3066                         drain slot; that recoverable copy is kept untouched and \
3067                         promotion will be retried next tick.",
3068                        state_dir.display()
3069                    );
3070                    return false;
3071                }
3072                if old_active.is_some() {
3073                    if let Some(ref prev) = old_drain {
3074                        if let Err(drain_error) = persist_ml_dsa_slot(state_dir, "drain", prev) {
3075                            error!(
3076                                "ML-DSA: could not restore previous drain slot to '{}': \
3077                                 {drain_error}. The old active signer is restored in the \
3078                                 active slot, but the durable drain slot may hold a \
3079                                 partial write; promotion will be retried next tick.",
3080                                state_dir.display()
3081                            );
3082                            return false;
3083                        }
3084                    } else {
3085                        delete_ml_dsa_slot(state_dir, "drain");
3086                    }
3087                }
3088                error!(
3089                    "ML-DSA: failed to persist promoted active to '{}': {e}. Retaining \
3090                     the old active in memory; promotion will be retried next tick.",
3091                    state_dir.display()
3092                );
3093                return true;
3094            }
3095
3096            delete_ml_dsa_slot(state_dir, "lead");
3097            if let Some(old) = old_active {
3098                slots.drain = Some(old);
3099            }
3100            slots.active = Some(new_active);
3101            slots.lead = None;
3102            info!("ML-DSA: promoted lead → active");
3103        }
3104
3105        // Phase 2: remove expired drain
3106        if let Some(ref drain) = slots.drain {
3107            if now >= drain.exp + drain_secs {
3108                delete_ml_dsa_slot(state_dir, "drain");
3109                slots.drain = None;
3110                info!("ML-DSA: removed expired drain slot");
3111            }
3112        }
3113
3114        // Phase 3: generate lead if active is near expiry
3115        if let Some(ref active) = slots.active {
3116            if slots.lead.is_none() && active.exp - now < lead_secs {
3117                let lead_nbf = active.exp - lead_secs;
3118                let lead_exp = lead_nbf + active_secs;
3119                let new_lead = generate_ml_dsa_slot(lead_nbf, lead_exp);
3120                if let Err(e) = persist_ml_dsa_slot(state_dir, "lead", &new_lead) {
3121                    error!(
3122                        "ML-DSA: failed to persist new lead to '{}': {e}. Rotation state \
3123                         will not survive restart.",
3124                        state_dir.display()
3125                    );
3126                } else {
3127                    info!("ML-DSA: generated new lead key");
3128                }
3129                slots.lead = Some(new_lead);
3130            }
3131        }
3132        true
3133    }
3134}
3135
3136pub use ml_dsa_rotation::{
3137    load_or_init_ml_dsa_key_store, rotate_ml_dsa_keys, MlDsaKeySlot, MlDsaKeySlots,
3138    MlDsaSigningKeyStore,
3139};
3140
3141// ── Tests ────────────────────────────────────────────────────────────────────
3142
3143#[cfg(test)]
3144#[allow(clippy::unwrap_used, clippy::expect_used)]
3145mod tests {
3146    use super::*;
3147    use std::io::Write as _;
3148    use tempfile::TempDir;
3149
3150    fn durability_ledger(version: u64) -> CompositeLedger {
3151        // Public synthetic metadata only; these tests exercise persistence and
3152        // marker selection, not signing-key generation or cryptographic trust.
3153        CompositeLedger {
3154            version,
3155            component_digest: format!("test-components-{version}"),
3156            pairs: Vec::new(),
3157        }
3158    }
3159
3160    fn durability_ledger_name(dir: &Path, ledger: &CompositeLedger) -> String {
3161        composite_committed_ledger_path(
3162            dir,
3163            &CompositeCommit {
3164                version: ledger.version,
3165                component_digest: ledger.component_digest.clone(),
3166            },
3167        )
3168        .file_name()
3169        .unwrap()
3170        .to_str()
3171        .unwrap()
3172        .to_owned()
3173    }
3174
3175    #[test]
3176    fn composite_durable_commit_orders_generation_before_marker() {
3177        use super::super::identity_store::durable_write_test::{self, Stage};
3178        let dir = TempDir::new().unwrap();
3179        let ledger = durability_ledger(1);
3180        let name = durability_ledger_name(dir.path(), &ledger);
3181        let _scope = durable_write_test::install(None);
3182        persist_committed_composite_ledger(dir.path(), &ledger).unwrap();
3183        assert_eq!(
3184            durable_write_test::events(),
3185            vec![
3186                (name.clone(), Stage::FileSync),
3187                (name, Stage::DirectorySync),
3188                ("jwt-composite-pairs.committed".into(), Stage::FileSync),
3189                ("jwt-composite-pairs.committed".into(), Stage::DirectorySync),
3190            ]
3191        );
3192        let (commit, selected) = read_committed_composite_ledger(dir.path()).unwrap();
3193        assert_eq!(commit.version, 1);
3194        assert_eq!(selected.component_digest, ledger.component_digest);
3195    }
3196
3197    #[test]
3198    fn composite_durable_commit_sync_failure_preserves_selected_authority() {
3199        use super::super::identity_store::durable_write_test::{self, Stage};
3200        for (marker_fault, stage) in [
3201            (false, Stage::FileSync),
3202            (false, Stage::DirectorySync),
3203            (true, Stage::FileSync),
3204        ] {
3205            let dir = TempDir::new().unwrap();
3206            let old = durability_ledger(1);
3207            persist_committed_composite_ledger(dir.path(), &old).unwrap();
3208            let old_marker = std::fs::read(composite_committed_path(dir.path())).unwrap();
3209            let old_snapshot = std::fs::read(composite_committed_ledger_path(
3210                dir.path(),
3211                &CompositeCommit {
3212                    version: 1,
3213                    component_digest: old.component_digest.clone(),
3214                },
3215            ))
3216            .unwrap();
3217            let next = durability_ledger(2);
3218            let name = if marker_fault {
3219                "jwt-composite-pairs.committed".to_owned()
3220            } else {
3221                durability_ledger_name(dir.path(), &next)
3222            };
3223            let _scope = durable_write_test::install(Some((name, stage)));
3224            assert!(persist_committed_composite_ledger(dir.path(), &next).is_err());
3225            let (commit, selected) = read_committed_composite_ledger(dir.path()).unwrap();
3226            assert_eq!(
3227                commit.version, 1,
3228                "a failed sync selected pending authority"
3229            );
3230            assert_eq!(selected.component_digest, old.component_digest);
3231            assert_eq!(
3232                std::fs::read(composite_committed_path(dir.path())).unwrap(),
3233                old_marker
3234            );
3235            assert_eq!(
3236                std::fs::read(composite_committed_ledger_path(dir.path(), &commit)).unwrap(),
3237                old_snapshot
3238            );
3239            if !marker_fault {
3240                assert!(durable_write_test::events()
3241                    .iter()
3242                    .all(|(file, _)| file != "jwt-composite-pairs.committed"));
3243            }
3244        }
3245    }
3246
3247    #[test]
3248    fn composite_durable_commit_marker_directory_failure_never_selects_partial_ledger() {
3249        use super::super::identity_store::durable_write_test::{self, Stage};
3250        let dir = TempDir::new().unwrap();
3251        let old = durability_ledger(1);
3252        persist_committed_composite_ledger(dir.path(), &old).unwrap();
3253        let old_marker = std::fs::read(composite_committed_path(dir.path())).unwrap();
3254        let next = durability_ledger(2);
3255        let _scope = durable_write_test::install(Some((
3256            "jwt-composite-pairs.committed".into(),
3257            Stage::DirectorySync,
3258        )));
3259        assert!(persist_committed_composite_ledger(dir.path(), &next).is_err());
3260        // Rename is already visible, but no durable-success claim is made.
3261        // If the new marker survives, its complete generation was synced first.
3262        let (commit, selected) = read_committed_composite_ledger(dir.path()).unwrap();
3263        assert_eq!(commit.version, 2);
3264        assert_eq!(selected.component_digest, next.component_digest);
3265        // Model loss of that last unsynchronized directory update: the old
3266        // marker still selects its unchanged complete generation as well.
3267        std::fs::write(composite_committed_path(dir.path()), old_marker).unwrap();
3268        let (commit, selected) = read_committed_composite_ledger(dir.path()).unwrap();
3269        assert_eq!(commit.version, 1);
3270        assert_eq!(selected.component_digest, old.component_digest);
3271    }
3272
3273    #[test]
3274    fn composite_durable_commit_legacy_migration_sync_failure_propagates() {
3275        use super::super::identity_store::durable_write_test::{self, Stage};
3276        let dir = TempDir::new().unwrap();
3277        let ledger = durability_ledger(1);
3278        let commit = CompositeCommit {
3279            version: ledger.version,
3280            component_digest: ledger.component_digest.clone(),
3281        };
3282        std::fs::write(
3283            composite_ledger_path(dir.path()),
3284            serde_json::to_vec(&ledger).unwrap(),
3285        )
3286        .unwrap();
3287        std::fs::write(
3288            composite_committed_path(dir.path()),
3289            serde_json::to_vec(&commit).unwrap(),
3290        )
3291        .unwrap();
3292        let marker = std::fs::read(composite_committed_path(dir.path())).unwrap();
3293        let _scope = durable_write_test::install(Some((
3294            durability_ledger_name(dir.path(), &ledger),
3295            Stage::DirectorySync,
3296        )));
3297        assert!(load_or_migrate_committed_composite_ledger(dir.path()).is_err());
3298        assert_eq!(
3299            std::fs::read(composite_committed_path(dir.path())).unwrap(),
3300            marker
3301        );
3302    }
3303
3304    #[tokio::test]
3305    async fn composite_durable_commit_initializer_retry_reestablishes_both_barriers() {
3306        use super::super::identity_store::durable_write_test::{self, Stage};
3307        const MODE: &str = "HYPRSTREAM_COMPOSITE_DURABILITY_RETRY_MODE";
3308        const DIRECTORY: &str = "HYPRSTREAM_COMPOSITE_DURABILITY_RETRY_DIR";
3309        let mode = std::env::var(MODE).unwrap_or_default();
3310        if mode.is_empty() {
3311            for retry_mode in ["migration", "marker"] {
3312                let dir = TempDir::new().unwrap();
3313                let phases: &[&str] = if retry_mode == "migration" {
3314                    &["bootstrap", "migration"]
3315                } else {
3316                    &["marker"]
3317                };
3318                for phase in phases {
3319                    let status = std::process::Command::new(std::env::current_exe().unwrap())
3320                        .args([
3321                            "--exact",
3322                            "auth::key_rotation::tests::composite_durable_commit_initializer_retry_reestablishes_both_barriers",
3323                            "--nocapture",
3324                        ])
3325                        .env(MODE, phase)
3326                        .env(DIRECTORY, dir.path())
3327                        .status()
3328                        .unwrap();
3329                    assert!(status.success(), "{phase} retry subprocess failed");
3330                }
3331            }
3332            return;
3333        }
3334        let dir = PathBuf::from(std::env::var_os(DIRECTORY).unwrap());
3335        let config = test_config();
3336        let ca = Arc::new(SigningKey::from_bytes(&[0x6b; 32]));
3337        let ed = load_or_init_key_store(&dir, &config);
3338        let pq = load_or_init_ml_dsa_key_store(&dir, &config);
3339        if mode == "bootstrap" {
3340            initialize_composite_key_set(&dir, &ed, &pq, ca, 300)
3341                .await
3342                .unwrap();
3343            return;
3344        }
3345        let fault_name = if mode == "migration" {
3346            let (commit, ledger) = read_committed_composite_ledger(&dir).unwrap();
3347            let name = durability_ledger_name(&dir, &ledger);
3348            std::fs::remove_file(composite_committed_ledger_path(&dir, &commit)).unwrap();
3349            name
3350        } else {
3351            "jwt-composite-pairs.committed".to_owned()
3352        };
3353        let key_set = hyprstream_rpc::auth::global_composite_key_set();
3354        assert_eq!(key_set.snapshot().version(), 0);
3355        let failed = durable_write_test::install(Some((fault_name, Stage::DirectorySync)));
3356        // The first attempt leaves a generation/marker rename visible. The
3357        // second attempt must retry its barrier and still fail closed, before
3358        // restoring even the valid persisted signing authority.
3359        for _ in 0..2 {
3360            let mutable_before = std::fs::read(composite_ledger_path(&dir)).ok();
3361            assert!(
3362                initialize_composite_key_set(&dir, &ed, &pq, Arc::clone(&ca), 300)
3363                    .await
3364                    .is_err()
3365            );
3366            assert_eq!(key_set.snapshot().version(), 0);
3367            if let Some(before) = mutable_before {
3368                assert_eq!(std::fs::read(composite_ledger_path(&dir)).unwrap(), before);
3369            }
3370        }
3371        let (commit, ledger) = read_committed_composite_ledger(&dir).unwrap();
3372        let name = durability_ledger_name(&dir, &ledger);
3373        drop(failed);
3374        let _success = durable_write_test::install(None);
3375        initialize_composite_key_set(&dir, &ed, &pq, ca, 300)
3376            .await
3377            .unwrap();
3378        assert_eq!(key_set.snapshot().version(), commit.version);
3379        assert_eq!(
3380            durable_write_test::events(),
3381            vec![
3382                (name.clone(), Stage::FileSync),
3383                (name, Stage::DirectorySync),
3384                ("jwt-composite-pairs.committed".into(), Stage::FileSync),
3385                ("jwt-composite-pairs.committed".into(), Stage::DirectorySync),
3386            ]
3387        );
3388    }
3389
3390    #[tokio::test]
3391    async fn committed_active_pair_failure_reports_safe_component_availability() {
3392        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_DIAGNOSTIC_TEST";
3393        if std::env::var_os(ISOLATED).is_none() {
3394            let status = std::process::Command::new(std::env::current_exe().unwrap())
3395                .args([
3396                    "--exact",
3397                    "auth::key_rotation::tests::committed_active_pair_failure_reports_safe_component_availability",
3398                    "--nocapture",
3399                ])
3400                .env(ISOLATED, "1")
3401                .status()
3402                .unwrap();
3403            assert!(status.success());
3404            return;
3405        }
3406
3407        let dir = TempDir::new().unwrap();
3408        let now = chrono::Utc::now().timestamp();
3409        let ca = Arc::new(SigningKey::from_bytes(&[0x61; 32]));
3410        let ed = SigningKeyStore::new(KeySlots {
3411            active: Some(KeySlot::new(
3412                SigningKey::from_bytes(&[0x62; 32]),
3413                now - 60,
3414                now + 3600,
3415            )),
3416            ..KeySlots::default()
3417        });
3418        let pq = MlDsaSigningKeyStore::new(MlDsaKeySlots {
3419            active: Some(ml_dsa_rotation::generate_ml_dsa_slot(now - 60, now + 3600)),
3420            ..MlDsaKeySlots::default()
3421        });
3422        initialize_composite_key_set(dir.path(), &ed, &pq, Arc::clone(&ca), 300)
3423            .await
3424            .unwrap();
3425        let commit: CompositeCommit =
3426            serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
3427                .unwrap();
3428        let immutable = composite_committed_ledger_path(dir.path(), &commit);
3429        let mut ledger: CompositeLedger =
3430            serde_json::from_slice(&std::fs::read(&immutable).unwrap()).unwrap();
3431        let oauth = ledger
3432            .pairs
3433            .iter()
3434            .find(|pair| pair.role == "oauth" && pair.state == "active")
3435            .unwrap();
3436
3437        let missing_ed = SigningKeyStore::new(KeySlots::default());
3438        let error =
3439            initialize_composite_key_set(dir.path(), &missing_ed, &pq, Arc::clone(&ca), 300)
3440                .await
3441                .expect_err("missing committed OAuth Ed component must fail closed");
3442        assert_eq!(
3443            error.to_string(),
3444            format!(
3445                "committed active composite signing pair is unavailable after restart: \
3446                 kid={:?} role={:?} ed_component_available=false pq_component_available=true \
3447                 policy_ca_ed_matched=false",
3448                oauth.kid, oauth.role,
3449            )
3450        );
3451        assert!(!error.to_string().contains(&oauth.ed25519_public));
3452        assert!(!error.to_string().contains(&oauth.ml_dsa_public));
3453
3454        // Put Policy first in this synthetic committed snapshot so the same
3455        // restore check reports its CA match when the PQ component is absent.
3456        ledger.pairs.sort_by_key(|pair| pair.role != "policy");
3457        std::fs::write(&immutable, serde_json::to_vec(&ledger).unwrap()).unwrap();
3458        let policy = ledger
3459            .pairs
3460            .iter()
3461            .find(|pair| pair.role == "policy" && pair.state == "active")
3462            .unwrap();
3463        let missing_pq = MlDsaSigningKeyStore::new(MlDsaKeySlots::default());
3464        let error = initialize_composite_key_set(dir.path(), &ed, &missing_pq, ca, 300)
3465            .await
3466            .expect_err("missing committed Policy PQ component must fail closed");
3467        assert_eq!(
3468            error.to_string(),
3469            format!(
3470                "committed active composite signing pair is unavailable after restart: \
3471                 kid={:?} role={:?} ed_component_available=true pq_component_available=false \
3472                 policy_ca_ed_matched=true",
3473                policy.kid, policy.role,
3474            )
3475        );
3476        assert!(!error.to_string().contains(&policy.ed25519_public));
3477        assert!(!error.to_string().contains(&policy.ml_dsa_public));
3478    }
3479
3480    #[test]
3481    fn rejected_composite_proposal_rechecks_replaced_component_slot() {
3482        let dir = TempDir::new().unwrap();
3483        let fingerprint = || {
3484            let mut digest = Sha256::new();
3485            fingerprint_component_slot_files(dir.path(), &mut digest).unwrap();
3486            digest.finalize().to_vec()
3487        };
3488        let missing = fingerprint();
3489        super::super::identity_store::write_secret(
3490            dir.path(),
3491            "jwt-signing-key.active",
3492            b"first-component",
3493        )
3494        .unwrap();
3495        let first = fingerprint();
3496        assert_ne!(missing, first);
3497        super::super::identity_store::write_secret(
3498            dir.path(),
3499            "jwt-signing-key.active",
3500            b"second-component",
3501        )
3502        .unwrap();
3503        assert_ne!(first, fingerprint());
3504    }
3505
3506    struct PermitFixtureAccountReads;
3507
3508    impl hyprstream_pds_service::AccountRecordReadAuthorizer for PermitFixtureAccountReads {
3509        fn check_read(
3510            &self,
3511            _subject: &hyprstream_rpc::Subject,
3512            _verified_tenant: Option<&str>,
3513            _security_context: Option<&hyprstream_rpc::auth::mac::SecurityContext>,
3514            _object_id: &str,
3515        ) -> hyprstream_rpc::auth::mac::MacDecision {
3516            hyprstream_rpc::auth::mac::MacDecision::Permit
3517        }
3518    }
3519
3520    fn hosted_account_store(
3521        label: &str,
3522        zone: &str,
3523    ) -> anyhow::Result<Arc<hyprstream_pds_service::AccountRecordStore>> {
3524        let ed = ed25519_dalek::SigningKey::generate(&mut rand::rngs::OsRng);
3525        let (pq, pq_vk) = hyprstream_crypto::pq::ml_dsa_generate_keypair();
3526        let hybrid = hyprstream_pds::did_op::HybridRotationKey::new(
3527            ed.verifying_key().to_bytes(),
3528            hyprstream_crypto::pq::ml_dsa_vk_bytes(&pq_vk),
3529        )?;
3530        let rotations = hyprstream_pds::did_op::GenesisRotationKeys::new(
3531            hyprstream_pds::did_op::UserRotationKey::new(hybrid),
3532            hyprstream_pds::did_op::RecoveryKeyEnrollment::Declined,
3533            hyprstream_pds::did_op::HostKeyEnrollment::Absent,
3534        )?;
3535        let name =
3536            hyprstream_pds::AllocatedAccountName::new(label, format!("did:web:{label}.{zone}"))?;
3537        let mint = hyprstream_pds::HostedAccountMint::begin(name, rotations)?;
3538        let document = mint.seal_did_document(&format!("https://{zone}"))?;
3539        let pending =
3540            mint.prepare_genesis(document, hyprstream_pds::did_op::GenesisRepoHead::EmptyRepo)?;
3541        let signature = hyprstream_pds::did_op::sign_genesis(pending.unsigned_genesis(), &ed, &pq)?;
3542        let record = pending.seal(signature)?.record_bytes().to_vec();
3543        let root = hyprstream_vfs::SyntheticNode::dir().with_child(
3544            zone,
3545            hyprstream_vfs::SyntheticNode::dir().with_child(
3546                "accounts",
3547                hyprstream_vfs::SyntheticNode::dir().with_child(
3548                    label,
3549                    hyprstream_vfs::SyntheticNode::dir().with_child(
3550                        "account-record.cbor",
3551                        hyprstream_vfs::SyntheticNode::file(record),
3552                    ),
3553                ),
3554            ),
3555        );
3556        Ok(Arc::new(hyprstream_pds_service::AccountRecordStore::new(
3557            Arc::new(hyprstream_vfs::SyntheticMount::new(root)),
3558            Arc::new(PermitFixtureAccountReads),
3559        )))
3560    }
3561
3562    fn test_config() -> OAuthConfig {
3563        OAuthConfig {
3564            jwt_key_active_days: 14,
3565            jwt_key_lead_days: 7,
3566            jwt_key_drain_days: 30,
3567            ..OAuthConfig::default()
3568        }
3569    }
3570
3571    struct ProductionAuthorityVerifier {
3572        transport: hyprstream_rpc::transport::TransportConfig,
3573        signing_key: hyprstream_rpc::prelude::SigningKey,
3574        key_source: Arc<hyprstream_rpc::auth::ClusterKeySource>,
3575        accepted: Arc<std::sync::atomic::AtomicBool>,
3576        error: Arc<parking_lot::Mutex<Option<String>>>,
3577    }
3578
3579    #[async_trait::async_trait(?Send)]
3580    impl crate::services::RequestService for ProductionAuthorityVerifier {
3581        fn decode_request_body(
3582            &self,
3583            signed_body: &[u8],
3584        ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
3585            Ok(hyprstream_rpc::service::DecodedRequestBody::opaque(
3586                signed_body.to_vec(),
3587            ))
3588        }
3589
3590        async fn handle_request(
3591            &self,
3592            _ctx: &crate::services::EnvelopeContext,
3593            _body: &hyprstream_rpc::service::DecodedRequestBody,
3594        ) -> anyhow::Result<(Vec<u8>, Option<crate::services::Continuation>)> {
3595            self.accepted.store(true, Ordering::Release);
3596            Ok((Vec::new(), None))
3597        }
3598
3599        fn name(&self) -> &str {
3600            "multiprocess-authority-verifier"
3601        }
3602
3603        fn transport(&self) -> &hyprstream_rpc::transport::TransportConfig {
3604            &self.transport
3605        }
3606
3607        fn signing_key(&self) -> hyprstream_rpc::prelude::SigningKey {
3608            self.signing_key.clone()
3609        }
3610
3611        fn jwt_key_source(&self) -> Option<Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
3612            Some(self.key_source.clone())
3613        }
3614
3615        fn expected_audience(&self) -> Option<&str> {
3616            Some("multiprocess")
3617        }
3618
3619        fn require_cnf_binding(&self) -> bool {
3620            false
3621        }
3622
3623        fn jwt_verify_policy(&self) -> hyprstream_rpc::crypto::CryptoPolicy {
3624            hyprstream_rpc::crypto::CryptoPolicy::Hybrid
3625        }
3626
3627        fn build_error_payload(&self, _request_id: u64, error: &str) -> Vec<u8> {
3628            *self.error.lock() = Some(error.to_owned());
3629            Vec::new()
3630        }
3631    }
3632
3633    async fn verify_through_rpc_endpoint(dir: &Path, token: String) -> anyhow::Result<()> {
3634        use hyprstream_rpc::rpc_client::RpcClientImpl;
3635        use hyprstream_rpc::signer::LocalSigner;
3636        use hyprstream_rpc::transport::lazy_uds::LazyUdsTransport;
3637        let rpc = RpcClientImpl::new(
3638            LocalSigner::new(SigningKey::from_bytes(&[0x75; 32])),
3639            LazyUdsTransport::new(dir.join("authority-rpc.sock")),
3640            Some(SigningKey::from_bytes(&[0x74; 32]).verifying_key()),
3641        )
3642        .with_response_verify_policy(hyprstream_rpc::crypto::CryptoPolicy::Classical)
3643        .with_default_jwt(token);
3644        rpc.call(b"verify-composite-authority".to_vec()).await?;
3645        Ok(())
3646    }
3647
3648    fn policy_client_for_socket(
3649        dir: &Path,
3650    ) -> anyhow::Result<hyprstream_rpc_std::policy_client::PolicyClient> {
3651        policy_client_for_named_socket(dir, "authority-policy.sock")
3652    }
3653
3654    fn policy_client_for_named_socket(
3655        dir: &Path,
3656        socket: &str,
3657    ) -> anyhow::Result<hyprstream_rpc_std::policy_client::PolicyClient> {
3658        use hyprstream_rpc::rpc_client::RpcClientImpl;
3659        use hyprstream_rpc::signer::LocalSigner;
3660        use hyprstream_rpc::transport::lazy_uds::LazyUdsTransport;
3661        let rpc = RpcClientImpl::new(
3662            LocalSigner::new(SigningKey::from_bytes(&[0x76; 32])),
3663            LazyUdsTransport::new(dir.join(socket)),
3664            Some(SigningKey::from_bytes(&[0x73; 32]).verifying_key()),
3665        )
3666        .with_response_verify_policy(hyprstream_rpc::crypto::CryptoPolicy::Classical);
3667        Ok(hyprstream_rpc_std::policy_client::PolicyClient::new(
3668            Arc::new(rpc),
3669        ))
3670    }
3671
3672    fn authority_process_dir() -> Option<PathBuf> {
3673        let dir = std::env::var_os("HYPRSTREAM_COMPOSITE_PRODUCTION_TEST_DIR")?;
3674        // Each authority role runs in a fresh process, so the dispatch fixture
3675        // must be installed here rather than in the parent orchestrator.
3676        crate::mac::install_explicit_test_dispatch_pep();
3677        Some(PathBuf::from(dir))
3678    }
3679
3680    fn authority_ca_key() -> Arc<SigningKey> {
3681        Arc::new(SigningKey::from_bytes(&[0x5a; 32]))
3682    }
3683
3684    fn install_hybrid_request_anchor(client_key: &SigningKey) -> anyhow::Result<()> {
3685        let mut store = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
3686        let pq_key = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(client_key);
3687        store.bind(
3688            client_key.verifying_key().to_bytes(),
3689            &ml_dsa::Keypair::verifying_key(&pq_key),
3690        );
3691        hyprstream_rpc::envelope::install_verify_config(
3692            hyprstream_rpc::envelope::EnvelopeVerifyConfig {
3693                policy: hyprstream_rpc::crypto::CryptoPolicy::Hybrid,
3694                pq_store: Some(Arc::new(store)),
3695            },
3696        )
3697    }
3698
3699    async fn install_signing_authority(
3700        dir: &Path,
3701        subscribe: bool,
3702    ) -> anyhow::Result<(Arc<SigningKeyStore>, Arc<MlDsaSigningKeyStore>)> {
3703        let config = test_config();
3704        let ca = authority_ca_key();
3705        let ed = Arc::new(load_or_init_key_store(dir, &config));
3706        let pq = Arc::new(load_or_init_ml_dsa_key_store(dir, &config));
3707        *composite_ca_signing_key().write() = Some(Arc::clone(&ca));
3708        configure_composite_authority(dir);
3709        initialize_composite_key_set(dir, &ed, &pq, ca, 300).await?;
3710        if subscribe {
3711            start_composite_authority_subscription(
3712                dir.to_path_buf(),
3713                authority_ca_key().verifying_key(),
3714            )?;
3715        }
3716        Ok((ed, pq))
3717    }
3718
3719    async fn install_verifying_authority(dir: &Path) -> anyhow::Result<()> {
3720        let config = test_config();
3721        let ca = authority_ca_key();
3722        let ed = load_or_init_key_store(dir, &config);
3723        let pq = load_or_init_ml_dsa_key_store(dir, &config);
3724        restore_composite_verifying_key_set(dir, &ed, &pq, ca.verifying_key()).await?;
3725        start_composite_authority_subscription(dir.to_path_buf(), ca.verifying_key())?;
3726        Ok(())
3727    }
3728
3729    const PRODUCTION_AUTHORITY_BARRIER_TIMEOUT: std::time::Duration =
3730        std::time::Duration::from_secs(120);
3731
3732    fn wait_path(path: &Path) {
3733        let deadline = std::time::Instant::now() + PRODUCTION_AUTHORITY_BARRIER_TIMEOUT;
3734        while !path.exists() {
3735            assert!(
3736                std::time::Instant::now() < deadline,
3737                "timeout: {}",
3738                path.display()
3739            );
3740            std::thread::sleep(std::time::Duration::from_millis(10));
3741        }
3742    }
3743
3744    fn wait_for_target_authority(dir: &Path) -> CompositeCommit {
3745        let target_path = dir.join("target-authority.json");
3746        wait_path(&target_path);
3747        let target: CompositeCommit =
3748            serde_json::from_slice(&std::fs::read(target_path).unwrap()).unwrap();
3749        let deadline = std::time::Instant::now() + PRODUCTION_AUTHORITY_BARRIER_TIMEOUT;
3750        loop {
3751            let snapshot = hyprstream_rpc::auth::global_composite_key_set().snapshot();
3752            if snapshot.version() == target.version
3753                && snapshot.component_digest() == target.component_digest
3754            {
3755                return target;
3756            }
3757            assert!(
3758                std::time::Instant::now() < deadline,
3759                "process did not converge to {} {}",
3760                target.version,
3761                target.component_digest
3762            );
3763            std::thread::sleep(std::time::Duration::from_millis(10));
3764        }
3765    }
3766
3767    async fn verify_returned_tokens(dir: &Path, process: &str) -> anyhow::Result<()> {
3768        wait_path(&dir.join("tokens-ready"));
3769        for token_file in ["oauth-token", "policy-token"] {
3770            verify_through_rpc_endpoint(dir, std::fs::read_to_string(dir.join(token_file))?)
3771                .await?;
3772        }
3773        std::fs::write(dir.join(format!("verified-{process}")), b"1")?;
3774        Ok(())
3775    }
3776
3777    fn wait_for_done(dir: &Path) {
3778        wait_path(&dir.join("done"));
3779    }
3780
3781    /// Publish the completed target snapshot in one rename-visible operation.
3782    ///
3783    /// The child readers use file existence as their readiness signal, so a
3784    /// direct write can expose an empty file between create and completion.
3785    /// Keep this helper test-only: it models the completed fixture handoff
3786    /// without changing the production authority or rollback paths.
3787    fn publish_target_authority(dir: &Path, target: &CompositeCommit) {
3788        let bytes = serde_json::to_vec(target).unwrap();
3789        let mut temporary = tempfile::NamedTempFile::new_in(dir).unwrap();
3790        temporary.write_all(&bytes).unwrap();
3791        temporary.flush().unwrap();
3792        temporary
3793            .persist(dir.join("target-authority.json"))
3794            .unwrap();
3795    }
3796
3797    async fn mutate_authority_for_failure(
3798        ed: &SigningKeyStore,
3799        pq: &MlDsaSigningKeyStore,
3800        persist_dir: Option<&Path>,
3801    ) {
3802        let now = chrono::Utc::now().timestamp();
3803        let new_ed = KeySlot::new(
3804            SigningKey::generate(&mut rand::rngs::OsRng),
3805            now,
3806            now + 1200,
3807        );
3808        let mut ed_slots = ed.0.write().await;
3809        ed_slots.drain = ed_slots.active.take();
3810        ed_slots.active = Some(new_ed.clone());
3811        if let Some(dir) = persist_dir {
3812            if let Some(drain) = &ed_slots.drain {
3813                persist_slot(dir, "drain", drain).unwrap();
3814            }
3815            persist_slot(dir, "active", &new_ed).unwrap();
3816        }
3817        drop(ed_slots);
3818
3819        let (pq_key, _) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
3820        let new_pq = MlDsaKeySlot::new(pq_key, now, now + 1200);
3821        let mut pq_slots = pq.0.write().await;
3822        pq_slots.drain = pq_slots.active.take();
3823        pq_slots.active = Some(new_pq.clone());
3824        if let Some(dir) = persist_dir {
3825            if let Some(drain) = &pq_slots.drain {
3826                ml_dsa_rotation::persist_ml_dsa_slot(dir, "drain", drain).unwrap();
3827            }
3828            ml_dsa_rotation::persist_ml_dsa_slot(dir, "active", &new_pq).unwrap();
3829        }
3830    }
3831
3832    /// Fixed classical primary key + base64url for the multiprocess issuance
3833    /// requests (they mint user at+jwt, which v16 binds to an authoritative
3834    /// primary suite).
3835    fn test_primary_key() -> [u8; 32] {
3836        [0x66; 32]
3837    }
3838    fn test_user_pub_key_b64() -> String {
3839        use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
3840        URL_SAFE_NO_PAD.encode(test_primary_key())
3841    }
3842    /// A permissive isolated primary resolver (any subject → the fixed classical
3843    /// key) — the fixture equivalent of WS-C's enrollment store for these
3844    /// multiprocess authority tests.
3845    fn test_permissive_primary_resolver(
3846    ) -> Arc<dyn crate::services::policy::PrimaryEnrollmentResolver> {
3847        struct R;
3848        impl crate::services::policy::PrimaryEnrollmentResolver for R {
3849            fn primary_group(
3850                &self,
3851                _principal: &str,
3852                _tenant: &str,
3853            ) -> Option<crate::services::policy::PrimaryGroup> {
3854                Some(crate::services::policy::PrimaryGroup {
3855                    suite_id: hyprstream_rpc::auth::SUITE_CLASSICAL_ED25519.to_owned(),
3856                    ordered_component_keys: vec![test_primary_key().to_vec()],
3857                })
3858            }
3859        }
3860        Arc::new(R)
3861    }
3862
3863    #[test]
3864    fn composite_oauth_production_process() {
3865        let Some(dir) = authority_process_dir() else {
3866            return;
3867        };
3868        let runtime = tokio::runtime::Runtime::new().unwrap();
3869        runtime
3870            .block_on(async {
3871                install_signing_authority(&dir, true).await?;
3872                wait_path(&dir.join("authority-policy.sock"));
3873                // Mirror production `init_process_authority_stores`: a non-policy
3874                // process (this OAuth child) publishes RPC-client authority
3875                // stores that DELEGATE to the canonical policy-owned authority
3876                // over the policy socket — never private in-memory stores, which
3877                // the separate policy process (where `issue_token` actually runs)
3878                // could not see. A fresh sid registered here therefore crosses
3879                // RPC into the canonical registry that `handle_issue_token`
3880                // validates against, and issued-token verification checks the
3881                // same canonical revocation store. Fail-closed is preserved: an
3882                // unreachable authority answers not-active / revoked. Isolated to
3883                // this re-exec'd single-test subprocess — the main-binary
3884                // invocation of this fn early-returns above before reaching here.
3885                let _ = hyprstream_rpc::auth::set_global_session_registry(Arc::new(
3886                    crate::services::revocation::PolicyAuthoritySessionRegistry::new(
3887                        policy_client_for_socket(&dir)?,
3888                    ),
3889                ));
3890                let _ = hyprstream_rpc::auth::set_global_credential_revocation_store(Arc::new(
3891                    crate::services::revocation::PolicyAuthorityRevocationStore::new(
3892                        policy_client_for_socket(&dir)?,
3893                    ),
3894                ));
3895                let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
3896                let address = listener.local_addr()?;
3897                let mut config = test_config();
3898                config.external_url = Some(format!("http://{address}"));
3899                config.token_ttl_seconds = 60;
3900                let unused: Arc<dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor> =
3901                    Arc::new(hyprstream_rpc::transport::rpc_session::from_fn(|_| async {
3902                        Ok(bytes::Bytes::new())
3903                    }));
3904                hyprstream_rpc::dial::register_inproc("multiprocess-unused-discovery", &unused);
3905                let discovery = hyprstream_rpc_std::discovery_client::DiscoveryClient::for_local_endpoint_bootstrap(
3906                    "inproc://multiprocess-unused-discovery",
3907                    SigningKey::from_bytes(&[0x76; 32]),
3908                    SigningKey::from_bytes(&[0x77; 32]).verifying_key(),
3909                    None,
3910                )?;
3911                let user_store = Arc::new(crate::auth::RocksDbUserStore::open(
3912                    &dir.join("oauth-users"),
3913                )?);
3914                crate::auth::UserStore::register(user_store.as_ref(), "multiprocess-oauth").await?;
3915                crate::auth::UserStore::set_profile(
3916                    user_store.as_ref(),
3917                    "multiprocess-oauth",
3918                    crate::auth::UserProfilePatch {
3919                        atproto_did: Some(Some(
3920                            "did:web:multiprocess-oauth.example.test".to_owned(),
3921                        )),
3922                        ..Default::default()
3923                    },
3924                )
3925                .await?;
3926                let hosted_store = hosted_account_store("multiprocess-oauth", "example.test")?;
3927                // The production account store warms its signed hosted-DID
3928                // index during startup.  This isolated multi-process fixture
3929                // constructs the store directly, so warm the same index before
3930                // the OAuth child accepts its first authorization-code exchange.
3931                hosted_store
3932                    .refresh_hosted_did_index(&hyprstream_rpc::Subject::new(
3933                        hyprstream_pds_service::OAUTH_ACCOUNT_RESOLVER_SUBJECT,
3934                    ))
3935                    .await?;
3936                let state = Arc::new(
3937                    crate::services::oauth::state::OAuthState::new(
3938                        &config,
3939                        policy_client_for_socket(&dir)?,
3940                        discovery,
3941                        authority_ca_key().verifying_key().to_bytes(),
3942                    )
3943                    .with_user_store(crate::auth::ProductionUserStore::for_test(user_store))
3944                    .with_hosted_account_zone(crate::account::AccountZone::new(
3945                        "example.test",
3946                    )?)
3947                    .with_hosted_account_store(hosted_store),
3948                );
3949                let verifier = "multiprocess-pkce-verifier";
3950                let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes()));
3951                for code in [
3952                    "multiprocess-code",
3953                    "post-failure-code",
3954                    "timeout-code",
3955                    "post-crash-code",
3956                ] {
3957                    state.pending_codes.write().await.insert(
3958                        code.to_owned(),
3959                        crate::services::oauth::state::PendingAuthCode {
3960                            code: code.to_owned(),
3961                            client_id: "multiprocess-client".to_owned(),
3962                            redirect_uri: "https://client.test/callback".to_owned(),
3963                            code_challenge: challenge.clone(),
3964                            scopes: vec!["read".to_owned()],
3965                            resource: Some("multiprocess".to_owned()),
3966                            oidc_nonce: None,
3967                            created_at: std::time::Instant::now(),
3968                            expires_at: std::time::Instant::now()
3969                                + std::time::Duration::from_secs(60),
3970                            username: "multiprocess-oauth".to_owned(),
3971                            verifying_key: None,
3972                            dpop_jkt: None,
3973                            client_assertion_jkt: None,
3974                        },
3975                    );
3976                }
3977                let app = crate::services::oauth::create_app(
3978                    state,
3979                    &crate::config::CorsConfig::default(),
3980                );
3981                let server = tokio::spawn(async move { axum::serve(listener, app).await });
3982                std::fs::write(dir.join("oauth-http-url"), format!("http://{address}"))?;
3983                std::fs::write(dir.join("ready-oauth"), b"1")?;
3984                wait_for_target_authority(&dir);
3985                std::fs::write(dir.join("converged-oauth"), b"1")?;
3986                verify_returned_tokens(&dir, "oauth").await?;
3987                wait_for_done(&dir);
3988                server.abort();
3989                anyhow::Ok(())
3990            })
3991            .unwrap();
3992    }
3993
3994    #[test]
3995    fn composite_policy_production_process() {
3996        let Some(dir) = authority_process_dir() else {
3997            return;
3998        };
3999        let runtime = tokio::runtime::Runtime::new().unwrap();
4000        runtime
4001            .block_on(async {
4002                install_signing_authority(&dir, true).await?;
4003                // Canonical policy-owned authority stores (v16 §3.3): the policy
4004                // process owns the single session registry and credential
4005                // revocation store. Every other process delegates to it over RPC
4006                // (see the OAuth child's PolicyAuthority* stores), so a sid the
4007                // OAuth child registers crosses RPC into THIS registry, which
4008                // `handle_issue_token` then validates against. Published before
4009                // the PolicyService serves. Isolated to this re-exec'd
4010                // single-test subprocess.
4011                let _ = hyprstream_rpc::auth::set_global_session_registry(Arc::new(
4012                    hyprstream_rpc::auth::InMemorySessionRegistry::new(),
4013                ));
4014                let _ = hyprstream_rpc::auth::set_global_credential_revocation_store(Arc::new(
4015                    hyprstream_rpc::auth::InMemoryCredentialRevocationStore::new(),
4016                ));
4017                let db = dir.join(format!("policy-db-{}", std::process::id()));
4018                std::fs::create_dir_all(&db)?;
4019                let policy_manager = Arc::new(
4020                    crate::auth::PolicyManager::new(&db.join(".registry/policies")).await?,
4021                );
4022                let git2db = Arc::new(tokio::sync::RwLock::new(git2db::Git2DB::open(&db).await?));
4023                let client_key = SigningKey::from_bytes(&[0x76; 32]);
4024                install_hybrid_request_anchor(&client_key)?;
4025                hyprstream_service::global_trust_store().insert(
4026                    client_key.verifying_key(),
4027                    hyprstream_service::Attestation {
4028                        scopes: std::iter::once("oauth".to_owned()).collect(),
4029                        subject: None,
4030                        jwt: None,
4031                        expires_at: chrono::Utc::now().timestamp() + 300,
4032                        attested_by: None,
4033                    },
4034                );
4035                let service = crate::services::PolicyService::new(
4036                    policy_manager,
4037                    Arc::new(SigningKey::from_bytes(&[0x73; 32])),
4038                    crate::config::TokenConfig::default(),
4039                    git2db,
4040                    hyprstream_rpc::transport::TransportConfig::ipc(
4041                        dir.join("authority-policy.sock"),
4042                    ),
4043                )
4044                .with_default_audience("multiprocess".to_owned())
4045                // Production-equivalent key source: signing resolves the
4046                // composite authority through the configured JwtKeySource (the
4047                // `ClusterKeySource` ledger defaults to the process-global
4048                // authority this subprocess configures).
4049                .with_jwt_key_source(Arc::new(hyprstream_rpc::auth::ClusterKeySource::new(
4050                    SigningKey::from_bytes(&[0x73; 32]).verifying_key(),
4051                    "multiprocess".to_owned(),
4052                )))
4053                .with_primary_enrollment_resolver(test_permissive_primary_resolver());
4054                let shutdown = Arc::new(tokio::sync::Notify::new());
4055                let shutdown_server = Arc::clone(&shutdown);
4056                std::thread::spawn(move || {
4057                    use hyprstream_rpc::service::Spawnable as _;
4058                    Box::new(service).run(shutdown_server, None).unwrap();
4059                });
4060                wait_path(&dir.join("authority-policy.sock"));
4061                std::fs::write(dir.join("ready-policy"), b"1")?;
4062                wait_for_target_authority(&dir);
4063                std::fs::write(dir.join("converged-policy"), b"1")?;
4064                let token = policy_client_for_socket(&dir)?
4065                    .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4066                        requested_scopes: Some(vec!["read".to_owned()]),
4067                        ttl: Some(60),
4068                        audience: Some("multiprocess".to_owned()),
4069                        subject: Some("multiprocess-policy".to_owned()),
4070                        user_pub_key: Some(test_user_pub_key_b64()),
4071                        dpop_jkt: None,
4072                        issuer: None,
4073                        tenant: None,
4074                        require_clearance: false,
4075                        session_id: None,
4076                        issuance_profile:
4077                            hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4078                        client_id: Some("hyprstream-oauth-client-1".to_owned()),
4079                    })
4080                    .await?
4081                    .token;
4082                std::fs::write(dir.join("policy-token"), token)?;
4083                verify_returned_tokens(&dir, "policy").await?;
4084                wait_for_done(&dir);
4085                shutdown.notify_waiters();
4086                anyhow::Ok(())
4087            })
4088            .unwrap();
4089    }
4090
4091    #[test]
4092    fn composite_jwks_production_process() {
4093        let Some(dir) = authority_process_dir() else {
4094            return;
4095        };
4096        let runtime = tokio::runtime::Runtime::new().unwrap();
4097        runtime
4098            .block_on(async {
4099                install_verifying_authority(&dir).await?;
4100                std::fs::write(dir.join("ready-jwks"), b"1")?;
4101                wait_for_target_authority(&dir);
4102                std::fs::write(dir.join("converged-jwks"), b"1")?;
4103                wait_path(&dir.join("jwks-response.json"));
4104                verify_returned_tokens(&dir, "jwks").await?;
4105                anyhow::Ok(())
4106            })
4107            .unwrap();
4108        wait_for_done(&dir);
4109    }
4110
4111    #[test]
4112    fn composite_rpc_production_process() {
4113        let Some(dir) = authority_process_dir() else {
4114            return;
4115        };
4116        let runtime = tokio::runtime::Runtime::new().unwrap();
4117        runtime
4118            .block_on(async {
4119                install_verifying_authority(&dir).await?;
4120                install_hybrid_request_anchor(&SigningKey::from_bytes(&[0x75; 32]))?;
4121                let accepted = Arc::new(std::sync::atomic::AtomicBool::new(false));
4122                let service = ProductionAuthorityVerifier {
4123                    transport: hyprstream_rpc::transport::TransportConfig::ipc(
4124                        dir.join("authority-rpc.sock"),
4125                    ),
4126                    signing_key: SigningKey::from_bytes(&[0x74; 32]),
4127                    key_source: Arc::new(hyprstream_rpc::auth::ClusterKeySource::new(
4128                        authority_ca_key().verifying_key(),
4129                        "https://multiprocess.test".to_owned(),
4130                    )),
4131                    accepted,
4132                    error: Arc::new(parking_lot::Mutex::new(None)),
4133                };
4134                let shutdown = Arc::new(tokio::sync::Notify::new());
4135                let shutdown_server = Arc::clone(&shutdown);
4136                std::thread::spawn(move || {
4137                    use hyprstream_rpc::service::Spawnable as _;
4138                    Box::new(service).run(shutdown_server, None).unwrap();
4139                });
4140                wait_path(&dir.join("authority-rpc.sock"));
4141                std::fs::write(dir.join("ready-rpc"), b"1")?;
4142                wait_for_target_authority(&dir);
4143                std::fs::write(dir.join("converged-rpc"), b"1")?;
4144                verify_returned_tokens(&dir, "rpc").await?;
4145                wait_for_done(&dir);
4146                shutdown.notify_waiters();
4147                anyhow::Ok(())
4148            })
4149            .unwrap();
4150    }
4151
4152    #[test]
4153    fn composite_stale_policy_production_process() {
4154        let Some(dir) = authority_process_dir() else {
4155            return;
4156        };
4157        let runtime = tokio::runtime::Runtime::new().unwrap();
4158        runtime
4159            .block_on(async {
4160                install_signing_authority(&dir, false).await?;
4161                std::fs::write(dir.join("ready-stale-policy"), b"1")?;
4162                wait_path(&dir.join("stale-attempt-gate"));
4163                let db = dir.join(format!("stale-policy-db-{}", std::process::id()));
4164                std::fs::create_dir_all(&db)?;
4165                let policy_manager = Arc::new(
4166                    crate::auth::PolicyManager::new(&db.join(".registry/policies")).await?,
4167                );
4168                let git2db = Arc::new(tokio::sync::RwLock::new(git2db::Git2DB::open(&db).await?));
4169                let client_key = SigningKey::from_bytes(&[0x76; 32]);
4170                install_hybrid_request_anchor(&client_key)?;
4171                hyprstream_service::global_trust_store().insert(
4172                    client_key.verifying_key(),
4173                    hyprstream_service::Attestation {
4174                        scopes: std::iter::once("oauth".to_owned()).collect(),
4175                        subject: None,
4176                        jwt: None,
4177                        expires_at: chrono::Utc::now().timestamp() + 300,
4178                        attested_by: None,
4179                    },
4180                );
4181                let service = crate::services::PolicyService::new(
4182                    policy_manager,
4183                    Arc::new(SigningKey::from_bytes(&[0x73; 32])),
4184                    crate::config::TokenConfig::default(),
4185                    git2db,
4186                    hyprstream_rpc::transport::TransportConfig::ipc(dir.join("stale-policy.sock")),
4187                )
4188                .with_default_audience("multiprocess".to_owned())
4189                .with_jwt_key_source(Arc::new(hyprstream_rpc::auth::ClusterKeySource::new(
4190                    SigningKey::from_bytes(&[0x73; 32]).verifying_key(),
4191                    "multiprocess".to_owned(),
4192                )))
4193                .with_primary_enrollment_resolver(test_permissive_primary_resolver());
4194                let shutdown = Arc::new(tokio::sync::Notify::new());
4195                let shutdown_server = Arc::clone(&shutdown);
4196                std::thread::spawn(move || {
4197                    use hyprstream_rpc::service::Spawnable as _;
4198                    Box::new(service).run(shutdown_server, None).unwrap();
4199                });
4200                wait_path(&dir.join("stale-policy.sock"));
4201                let result = policy_client_for_named_socket(&dir, "stale-policy.sock")?
4202                    .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4203                        requested_scopes: Some(vec!["read".to_owned()]),
4204                        ttl: Some(60),
4205                        audience: Some("multiprocess".to_owned()),
4206                        subject: Some("stale-policy".to_owned()),
4207                        user_pub_key: Some(test_user_pub_key_b64()),
4208                        dpop_jkt: None,
4209                        issuer: None,
4210                        tenant: None,
4211                        require_clearance: false,
4212                        session_id: None,
4213                        issuance_profile:
4214                            hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4215                        client_id: Some("hyprstream-oauth-client-1".to_owned()),
4216                    })
4217                    .await;
4218                anyhow::ensure!(result.is_err(), "stale PolicyService minted a token");
4219                shutdown.notify_waiters();
4220                std::fs::write(dir.join("stale-policy-refused"), b"1")?;
4221                anyhow::Ok(())
4222            })
4223            .unwrap();
4224    }
4225
4226    #[test]
4227    fn composite_stale_writer_process() {
4228        let Some(dir) = authority_process_dir() else {
4229            return;
4230        };
4231        let runtime = tokio::runtime::Runtime::new().unwrap();
4232        runtime
4233            .block_on(async {
4234                let (ed, pq) = install_signing_authority(&dir, false).await?;
4235                let expected = hyprstream_rpc::auth::global_composite_key_set()
4236                    .snapshot()
4237                    .component_digest()
4238                    .to_owned();
4239                std::fs::write(dir.join("ready-stale-writer"), b"1")?;
4240                wait_path(&dir.join("stale-attempt-gate"));
4241                let result =
4242                    refresh_composite_key_set(&dir, &ed, &pq, authority_ca_key(), 300, &expected)
4243                        .await;
4244                let error = result.expect_err("stale writer publication unexpectedly succeeded");
4245                anyhow::ensure!(
4246                    error
4247                        .to_string()
4248                        .contains("stale composite component authority"),
4249                    "unexpected stale writer error: {error:#}"
4250                );
4251                std::fs::write(dir.join("stale-writer-refused"), error.to_string())?;
4252                anyhow::Ok(())
4253            })
4254            .unwrap();
4255    }
4256
4257    #[test]
4258    fn composite_crashing_writer_process() {
4259        let Some(dir) = authority_process_dir() else {
4260            return;
4261        };
4262        let runtime = tokio::runtime::Runtime::new().unwrap();
4263        runtime
4264            .block_on(async {
4265                let config = test_config();
4266                let ed = load_or_init_key_store(&dir, &config);
4267                let pq = load_or_init_ml_dsa_key_store(&dir, &config);
4268                mutate_authority_for_failure(&ed, &pq, Some(&dir)).await;
4269                let expected = std::fs::read_to_string(dir.join("crash-expected-digest"))?;
4270                refresh_composite_key_set(&dir, &ed, &pq, authority_ca_key(), 300, &expected).await
4271            })
4272            .unwrap();
4273        panic!("crash mutation returned instead of exiting after stage");
4274    }
4275
4276    #[test]
4277    fn composite_timing_out_writer_process() {
4278        let Some(dir) = authority_process_dir() else {
4279            return;
4280        };
4281        let runtime = tokio::runtime::Runtime::new().unwrap();
4282        runtime
4283            .block_on(async {
4284                let config = test_config();
4285                let ed = load_or_init_key_store(&dir, &config);
4286                let pq = load_or_init_ml_dsa_key_store(&dir, &config);
4287                mutate_authority_for_failure(&ed, &pq, None).await;
4288                let expected = std::fs::read_to_string(dir.join("timeout-expected-digest"))?;
4289                let error =
4290                    refresh_composite_key_set(&dir, &ed, &pq, authority_ca_key(), 300, &expected)
4291                        .await
4292                        .expect_err("unacknowledged composite generation committed");
4293                anyhow::ensure!(
4294                    error.to_string().contains("was not acknowledged"),
4295                    "unexpected acknowledgement-timeout error: {error:#}"
4296                );
4297                std::fs::write(dir.join("timeout-writer-refused"), error.to_string())?;
4298                anyhow::Ok(())
4299            })
4300            .unwrap();
4301    }
4302
4303    #[test]
4304    fn composite_restart_rpc_production_process() {
4305        let Some(dir) = authority_process_dir() else {
4306            return;
4307        };
4308        let runtime = tokio::runtime::Runtime::new().unwrap();
4309        runtime
4310            .block_on(async {
4311                install_verifying_authority(&dir).await?;
4312                verify_through_rpc_endpoint(
4313                    &dir,
4314                    std::fs::read_to_string(dir.join("old-oauth-token"))?,
4315                )
4316                .await?;
4317                std::fs::write(dir.join("restart-drain-verified"), b"1")?;
4318                anyhow::Ok(())
4319            })
4320            .unwrap();
4321    }
4322
4323    #[test]
4324    fn composite_restart_policy_production_process() {
4325        let Some(dir) = authority_process_dir() else {
4326            return;
4327        };
4328        let runtime = tokio::runtime::Runtime::new().unwrap();
4329        runtime
4330            .block_on(async {
4331                install_signing_authority(&dir, false).await?;
4332                let db = dir.join(format!("restart-policy-db-{}", std::process::id()));
4333                std::fs::create_dir_all(&db)?;
4334                let policy_manager = Arc::new(
4335                    crate::auth::PolicyManager::new(&db.join(".registry/policies")).await?,
4336                );
4337                let git2db = Arc::new(tokio::sync::RwLock::new(git2db::Git2DB::open(&db).await?));
4338                let client_key = SigningKey::from_bytes(&[0x76; 32]);
4339                install_hybrid_request_anchor(&client_key)?;
4340                hyprstream_service::global_trust_store().insert(
4341                    client_key.verifying_key(),
4342                    hyprstream_service::Attestation {
4343                        scopes: std::iter::once("oauth".to_owned()).collect(),
4344                        subject: None,
4345                        jwt: None,
4346                        expires_at: chrono::Utc::now().timestamp() + 300,
4347                        attested_by: None,
4348                    },
4349                );
4350                let socket = "restart-policy.sock";
4351                let service = crate::services::PolicyService::new(
4352                    policy_manager,
4353                    Arc::new(SigningKey::from_bytes(&[0x73; 32])),
4354                    crate::config::TokenConfig::default(),
4355                    git2db,
4356                    hyprstream_rpc::transport::TransportConfig::ipc(dir.join(socket)),
4357                )
4358                .with_default_audience("multiprocess".to_owned())
4359                .with_jwt_key_source(Arc::new(hyprstream_rpc::auth::ClusterKeySource::new(
4360                    SigningKey::from_bytes(&[0x73; 32]).verifying_key(),
4361                    "multiprocess".to_owned(),
4362                )))
4363                .with_primary_enrollment_resolver(test_permissive_primary_resolver());
4364                let shutdown = Arc::new(tokio::sync::Notify::new());
4365                let shutdown_server = Arc::clone(&shutdown);
4366                std::thread::spawn(move || {
4367                    use hyprstream_rpc::service::Spawnable as _;
4368                    Box::new(service).run(shutdown_server, None).unwrap();
4369                });
4370                wait_path(&dir.join(socket));
4371                let token = policy_client_for_named_socket(&dir, socket)?
4372                    .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4373                        requested_scopes: Some(vec!["read".to_owned()]),
4374                        ttl: Some(60),
4375                        audience: Some("multiprocess".to_owned()),
4376                        subject: Some("restart-policy".to_owned()),
4377                        user_pub_key: Some(test_user_pub_key_b64()),
4378                        dpop_jkt: None,
4379                        issuer: None,
4380                        tenant: None,
4381                        require_clearance: false,
4382                        session_id: None,
4383                        issuance_profile:
4384                            hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4385                        client_id: Some("hyprstream-oauth-client-1".to_owned()),
4386                    })
4387                    .await?
4388                    .token;
4389                verify_through_rpc_endpoint(&dir, token).await?;
4390                std::fs::write(dir.join("restart-policy-minted"), b"1")?;
4391                shutdown.notify_waiters();
4392                anyhow::Ok(())
4393            })
4394            .unwrap();
4395    }
4396
4397    fn spawn_production_authority_process(dir: &Path, test_name: &str) -> std::process::Child {
4398        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
4399        command
4400            .args(["--exact", test_name, "--nocapture"])
4401            .env("HYPRSTREAM_COMPOSITE_PRODUCTION_TEST_DIR", dir);
4402        if test_name.ends_with("composite_crashing_writer_process") {
4403            command.env("HYPRSTREAM_COMPOSITE_CRASH_AFTER_STAGE", "1");
4404        }
4405        if test_name.ends_with("composite_timing_out_writer_process") {
4406            command.env(
4407                "HYPRSTREAM_COMPOSITE_STAGE_MARKER",
4408                dir.join("timeout-staged"),
4409            );
4410        }
4411        command.spawn().unwrap()
4412    }
4413
4414    #[test]
4415    fn composite_authority_production_paths_reject_semantic_rollback() {
4416        const ORCHESTRATOR_DIR: &str = "HYPRSTREAM_COMPOSITE_ORCHESTRATOR_TEST_DIR";
4417        if std::env::var_os(ORCHESTRATOR_DIR).is_none() {
4418            // UDS socket paths are limited to ~108 bytes; keep the parent
4419            // short and unique even when the worktree has a long absolute path.
4420            let dir = TempDir::new().unwrap();
4421            let status = std::process::Command::new(std::env::current_exe().unwrap())
4422                .args([
4423                    "--exact",
4424                    "auth::key_rotation::tests::composite_authority_production_paths_reject_semantic_rollback",
4425                    "--nocapture",
4426                ])
4427                .env(ORCHESTRATOR_DIR, dir.path())
4428                .status()
4429                .unwrap();
4430            assert!(status.success());
4431            return;
4432        }
4433
4434        struct AuthorityTestDir(PathBuf);
4435        impl AuthorityTestDir {
4436            fn path(&self) -> &Path {
4437                &self.0
4438            }
4439        }
4440        let dir = AuthorityTestDir(PathBuf::from(std::env::var_os(ORCHESTRATOR_DIR).unwrap()));
4441        let config = test_config();
4442        let now = chrono::Utc::now().timestamp();
4443        let old_ed = KeySlot::new(SigningKey::from_bytes(&[0x11; 32]), now - 10, now + 600);
4444        let (old_pq_key, _) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
4445        let old_pq = MlDsaKeySlot::new(old_pq_key, now - 10, now + 600);
4446        persist_slot(dir.path(), "active", &old_ed).unwrap();
4447        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &old_pq).unwrap();
4448        let ca = authority_ca_key();
4449        let runtime = tokio::runtime::Runtime::new().unwrap();
4450        let old_ed_store = load_or_init_key_store(dir.path(), &config);
4451        let old_pq_store = load_or_init_ml_dsa_key_store(dir.path(), &config);
4452        runtime
4453            .block_on(initialize_composite_key_set(
4454                dir.path(),
4455                &old_ed_store,
4456                &old_pq_store,
4457                Arc::clone(&ca),
4458                300,
4459            ))
4460            .unwrap();
4461        let old_digest = hyprstream_rpc::auth::global_composite_key_set()
4462            .snapshot()
4463            .component_digest()
4464            .to_owned();
4465
4466        let process_tests = [
4467            (
4468                "oauth",
4469                "auth::key_rotation::tests::composite_oauth_production_process",
4470            ),
4471            (
4472                "policy",
4473                "auth::key_rotation::tests::composite_policy_production_process",
4474            ),
4475            (
4476                "jwks",
4477                "auth::key_rotation::tests::composite_jwks_production_process",
4478            ),
4479            (
4480                "rpc",
4481                "auth::key_rotation::tests::composite_rpc_production_process",
4482            ),
4483        ];
4484        let mut services: Vec<_> = process_tests
4485            .iter()
4486            .map(|(_, test)| spawn_production_authority_process(dir.path(), test))
4487            .collect();
4488        let mut stale_policy = spawn_production_authority_process(
4489            dir.path(),
4490            "auth::key_rotation::tests::composite_stale_policy_production_process",
4491        );
4492        let mut stale_writer = spawn_production_authority_process(
4493            dir.path(),
4494            "auth::key_rotation::tests::composite_stale_writer_process",
4495        );
4496        for process in [
4497            "oauth",
4498            "policy",
4499            "jwks",
4500            "rpc",
4501            "stale-policy",
4502            "stale-writer",
4503        ] {
4504            wait_path(&dir.path().join(format!("ready-{process}")));
4505        }
4506
4507        // Capture a pre-rotation token through the booted OAuth HTTP endpoint;
4508        // it is used later to prove the committed drain remains usable.
4509        let oauth_url = std::fs::read_to_string(dir.path().join("oauth-http-url")).unwrap();
4510        let old_token = runtime
4511            .block_on(async {
4512                policy_client_for_socket(dir.path())?
4513                    .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4514                        requested_scopes: Some(vec!["read".to_owned()]),
4515                        ttl: Some(60),
4516                        audience: Some("multiprocess".to_owned()),
4517                        subject: Some("pre-rotation-policy".to_owned()),
4518                        user_pub_key: Some(test_user_pub_key_b64()),
4519                        dpop_jkt: None,
4520                        issuer: None,
4521                        tenant: None,
4522                        require_clearance: false,
4523                        session_id: None,
4524                        issuance_profile:
4525                            hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4526                        client_id: Some("hyprstream-oauth-client-1".to_owned()),
4527                    })
4528                    .await?;
4529                let client = reqwest::Client::new();
4530                let authorization_code_form = [
4531                    ("grant_type", "authorization_code"),
4532                    ("client_id", "multiprocess-client"),
4533                    ("code", "multiprocess-code"),
4534                    ("redirect_uri", "https://client.test/callback"),
4535                    ("code_verifier", "multiprocess-pkce-verifier"),
4536                ];
4537                let response = client
4538                    .post(format!("{oauth_url}/oauth/token"))
4539                    .form(&authorization_code_form)
4540                    .send()
4541                    .await?;
4542                if !response.status().is_success() {
4543                    let status = response.status();
4544                    let body = response.text().await.unwrap_or_default();
4545                    anyhow::bail!("pre-rotation OAuth endpoint returned {status}: {body}");
4546                }
4547                let body: serde_json::Value = response.json().await?;
4548                let token = body
4549                    .get("access_token")
4550                    .and_then(serde_json::Value::as_str)
4551                    .map(str::to_owned)
4552                    .ok_or_else(|| {
4553                        anyhow::anyhow!("pre-rotation OAuth response omitted access_token")
4554                    })?;
4555                let replay = client
4556                    .post(format!("{oauth_url}/oauth/token"))
4557                    .form(&authorization_code_form)
4558                    .send()
4559                    .await?;
4560                anyhow::ensure!(
4561                    replay.status() == reqwest::StatusCode::BAD_REQUEST,
4562                    "authorization-code replay returned {}, expected 400",
4563                    replay.status()
4564                );
4565                let replay_body: serde_json::Value = replay.json().await?;
4566                anyhow::ensure!(
4567                    replay_body.get("error").and_then(serde_json::Value::as_str)
4568                        == Some("invalid_grant"),
4569                    "authorization-code replay was not rejected as invalid_grant: {replay_body}"
4570                );
4571                anyhow::Ok(token)
4572            })
4573            .unwrap();
4574        std::fs::write(dir.path().join("old-oauth-token"), old_token).unwrap();
4575
4576        persist_slot(dir.path(), "drain", &old_ed).unwrap();
4577        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "drain", &old_pq).unwrap();
4578        let new_ed = KeySlot::new(SigningKey::from_bytes(&[0x22; 32]), now, now + 1200);
4579        let (new_pq_key, _) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
4580        let new_pq = MlDsaKeySlot::new(new_pq_key, now, now + 1200);
4581        persist_slot(dir.path(), "active", &new_ed).unwrap();
4582        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &new_pq).unwrap();
4583        let new_ed_store = load_or_init_key_store(dir.path(), &config);
4584        let new_pq_store = load_or_init_ml_dsa_key_store(dir.path(), &config);
4585        runtime
4586            .block_on(refresh_composite_key_set(
4587                dir.path(),
4588                &new_ed_store,
4589                &new_pq_store,
4590                Arc::clone(&ca),
4591                300,
4592                &old_digest,
4593            ))
4594            .unwrap();
4595        let committed: CompositeCommit =
4596            serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
4597                .unwrap();
4598        let ledger_after_rotation: CompositeLedger =
4599            serde_json::from_slice(&std::fs::read(composite_ledger_path(dir.path())).unwrap())
4600                .unwrap();
4601        assert_eq!(committed.version, ledger_after_rotation.version);
4602        assert_eq!(
4603            committed.component_digest,
4604            ledger_after_rotation.component_digest
4605        );
4606        assert_ne!(committed.component_digest, old_digest);
4607
4608        // Recreate a round-three committed-B layout (marker + matching mutable
4609        // ledger, no immutable B). The production initializer must migrate B
4610        // while holding the ledger lock, before either failure writer can
4611        // replace the mutable file with pending C.
4612        let committed_immutable = composite_committed_ledger_path(dir.path(), &committed);
4613        std::fs::remove_file(&committed_immutable).unwrap();
4614        runtime
4615            .block_on(initialize_composite_key_set(
4616                dir.path(),
4617                &new_ed_store,
4618                &new_pq_store,
4619                Arc::clone(&ca),
4620                300,
4621            ))
4622            .unwrap();
4623        let migrated: CompositeLedger =
4624            serde_json::from_slice(&std::fs::read(&committed_immutable).unwrap()).unwrap();
4625        assert_eq!(migrated.version, committed.version);
4626        assert_eq!(migrated.component_digest, committed.component_digest);
4627        assert_eq!(
4628            std::fs::read(composite_ledger_path(dir.path())).unwrap(),
4629            serde_json::to_vec(&ledger_after_rotation).unwrap(),
4630            "legacy migration changed mutable B before pending C was staged"
4631        );
4632        publish_target_authority(dir.path(), &committed);
4633        for process in ["oauth", "policy", "jwks", "rpc"] {
4634            wait_path(&dir.path().join(format!("converged-{process}")));
4635        }
4636
4637        let oauth_url = std::fs::read_to_string(dir.path().join("oauth-http-url")).unwrap();
4638        let (oauth_token, jwks) = runtime
4639            .block_on(async {
4640                let client = reqwest::Client::new();
4641                let response = client
4642                    .post(format!("{oauth_url}/oauth/token"))
4643                    .form(&[
4644                        ("grant_type", "authorization_code"),
4645                        ("client_id", "multiprocess-client"),
4646                        ("code", "post-failure-code"),
4647                        ("redirect_uri", "https://client.test/callback"),
4648                        ("code_verifier", "multiprocess-pkce-verifier"),
4649                    ])
4650                    .send()
4651                    .await?;
4652                anyhow::ensure!(
4653                    response.status().is_success(),
4654                    "ordinary OAuth token endpoint returned {}: {}",
4655                    response.status(),
4656                    response.text().await?
4657                );
4658                let body: serde_json::Value = response.json().await?;
4659                let token = body
4660                    .get("access_token")
4661                    .and_then(serde_json::Value::as_str)
4662                    .ok_or_else(|| anyhow::anyhow!("OAuth response omitted access_token"))?
4663                    .to_owned();
4664                let jwks_response = client.get(format!("{oauth_url}/oauth/jwks")).send().await?;
4665                anyhow::ensure!(
4666                    jwks_response.status().is_success(),
4667                    "HTTP JWKS endpoint returned {}",
4668                    jwks_response.status()
4669                );
4670                let jwks = jwks_response.json::<serde_json::Value>().await?;
4671                anyhow::Ok((token, jwks))
4672            })
4673            .unwrap();
4674        std::fs::write(dir.path().join("oauth-token"), oauth_token).unwrap();
4675        std::fs::write(
4676            dir.path().join("jwks-response.json"),
4677            serde_json::to_vec(&jwks).unwrap(),
4678        )
4679        .unwrap();
4680
4681        wait_path(&dir.path().join("oauth-token"));
4682        wait_path(&dir.path().join("policy-token"));
4683        std::fs::write(dir.path().join("tokens-ready"), b"1").unwrap();
4684        for process in ["oauth", "policy", "jwks", "rpc"] {
4685            wait_path(&dir.path().join(format!("verified-{process}")));
4686        }
4687        wait_path(&dir.path().join("jwks-response.json"));
4688        let jwks: serde_json::Value =
4689            serde_json::from_slice(&std::fs::read(dir.path().join("jwks-response.json")).unwrap())
4690                .unwrap();
4691        assert_eq!(jwks["composite_version"].as_u64(), Some(committed.version));
4692        assert_eq!(
4693            jwks["composite_component_digest"].as_str(),
4694            Some(committed.component_digest.as_str())
4695        );
4696        let active_kids: Vec<&str> = ledger_after_rotation
4697            .pairs
4698            .iter()
4699            .filter(|pair| pair.state == "active")
4700            .map(|pair| pair.kid.as_str())
4701            .collect();
4702        for kid in active_kids {
4703            assert!(jwks["keys"]
4704                .as_array()
4705                .unwrap()
4706                .iter()
4707                .any(|key| { key.get("kid").and_then(serde_json::Value::as_str) == Some(kid) }));
4708        }
4709
4710        // Hold a writer at the acknowledgement barrier with an in-memory-only
4711        // component generation. Live services must keep serving and minting
4712        // exclusively from the previous committed snapshot throughout timeout.
4713        std::fs::write(
4714            dir.path().join("timeout-expected-digest"),
4715            &committed.component_digest,
4716        )
4717        .unwrap();
4718        let mut timing_out_writer = spawn_production_authority_process(
4719            dir.path(),
4720            "auth::key_rotation::tests::composite_timing_out_writer_process",
4721        );
4722        wait_path(&dir.path().join("timeout-staged"));
4723        let (timeout_oauth_token, timeout_policy_token, timeout_jwks) = runtime
4724            .block_on(async {
4725                let client = reqwest::Client::new();
4726                let response = client
4727                    .post(format!("{oauth_url}/oauth/token"))
4728                    .form(&[
4729                        ("grant_type", "authorization_code"),
4730                        ("client_id", "multiprocess-client"),
4731                        ("code", "timeout-code"),
4732                        ("redirect_uri", "https://client.test/callback"),
4733                        ("code_verifier", "multiprocess-pkce-verifier"),
4734                    ])
4735                    .send()
4736                    .await?;
4737                anyhow::ensure!(
4738                    response.status().is_success(),
4739                    "OAuth mint failed while publication was pending: {}",
4740                    response.text().await?
4741                );
4742                let oauth = response.json::<serde_json::Value>().await?["access_token"]
4743                    .as_str()
4744                    .ok_or_else(|| anyhow::anyhow!("pending-timeout OAuth token missing"))?
4745                    .to_owned();
4746                let policy = policy_client_for_socket(dir.path())?
4747                    .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4748                        requested_scopes: Some(vec!["read".to_owned()]),
4749                        ttl: Some(60),
4750                        audience: Some("multiprocess".to_owned()),
4751                        subject: Some("timeout-policy".to_owned()),
4752                        user_pub_key: Some(test_user_pub_key_b64()),
4753                        dpop_jkt: None,
4754                        issuer: None,
4755                        tenant: None,
4756                        require_clearance: false,
4757                        session_id: None,
4758                        issuance_profile:
4759                            hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4760                        client_id: Some("hyprstream-oauth-client-1".to_owned()),
4761                    })
4762                    .await?
4763                    .token;
4764                let jwks = client
4765                    .get(format!("{oauth_url}/oauth/jwks"))
4766                    .send()
4767                    .await?
4768                    .json::<serde_json::Value>()
4769                    .await?;
4770                verify_through_rpc_endpoint(dir.path(), oauth.clone()).await?;
4771                verify_through_rpc_endpoint(dir.path(), policy.clone()).await?;
4772                anyhow::Ok((oauth, policy, jwks))
4773            })
4774            .unwrap();
4775        assert_eq!(
4776            timeout_jwks["composite_version"].as_u64(),
4777            Some(committed.version)
4778        );
4779        assert_eq!(
4780            timeout_jwks["composite_component_digest"].as_str(),
4781            Some(committed.component_digest.as_str())
4782        );
4783        wait_path(&dir.path().join("timeout-writer-refused"));
4784        assert!(timing_out_writer.wait().unwrap().success());
4785        let committed_after_timeout: CompositeCommit =
4786            serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
4787                .unwrap();
4788        assert_eq!(committed_after_timeout.version, committed.version);
4789        assert_eq!(
4790            committed_after_timeout.component_digest,
4791            committed.component_digest
4792        );
4793
4794        // A distinct writer now dies immediately after persisting and staging
4795        // another generation. Pending-only pairs must never enter JWKS,
4796        // verification, or mint authority, and restart must select the marker.
4797        std::fs::write(
4798            dir.path().join("crash-expected-digest"),
4799            &committed.component_digest,
4800        )
4801        .unwrap();
4802        let mut crashing_writer = spawn_production_authority_process(
4803            dir.path(),
4804            "auth::key_rotation::tests::composite_crashing_writer_process",
4805        );
4806        assert_eq!(crashing_writer.wait().unwrap().code(), Some(86));
4807        let pending_after_crash: CompositeLedger =
4808            serde_json::from_slice(&std::fs::read(composite_ledger_path(dir.path())).unwrap())
4809                .unwrap();
4810        assert_ne!(
4811            pending_after_crash.component_digest,
4812            committed.component_digest
4813        );
4814        let (post_crash_token, post_crash_jwks) = runtime
4815            .block_on(async {
4816                let client = reqwest::Client::new();
4817                let response = client
4818                    .post(format!("{oauth_url}/oauth/token"))
4819                    .form(&[
4820                        ("grant_type", "authorization_code"),
4821                        ("client_id", "multiprocess-client"),
4822                        ("code", "post-crash-code"),
4823                        ("redirect_uri", "https://client.test/callback"),
4824                        ("code_verifier", "multiprocess-pkce-verifier"),
4825                    ])
4826                    .send()
4827                    .await?;
4828                anyhow::ensure!(
4829                    response.status().is_success(),
4830                    "OAuth mint failed after writer crash: {}",
4831                    response.text().await?
4832                );
4833                let token = response.json::<serde_json::Value>().await?["access_token"]
4834                    .as_str()
4835                    .ok_or_else(|| anyhow::anyhow!("post-crash OAuth token missing"))?
4836                    .to_owned();
4837                let jwks = client
4838                    .get(format!("{oauth_url}/oauth/jwks"))
4839                    .send()
4840                    .await?
4841                    .json::<serde_json::Value>()
4842                    .await?;
4843                verify_through_rpc_endpoint(dir.path(), token.clone()).await?;
4844                anyhow::Ok((token, jwks))
4845            })
4846            .unwrap();
4847        assert_eq!(
4848            post_crash_jwks["composite_version"].as_u64(),
4849            Some(committed.version)
4850        );
4851        assert_eq!(
4852            post_crash_jwks["composite_component_digest"].as_str(),
4853            Some(committed.component_digest.as_str())
4854        );
4855        for pending_only in pending_after_crash.pairs.iter().filter(|candidate| {
4856            !ledger_after_rotation
4857                .pairs
4858                .iter()
4859                .any(|committed_pair| committed_pair.kid == candidate.kid)
4860        }) {
4861            assert!(!post_crash_jwks["keys"]
4862                .as_array()
4863                .unwrap()
4864                .iter()
4865                .any(|key| key["kid"].as_str() == Some(pending_only.kid.as_str())));
4866        }
4867        let _ = (timeout_oauth_token, timeout_policy_token, post_crash_token);
4868
4869        let mut restart = spawn_production_authority_process(
4870            dir.path(),
4871            "auth::key_rotation::tests::composite_restart_rpc_production_process",
4872        );
4873        wait_path(&dir.path().join("restart-drain-verified"));
4874        assert!(restart.wait().unwrap().success());
4875
4876        let mut restart_policy = spawn_production_authority_process(
4877            dir.path(),
4878            "auth::key_rotation::tests::composite_restart_policy_production_process",
4879        );
4880        wait_path(&dir.path().join("restart-policy-minted"));
4881        assert!(restart_policy.wait().unwrap().success());
4882
4883        std::fs::write(dir.path().join("stale-attempt-gate"), b"1").unwrap();
4884        wait_path(&dir.path().join("stale-policy-refused"));
4885        wait_path(&dir.path().join("stale-writer-refused"));
4886        assert!(stale_policy.wait().unwrap().success());
4887        assert!(stale_writer.wait().unwrap().success());
4888        let ledger_after_stale: CompositeLedger =
4889            serde_json::from_slice(&std::fs::read(composite_ledger_path(dir.path())).unwrap())
4890                .unwrap();
4891        assert_eq!(ledger_after_stale.version, pending_after_crash.version);
4892        assert_eq!(
4893            ledger_after_stale.component_digest,
4894            pending_after_crash.component_digest
4895        );
4896        assert_eq!(
4897            ledger_after_stale.pairs.len(),
4898            pending_after_crash.pairs.len()
4899        );
4900        for pair in &pending_after_crash.pairs {
4901            assert!(ledger_after_stale.pairs.iter().any(|candidate| {
4902                candidate.kid == pair.kid
4903                    && candidate.role == pair.role
4904                    && candidate.state == pair.state
4905            }));
4906        }
4907        let committed_after_stale: CompositeCommit =
4908            serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
4909                .unwrap();
4910        assert_eq!(committed_after_stale.version, committed.version);
4911        assert_eq!(
4912            committed_after_stale.component_digest,
4913            committed.component_digest
4914        );
4915        runtime
4916            .block_on(async {
4917                for token_file in ["oauth-token", "policy-token"] {
4918                    verify_through_rpc_endpoint(
4919                        dir.path(),
4920                        std::fs::read_to_string(dir.path().join(token_file))?,
4921                    )
4922                    .await?;
4923                }
4924                anyhow::Ok(())
4925            })
4926            .unwrap();
4927
4928        std::fs::write(dir.path().join("done"), b"1").unwrap();
4929        for service in &mut services {
4930            assert!(service.wait().unwrap().success());
4931        }
4932    }
4933
4934    #[test]
4935    fn load_or_init_creates_active_key_on_first_boot() {
4936        let dir = TempDir::new().unwrap();
4937        let config = test_config();
4938        let store = load_or_init_key_store(dir.path(), &config);
4939        let rt = tokio::runtime::Runtime::new().unwrap();
4940        let vk = rt.block_on(store.active_verifying_key_bytes());
4941        assert!(
4942            vk.is_some(),
4943            "active key should be present after first boot"
4944        );
4945    }
4946
4947    #[test]
4948    fn committed_marker_with_missing_slots_does_not_bootstrap_replacement_keys() {
4949        let dir = TempDir::new().unwrap();
4950        let config = test_config();
4951        let marker = composite_committed_path(dir.path());
4952        // The guard is intentionally based on marker presence, not successful
4953        // parsing: malformed committed metadata is not permission to mint a
4954        // different authority.
4955        std::fs::write(&marker, b"committed but unreadable").unwrap();
4956
4957        let ed = load_or_init_key_store(dir.path(), &config);
4958        let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
4959        let rt = tokio::runtime::Runtime::new().unwrap();
4960        assert!(rt.block_on(ed.active_verifying_key_bytes()).is_none());
4961        assert!(rt.block_on(pq.active_key()).is_none());
4962        assert_eq!(std::fs::read(&marker).unwrap(), b"committed but unreadable");
4963        for name in [
4964            "jwt-signing-key.active",
4965            "jwt-signing-key.active.meta",
4966            "jwt-signing-key.lead",
4967            "jwt-signing-key.lead.meta",
4968            "ml-dsa-signing-key.active",
4969            "ml-dsa-signing-key.active.meta",
4970            "ml-dsa-signing-key.lead",
4971            "ml-dsa-signing-key.lead.meta",
4972        ] {
4973            assert!(
4974                !dir.path().join(name).exists(),
4975                "committed store unexpectedly created {name}"
4976            );
4977        }
4978    }
4979
4980    #[tokio::test]
4981    async fn committed_marker_does_not_generate_missing_lead_slots() {
4982        let dir = TempDir::new().unwrap();
4983        let config = test_config();
4984        let now = chrono::Utc::now().timestamp();
4985        let marker = composite_committed_path(dir.path());
4986        std::fs::write(&marker, b"committed").unwrap();
4987        persist_slot(
4988            dir.path(),
4989            "active",
4990            &KeySlot::new(SigningKey::from_bytes(&[0x71; 32]), now - 60, now + 1),
4991        )
4992        .unwrap();
4993        ml_dsa_rotation::persist_ml_dsa_slot(
4994            dir.path(),
4995            "active",
4996            &ml_dsa_rotation::generate_ml_dsa_slot(now - 60, now + 1),
4997        )
4998        .unwrap();
4999
5000        let ed = load_or_init_key_store(dir.path(), &config);
5001        let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
5002        assert!(ed.0.read().await.active.is_some());
5003        assert!(ed.0.read().await.lead.is_none());
5004        assert!(pq.0.read().await.active.is_some());
5005        assert!(pq.0.read().await.lead.is_none());
5006        for name in [
5007            "jwt-signing-key.lead",
5008            "jwt-signing-key.lead.meta",
5009            "ml-dsa-signing-key.lead",
5010            "ml-dsa-signing-key.lead.meta",
5011        ] {
5012            assert!(
5013                !dir.path().join(name).exists(),
5014                "committed store unexpectedly created {name}"
5015            );
5016        }
5017    }
5018
5019    #[test]
5020    fn persist_and_reload_slot() {
5021        let dir = TempDir::new().unwrap();
5022        let config = test_config();
5023        {
5024            let store = load_or_init_key_store(dir.path(), &config);
5025            let rt = tokio::runtime::Runtime::new().unwrap();
5026            let vk1 = rt.block_on(store.active_verifying_key_bytes()).unwrap();
5027            // Second load should return same key
5028            let store2 = load_or_init_key_store(dir.path(), &config);
5029            let vk2 = rt.block_on(store2.active_verifying_key_bytes()).unwrap();
5030            assert_eq!(vk1, vk2, "persisted key must reload identically");
5031        }
5032    }
5033
5034    #[tokio::test]
5035    async fn rotate_promotes_lead_to_active() {
5036        let dir = TempDir::new().unwrap();
5037        let config = test_config();
5038        let now = chrono::Utc::now().timestamp();
5039
5040        // Build a store with a lead whose nbf is in the past.
5041        let active_slot = KeySlot::new(
5042            SigningKey::generate(&mut rand::rngs::OsRng),
5043            now - 14 * 86400,
5044            now + 1,
5045        );
5046        let lead_slot = KeySlot::new(
5047            SigningKey::generate(&mut rand::rngs::OsRng),
5048            now - 1, // nbf already passed
5049            now + 14 * 86400,
5050        );
5051        persist_slot(dir.path(), "active", &active_slot).unwrap();
5052        persist_slot(dir.path(), "lead", &lead_slot).unwrap();
5053        let lead_vk = lead_slot.verifying_key_bytes();
5054
5055        let store = SigningKeyStore::new(KeySlots {
5056            drain: None,
5057            active: Some(active_slot),
5058            lead: Some(lead_slot),
5059        });
5060
5061        rotate_jwt_keys(&config, dir.path(), &store, now).await;
5062
5063        let new_active_vk = store.active_verifying_key_bytes().await.unwrap();
5064        assert_eq!(
5065            new_active_vk, lead_vk,
5066            "lead must become active after promotion"
5067        );
5068
5069        // Old active must now be drain
5070        let slots = store.0.read().await;
5071        assert!(slots.drain.is_some(), "old active must become drain");
5072        assert!(
5073            slots.lead.is_none(),
5074            "lead slot must be cleared after promotion"
5075        );
5076    }
5077
5078    #[tokio::test]
5079    async fn drain_is_removed_after_drain_window() {
5080        let dir = TempDir::new().unwrap();
5081        let config = test_config();
5082        let now = chrono::Utc::now().timestamp();
5083
5084        let drain_slot = KeySlot::new(
5085            SigningKey::generate(&mut rand::rngs::OsRng),
5086            now - 60 * 86400,
5087            now - 31 * 86400, // exp + 30 drain days < now
5088        );
5089        let active_slot = KeySlot::new(
5090            SigningKey::generate(&mut rand::rngs::OsRng),
5091            now - 86400,
5092            now + 13 * 86400,
5093        );
5094        persist_slot(dir.path(), "drain", &drain_slot).unwrap();
5095        persist_slot(dir.path(), "active", &active_slot).unwrap();
5096
5097        let store = SigningKeyStore::new(KeySlots {
5098            drain: Some(drain_slot),
5099            active: Some(active_slot),
5100            lead: None,
5101        });
5102
5103        rotate_jwt_keys(&config, dir.path(), &store, now).await;
5104
5105        let slots = store.0.read().await;
5106        assert!(
5107            slots.drain.is_none(),
5108            "drain must be removed after drain window closes"
5109        );
5110    }
5111
5112    #[tokio::test]
5113    async fn lead_is_generated_when_active_near_expiry() {
5114        let dir = TempDir::new().unwrap();
5115        let config = test_config();
5116        let now = chrono::Utc::now().timestamp();
5117
5118        // Active expires in 6 days — less than lead_days (7)
5119        let active_slot = KeySlot::new(
5120            SigningKey::generate(&mut rand::rngs::OsRng),
5121            now - 8 * 86400,
5122            now + 6 * 86400,
5123        );
5124        persist_slot(dir.path(), "active", &active_slot).unwrap();
5125
5126        let store = SigningKeyStore::new(KeySlots {
5127            drain: None,
5128            active: Some(active_slot),
5129            lead: None,
5130        });
5131
5132        rotate_jwt_keys(&config, dir.path(), &store, now).await;
5133
5134        let slots = store.0.read().await;
5135        assert!(
5136            slots.lead.is_some(),
5137            "lead must be generated when active is within lead window"
5138        );
5139    }
5140
5141    // ── Disk-before-memory promotion ───────────────────────────────────────
5142    //
5143    // Operator invariant under repair: a generated or promoted signing key
5144    // becomes active in memory only after its slot persistence reports
5145    // success. Failure injection is a filesystem fixture — a directory
5146    // planted at the slot path, or a state dir that is a regular file — which
5147    // stays deterministic under root, unlike permission-bit tricks.
5148
5149    #[tokio::test]
5150    async fn first_boot_persistence_failure_leaves_no_active_ed25519_key() {
5151        // The production composite boundary mutates the process-global
5152        // key-set version/snapshot; a mutex cannot reset that state between
5153        // tests. Re-execute this test in its own process (fresh globals),
5154        // mirroring the other composite-global tests in this module.
5155        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_FIRST_BOOT_ED_TEST";
5156        if std::env::var_os(ISOLATED).is_none() {
5157            let status = std::process::Command::new(std::env::current_exe().unwrap())
5158                .args([
5159                    "--exact",
5160                    "auth::key_rotation::tests::first_boot_persistence_failure_leaves_no_active_ed25519_key",
5161                    "--nocapture",
5162                ])
5163                .env(ISOLATED, "1")
5164                .status()
5165                .unwrap();
5166            assert!(status.success());
5167            return;
5168        }
5169        let dir = TempDir::new().unwrap();
5170        let config = test_config();
5171        // write_secret persists via rename onto this path; renaming a file
5172        // onto a directory fails for every user, including root.
5173        std::fs::create_dir(dir.path().join("jwt-signing-key.active")).unwrap();
5174
5175        let store = load_or_init_key_store(dir.path(), &config);
5176        assert!(
5177            store.active_verifying_key_bytes().await.is_none(),
5178            "an unpersisted generated key must not be exposed as active"
5179        );
5180        assert!(
5181            !dir.path().join("jwt-signing-key.active.meta").exists(),
5182            "no slot metadata may exist when the key bytes failed to persist"
5183        );
5184
5185        // Production composite-initialization boundary with a healthy ML-DSA
5186        // counterpart: an incomplete first-boot store must fail the
5187        // initialization instead of committing partial authority.
5188        let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
5189        let error = initialize_composite_key_set(
5190            dir.path(),
5191            &store,
5192            &pq,
5193            Arc::new(SigningKey::from_bytes(&[0x5A; 32])),
5194            30,
5195        )
5196        .await
5197        .expect_err("incomplete first-boot store must not create composite authority");
5198        assert!(
5199            error.to_string().contains("incomplete local signer set"),
5200            "unexpected initialization error: {error:#}"
5201        );
5202        assert!(
5203            !dir.path().join("jwt-composite-pairs.json").exists(),
5204            "no pending ledger may be created from an incomplete store"
5205        );
5206        assert!(
5207            !dir.path().join("jwt-composite-pairs.committed").exists(),
5208            "no commit marker may be created from an incomplete store"
5209        );
5210        let snapshots = std::fs::read_dir(dir.path())
5211            .unwrap()
5212            .filter_map(Result::ok)
5213            .filter(|entry| {
5214                entry
5215                    .file_name()
5216                    .to_string_lossy()
5217                    .starts_with("jwt-composite-pairs.committed-")
5218            })
5219            .count();
5220        assert_eq!(
5221            snapshots, 0,
5222            "no immutable committed snapshot may be created from an incomplete store"
5223        );
5224
5225        // Remove the injected failure and recover with fresh stores through
5226        // the same production boundary.
5227        std::fs::remove_dir(dir.path().join("jwt-signing-key.active")).unwrap();
5228        let fresh_ed = load_or_init_key_store(dir.path(), &config);
5229        let fresh_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
5230        initialize_composite_key_set(
5231            dir.path(),
5232            &fresh_ed,
5233            &fresh_pq,
5234            Arc::new(SigningKey::from_bytes(&[0x5A; 32])),
5235            30,
5236        )
5237        .await
5238        .expect("normal initialization must succeed once persistence recovers");
5239        assert!(dir.path().join("jwt-composite-pairs.json").exists());
5240        assert!(dir.path().join("jwt-composite-pairs.committed").exists());
5241    }
5242
5243    #[tokio::test]
5244    async fn ed25519_rotation_keeps_active_when_drain_persist_fails() {
5245        let dir = TempDir::new().unwrap();
5246        let config = test_config();
5247        let now = chrono::Utc::now().timestamp();
5248
5249        let active_slot = KeySlot::new(
5250            SigningKey::generate(&mut rand::rngs::OsRng),
5251            now - 14 * 86400,
5252            now + 1,
5253        );
5254        let lead_slot = KeySlot::new(
5255            SigningKey::generate(&mut rand::rngs::OsRng),
5256            now - 1, // nbf already passed
5257            now + 14 * 86400,
5258        );
5259        let drain_slot = KeySlot::new(
5260            SigningKey::generate(&mut rand::rngs::OsRng),
5261            now - 45 * 86400,
5262            now + 5 * 86400,
5263        );
5264        let active_vk = active_slot.verifying_key_bytes();
5265        let lead_vk = lead_slot.verifying_key_bytes();
5266        let drain_vk = drain_slot.verifying_key_bytes();
5267
5268        let store = SigningKeyStore::new(KeySlots {
5269            drain: Some(drain_slot),
5270            active: Some(active_slot),
5271            lead: Some(lead_slot),
5272        });
5273
5274        // Deterministic seam: this state dir is a regular file, so the first
5275        // promotion write (old active → drain) fails.
5276        let broken_state = dir.path().join("not-a-directory");
5277        std::fs::write(&broken_state, b"file").unwrap();
5278        rotate_jwt_keys_in_state_dir(&config, &broken_state, &store, now).await;
5279
5280        let slots = store.0.read().await;
5281        assert_eq!(
5282            slots.active.as_ref().map(KeySlot::verifying_key_bytes),
5283            Some(active_vk),
5284            "the old active must remain the in-memory active signer"
5285        );
5286        assert_eq!(
5287            slots.lead.as_ref().map(KeySlot::verifying_key_bytes),
5288            Some(lead_vk),
5289            "the queued lead must stay in memory for the next tick"
5290        );
5291        assert_eq!(
5292            slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5293            Some(drain_vk),
5294            "a failed promotion must not evict the in-memory drain"
5295        );
5296        drop(slots);
5297
5298        // Persistence recovers: the queued lead promotes and the promoted
5299        // active is on disk before it is served from memory.
5300        rotate_jwt_keys_in_state_dir(&config, dir.path(), &store, now).await;
5301        assert_eq!(
5302            store.active_verifying_key_bytes().await,
5303            Some(lead_vk),
5304            "the queued lead must promote once persistence succeeds"
5305        );
5306        let persisted = load_slot(dir.path(), "active").expect("promoted active must be persisted");
5307        assert_eq!(
5308            persisted.verifying_key_bytes(),
5309            lead_vk,
5310            "memory promotion must follow the durable active write"
5311        );
5312        let slots = store.0.read().await;
5313        assert_eq!(
5314            slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5315            Some(active_vk),
5316            "the old active becomes drain only after its persist succeeded"
5317        );
5318        assert!(slots.lead.is_none());
5319        assert!(!dir.path().join("jwt-signing-key.lead").exists());
5320    }
5321
5322    #[tokio::test]
5323    async fn ed25519_rotation_keeps_active_when_active_persist_fails() {
5324        let dir = TempDir::new().unwrap();
5325        let config = test_config();
5326        let now = chrono::Utc::now().timestamp();
5327
5328        let active_slot = KeySlot::new(
5329            SigningKey::generate(&mut rand::rngs::OsRng),
5330            now - 14 * 86400,
5331            now + 1,
5332        );
5333        let lead_slot = KeySlot::new(
5334            SigningKey::generate(&mut rand::rngs::OsRng),
5335            now - 1,
5336            now + 14 * 86400,
5337        );
5338        let active_vk = active_slot.verifying_key_bytes();
5339        let lead_vk = lead_slot.verifying_key_bytes();
5340        persist_slot(dir.path(), "lead", &lead_slot).unwrap();
5341
5342        let store = SigningKeyStore::new(KeySlots {
5343            drain: None,
5344            active: Some(active_slot),
5345            lead: Some(lead_slot),
5346        });
5347
5348        // The drain write succeeds (old active → drain) but the promoted
5349        // active write fails before its seed lands: its slot path is a
5350        // directory. The rollback restore fails the same way, so the cycle
5351        // reports not-restored and keeps the durable drain copy of the old
5352        // active signer for restart recovery.
5353        std::fs::create_dir(dir.path().join("jwt-signing-key.active")).unwrap();
5354        let restored = rotate_jwt_keys_in_state_dir(&config, dir.path(), &store, now).await;
5355        assert!(
5356            !restored,
5357            "a failed rollback must report the stores as not fully restored"
5358        );
5359
5360        assert_eq!(
5361            store.active_verifying_key_bytes().await,
5362            Some(active_vk),
5363            "the failed new key must not be exposed as active in memory"
5364        );
5365        {
5366            let slots = store.0.read().await;
5367            assert!(slots.lead.is_some(), "the lead stays queued for retry");
5368            assert!(
5369                slots.drain.is_none(),
5370                "in-memory drain must not change before every write succeeded"
5371            );
5372        }
5373        assert_eq!(
5374            load_slot(dir.path(), "drain")
5375                .expect("the drain copy of the old active must survive a failed rollback")
5376                .verifying_key_bytes(),
5377            active_vk,
5378            "a failed rollback must not erase the recoverable old active signer"
5379        );
5380        assert!(
5381            dir.path().join("jwt-signing-key.active").is_dir()
5382                && load_slot(dir.path(), "active").is_none(),
5383            "the active slot must not be durably replaced by the failed key; until the retry lands, the old active survives in memory and in drain"
5384        );
5385
5386        // Remove the obstruction; the queued lead promotes on the next tick.
5387        std::fs::remove_dir(dir.path().join("jwt-signing-key.active")).unwrap();
5388        rotate_jwt_keys_in_state_dir(&config, dir.path(), &store, now).await;
5389        assert_eq!(store.active_verifying_key_bytes().await, Some(lead_vk));
5390        assert_eq!(
5391            load_slot(dir.path(), "active")
5392                .expect("promoted active must be persisted")
5393                .verifying_key_bytes(),
5394            lead_vk,
5395            "memory promotion must be preceded by the durable active write"
5396        );
5397        let slots = store.0.read().await;
5398        assert_eq!(
5399            slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5400            Some(active_vk)
5401        );
5402        assert!(slots.lead.is_none());
5403    }
5404
5405    #[tokio::test]
5406    async fn ed25519_failed_rollback_preserves_recoverable_active_in_drain() {
5407        // The production composite boundary mutates the process-global
5408        // key-set version/snapshot; a mutex cannot reset that state between
5409        // tests. Re-execute this test in its own process (fresh globals),
5410        // mirroring the other composite-global tests in this module.
5411        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_ROLLBACK_RECOVERY_TEST";
5412        if std::env::var_os(ISOLATED).is_none() {
5413            let status = std::process::Command::new(std::env::current_exe().unwrap())
5414                .args([
5415                    "--exact",
5416                    "auth::key_rotation::tests::ed25519_failed_rollback_preserves_recoverable_active_in_drain",
5417                    "--nocapture",
5418                ])
5419                .env(ISOLATED, "1")
5420                .status()
5421                .unwrap();
5422            assert!(status.success());
5423            return;
5424        }
5425        for prev_drain_present in [false, true] {
5426            let dir = TempDir::new().unwrap();
5427            let config = test_config();
5428            let now = chrono::Utc::now().timestamp();
5429
5430            let active_slot = KeySlot::new(
5431                SigningKey::generate(&mut rand::rngs::OsRng),
5432                now - 14 * 86400,
5433                now + 6 * 3600,
5434            );
5435            let lead_slot = KeySlot::new(
5436                SigningKey::generate(&mut rand::rngs::OsRng),
5437                now - 1, // nbf already passed
5438                now + 14 * 86400,
5439            );
5440            let active_seed = active_slot.key.to_bytes();
5441            let active_vk = active_slot.verifying_key_bytes();
5442            let lead_vk = lead_slot.verifying_key_bytes();
5443            persist_slot(dir.path(), "active", &active_slot).unwrap();
5444            persist_slot(dir.path(), "lead", &lead_slot).unwrap();
5445            let prev_drain = if prev_drain_present {
5446                let drain = KeySlot::new(
5447                    SigningKey::generate(&mut rand::rngs::OsRng),
5448                    now - 60 * 86400,
5449                    now - 45 * 86400,
5450                );
5451                persist_slot(dir.path(), "drain", &drain).unwrap();
5452                Some(drain)
5453            } else {
5454                None
5455            };
5456
5457            let store = SigningKeyStore::new(KeySlots {
5458                drain: prev_drain.clone(),
5459                active: Some(active_slot),
5460                lead: Some(lead_slot),
5461            });
5462
5463            // Commit real authority through the production writer: the
5464            // committed OAuth active pair names exactly this Ed25519 signer.
5465            let pq_store = load_or_init_ml_dsa_key_store(dir.path(), &config);
5466            let ca_key = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
5467            initialize_composite_key_set(dir.path(), &store, &pq_store, Arc::clone(&ca_key), 30)
5468                .await
5469                .expect("healthy starting authority must initialize composite");
5470            let (commit_before, ledger_before) =
5471                read_committed_composite_ledger(dir.path()).unwrap();
5472            let oauth_active_ed = ledger_before
5473                .pairs
5474                .iter()
5475                .find(|record| record.role == "oauth" && record.state == "active")
5476                .expect("initialized authority must carry an active OAuth pair");
5477            assert_eq!(
5478                oauth_active_ed.ed25519_public,
5479                URL_SAFE_NO_PAD.encode(active_vk),
5480                "prior_drain={prev_drain_present}: the committed active pair must \
5481                 name the old active Ed25519 signer"
5482            );
5483
5484            // Deterministic partial write: the promotion's seed rename onto
5485            // `jwt-signing-key.active` succeeds but every metadata persistence
5486            // at `jwt-signing-key.active.meta` fails (directory obstruction).
5487            // The promoted active fails on its metadata write, and the
5488            // rollback of the old active fails the same way — after its seed
5489            // has been restored.
5490            let meta_path = dir.path().join("jwt-signing-key.active.meta");
5491            std::fs::remove_file(&meta_path).unwrap();
5492            std::fs::create_dir(&meta_path).unwrap();
5493
5494            let restored = rotate_jwt_keys_in_state_dir(&config, dir.path(), &store, now).await;
5495            assert!(
5496                !restored,
5497                "prior_drain={prev_drain_present}: a failed rollback must report \
5498                 the stores as not fully restored"
5499            );
5500
5501            // Memory is unchanged: old active still active, lead queued.
5502            let slots = store.0.read().await;
5503            assert_eq!(
5504                slots.active.as_ref().map(KeySlot::verifying_key_bytes),
5505                Some(active_vk),
5506                "prior_drain={prev_drain_present}: the old active must remain the \
5507                 in-memory active signer"
5508            );
5509            assert_eq!(
5510                slots.lead.as_ref().map(KeySlot::verifying_key_bytes),
5511                Some(lead_vk),
5512                "prior_drain={prev_drain_present}: the lead must stay queued"
5513            );
5514            assert_eq!(
5515                slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5516                prev_drain.as_ref().map(KeySlot::verifying_key_bytes),
5517                "prior_drain={prev_drain_present}: memory drain must not change \
5518                 before the rollback is confirmed"
5519            );
5520            drop(slots);
5521
5522            // The drain slot keeps the only complete durable copy of the old
5523            // active signer — not deleted, not overwritten with the previous
5524            // drain generation.
5525            let recovered = load_slot(dir.path(), "drain").expect(
5526                "prior_drain={prev_drain_present}: drain copy of the old active \
5527                 must survive a failed rollback",
5528            );
5529            assert_eq!(
5530                recovered.verifying_key_bytes(),
5531                active_vk,
5532                "prior_drain={prev_drain_present}: failed rollback must not erase \
5533                 the recoverable old active signer"
5534            );
5535            assert_eq!(
5536                std::fs::read(dir.path().join("jwt-signing-key.active")).unwrap(),
5537                active_seed,
5538                "prior_drain={prev_drain_present}: the rollback must have restored \
5539                 the old active seed before failing on metadata"
5540            );
5541            assert!(
5542                load_slot(dir.path(), "active").is_none(),
5543                "prior_drain={prev_drain_present}: the active slot is incomplete \
5544                 without its metadata and must not load"
5545            );
5546
5547            // A fresh reload — as after a restart, with the real commit marker
5548            // present so no implicit generation may run — still recovers the
5549            // signer required by committed authority from the drain slot,
5550            // before any retry is attempted and without reusing the in-memory
5551            // store.
5552            let fresh = load_or_init_key_store(dir.path(), &config);
5553            {
5554                let fresh_slots = fresh.0.read().await;
5555                assert_eq!(
5556                    fresh_slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5557                    Some(active_vk),
5558                    "prior_drain={prev_drain_present}: fresh reload must still have \
5559                     the committed old active signer recoverable in drain"
5560                );
5561                assert!(
5562                    fresh_slots.active.is_none(),
5563                    "prior_drain={prev_drain_present}: the torn active slot must not \
5564                     load as a usable signer"
5565                );
5566            }
5567
5568            // The production restore resolver — the same initialization the
5569            // oauth/policy services run at startup — must rebuild the
5570            // committed active pair from the recovered slots. The only
5571            // surviving Ed25519 copy of the committed signer is the drain
5572            // slot, so this fails (with the #1713-style restart error) if a
5573            // rollback erased it.
5574            let fresh_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
5575            initialize_composite_key_set(dir.path(), &fresh, &fresh_pq, Arc::clone(&ca_key), 30)
5576                .await
5577                .expect(
5578                    "prior_drain={prev_drain_present}: committed authority must restore \
5579                 from the recovered drain slot after a failed rollback",
5580                );
5581            let (commit_after, ledger_after) = read_committed_composite_ledger(dir.path()).unwrap();
5582            assert_eq!(
5583                commit_after.version, commit_before.version,
5584                "prior_drain={prev_drain_present}: recovery must not need a new \
5585                 committed generation"
5586            );
5587            assert_eq!(
5588                commit_after.component_digest,
5589                commit_before.component_digest
5590            );
5591            let oauth_active_ed_after = ledger_after
5592                .pairs
5593                .iter()
5594                .find(|record| record.role == "oauth" && record.state == "active")
5595                .expect("restored authority must still carry an active OAuth pair");
5596            assert_eq!(
5597                oauth_active_ed_after.ed25519_public,
5598                URL_SAFE_NO_PAD.encode(active_vk),
5599                "prior_drain={prev_drain_present}: restored committed authority \
5600                 must still name the recovered old active signer"
5601            );
5602
5603            // Only now remove the injected failure: a retry from the fresh
5604            // store promotes the queued lead while the recovered old active
5605            // survives in drain.
5606            std::fs::remove_dir(&meta_path).unwrap();
5607            let retried = rotate_jwt_keys_in_state_dir(&config, dir.path(), &fresh, now).await;
5608            assert!(
5609                retried,
5610                "prior_drain={prev_drain_present}: the retry must fully restore and \
5611                 promote"
5612            );
5613            assert_eq!(
5614                fresh.active_verifying_key_bytes().await,
5615                Some(lead_vk),
5616                "prior_drain={prev_drain_present}: the queued lead promotes after \
5617                 recovery"
5618            );
5619            let slots = fresh.0.read().await;
5620            assert_eq!(
5621                slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5622                Some(active_vk),
5623                "prior_drain={prev_drain_present}: the recovered old active must \
5624                 survive the retry in drain"
5625            );
5626            assert!(slots.lead.is_none());
5627        }
5628    }
5629
5630    // ── ES256 store tests ──────────────────────────────────────────────────
5631
5632    #[test]
5633    fn es256_load_or_init_creates_active_key() {
5634        let dir = TempDir::new().unwrap();
5635        let config = test_config();
5636        let store = load_or_init_es256_key_store(dir.path(), &config);
5637        let key = store.active_key();
5638        assert!(key.is_some());
5639    }
5640
5641    #[test]
5642    fn es256_missing_active_with_durable_oplog_authority_does_not_regenerate() {
5643        for marker in [
5644            super::super::op_log::SEALED_HEAD_FILENAME,
5645            super::super::op_log::SEALED_HEAD_MAX_SEQ_FILENAME,
5646            super::super::op_log::HEAD_VERIFYING_KEY_FILENAME,
5647        ] {
5648            let root = TempDir::new().unwrap();
5649            let secrets_dir = root.path().join("credentials");
5650            let oplog_dir = root.path().join("oplog-state");
5651            std::fs::create_dir(&secrets_dir).unwrap();
5652            std::fs::create_dir(&oplog_dir).unwrap();
5653            std::fs::write(oplog_dir.join(marker), b"existing authority marker").unwrap();
5654
5655            let store = load_or_init_es256_key_store_with_oplog_state_dir(
5656                &secrets_dir,
5657                &test_config(),
5658                &oplog_dir,
5659            );
5660            assert!(store.active_key().is_none(), "marker={marker}");
5661            assert!(
5662                load_es256_slot(&secrets_dir, "active").is_none(),
5663                "marker={marker}"
5664            );
5665            assert!(
5666                load_es256_slot(&secrets_dir, "drain").is_none(),
5667                "marker={marker}"
5668            );
5669            assert!(
5670                load_es256_slot(&secrets_dir, "lead").is_none(),
5671                "marker={marker}"
5672            );
5673        }
5674    }
5675
5676    #[tokio::test]
5677    async fn es256_failed_first_boot_persistence_can_recover_without_published_marker() {
5678        let root = TempDir::new().unwrap();
5679        let secrets_dir = root.path().join("credentials");
5680        let oplog_dir = root.path().join("oplog-state");
5681        std::fs::create_dir(&secrets_dir).unwrap();
5682        std::fs::create_dir(&oplog_dir).unwrap();
5683
5684        // The seed write succeeds, but making the metadata destination a
5685        // directory forces the second half of the active-slot write to fail.
5686        let blocked_metadata = secrets_dir.join("es256-signing-key.active.meta");
5687        std::fs::create_dir(&blocked_metadata).unwrap();
5688        let failed_boot = load_or_init_es256_key_store_with_oplog_state_dir(
5689            &secrets_dir,
5690            &test_config(),
5691            &oplog_dir,
5692        );
5693        assert!(failed_boot.active_key().is_none());
5694
5695        assert!(super::super::op_log::advance_sealed_head_with_state_dir(
5696            &secrets_dir,
5697            &oplog_dir,
5698            &failed_boot,
5699        )
5700        .await
5701        .is_err());
5702        assert!(!oplog_dir
5703            .join(super::super::op_log::HEAD_VERIFYING_KEY_FILENAME)
5704            .exists());
5705        assert!(!oplog_dir
5706            .join(super::super::op_log::SEALED_HEAD_FILENAME)
5707            .exists());
5708
5709        // After the transient obstruction is repaired, first boot can
5710        // complete normally without deleting any operation-log marker.
5711        std::fs::remove_dir(&blocked_metadata).unwrap();
5712        let recovered = load_or_init_es256_key_store_with_oplog_state_dir(
5713            &secrets_dir,
5714            &test_config(),
5715            &oplog_dir,
5716        );
5717        assert!(recovered.active_key().is_some());
5718        super::super::op_log::advance_sealed_head_with_state_dir(
5719            &secrets_dir,
5720            &oplog_dir,
5721            &recovered,
5722        )
5723        .await
5724        .unwrap();
5725        assert!(oplog_dir
5726            .join(super::super::op_log::HEAD_VERIFYING_KEY_FILENAME)
5727            .exists());
5728        assert!(oplog_dir
5729            .join(super::super::op_log::SEALED_HEAD_FILENAME)
5730            .exists());
5731    }
5732
5733    #[tokio::test]
5734    async fn advance_oplog_does_not_reload_unpromoted_es256_disk_candidate() {
5735        let root = TempDir::new().unwrap();
5736        let secrets_dir = root.path().join("credentials");
5737        let oplog_dir = root.path().join("oplog-state");
5738        std::fs::create_dir(&secrets_dir).unwrap();
5739        std::fs::create_dir(&oplog_dir).unwrap();
5740
5741        let now = chrono::Utc::now().timestamp();
5742        let retained = generate_es256_slot(now - 120, now + 86_400);
5743        let rejected_candidate = generate_es256_slot(now, now + 172_800);
5744        let retained_kid = retained.kid();
5745        persist_es256_slot(&secrets_dir, "active", &rejected_candidate).unwrap();
5746        persist_es256_slot(&secrets_dir, "drain", &retained).unwrap();
5747
5748        // Simulate a failed rotation rollback: the candidate remains in the
5749        // durable active path, but the store correctly retains the old key.
5750        let store = Es256SigningKeyStore::new(Es256KeySlots {
5751            drain: None,
5752            active: Some(retained),
5753            lead: Some(rejected_candidate),
5754        });
5755        store.bind_secrets_dir(secrets_dir.clone());
5756        assert_eq!(store.active_slot().unwrap().kid(), retained_kid);
5757
5758        super::super::op_log::advance_sealed_head_with_state_dir(&secrets_dir, &oplog_dir, &store)
5759            .await
5760            .unwrap();
5761
5762        assert_eq!(
5763            store.active_slot().unwrap().kid(),
5764            retained_kid,
5765            "op-log advancement must not reload an unpromoted disk candidate"
5766        );
5767        let head: super::super::op_log::SealedOpLogHead = serde_json::from_slice(
5768            &std::fs::read(oplog_dir.join(super::super::op_log::SEALED_HEAD_FILENAME)).unwrap(),
5769        )
5770        .unwrap();
5771        assert_eq!(head.active_kid, retained_kid);
5772    }
5773
5774    #[test]
5775    fn es256_persist_and_reload() {
5776        let dir = TempDir::new().unwrap();
5777        let config = test_config();
5778        let store1 = load_or_init_es256_key_store(dir.path(), &config);
5779        let kid1 = {
5780            let slots = store1.all_slots_snapshot();
5781            slots[0].kid()
5782        };
5783        let store2 = load_or_init_es256_key_store(dir.path(), &config);
5784        let kid2 = {
5785            let slots = store2.all_slots_snapshot();
5786            slots[0].kid()
5787        };
5788        assert_eq!(kid1, kid2, "ES256 key must survive reload from disk");
5789    }
5790
5791    #[test]
5792    fn es256_startup_removes_expired_bounded_slots() {
5793        let dir = TempDir::new().unwrap();
5794        let config = test_config();
5795        let now = chrono::Utc::now().timestamp();
5796        let active = generate_es256_slot(now - 60, now + 86_400);
5797        let expired_drain = generate_es256_slot(now - 120, now);
5798        persist_es256_slot(dir.path(), "active", &active).unwrap();
5799        persist_es256_slot(dir.path(), "drain", &expired_drain).unwrap();
5800
5801        let store = load_or_init_es256_key_store(dir.path(), &config);
5802        assert!(store.drain_slot().is_none());
5803        assert!(load_es256_slot(dir.path(), "drain").is_none());
5804    }
5805
5806    #[tokio::test]
5807    async fn es256_rotate_promotes_lead() {
5808        let dir = TempDir::new().unwrap();
5809        let config = test_config();
5810        let now = chrono::Utc::now().timestamp();
5811
5812        let active = generate_es256_slot(now - 14 * 86400, now + 1);
5813        let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5814        persist_es256_slot(dir.path(), "active", &active).unwrap();
5815        persist_es256_slot(dir.path(), "lead", &lead).unwrap();
5816        let lead_kid = lead.kid();
5817
5818        let store = Es256SigningKeyStore::new(Es256KeySlots {
5819            drain: None,
5820            active: Some(active),
5821            lead: Some(lead),
5822        });
5823        store.set_promotion_hook(Arc::new(|_| Ok(())));
5824
5825        rotate_es256_keys(&config, dir.path(), &store, now).await;
5826
5827        let slots = store.0.read();
5828        assert_eq!(slots.active.as_ref().unwrap().kid(), lead_kid);
5829        assert!(slots.drain.is_some());
5830        assert!(slots.lead.is_none());
5831    }
5832
5833    #[tokio::test]
5834    async fn es256_drain_is_removed_at_its_published_expiry() {
5835        let dir = TempDir::new().unwrap();
5836        let config = test_config();
5837        let now = chrono::Utc::now().timestamp();
5838        let active = generate_es256_slot(now - 14 * 86400, now + 1);
5839        let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5840        let store = Es256SigningKeyStore::new(Es256KeySlots {
5841            drain: None,
5842            active: Some(active),
5843            lead: Some(lead),
5844        });
5845        store.set_promotion_hook(Arc::new(|_| Ok(())));
5846
5847        assert!(rotate_es256_keys(&config, dir.path(), &store, now).await);
5848        let published_expiry = store.drain_slot().expect("drain after promotion").exp;
5849        assert!(
5850            rotate_es256_keys(&config, dir.path(), &store, published_expiry).await
5851                || store.active_slot().is_some(),
5852            "cleanup tick remains operational"
5853        );
5854        assert!(
5855            store.drain_slot().is_none(),
5856            "the drain slot must leave the publication source at its expiry"
5857        );
5858    }
5859
5860    #[tokio::test]
5861    async fn es256_rotate_without_hook_publishes_drain_for_ipc_workers() {
5862        let dir = TempDir::new().unwrap();
5863        let config = test_config();
5864        let now = chrono::Utc::now().timestamp();
5865
5866        let active = generate_es256_slot(now - 14 * 86400, now + 1);
5867        let active_kid = active.kid();
5868        let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5869        let lead_kid = lead.kid();
5870        persist_es256_slot(dir.path(), "active", &active).unwrap();
5871        persist_es256_slot(dir.path(), "lead", &lead).unwrap();
5872        let store = Es256SigningKeyStore::new(Es256KeySlots {
5873            drain: None,
5874            active: Some(active),
5875            lead: Some(lead),
5876        });
5877
5878        assert!(rotate_es256_keys(&config, dir.path(), &store, now).await);
5879        let slots = store.0.read();
5880        assert_eq!(slots.active.as_ref().unwrap().kid(), lead_kid);
5881        assert!(slots.lead.is_none());
5882        assert_eq!(slots.drain.as_ref().unwrap().kid(), active_kid);
5883        assert_eq!(
5884            load_es256_slot(dir.path(), "active").unwrap().kid(),
5885            lead_kid
5886        );
5887    }
5888
5889    #[tokio::test]
5890    async fn es256_rotation_retries_after_promotion_hook_failure() {
5891        let dir = TempDir::new().unwrap();
5892        let config = test_config();
5893        let now = chrono::Utc::now().timestamp();
5894
5895        let active = generate_es256_slot(now - 14 * 86400, now + 1);
5896        let active_kid = active.kid();
5897        let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5898        let lead_kid = lead.kid();
5899        persist_es256_slot(dir.path(), "active", &active).unwrap();
5900        persist_es256_slot(dir.path(), "lead", &lead).unwrap();
5901
5902        let store = Es256SigningKeyStore::new(Es256KeySlots {
5903            drain: None,
5904            active: Some(active),
5905            lead: Some(lead),
5906        });
5907        let fail_once = Arc::new(std::sync::atomic::AtomicBool::new(true));
5908        let injected_failure = Arc::clone(&fail_once);
5909        store.set_promotion_hook(Arc::new(move |_| {
5910            if injected_failure.swap(false, Ordering::AcqRel) {
5911                anyhow::bail!("injected head re-sign failure");
5912            }
5913            Ok(())
5914        }));
5915
5916        assert!(
5917            !rotate_es256_keys(&config, dir.path(), &store, now).await,
5918            "a failed promotion hook must leave the candidate queued"
5919        );
5920        {
5921            let slots = store.0.read();
5922            assert_eq!(slots.active.as_ref().unwrap().kid(), active_kid);
5923            assert_eq!(slots.lead.as_ref().unwrap().kid(), lead_kid);
5924            assert!(slots.drain.is_none());
5925        }
5926        assert_eq!(
5927            load_es256_slot(dir.path(), "active").unwrap().kid(),
5928            active_kid,
5929            "failed promotion must restore the durable active key"
5930        );
5931
5932        assert!(rotate_es256_keys(&config, dir.path(), &store, now).await);
5933        let slots = store.0.read();
5934        assert_eq!(slots.active.as_ref().unwrap().kid(), lead_kid);
5935        assert!(slots.lead.is_none());
5936    }
5937
5938    #[tokio::test]
5939    async fn es256_rotate_retries_after_active_persistence_failure() {
5940        let dir = TempDir::new().unwrap();
5941        let invalid_secrets_dir = dir.path().join("not-a-directory");
5942        std::fs::write(&invalid_secrets_dir, b"file").unwrap();
5943        let valid_secrets_dir = dir.path().join("secrets");
5944        let config = test_config();
5945        let now = chrono::Utc::now().timestamp();
5946
5947        let active = generate_es256_slot(now - 14 * 86400, now + 1);
5948        let active_kid = active.kid();
5949        let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5950        let lead_kid = lead.kid();
5951        let store = Es256SigningKeyStore::new(Es256KeySlots {
5952            drain: None,
5953            active: Some(active),
5954            lead: Some(lead),
5955        });
5956        assert!(
5957            !rotate_es256_keys_in_state_dir(&config, &invalid_secrets_dir, &store, now).await,
5958            "failed active persistence must not report promotion"
5959        );
5960        {
5961            let slots = store.0.read();
5962            assert_eq!(slots.active.as_ref().unwrap().kid(), active_kid);
5963            assert_eq!(slots.lead.as_ref().unwrap().kid(), lead_kid);
5964        }
5965        assert!(
5966            rotate_es256_keys(&config, &valid_secrets_dir, &store, now).await,
5967            "queued lead must be retried after persistence recovers"
5968        );
5969        assert_eq!(store.active_slot().unwrap().kid(), lead_kid);
5970    }
5971
5972    // ── ML-DSA store tests ─────────────────────────────────────────────────
5973
5974    #[test]
5975    fn ml_dsa_load_or_init_creates_active_key() {
5976        let dir = TempDir::new().unwrap();
5977        let config = test_config();
5978        let store = load_or_init_ml_dsa_key_store(dir.path(), &config);
5979        let rt = tokio::runtime::Runtime::new().unwrap();
5980        let key = rt.block_on(store.active_key());
5981        assert!(key.is_some());
5982    }
5983
5984    #[test]
5985    fn ml_dsa_persist_and_reload() {
5986        let dir = TempDir::new().unwrap();
5987        let config = test_config();
5988        let store1 = load_or_init_ml_dsa_key_store(dir.path(), &config);
5989        let rt = tokio::runtime::Runtime::new().unwrap();
5990        let vk1 = {
5991            let key = rt.block_on(store1.active_key()).unwrap();
5992            hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
5993                &ml_dsa::Keypair::verifying_key(&*key).clone(),
5994            )
5995        };
5996        let store2 = load_or_init_ml_dsa_key_store(dir.path(), &config);
5997        let vk2 = {
5998            let key = rt.block_on(store2.active_key()).unwrap();
5999            hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
6000                &ml_dsa::Keypair::verifying_key(&*key).clone(),
6001            )
6002        };
6003        assert_eq!(vk1, vk2, "ML-DSA key must survive reload from disk");
6004    }
6005
6006    #[tokio::test]
6007    async fn ml_dsa_rotate_promotes_lead() {
6008        use super::ml_dsa_rotation::*;
6009        let dir = TempDir::new().unwrap();
6010        let config = test_config();
6011        let now = chrono::Utc::now().timestamp();
6012
6013        let active = generate_ml_dsa_slot(now - 14 * 86400, now + 1);
6014        let lead = generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6015        persist_ml_dsa_slot(dir.path(), "active", &active).unwrap();
6016        persist_ml_dsa_slot(dir.path(), "lead", &lead).unwrap();
6017        let lead_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
6018            &ml_dsa::Keypair::verifying_key(&*lead.key).clone(),
6019        );
6020
6021        let store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
6022            drain: None,
6023            active: Some(active),
6024            lead: Some(lead),
6025        });
6026
6027        rotate_ml_dsa_keys(&config, dir.path(), &store, now).await;
6028
6029        let slots = store.0.read().await;
6030        let active_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
6031            &ml_dsa::Keypair::verifying_key(&*slots.active.as_ref().unwrap().key).clone(),
6032        );
6033        assert_eq!(
6034            active_vk, lead_vk,
6035            "lead must become active after promotion"
6036        );
6037        assert!(slots.drain.is_some());
6038        assert!(slots.lead.is_none());
6039    }
6040
6041    fn ml_dsa_slot_vk_bytes(slot: &ml_dsa_rotation::MlDsaKeySlot) -> Vec<u8> {
6042        hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&slot.verifying_key())
6043    }
6044
6045    fn ml_dsa_key_vk_bytes(key: &hyprstream_rpc::crypto::pq::MlDsaSigningKey) -> Vec<u8> {
6046        hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&ml_dsa::Keypair::verifying_key(key).clone())
6047    }
6048
6049    #[tokio::test]
6050    async fn ml_dsa_first_boot_persistence_failure_leaves_no_active_key() {
6051        // The production composite boundary mutates the process-global
6052        // key-set version/snapshot; a mutex cannot reset that state between
6053        // tests. Re-execute this test in its own process (fresh globals),
6054        // mirroring the other composite-global tests in this module.
6055        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_FIRST_BOOT_ML_DSA_TEST";
6056        if std::env::var_os(ISOLATED).is_none() {
6057            let status = std::process::Command::new(std::env::current_exe().unwrap())
6058                .args([
6059                    "--exact",
6060                    "auth::key_rotation::tests::ml_dsa_first_boot_persistence_failure_leaves_no_active_key",
6061                    "--nocapture",
6062                ])
6063                .env(ISOLATED, "1")
6064                .status()
6065                .unwrap();
6066            assert!(status.success());
6067            return;
6068        }
6069        let dir = TempDir::new().unwrap();
6070        let config = test_config();
6071        // write_secret persists via rename onto this path; renaming a file
6072        // onto a directory fails for every user, including root.
6073        std::fs::create_dir(dir.path().join("ml-dsa-signing-key.active")).unwrap();
6074
6075        let store = load_or_init_ml_dsa_key_store(dir.path(), &config);
6076        assert!(
6077            store.active_key().await.is_none(),
6078            "an unpersisted generated ML-DSA key must not be exposed as active"
6079        );
6080        assert!(
6081            !dir.path().join("ml-dsa-signing-key.active.meta").exists(),
6082            "no slot metadata may exist when the key bytes failed to persist"
6083        );
6084
6085        // Production composite-initialization boundary with a healthy Ed25519
6086        // counterpart: the symmetric first-boot failure must fail the
6087        // initialization instead of committing authority without an active
6088        // ML-DSA component.
6089        let ed = load_or_init_key_store(dir.path(), &config);
6090        let error = initialize_composite_key_set(
6091            dir.path(),
6092            &ed,
6093            &store,
6094            Arc::new(SigningKey::from_bytes(&[0x5A; 32])),
6095            30,
6096        )
6097        .await
6098        .expect_err("incomplete first-boot store must not create composite authority");
6099        assert!(
6100            error.to_string().contains("incomplete local signer set"),
6101            "unexpected initialization error: {error:#}"
6102        );
6103        assert!(
6104            !dir.path().join("jwt-composite-pairs.json").exists(),
6105            "no pending ledger may be created from an incomplete store"
6106        );
6107        assert!(
6108            !dir.path().join("jwt-composite-pairs.committed").exists(),
6109            "no commit marker may be created from an incomplete store"
6110        );
6111        let snapshots = std::fs::read_dir(dir.path())
6112            .unwrap()
6113            .filter_map(Result::ok)
6114            .filter(|entry| {
6115                entry
6116                    .file_name()
6117                    .to_string_lossy()
6118                    .starts_with("jwt-composite-pairs.committed-")
6119            })
6120            .count();
6121        assert_eq!(
6122            snapshots, 0,
6123            "no immutable committed snapshot may be created from an incomplete store"
6124        );
6125
6126        // Remove the injected failure and recover with fresh stores through
6127        // the same production boundary.
6128        std::fs::remove_dir(dir.path().join("ml-dsa-signing-key.active")).unwrap();
6129        let fresh_ed = load_or_init_key_store(dir.path(), &config);
6130        let fresh_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
6131        initialize_composite_key_set(
6132            dir.path(),
6133            &fresh_ed,
6134            &fresh_pq,
6135            Arc::new(SigningKey::from_bytes(&[0x5A; 32])),
6136            30,
6137        )
6138        .await
6139        .expect("normal initialization must succeed once persistence recovers");
6140        assert!(dir.path().join("jwt-composite-pairs.json").exists());
6141        assert!(dir.path().join("jwt-composite-pairs.committed").exists());
6142    }
6143
6144    #[tokio::test]
6145    async fn ml_dsa_rotation_keeps_active_when_drain_persist_fails() {
6146        let dir = TempDir::new().unwrap();
6147        let config = test_config();
6148        let now = chrono::Utc::now().timestamp();
6149
6150        let active = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 1);
6151        let lead = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6152        let drain = ml_dsa_rotation::generate_ml_dsa_slot(now - 45 * 86400, now + 5 * 86400);
6153        let active_vk = ml_dsa_slot_vk_bytes(&active);
6154        let lead_vk = ml_dsa_slot_vk_bytes(&lead);
6155        let drain_vk = ml_dsa_slot_vk_bytes(&drain);
6156
6157        let store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
6158            drain: Some(drain),
6159            active: Some(active),
6160            lead: Some(lead),
6161        });
6162
6163        // Deterministic seam: this state dir is a regular file, so the first
6164        // promotion write (old active → drain) fails.
6165        let broken_state = dir.path().join("not-a-directory");
6166        std::fs::write(&broken_state, b"file").unwrap();
6167        ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, &broken_state, &store, now).await;
6168
6169        let slots = store.0.read().await;
6170        assert_eq!(
6171            slots.active.as_ref().map(ml_dsa_slot_vk_bytes),
6172            Some(active_vk.clone()),
6173            "the old active must remain the in-memory active signer"
6174        );
6175        assert_eq!(
6176            slots.lead.as_ref().map(ml_dsa_slot_vk_bytes),
6177            Some(lead_vk.clone()),
6178            "the queued lead must stay in memory for the next tick"
6179        );
6180        assert_eq!(
6181            slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6182            Some(drain_vk.clone()),
6183            "a failed promotion must not evict the in-memory drain"
6184        );
6185        drop(slots);
6186
6187        // Persistence recovers: the queued lead promotes and the promoted
6188        // active is on disk before it is served from memory.
6189        ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &store, now).await;
6190        let persisted = ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "active")
6191            .expect("promoted active must be persisted");
6192        assert_eq!(
6193            ml_dsa_slot_vk_bytes(&persisted),
6194            lead_vk,
6195            "memory promotion must follow the durable active write"
6196        );
6197        let slots = store.0.read().await;
6198        assert_eq!(
6199            slots.active.as_ref().map(ml_dsa_slot_vk_bytes),
6200            Some(lead_vk),
6201            "the queued lead must promote once persistence succeeds"
6202        );
6203        assert_eq!(
6204            slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6205            Some(active_vk),
6206            "the old active becomes drain only after its persist succeeded"
6207        );
6208        assert!(slots.lead.is_none());
6209        assert!(!dir.path().join("ml-dsa-signing-key.lead").exists());
6210    }
6211
6212    #[tokio::test]
6213    async fn ml_dsa_rotation_keeps_active_when_active_persist_fails() {
6214        let dir = TempDir::new().unwrap();
6215        let config = test_config();
6216        let now = chrono::Utc::now().timestamp();
6217
6218        let active = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 1);
6219        let lead = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6220        let active_vk = ml_dsa_slot_vk_bytes(&active);
6221        let lead_vk = ml_dsa_slot_vk_bytes(&lead);
6222        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &lead).unwrap();
6223
6224        let store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
6225            drain: None,
6226            active: Some(active),
6227            lead: Some(lead),
6228        });
6229
6230        // The drain write succeeds (old active → drain) but the promoted
6231        // active write fails: its slot path is a directory. The old active
6232        // lives only in memory here, which is exactly what the rollback and
6233        // the retry rely on.
6234        std::fs::create_dir(dir.path().join("ml-dsa-signing-key.active")).unwrap();
6235        let restored =
6236            ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &store, now)
6237                .await;
6238        assert!(
6239            !restored,
6240            "a failed rollback must report the stores as not fully restored"
6241        );
6242
6243        assert_eq!(
6244            store.active_key().await.as_deref().map(ml_dsa_key_vk_bytes),
6245            Some(active_vk.clone()),
6246            "the failed new key must not be exposed as active in memory"
6247        );
6248        {
6249            let slots = store.0.read().await;
6250            assert!(slots.lead.is_some(), "the lead stays queued for retry");
6251            assert!(
6252                slots.drain.is_none(),
6253                "in-memory drain must not change before every write succeeded"
6254            );
6255        }
6256        assert_eq!(
6257            ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "drain")
6258                .map(|slot| ml_dsa_slot_vk_bytes(&slot)),
6259            Some(active_vk.clone()),
6260            "a failed rollback must not erase the drain copy of the old active \
6261             signer"
6262        );
6263        assert!(
6264            dir.path().join("ml-dsa-signing-key.active").is_dir()
6265                && ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "active").is_none(),
6266            "the active slot must not be durably replaced by the failed key; \
6267             until the retry lands, the old active survives in memory and in \
6268             drain"
6269        );
6270
6271        // Remove the obstruction; the queued lead promotes on the next tick.
6272        std::fs::remove_dir(dir.path().join("ml-dsa-signing-key.active")).unwrap();
6273        ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &store, now).await;
6274        assert_eq!(
6275            store.active_key().await.as_deref().map(ml_dsa_key_vk_bytes),
6276            Some(lead_vk.clone()),
6277            "the queued lead must promote once persistence succeeds"
6278        );
6279        let persisted = ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "active")
6280            .expect("promoted active must be persisted");
6281        assert_eq!(
6282            ml_dsa_key_vk_bytes(persisted.key.as_ref()),
6283            lead_vk,
6284            "memory promotion must be preceded by the durable active write"
6285        );
6286        let slots = store.0.read().await;
6287        assert_eq!(
6288            slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6289            Some(active_vk),
6290            "the old active becomes drain only after its persist succeeded"
6291        );
6292        assert!(slots.lead.is_none());
6293    }
6294
6295    #[tokio::test]
6296    async fn ml_dsa_failed_rollback_preserves_recoverable_active_in_drain() {
6297        // The production composite boundary mutates the process-global
6298        // key-set version/snapshot; a mutex cannot reset that state between
6299        // tests. Re-execute this test in its own process (fresh globals),
6300        // mirroring the other composite-global tests in this module.
6301        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ML_DSA_ROLLBACK_RECOVERY_TEST";
6302        if std::env::var_os(ISOLATED).is_none() {
6303            let status = std::process::Command::new(std::env::current_exe().unwrap())
6304                .args([
6305                    "--exact",
6306                    "auth::key_rotation::tests::ml_dsa_failed_rollback_preserves_recoverable_active_in_drain",
6307                    "--nocapture",
6308                ])
6309                .env(ISOLATED, "1")
6310                .status()
6311                .unwrap();
6312            assert!(status.success());
6313            return;
6314        }
6315        for prev_drain_present in [false, true] {
6316            let dir = TempDir::new().unwrap();
6317            let config = test_config();
6318            let now = chrono::Utc::now().timestamp();
6319
6320            // Generous validity margins: this test may run on a slow or
6321            // suspended machine, and restore resolution skips expired
6322            // committed records.
6323            let active = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 6 * 3600);
6324            let lead = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6325            let active_vk = ml_dsa_slot_vk_bytes(&active);
6326            let lead_vk = ml_dsa_slot_vk_bytes(&lead);
6327            ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &active).unwrap();
6328            ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &lead).unwrap();
6329            let prev_drain = if prev_drain_present {
6330                let drain =
6331                    ml_dsa_rotation::generate_ml_dsa_slot(now - 60 * 86400, now - 45 * 86400);
6332                ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "drain", &drain).unwrap();
6333                Some(drain)
6334            } else {
6335                None
6336            };
6337
6338            let store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
6339                drain: prev_drain.clone(),
6340                active: Some(active),
6341                lead: Some(lead),
6342            });
6343
6344            // Commit real authority through the production writer: the
6345            // committed OAuth active pair names exactly this ML-DSA signer.
6346            let ed_store = load_or_init_key_store(dir.path(), &config);
6347            let ca_key = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
6348            initialize_composite_key_set(dir.path(), &ed_store, &store, Arc::clone(&ca_key), 30)
6349                .await
6350                .expect("healthy starting authority must initialize composite");
6351            let (commit_before, ledger_before) =
6352                read_committed_composite_ledger(dir.path()).unwrap();
6353            let oauth_active_pq = ledger_before
6354                .pairs
6355                .iter()
6356                .find(|record| record.role == "oauth" && record.state == "active")
6357                .expect("initialized authority must carry an active OAuth pair");
6358            assert_eq!(
6359                oauth_active_pq.ml_dsa_public,
6360                URL_SAFE_NO_PAD.encode(&active_vk),
6361                "prior_drain={prev_drain_present}: the committed active pair must \
6362                 name the old active ML-DSA signer"
6363            );
6364
6365            // Deterministic partial write, mirroring the Ed25519 scenario: the
6366            // promotion's seed rename succeeds but every metadata persistence
6367            // at `ml-dsa-signing-key.active.meta` fails, so the promoted
6368            // active and its rollback both fail after the seed write.
6369            let meta_path = dir.path().join("ml-dsa-signing-key.active.meta");
6370            std::fs::remove_file(&meta_path).unwrap();
6371            std::fs::create_dir(&meta_path).unwrap();
6372
6373            let restored =
6374                ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &store, now)
6375                    .await;
6376            assert!(
6377                !restored,
6378                "prior_drain={prev_drain_present}: a failed rollback must report \
6379                 the stores as not fully restored"
6380            );
6381
6382            let slots = store.0.read().await;
6383            assert_eq!(
6384                slots.active.as_ref().map(ml_dsa_slot_vk_bytes),
6385                Some(active_vk.clone()),
6386                "prior_drain={prev_drain_present}: the old active must remain the \
6387                 in-memory active signer"
6388            );
6389            assert_eq!(
6390                slots.lead.as_ref().map(ml_dsa_slot_vk_bytes),
6391                Some(lead_vk.clone()),
6392                "prior_drain={prev_drain_present}: the lead must stay queued"
6393            );
6394            assert_eq!(
6395                slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6396                prev_drain.as_ref().map(ml_dsa_slot_vk_bytes),
6397                "prior_drain={prev_drain_present}: memory drain must not change \
6398                 before the rollback is confirmed"
6399            );
6400            drop(slots);
6401
6402            let recovered = ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "drain").expect(
6403                "prior_drain={prev_drain_present}: drain copy of the old active \
6404                 must survive a failed rollback",
6405            );
6406            assert_eq!(
6407                ml_dsa_slot_vk_bytes(&recovered),
6408                active_vk,
6409                "prior_drain={prev_drain_present}: failed rollback must not erase \
6410                 the recoverable old active signer"
6411            );
6412            assert!(
6413                ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "active").is_none(),
6414                "prior_drain={prev_drain_present}: the active slot is incomplete \
6415                 without its metadata and must not load"
6416            );
6417
6418            // Fresh reload with the real commit marker present: the old active
6419            // signer stays recoverable from drain, before any retry.
6420            let fresh = load_or_init_ml_dsa_key_store(dir.path(), &config);
6421            {
6422                let fresh_slots = fresh.0.read().await;
6423                assert_eq!(
6424                    fresh_slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6425                    Some(active_vk.clone()),
6426                    "prior_drain={prev_drain_present}: fresh reload must still have \
6427                     the committed old active signer recoverable in drain"
6428                );
6429                assert!(
6430                    fresh_slots.active.is_none(),
6431                    "prior_drain={prev_drain_present}: the torn active slot must not \
6432                     load as a usable signer"
6433                );
6434            }
6435
6436            // The production restore resolver must rebuild the committed
6437            // active pair from the recovered slots: the only surviving ML-DSA
6438            // copy of the committed signer is the drain slot, so this fails
6439            // (with the #1713-style restart error) if a rollback erased it.
6440            let fresh_ed = load_or_init_key_store(dir.path(), &config);
6441            initialize_composite_key_set(dir.path(), &fresh_ed, &fresh, Arc::clone(&ca_key), 30)
6442                .await
6443                .expect(
6444                    "prior_drain={prev_drain_present}: committed authority must restore \
6445                 from the recovered drain slot after a failed rollback",
6446                );
6447            let (commit_after, ledger_after) = read_committed_composite_ledger(dir.path()).unwrap();
6448            assert_eq!(
6449                commit_after.version, commit_before.version,
6450                "prior_drain={prev_drain_present}: recovery must not need a new \
6451                 committed generation"
6452            );
6453            assert_eq!(
6454                commit_after.component_digest,
6455                commit_before.component_digest
6456            );
6457            let oauth_active_pq_after = ledger_after
6458                .pairs
6459                .iter()
6460                .find(|record| record.role == "oauth" && record.state == "active")
6461                .expect("restored authority must still carry an active OAuth pair");
6462            assert_eq!(
6463                oauth_active_pq_after.ml_dsa_public,
6464                URL_SAFE_NO_PAD.encode(&active_vk),
6465                "prior_drain={prev_drain_present}: restored committed authority \
6466                 must still name the recovered old active signer"
6467            );
6468
6469            // Remove the injected failure and retry from the fresh store.
6470            std::fs::remove_dir(&meta_path).unwrap();
6471            let retried =
6472                ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &fresh, now)
6473                    .await;
6474            assert!(
6475                retried,
6476                "prior_drain={prev_drain_present}: the retry must fully restore and \
6477                 promote"
6478            );
6479            assert_eq!(
6480                fresh.active_key().await.as_deref().map(ml_dsa_key_vk_bytes),
6481                Some(lead_vk),
6482                "prior_drain={prev_drain_present}: the queued lead promotes after \
6483                 recovery"
6484            );
6485            let slots = fresh.0.read().await;
6486            assert_eq!(
6487                slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6488                Some(active_vk),
6489                "prior_drain={prev_drain_present}: the recovered old active must \
6490                 survive the retry in drain"
6491            );
6492            assert!(slots.lead.is_none());
6493        }
6494    }
6495
6496    #[tokio::test]
6497    async fn failed_rollback_defers_composite_publication() {
6498        // The production composite boundary mutates the process-global
6499        // key-set version/snapshot; a mutex cannot reset that state between
6500        // tests. Re-execute this test in its own process (fresh globals),
6501        // mirroring the other composite-global tests in this module.
6502        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_PUBLICATION_GATE_TEST";
6503        if std::env::var_os(ISOLATED).is_none() {
6504            let status = std::process::Command::new(std::env::current_exe().unwrap())
6505                .args([
6506                    "--exact",
6507                    "auth::key_rotation::tests::failed_rollback_defers_composite_publication",
6508                    "--nocapture",
6509                ])
6510                .env(ISOLATED, "1")
6511                .status()
6512                .unwrap();
6513            assert!(status.success());
6514            return;
6515        }
6516        let dir = TempDir::new().unwrap();
6517        let config = test_config();
6518        let now = chrono::Utc::now().timestamp();
6519
6520        // Healthy ML-DSA store; the Ed25519 promotion below fails its rollback
6521        // via a directory obstruction at the active slot metadata path.
6522        let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6523        let active_slot = KeySlot::new(
6524            SigningKey::generate(&mut rand::rngs::OsRng),
6525            now - 14 * 86400,
6526            now + 1,
6527        );
6528        let lead_slot = KeySlot::new(
6529            SigningKey::generate(&mut rand::rngs::OsRng),
6530            now - 1,
6531            now + 14 * 86400,
6532        );
6533        persist_slot(dir.path(), "active", &active_slot).unwrap();
6534        persist_slot(dir.path(), "lead", &lead_slot).unwrap();
6535        let ed_store = Arc::new(SigningKeyStore::new(KeySlots {
6536            drain: None,
6537            active: Some(active_slot),
6538            lead: Some(lead_slot),
6539        }));
6540
6541        let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
6542        initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
6543            .await
6544            .expect("a fully available signer set must initialize composite authority");
6545        let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
6546        let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
6547        let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
6548            .snapshot()
6549            .version();
6550
6551        let meta_path = dir.path().join("jwt-signing-key.active.meta");
6552        std::fs::remove_file(&meta_path).unwrap();
6553        std::fs::create_dir(&meta_path).unwrap();
6554
6555        let extra = RotationStores {
6556            es256: None,
6557            ml_dsa: Some(Arc::clone(&pq_store)),
6558            composite_ca_key: Arc::clone(&ca),
6559        };
6560        run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
6561
6562        // The cycle must not publish as successfully restored: committed
6563        // authority, the pending ledger, and the live composite snapshot all
6564        // retain the prior generation.
6565        let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
6566        assert_eq!(
6567            commit_after.version, commit_before.version,
6568            "a not-fully-restored cycle must not advance the commit marker"
6569        );
6570        assert_eq!(
6571            commit_after.component_digest, commit_before.component_digest,
6572            "a not-fully-restored cycle must not change the committed digest"
6573        );
6574        assert_eq!(
6575            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6576            pending_before,
6577            "a not-fully-restored cycle must not restate the pending ledger"
6578        );
6579        assert_eq!(
6580            hyprstream_rpc::auth::global_composite_key_set()
6581                .snapshot()
6582                .version(),
6583            snapshot_version_before,
6584            "a not-fully-restored cycle must not publish to the live key set"
6585        );
6586        // The committed predecessor is durably recorded for the recovery tick.
6587        assert_eq!(
6588            read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
6589            Some(commit_before.component_digest.clone()),
6590            "a deferring cycle must retain the committed predecessor digest"
6591        );
6592    }
6593
6594    #[tokio::test]
6595    async fn ed25519_restart_mismatch_fails_closed() {
6596        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_RESTART_MISMATCH_TEST";
6597        if std::env::var_os(ISOLATED).is_none() {
6598            let status = std::process::Command::new(std::env::current_exe().unwrap())
6599                .args([
6600                    "--exact",
6601                    "auth::key_rotation::tests::ed25519_restart_mismatch_fails_closed",
6602                    "--nocapture",
6603                ])
6604                .env(ISOLATED, "1")
6605                .status()
6606                .unwrap();
6607            assert!(status.success());
6608            return;
6609        }
6610
6611        let dir = TempDir::new().unwrap();
6612        let config = test_config();
6613        let now = chrono::Utc::now().timestamp();
6614
6615        // Healthy ML-DSA store with its own ready lead; Ed25519 store whose
6616        // promotion will fail its rollback in the first cycle.
6617        let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6618        let new_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6619        let pq_lead = ml_dsa_rotation::MlDsaSigningKeyStore::new(MlDsaKeySlots {
6620            drain: None,
6621            active: pq_store.active_key().await.map(|key| {
6622                ml_dsa_rotation::MlDsaKeySlot::new((*key).clone(), now - 3600, now + 14 * 86400)
6623            }),
6624            lead: Some(new_pq.clone()),
6625        });
6626        let old_ed = KeySlot::new(
6627            SigningKey::generate(&mut rand::rngs::OsRng),
6628            now - 14 * 86400,
6629            now + 6 * 3600,
6630        );
6631        let new_ed = KeySlot::new(
6632            SigningKey::generate(&mut rand::rngs::OsRng),
6633            now - 1,
6634            now + 14 * 86400,
6635        );
6636        persist_slot(dir.path(), "active", &old_ed).unwrap();
6637        persist_slot(dir.path(), "lead", &new_ed).unwrap();
6638        let ed_store = Arc::new(SigningKeyStore::new(KeySlots {
6639            drain: None,
6640            active: Some(old_ed.clone()),
6641            lead: Some(new_ed.clone()),
6642        }));
6643
6644        let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
6645        // Initialize with exactly the stores cycle 1 consumes, so the
6646        // committed digest matches the cycle's pre-rotation component state
6647        // (queued leads included) and the predecessor seeding validates.
6648        initialize_composite_key_set(dir.path(), &ed_store, &pq_lead, Arc::clone(&ca), 30)
6649            .await
6650            .expect("healthy starting authority must initialize composite");
6651        let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
6652        let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
6653        let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
6654            .snapshot()
6655            .version();
6656
6657        // Cycle 1: the ML-DSA lead promotes while the Ed25519 promotion fails
6658        // its rollback. Publication is deferred and the committed predecessor
6659        // digest is retained for the recovery tick.
6660        let meta_path = dir.path().join("jwt-signing-key.active.meta");
6661        std::fs::remove_file(&meta_path).unwrap();
6662        std::fs::create_dir(&meta_path).unwrap();
6663        let extra = RotationStores {
6664            es256: None,
6665            ml_dsa: Some(Arc::new(pq_lead)),
6666            composite_ca_key: Arc::clone(&ca),
6667        };
6668        let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
6669        assert!(
6670            !all_restored,
6671            "the mixed cycle must report not fully restored"
6672        );
6673        let (commit_deferred, _) = read_committed_composite_ledger(dir.path()).unwrap();
6674        assert_eq!(commit_deferred.version, commit_before.version);
6675        assert_eq!(
6676            commit_deferred.component_digest,
6677            commit_before.component_digest
6678        );
6679        assert_eq!(
6680            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6681            pending_before,
6682            "a deferred cycle must not restate the pending ledger"
6683        );
6684        assert_eq!(
6685            hyprstream_rpc::auth::global_composite_key_set()
6686                .snapshot()
6687                .version(),
6688            snapshot_version_before,
6689            "a deferred cycle must not publish to the live key set"
6690        );
6691        assert_eq!(
6692            read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
6693            Some(commit_before.component_digest.clone()),
6694            "the committed predecessor digest must be retained across the \
6695             deferred cycle"
6696        );
6697
6698        // Clear the write obstruction and reload. The slot state may not
6699        // match what this process certified in memory, so restart must not
6700        // infer authority from the files. Even after the marker becomes
6701        // readable again, the ledger digest gate must keep this state from
6702        // being published.
6703        std::fs::remove_dir(&meta_path).unwrap();
6704        let fresh_ed = Arc::new(load_or_init_key_store(dir.path(), &config));
6705        let fresh_pq = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6706        let extra_fresh = RotationStores {
6707            es256: None,
6708            ml_dsa: Some(Arc::clone(&fresh_pq)),
6709            composite_ca_key: Arc::clone(&ca),
6710        };
6711        let initial_record = read_deferred_predecessor(dir.path()).unwrap();
6712        let reloaded_digest =
6713            component_state_digest(&fresh_ed, &fresh_pq, ca.verifying_key()).await;
6714        assert!(
6715            !initial_record.states.contains(&reloaded_digest),
6716            "a restart reload not certified by this process must remain untrusted"
6717        );
6718        let marker_path = dir.path().join("jwt-composite-pairs.committed");
6719        let marker_bytes = std::fs::read(&marker_path).unwrap();
6720        std::fs::write(&marker_path, b"corrupt").unwrap();
6721        // The rotation succeeds; the publication fails on the corrupt
6722        // committed marker and must not commit anything.
6723        let _failed_cycle =
6724            run_rotation_cycle(&config, dir.path(), &fresh_ed, &extra_fresh, now).await;
6725        // Ledger access restored: the committed generation is untouched, the
6726        // record is retained, and its certified states advanced to the
6727        // recovery tick's own post-rotation outcome.
6728        std::fs::write(&marker_path, &marker_bytes).unwrap();
6729        let (commit_failed, _) = read_committed_composite_ledger(dir.path()).unwrap();
6730        assert_eq!(
6731            commit_failed.version, commit_before.version,
6732            "a failed recovery publication must not advance the committed \
6733             generation"
6734        );
6735        let record_after_failure = read_deferred_predecessor(dir.path())
6736            .expect("a failed recovery publication must retain the record");
6737        assert_eq!(
6738            record_after_failure.committed_digest, commit_before.component_digest,
6739            "the retained record must keep the committed predecessor"
6740        );
6741        assert_eq!(
6742            record_after_failure.states, initial_record.states,
6743            "an untrusted restart state must not be added to the recovery record"
6744        );
6745        assert_eq!(
6746            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6747            pending_before,
6748            "the failed recovery publication must not restate the pending ledger"
6749        );
6750
6751        // A readable marker does not make the mismatching restart state
6752        // trustworthy. The attempt remains fail-closed and preserves the
6753        // last committed authority for operator recovery.
6754        std::fs::write(&marker_path, &marker_bytes).unwrap();
6755        let recovered = run_rotation_cycle(&config, dir.path(), &fresh_ed, &extra_fresh, now).await;
6756        assert!(
6757            recovered,
6758            "the tick after a failed recovery publication must publish"
6759        );
6760        let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
6761        assert_eq!(commit_after.version, commit_before.version);
6762        assert_eq!(
6763            commit_after.component_digest,
6764            commit_before.component_digest
6765        );
6766        assert_eq!(
6767            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6768            pending_before
6769        );
6770        assert!(read_deferred_predecessor(dir.path()).is_some());
6771        assert_eq!(
6772            hyprstream_rpc::auth::global_composite_key_set()
6773                .snapshot()
6774                .version(),
6775            snapshot_version_before
6776        );
6777    }
6778
6779    #[tokio::test]
6780    async fn ml_dsa_restart_mismatch_fails_closed() {
6781        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ML_DSA_RESTART_MISMATCH_TEST";
6782        if std::env::var_os(ISOLATED).is_none() {
6783            let status = std::process::Command::new(std::env::current_exe().unwrap())
6784                .args([
6785                    "--exact",
6786                    "auth::key_rotation::tests::ml_dsa_restart_mismatch_fails_closed",
6787                    "--nocapture",
6788                ])
6789                .env(ISOLATED, "1")
6790                .status()
6791                .unwrap();
6792            assert!(status.success());
6793            return;
6794        }
6795
6796        let dir = TempDir::new().unwrap();
6797        let config = test_config();
6798        let now = chrono::Utc::now().timestamp();
6799
6800        // Healthy Ed25519 store with its own ready lead; ML-DSA store whose
6801        // promotion will fail its rollback in the first cycle.
6802        let ed_store = Arc::new(load_or_init_key_store(dir.path(), &config));
6803        let new_ed = KeySlot::new(
6804            SigningKey::generate(&mut rand::rngs::OsRng),
6805            now - 1,
6806            now + 14 * 86400,
6807        );
6808        let ed_lead = SigningKeyStore::new(KeySlots {
6809            drain: None,
6810            active: ed_store
6811                .active_key()
6812                .await
6813                .map(|key| KeySlot::new((*key).clone(), now - 3600, now + 14 * 86400)),
6814            lead: Some(new_ed.clone()),
6815        });
6816        let old_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 6 * 3600);
6817        let new_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6818        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &old_pq).unwrap();
6819        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &new_pq).unwrap();
6820        let pq_store = Arc::new(MlDsaSigningKeyStore::new(MlDsaKeySlots {
6821            drain: None,
6822            active: Some(old_pq.clone()),
6823            lead: Some(new_pq.clone()),
6824        }));
6825
6826        let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
6827        // Initialize with exactly the stores cycle 1 consumes, so the
6828        // committed digest matches the cycle's pre-rotation component state
6829        // (queued leads included) and the predecessor seeding validates.
6830        initialize_composite_key_set(dir.path(), &ed_lead, &pq_store, Arc::clone(&ca), 30)
6831            .await
6832            .expect("healthy starting authority must initialize composite");
6833        let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
6834        let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
6835        let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
6836            .snapshot()
6837            .version();
6838
6839        // Cycle 1: the Ed25519 lead promotes while the ML-DSA promotion fails
6840        // its rollback. Publication is deferred.
6841        let meta_path = dir.path().join("ml-dsa-signing-key.active.meta");
6842        std::fs::remove_file(&meta_path).unwrap();
6843        std::fs::create_dir(&meta_path).unwrap();
6844        let extra = RotationStores {
6845            es256: None,
6846            ml_dsa: Some(Arc::clone(&pq_store)),
6847            composite_ca_key: Arc::clone(&ca),
6848        };
6849        let all_restored = run_rotation_cycle(&config, dir.path(), &ed_lead, &extra, now).await;
6850        assert!(
6851            !all_restored,
6852            "the mixed cycle must report not fully restored"
6853        );
6854        let (commit_deferred, _) = read_committed_composite_ledger(dir.path()).unwrap();
6855        assert_eq!(commit_deferred.version, commit_before.version);
6856        assert_eq!(
6857            commit_deferred.component_digest,
6858            commit_before.component_digest
6859        );
6860        assert_eq!(
6861            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6862            pending_before,
6863            "a deferred cycle must not restate the pending ledger"
6864        );
6865        assert_eq!(
6866            hyprstream_rpc::auth::global_composite_key_set()
6867                .snapshot()
6868                .version(),
6869            snapshot_version_before,
6870            "a deferred cycle must not publish to the live key set"
6871        );
6872        assert_eq!(
6873            read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
6874            Some(commit_before.component_digest.clone()),
6875            "the committed predecessor digest must be retained across the \
6876             deferred cycle"
6877        );
6878
6879        // Clear the write obstruction and reload. If the loaded state does
6880        // not match what this process certified, restart must not infer
6881        // authority from the slot files. A readable marker later does not
6882        // make that mismatching state trusted.
6883        std::fs::remove_dir(&meta_path).unwrap();
6884        let fresh_ed = Arc::new(load_or_init_key_store(dir.path(), &config));
6885        let fresh_pq = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6886        let extra_fresh = RotationStores {
6887            es256: None,
6888            ml_dsa: Some(Arc::clone(&fresh_pq)),
6889            composite_ca_key: Arc::clone(&ca),
6890        };
6891        let initial_record = read_deferred_predecessor(dir.path()).unwrap();
6892        let reloaded_digest =
6893            component_state_digest(&fresh_ed, &fresh_pq, ca.verifying_key()).await;
6894        assert!(
6895            !initial_record.states.contains(&reloaded_digest),
6896            "a restart reload not certified by this process must remain untrusted"
6897        );
6898        let marker_path = dir.path().join("jwt-composite-pairs.committed");
6899        let marker_bytes = std::fs::read(&marker_path).unwrap();
6900        std::fs::write(&marker_path, b"corrupt").unwrap();
6901        // The rotation succeeds; the publication fails on the corrupt
6902        // committed marker and must not commit anything.
6903        let _failed_cycle =
6904            run_rotation_cycle(&config, dir.path(), &fresh_ed, &extra_fresh, now).await;
6905        // Ledger access restored: the committed generation is untouched, the
6906        // record is retained, and its certified states advanced to the
6907        // recovery tick's own post-rotation outcome.
6908        std::fs::write(&marker_path, &marker_bytes).unwrap();
6909        let (commit_failed, _) = read_committed_composite_ledger(dir.path()).unwrap();
6910        assert_eq!(
6911            commit_failed.version, commit_before.version,
6912            "a failed recovery publication must not advance the committed \
6913             generation"
6914        );
6915        let record_after_failure = read_deferred_predecessor(dir.path())
6916            .expect("a failed recovery publication must retain the record");
6917        assert_eq!(
6918            record_after_failure.committed_digest, commit_before.component_digest,
6919            "the retained record must keep the committed predecessor"
6920        );
6921        assert_eq!(
6922            record_after_failure.states, initial_record.states,
6923            "an untrusted restart state must not be added to the recovery record"
6924        );
6925        assert_eq!(
6926            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6927            pending_before,
6928            "the failed recovery publication must not restate the pending ledger"
6929        );
6930
6931        // A readable marker does not make the mismatching restart state
6932        // trustworthy. The retry preserves the committed authority and waits
6933        // for explicit repair.
6934        std::fs::write(&marker_path, &marker_bytes).unwrap();
6935        let recovered = run_rotation_cycle(&config, dir.path(), &fresh_ed, &extra_fresh, now).await;
6936        assert!(
6937            recovered,
6938            "the rotation operation itself should remain usable"
6939        );
6940        let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
6941        assert_eq!(commit_after.version, commit_before.version);
6942        assert_eq!(
6943            commit_after.component_digest,
6944            commit_before.component_digest
6945        );
6946        assert_eq!(
6947            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6948            pending_before
6949        );
6950        assert!(read_deferred_predecessor(dir.path()).is_some());
6951        assert_eq!(
6952            hyprstream_rpc::auth::global_composite_key_set()
6953                .snapshot()
6954                .version(),
6955            snapshot_version_before,
6956            "the restart mismatch must not publish new signer authority"
6957        );
6958    }
6959
6960    #[tokio::test]
6961    async fn ed25519_failed_drain_rollback_defers_publication() {
6962        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_DRAIN_ROLLBACK_TEST";
6963        if std::env::var_os(ISOLATED).is_none() {
6964            let status = std::process::Command::new(std::env::current_exe().unwrap())
6965                .args([
6966                    "--exact",
6967                    "auth::key_rotation::tests::ed25519_failed_drain_rollback_defers_publication",
6968                    "--nocapture",
6969                ])
6970                .env(ISOLATED, "1")
6971                .status()
6972                .unwrap();
6973            assert!(status.success());
6974            return;
6975        }
6976
6977        for prev_drain_present in [false, true] {
6978            let dir = TempDir::new().unwrap();
6979            let config = test_config();
6980            let now = chrono::Utc::now().timestamp();
6981
6982            let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6983            let active_slot = KeySlot::new(
6984                SigningKey::generate(&mut rand::rngs::OsRng),
6985                now - 14 * 86400,
6986                now + 6 * 3600,
6987            );
6988            let lead_slot = KeySlot::new(
6989                SigningKey::generate(&mut rand::rngs::OsRng),
6990                now - 1,
6991                now + 14 * 86400,
6992            );
6993            let active_vk = active_slot.verifying_key_bytes();
6994            let lead_vk = lead_slot.verifying_key_bytes();
6995            let prev_drain = if prev_drain_present {
6996                let drain = KeySlot::new(
6997                    SigningKey::generate(&mut rand::rngs::OsRng),
6998                    now - 60 * 86400,
6999                    now - 45 * 86400,
7000                );
7001                persist_slot(dir.path(), "drain", &drain).unwrap();
7002                Some(drain)
7003            } else {
7004                None
7005            };
7006            persist_slot(dir.path(), "active", &active_slot).unwrap();
7007            persist_slot(dir.path(), "lead", &lead_slot).unwrap();
7008
7009            let ed_store = SigningKeyStore::new(KeySlots {
7010                drain: prev_drain.clone(),
7011                active: Some(active_slot),
7012                lead: Some(lead_slot),
7013            });
7014
7015            let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7016            initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7017                .await
7018                .expect("healthy starting authority must initialize composite");
7019            let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7020            let pending_before =
7021                std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7022            let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
7023                .snapshot()
7024                .version();
7025
7026            // Deterministic partial drain write: the old-active-to-drain seed
7027            // rename succeeds but the drain metadata persistence fails, and
7028            // the previous-drain restoration (prev present) or the partial
7029            // replacement removal (prev absent) fails on the same metadata
7030            // path, so restoration cannot be confirmed.
7031            let meta_path = dir.path().join("jwt-signing-key.drain.meta");
7032            if meta_path.exists() {
7033                std::fs::remove_file(&meta_path).unwrap();
7034            }
7035            std::fs::create_dir(&meta_path).unwrap();
7036
7037            let extra = RotationStores {
7038                es256: None,
7039                ml_dsa: Some(Arc::clone(&pq_store)),
7040                composite_ca_key: Arc::clone(&ca),
7041            };
7042            let all_restored =
7043                run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7044            assert!(
7045                !all_restored,
7046                "prior_drain={prev_drain_present}: an unconfirmed previous-drain \
7047                 restoration must report the stores as not fully restored"
7048            );
7049
7050            let slots = ed_store.0.read().await;
7051            assert_eq!(
7052                slots.active.as_ref().map(KeySlot::verifying_key_bytes),
7053                Some(active_vk),
7054                "prior_drain={prev_drain_present}: the old active must remain the \
7055                 in-memory active signer"
7056            );
7057            assert_eq!(
7058                slots.lead.as_ref().map(KeySlot::verifying_key_bytes),
7059                Some(lead_vk),
7060                "prior_drain={prev_drain_present}: the lead must stay queued"
7061            );
7062            assert_eq!(
7063                slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
7064                prev_drain.as_ref().map(KeySlot::verifying_key_bytes),
7065                "prior_drain={prev_drain_present}: memory drain must not change"
7066            );
7067            drop(slots);
7068
7069            assert!(
7070                load_slot(dir.path(), "drain").is_none(),
7071                "prior_drain={prev_drain_present}: the torn drain slot must not load"
7072            );
7073
7074            // The failed cleanup/restore must defer publication exactly like
7075            // the active-rollback failure: committed, pending, and live
7076            // authority all retain the prior generation.
7077            let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
7078            assert_eq!(commit_after.version, commit_before.version);
7079            assert_eq!(
7080                commit_after.component_digest,
7081                commit_before.component_digest
7082            );
7083            assert_eq!(
7084                std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7085                pending_before,
7086                "a not-fully-restored cycle must not restate the pending ledger"
7087            );
7088            assert_eq!(
7089                hyprstream_rpc::auth::global_composite_key_set()
7090                    .snapshot()
7091                    .version(),
7092                snapshot_version_before,
7093                "a not-fully-restored cycle must not publish to the live key set"
7094            );
7095            assert_eq!(
7096                read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
7097                Some(commit_before.component_digest.clone()),
7098                "a deferring cycle must retain the committed predecessor digest"
7099            );
7100        }
7101    }
7102
7103    #[tokio::test]
7104    async fn ml_dsa_failed_drain_rollback_defers_publication() {
7105        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ML_DSA_DRAIN_ROLLBACK_TEST";
7106        if std::env::var_os(ISOLATED).is_none() {
7107            let status = std::process::Command::new(std::env::current_exe().unwrap())
7108                .args([
7109                    "--exact",
7110                    "auth::key_rotation::tests::ml_dsa_failed_drain_rollback_defers_publication",
7111                    "--nocapture",
7112                ])
7113                .env(ISOLATED, "1")
7114                .status()
7115                .unwrap();
7116            assert!(status.success());
7117            return;
7118        }
7119
7120        for prev_drain_present in [false, true] {
7121            let dir = TempDir::new().unwrap();
7122            let config = test_config();
7123            let now = chrono::Utc::now().timestamp();
7124
7125            let ed_store = Arc::new(load_or_init_key_store(dir.path(), &config));
7126            let active = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 6 * 3600);
7127            let lead = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7128            let active_vk = ml_dsa_slot_vk_bytes(&active);
7129            let lead_vk = ml_dsa_slot_vk_bytes(&lead);
7130            let prev_drain = if prev_drain_present {
7131                let drain =
7132                    ml_dsa_rotation::generate_ml_dsa_slot(now - 60 * 86400, now - 45 * 86400);
7133                ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "drain", &drain).unwrap();
7134                Some(drain)
7135            } else {
7136                None
7137            };
7138            ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &active).unwrap();
7139            ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &lead).unwrap();
7140
7141            let pq_store = Arc::new(MlDsaSigningKeyStore::new(MlDsaKeySlots {
7142                drain: prev_drain.clone(),
7143                active: Some(active),
7144                lead: Some(lead),
7145            }));
7146
7147            let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7148            initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7149                .await
7150                .expect("healthy starting authority must initialize composite");
7151            let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7152            let pending_before =
7153                std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7154            let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
7155                .snapshot()
7156                .version();
7157
7158            // Deterministic partial drain write, mirroring the Ed25519 case.
7159            let meta_path = dir.path().join("ml-dsa-signing-key.drain.meta");
7160            if meta_path.exists() {
7161                std::fs::remove_file(&meta_path).unwrap();
7162            }
7163            std::fs::create_dir(&meta_path).unwrap();
7164
7165            let extra = RotationStores {
7166                es256: None,
7167                ml_dsa: Some(Arc::clone(&pq_store)),
7168                composite_ca_key: Arc::clone(&ca),
7169            };
7170            let all_restored =
7171                run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7172            assert!(
7173                !all_restored,
7174                "prior_drain={prev_drain_present}: an unconfirmed previous-drain \
7175                 restoration must report the stores as not fully restored"
7176            );
7177
7178            let slots = pq_store.0.read().await;
7179            assert_eq!(
7180                slots.active.as_ref().map(ml_dsa_slot_vk_bytes),
7181                Some(active_vk.clone()),
7182                "prior_drain={prev_drain_present}: the old active must remain the \
7183                 in-memory active signer"
7184            );
7185            assert_eq!(
7186                slots.lead.as_ref().map(ml_dsa_slot_vk_bytes),
7187                Some(lead_vk.clone()),
7188                "prior_drain={prev_drain_present}: the lead must stay queued"
7189            );
7190            assert_eq!(
7191                slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
7192                prev_drain.as_ref().map(ml_dsa_slot_vk_bytes),
7193                "prior_drain={prev_drain_present}: memory drain must not change"
7194            );
7195            drop(slots);
7196
7197            assert!(
7198                ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "drain").is_none(),
7199                "prior_drain={prev_drain_present}: the torn drain slot must not load"
7200            );
7201
7202            let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
7203            assert_eq!(commit_after.version, commit_before.version);
7204            assert_eq!(
7205                commit_after.component_digest,
7206                commit_before.component_digest
7207            );
7208            assert_eq!(
7209                std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7210                pending_before,
7211                "a not-fully-restored cycle must not restate the pending ledger"
7212            );
7213            assert_eq!(
7214                hyprstream_rpc::auth::global_composite_key_set()
7215                    .snapshot()
7216                    .version(),
7217                snapshot_version_before,
7218                "a not-fully-restored cycle must not publish to the live key set"
7219            );
7220            assert_eq!(
7221                read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
7222                Some(commit_before.component_digest.clone()),
7223                "a deferring cycle must retain the committed predecessor digest"
7224            );
7225        }
7226    }
7227
7228    #[tokio::test]
7229    async fn ed25519_stale_anchor_does_not_authorize_publication() {
7230        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_STALE_ANCHOR_TEST";
7231        if std::env::var_os(ISOLATED).is_none() {
7232            let status = std::process::Command::new(std::env::current_exe().unwrap())
7233                .args([
7234                    "--exact",
7235                    "auth::key_rotation::tests::ed25519_stale_anchor_does_not_authorize_publication",
7236                    "--nocapture",
7237                ])
7238                .env(ISOLATED, "1")
7239                .status()
7240                .unwrap();
7241            assert!(status.success());
7242            return;
7243        }
7244
7245        let dir = TempDir::new().unwrap();
7246        let config = test_config();
7247        let now = chrono::Utc::now().timestamp();
7248
7249        let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
7250        // The active slot sits outside the lead-generation window so this
7251        // no-op cycle does not publish a newly generated lead over the
7252        // foreign-record assertion below.
7253        let active_slot = KeySlot::new(
7254            SigningKey::generate(&mut rand::rngs::OsRng),
7255            now - 14 * 86400,
7256            now + 8 * 86400,
7257        );
7258        persist_slot(dir.path(), "active", &active_slot).unwrap();
7259        let ed_store = SigningKeyStore::new(KeySlots {
7260            drain: None,
7261            active: Some(active_slot),
7262            lead: None,
7263        });
7264
7265        let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7266        initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7267            .await
7268            .expect("healthy starting authority must initialize composite");
7269        let (commit_before, ledger_before) = read_committed_composite_ledger(dir.path()).unwrap();
7270        let oauth_before = ledger_before
7271            .pairs
7272            .iter()
7273            .find(|record| record.role == "oauth" && record.state == "active")
7274            .expect("initialized authority must carry an active OAuth pair")
7275            .clone();
7276
7277        // Another writer's recovery record: its certified deferred state is a
7278        // foreign stale component state, and its committed digest is the
7279        // current one. The writer of this test process holds stores that are
7280        // the committed generation itself, so the continuity check must
7281        // reject the record (its deferred state is not this process's
7282        // tick-start memory) and no stale publication may be staged.
7283        let foreign_ed = load_or_init_key_store(&dir.path().join("foreign-ed"), &config);
7284        let foreign_pq = load_or_init_ml_dsa_key_store(&dir.path().join("foreign-pq"), &config);
7285        let foreign_state =
7286            component_state_digest(&foreign_ed, &foreign_pq, ca.verifying_key()).await;
7287        let record = DeferredPredecessorRecord {
7288            committed_digest: commit_before.component_digest.clone(),
7289            states: vec![foreign_state],
7290        };
7291        assert!(
7292            write_deferred_predecessor(dir.path(), &record),
7293            "the test fixture must be able to record a foreign anchor"
7294        );
7295
7296        let extra = RotationStores {
7297            es256: None,
7298            ml_dsa: Some(Arc::clone(&pq_store)),
7299            composite_ca_key: Arc::clone(&ca),
7300        };
7301        let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7302        assert!(all_restored);
7303
7304        // No stale publication: the committed generation still names this
7305        // process's active OAuth pair, and the foreign component state never
7306        // entered the ledger.
7307        let (commit_after, ledger_after) = read_committed_composite_ledger(dir.path()).unwrap();
7308        let oauth_active_after = ledger_after
7309            .pairs
7310            .iter()
7311            .find(|record| record.role == "oauth" && record.state == "active")
7312            .expect("authority must still carry an active OAuth pair");
7313        assert_eq!(
7314            oauth_active_after.ed25519_public, oauth_before.ed25519_public,
7315            "a foreign certified state must not replace the committed OAuth \
7316             signer"
7317        );
7318        assert_ne!(
7319            oauth_active_after.ed25519_public,
7320            URL_SAFE_NO_PAD.encode(foreign_ed.active_verifying_key_bytes().await.unwrap()),
7321            "the foreign stale component state must never be published"
7322        );
7323        assert_eq!(
7324            commit_after.component_digest, commit_before.component_digest,
7325            "an ignored record must not change the committed digest"
7326        );
7327        assert!(
7328            read_deferred_predecessor(dir.path()).is_none(),
7329            "the successful publication clears the ignored record"
7330        );
7331    }
7332
7333    #[tokio::test]
7334    async fn ed25519_stale_writer_ticks_do_not_certify_or_publish() {
7335        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_STALE_WRITER_TEST";
7336        if std::env::var_os(ISOLATED).is_none() {
7337            let status = std::process::Command::new(std::env::current_exe().unwrap())
7338                .args([
7339                    "--exact",
7340                    "auth::key_rotation::tests::ed25519_stale_writer_ticks_do_not_certify_or_publish",
7341                    "--nocapture",
7342                ])
7343                .env(ISOLATED, "1")
7344                .status()
7345                .unwrap();
7346            assert!(status.success());
7347            return;
7348        }
7349
7350        let dir = TempDir::new().unwrap();
7351        let config = test_config();
7352        let now = chrono::Utc::now().timestamp();
7353
7354        // Writer A: healthy ML-DSA store; Ed25519 store whose promotion fails
7355        // its rollback in cycle 1 (mixed cycle), seeding a real deferred
7356        // record.
7357        let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
7358        let old_ed = KeySlot::new(
7359            SigningKey::generate(&mut rand::rngs::OsRng),
7360            now - 14 * 86400,
7361            now + 6 * 3600,
7362        );
7363        let new_ed = KeySlot::new(
7364            SigningKey::generate(&mut rand::rngs::OsRng),
7365            now - 1,
7366            now + 14 * 86400,
7367        );
7368        let new_ed_vk = new_ed.verifying_key_bytes();
7369        persist_slot(dir.path(), "active", &old_ed).unwrap();
7370        persist_slot(dir.path(), "lead", &new_ed).unwrap();
7371        let ed_store = SigningKeyStore::new(KeySlots {
7372            drain: None,
7373            active: Some(old_ed),
7374            lead: Some(new_ed),
7375        });
7376
7377        let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7378        initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7379            .await
7380            .expect("healthy starting authority must initialize composite");
7381        let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7382        let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7383        let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
7384            .snapshot()
7385            .version();
7386
7387        // Simulate a stale writer having changed the shared ML-DSA slot
7388        // files before this process records its first deferred cycle. The
7389        // current process's in-memory store still has the committed key; the
7390        // reload digest below must not be certified from disk.
7391        let foreign_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 30 * 86400, now + 8 * 86400);
7392        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &foreign_pq).unwrap();
7393
7394        let meta_path = dir.path().join("jwt-signing-key.active.meta");
7395        std::fs::remove_file(&meta_path).unwrap();
7396        std::fs::create_dir(&meta_path).unwrap();
7397        let extra = RotationStores {
7398            es256: None,
7399            ml_dsa: Some(Arc::clone(&pq_store)),
7400            composite_ca_key: Arc::clone(&ca),
7401        };
7402        let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7403        assert!(
7404            !all_restored,
7405            "the mixed cycle must defer and record its certified states"
7406        );
7407        let seeded = read_deferred_predecessor(dir.path())
7408            .expect("the deferring cycle must retain a recovery record");
7409        assert_eq!(
7410            seeded.committed_digest, commit_before.component_digest,
7411            "the record must anchor on the current committed generation"
7412        );
7413        assert_eq!(
7414            seeded.states.len(),
7415            1,
7416            "only this process's memory state is recorded"
7417        );
7418        let foreign_reload = {
7419            let reloaded_ed = load_or_init_key_store(dir.path(), &config);
7420            let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
7421            component_state_digest(&reloaded_ed, &reloaded_pq, ca.verifying_key()).await
7422        };
7423        assert_ne!(
7424            foreign_reload, seeded.states[0],
7425            "the mutable slot state predated this record and is not trusted"
7426        );
7427        assert!(
7428            !seeded.states.contains(&foreign_reload),
7429            "a foreign reload must never be certified into the first record"
7430        );
7431        let certified_states = seeded.states.clone();
7432
7433        // Stale writer B: stores from a different (non-member) component
7434        // generation, no due work, so both of its ticks are all-restored
7435        // no-ops whose tick-start digest is not certified.
7436        let stale_ed = KeySlot::new(
7437            SigningKey::generate(&mut rand::rngs::OsRng),
7438            now - 30 * 86400,
7439            now + 8 * 86400,
7440        );
7441        let stale_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 30 * 86400, now + 8 * 86400);
7442        let stale_ed_store = SigningKeyStore::new(KeySlots {
7443            drain: None,
7444            active: Some(stale_ed),
7445            lead: None,
7446        });
7447        let stale_pq_store = Arc::new(MlDsaSigningKeyStore::new(MlDsaKeySlots {
7448            drain: None,
7449            active: Some(stale_pq),
7450            lead: None,
7451        }));
7452        let stale_extra = RotationStores {
7453            es256: None,
7454            ml_dsa: Some(Arc::clone(&stale_pq_store)),
7455            composite_ca_key: Arc::clone(&ca),
7456        };
7457
7458        // Two all-restored stale-writer ticks: each supplies its own (stale)
7459        // expectation, fails the digest gate against the committed ledger,
7460        // and must neither publish nor certify its outcome into the record.
7461        for tick in 1..=2 {
7462            let all_restored =
7463                run_rotation_cycle(&config, dir.path(), &stale_ed_store, &stale_extra, now).await;
7464            assert!(
7465                all_restored,
7466                "stale tick {tick}: the no-op writer is restored"
7467            );
7468            let (commit_after_tick, _ledger_after_tick) =
7469                read_committed_composite_ledger(dir.path()).unwrap();
7470            assert_eq!(
7471                commit_after_tick.version, commit_before.version,
7472                "stale tick {tick}: a rejected stale writer must not advance the \
7473                 committed generation"
7474            );
7475            assert_eq!(
7476                commit_after_tick.component_digest, commit_before.component_digest,
7477                "stale tick {tick}: the committed digest must not change"
7478            );
7479            assert_eq!(
7480                std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7481                pending_before,
7482                "stale tick {tick}: the pending ledger must be untouched"
7483            );
7484            assert_eq!(
7485                hyprstream_rpc::auth::global_composite_key_set()
7486                    .snapshot()
7487                    .version(),
7488                snapshot_version_before,
7489                "stale tick {tick}: a rejected stale writer must not publish to \
7490                 the live key set"
7491            );
7492            let record = read_deferred_predecessor(dir.path())
7493                .expect("stale tick {tick}: the recovery record must survive");
7494            assert_eq!(
7495                record.committed_digest, seeded.committed_digest,
7496                "stale tick {tick}: the recorded committed predecessor must be \
7497                 preserved"
7498            );
7499            assert_eq!(
7500                record.states, certified_states,
7501                "stale tick {tick}: the stale writer's outcome must never be \
7502                 certified into the record"
7503            );
7504        }
7505
7506        // The legitimate writer still recovers: its certified state anchors
7507        // publication of the intended generation, and the record is cleared.
7508        std::fs::remove_dir(&meta_path).unwrap();
7509        let recovered = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7510        assert!(recovered, "the certified writer must recover and publish");
7511        let (commit_recovered, ledger_recovered) =
7512            read_committed_composite_ledger(dir.path()).unwrap();
7513        assert!(
7514            commit_recovered.version > commit_before.version,
7515            "the certified recovery must publish a new committed generation"
7516        );
7517        let oauth_active = ledger_recovered
7518            .pairs
7519            .iter()
7520            .find(|record| record.role == "oauth" && record.state == "active")
7521            .expect("the published generation must carry an active OAuth pair");
7522        assert_eq!(
7523            oauth_active.ed25519_public,
7524            URL_SAFE_NO_PAD.encode(new_ed_vk),
7525            "the published generation must carry the recovered Ed25519 signer"
7526        );
7527        assert!(read_deferred_predecessor(dir.path()).is_none());
7528    }
7529
7530    #[tokio::test]
7531    async fn ed25519_foreign_disk_state_never_certified() {
7532        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_FOREIGN_DISK_STATE_TEST";
7533        if std::env::var_os(ISOLATED).is_none() {
7534            let status = std::process::Command::new(std::env::current_exe().unwrap())
7535                .args([
7536                    "--exact",
7537                    "auth::key_rotation::tests::ed25519_foreign_disk_state_never_certified",
7538                    "--nocapture",
7539                ])
7540                .env(ISOLATED, "1")
7541                .status()
7542                .unwrap();
7543            assert!(status.success());
7544            return;
7545        }
7546
7547        let dir = TempDir::new().unwrap();
7548        let config = test_config();
7549        let now = chrono::Utc::now().timestamp();
7550
7551        // Writer A: healthy ML-DSA store with a ready lead (it promotes in
7552        // the mixed cycle); Ed25519 store whose promotion fails its rollback.
7553        let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
7554        let new_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7555        let pq_lead = Arc::new(MlDsaSigningKeyStore::new(MlDsaKeySlots {
7556            drain: None,
7557            active: pq_store.active_key().await.map(|key| {
7558                ml_dsa_rotation::MlDsaKeySlot::new((*key).clone(), now - 3600, now + 14 * 86400)
7559            }),
7560            lead: Some(new_pq.clone()),
7561        }));
7562        let old_ed = KeySlot::new(
7563            SigningKey::generate(&mut rand::rngs::OsRng),
7564            now - 14 * 86400,
7565            now + 6 * 3600,
7566        );
7567        let new_ed = KeySlot::new(
7568            SigningKey::generate(&mut rand::rngs::OsRng),
7569            now - 1,
7570            now + 14 * 86400,
7571        );
7572        let new_ed_vk = new_ed.verifying_key_bytes();
7573        let new_pq_vk = ml_dsa_slot_vk_bytes(&new_pq);
7574        persist_slot(dir.path(), "active", &old_ed).unwrap();
7575        persist_slot(dir.path(), "lead", &new_ed).unwrap();
7576        let ed_store = SigningKeyStore::new(KeySlots {
7577            drain: None,
7578            active: Some(old_ed),
7579            lead: Some(new_ed),
7580        });
7581
7582        let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7583        // Initialize with exactly the stores cycle 1 consumes (queued leads
7584        // included), so the committed digest matches cycle 1's pre-rotation
7585        // component state and the predecessor seeding validates.
7586        initialize_composite_key_set(dir.path(), &ed_store, &pq_lead, Arc::clone(&ca), 30)
7587            .await
7588            .expect("healthy starting authority must initialize composite");
7589        let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7590        let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7591        let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
7592            .snapshot()
7593            .version();
7594
7595        // Cycle 1: the mixed cycle defers and records the committed
7596        // predecessor plus its certified states.
7597        let meta_path = dir.path().join("jwt-signing-key.active.meta");
7598        std::fs::remove_file(&meta_path).unwrap();
7599        std::fs::create_dir(&meta_path).unwrap();
7600        let extra = RotationStores {
7601            es256: None,
7602            ml_dsa: Some(Arc::clone(&pq_lead)),
7603            composite_ca_key: Arc::clone(&ca),
7604        };
7605        let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7606        assert!(!all_restored, "the mixed cycle must defer");
7607        let seeded = read_deferred_predecessor(dir.path())
7608            .expect("the deferring cycle must retain a recovery record");
7609        assert_eq!(
7610            seeded.committed_digest, commit_before.component_digest,
7611            "the record must anchor on the current committed generation"
7612        );
7613        assert_eq!(
7614            seeded.states.len(),
7615            1,
7616            "only process-computed state recorded"
7617        );
7618
7619        // The transient Ed25519 obstruction clears before writer B runs.
7620        std::fs::remove_dir(&meta_path).unwrap();
7621
7622        // Writer B: the same pre-promotion Ed25519 slots as the committed
7623        // generation, but a different stale ML-DSA slot set with its own
7624        // ready lead. B promotes both families (successful due promotions),
7625        // writes its outcome to the shared disk, and its publication is
7626        // rejected by the digest gate (its captured predecessor differs).
7627        let new_ed_b = KeySlot::new(
7628            SigningKey::generate(&mut rand::rngs::OsRng),
7629            now - 1,
7630            now + 14 * 86400,
7631        );
7632        let new_pq_b = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7633        let new_pq_b_vk = ml_dsa_slot_vk_bytes(&new_pq_b);
7634        let ed_b = SigningKeyStore::new(KeySlots {
7635            drain: None,
7636            active: ed_store
7637                .active_key()
7638                .await
7639                .map(|key| KeySlot::new((*key).clone(), now - 3600, now + 14 * 86400)),
7640            lead: Some(new_ed_b.clone()),
7641        });
7642        let pq_b = MlDsaSigningKeyStore::new(MlDsaKeySlots {
7643            drain: None,
7644            active: Some(ml_dsa_rotation::generate_ml_dsa_slot(
7645                now - 3600,
7646                now + 14 * 86400,
7647            )),
7648            lead: Some(new_pq_b.clone()),
7649        });
7650        let extra_b = RotationStores {
7651            es256: None,
7652            ml_dsa: Some(Arc::new(pq_b)),
7653            composite_ca_key: Arc::clone(&ca),
7654        };
7655        let b_restored = run_rotation_cycle(&config, dir.path(), &ed_b, &extra_b, now).await;
7656        assert!(b_restored, "writer B's own rotations must succeed");
7657
7658        // Writer A's certified recovery tick: its memory is certified, but
7659        // its ML-DSA rotation has no due work and leaves writer B's disk
7660        // slots untouched. The publication is forced to fail (corrupt
7661        // committed marker), and the advance must certify only writer A's
7662        // own memory outcome — never writer B's durable outcome.
7663        let marker_path = dir.path().join("jwt-composite-pairs.committed");
7664        let marker_bytes = std::fs::read(&marker_path).unwrap();
7665        std::fs::write(&marker_path, b"corrupt").unwrap();
7666        let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7667        assert!(
7668            all_restored,
7669            "writer A's rotation is restored; only its publication fails"
7670        );
7671        let record = read_deferred_predecessor(dir.path())
7672            .expect("the recovery record must survive the failed publication");
7673        assert_eq!(
7674            record.committed_digest, commit_before.component_digest,
7675            "the record must keep the committed predecessor"
7676        );
7677        // Writer A's memory outcome includes the ML-DSA store the cycle
7678        // rotates (pq_lead), which advanced in the mixed cycle.
7679        let post_memory = component_state_digest(&ed_store, &pq_lead, ca.verifying_key()).await;
7680        assert_eq!(
7681            record.states,
7682            vec![post_memory.clone()],
7683            "the advance must certify only writer A's own memory outcome"
7684        );
7685        let foreign_reload = {
7686            let reloaded_ed = load_or_init_key_store(dir.path(), &config);
7687            let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
7688            component_state_digest(&reloaded_ed, &reloaded_pq, ca.verifying_key()).await
7689        };
7690        assert_ne!(
7691            foreign_reload, post_memory,
7692            "writer B's disk slots must still be on disk for this regression"
7693        );
7694        assert!(
7695            !record.states.contains(&foreign_reload),
7696            "the durable state left by the rejected writer must never be \
7697             certified into the record"
7698        );
7699        assert_eq!(
7700            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7701            pending_before,
7702            "the failed publication must not restate the pending ledger"
7703        );
7704        assert_eq!(
7705            hyprstream_rpc::auth::global_composite_key_set()
7706                .snapshot()
7707                .version(),
7708            snapshot_version_before,
7709            "the failed publication must not publish to the live key set"
7710        );
7711
7712        // Ledger access restored: the committed generation is untouched by
7713        // the failed publication.
7714        std::fs::write(&marker_path, &marker_bytes).unwrap();
7715        let (commit_recovered_check, _) = read_committed_composite_ledger(dir.path()).unwrap();
7716        assert_eq!(
7717            commit_recovered_check.version, commit_before.version,
7718            "the failed publication must not advance the committed generation"
7719        );
7720
7721        // Retry writer B: its tick-start digest is still not certified, so it
7722        // can neither publish nor advance the record; committed and live
7723        // authority remain the retained generation.
7724        let b_retry = run_rotation_cycle(&config, dir.path(), &ed_b, &extra_b, now).await;
7725        assert!(b_retry, "writer B's no-op tick is restored");
7726        let (commit_b_retry, ledger_b_retry) = read_committed_composite_ledger(dir.path()).unwrap();
7727        assert_eq!(commit_b_retry.version, commit_before.version);
7728        let oauth_b = ledger_b_retry
7729            .pairs
7730            .iter()
7731            .find(|record| record.role == "oauth" && record.state == "active")
7732            .expect("authority must still carry an active OAuth pair");
7733        assert_ne!(
7734            oauth_b.ml_dsa_public,
7735            URL_SAFE_NO_PAD.encode(&new_pq_b_vk),
7736            "writer B's stale ML-DSA outcome must never become committed \
7737             authority"
7738        );
7739        assert_eq!(
7740            read_deferred_predecessor(dir.path())
7741                .expect("the record must survive writer B's retry")
7742                .states,
7743            vec![post_memory.clone()],
7744            "writer B's retry must not certify its outcome into the record"
7745        );
7746
7747        // Ledger access restored: writer A's next tick anchors on the
7748        // certified memory state and publishes the recovered generation.
7749        std::fs::write(&marker_path, &marker_bytes).unwrap();
7750        let recovered = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7751        assert!(
7752            recovered,
7753            "writer A's certified tick must recover and publish"
7754        );
7755        let (commit_recovered, ledger_recovered) =
7756            read_committed_composite_ledger(dir.path()).unwrap();
7757        assert!(
7758            commit_recovered.version > commit_before.version,
7759            "the certified recovery must publish a new committed generation"
7760        );
7761        let oauth_recovered = ledger_recovered
7762            .pairs
7763            .iter()
7764            .find(|record| record.role == "oauth" && record.state == "active")
7765            .expect("the published generation must carry an active OAuth pair");
7766        assert_eq!(
7767            oauth_recovered.ed25519_public,
7768            URL_SAFE_NO_PAD.encode(new_ed_vk),
7769            "the published generation must carry the recovered Ed25519 signer"
7770        );
7771        assert_eq!(
7772            oauth_recovered.ml_dsa_public,
7773            URL_SAFE_NO_PAD.encode(&new_pq_vk),
7774            "the published generation must carry the promoted ML-DSA signer"
7775        );
7776        assert!(read_deferred_predecessor(dir.path()).is_none());
7777    }
7778
7779    #[tokio::test]
7780    async fn ed25519_chained_deferral_never_certifies_reload() {
7781        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_CHAINED_DEFER_TEST";
7782        if std::env::var_os(ISOLATED).is_none() {
7783            let status = std::process::Command::new(std::env::current_exe().unwrap())
7784                .args([
7785                    "--exact",
7786                    "auth::key_rotation::tests::ed25519_chained_deferral_never_certifies_reload",
7787                    "--nocapture",
7788                ])
7789                .env(ISOLATED, "1")
7790                .status()
7791                .unwrap();
7792            assert!(status.success());
7793            return;
7794        }
7795
7796        let dir = TempDir::new().unwrap();
7797        let config = test_config();
7798        let now = chrono::Utc::now().timestamp();
7799
7800        // Bootstrap the ML-DSA active slot on disk (the store handle itself
7801        // is rebuilt below with its ready lead).
7802        load_or_init_ml_dsa_key_store(dir.path(), &config);
7803        let old_ed = KeySlot::new(
7804            SigningKey::generate(&mut rand::rngs::OsRng),
7805            now - 14 * 86400,
7806            now + 6 * 3600,
7807        );
7808        let new_ed = KeySlot::new(
7809            SigningKey::generate(&mut rand::rngs::OsRng),
7810            now - 1,
7811            now + 14 * 86400,
7812        );
7813        let old_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 3600, now + 14 * 86400);
7814        let new_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7815        persist_slot(dir.path(), "active", &old_ed).unwrap();
7816        persist_slot(dir.path(), "lead", &new_ed).unwrap();
7817        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &old_pq).unwrap();
7818        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &new_pq).unwrap();
7819        let ed_store = SigningKeyStore::new(KeySlots {
7820            drain: None,
7821            active: Some(old_ed),
7822            lead: Some(new_ed),
7823        });
7824        let pq_store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
7825            drain: None,
7826            active: Some(old_pq),
7827            lead: Some(new_pq),
7828        });
7829
7830        let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7831        initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7832            .await
7833            .expect("healthy starting authority must initialize composite");
7834        let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7835        let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7836
7837        let meta_path = dir.path().join("jwt-signing-key.active.meta");
7838        std::fs::remove_file(&meta_path).unwrap();
7839        std::fs::create_dir(&meta_path).unwrap();
7840        let pq_store = Arc::new(pq_store);
7841        let extra = RotationStores {
7842            es256: None,
7843            ml_dsa: Some(Arc::clone(&pq_store)),
7844            composite_ca_key: Arc::clone(&ca),
7845        };
7846
7847        // Cycle 1: mixed cycle (Ed25519 rollback fails, ML-DSA promotes)
7848        // defers and seeds {committed, [memory]}.
7849        let deferred1 = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7850        assert!(!deferred1, "cycle 1 must defer");
7851        let seeded =
7852            read_deferred_predecessor(dir.path()).expect("cycle 1 must retain a recovery record");
7853        assert_eq!(seeded.states.len(), 1, "seed records process memory only");
7854        let seeded_mem_digest =
7855            component_state_digest(&ed_store, &pq_store, ca.verifying_key()).await;
7856        assert_eq!(
7857            seeded.states,
7858            vec![seeded_mem_digest.clone()],
7859            "the seed must match only this process's computed memory"
7860        );
7861
7862        // Cycle 2: consecutive not-fully-restored certified cycle. The
7863        // transition is unchanged, so the record must be kept as-is; nothing
7864        // new enters the certified states.
7865        let deferred2 = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7866        assert!(!deferred2, "cycle 2 must defer again (same obstruction)");
7867        let advanced = read_deferred_predecessor(dir.path())
7868            .expect("the record must survive the chained deferral");
7869        assert_eq!(
7870            advanced.committed_digest, commit_before.component_digest,
7871            "the chained deferral must keep the committed predecessor"
7872        );
7873        assert_eq!(
7874            advanced.states, seeded.states,
7875            "an unchanged chained deferral must keep the record as-is"
7876        );
7877
7878        // Writer B's contamination analogue: a foreign ML-DSA active slot
7879        // written to the shared state dir between certified cycles (the
7880        // Ed25519 side stays torn under its own obstruction, as in the
7881        // captured-transition state the record describes).
7882        let foreign_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7883        ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &foreign_pq).unwrap();
7884        let foreign_reload = {
7885            let reloaded_ed = load_or_init_key_store(dir.path(), &config);
7886            let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
7887            component_state_digest(&reloaded_ed, &reloaded_pq, ca.verifying_key()).await
7888        };
7889        assert_ne!(
7890            foreign_reload, seeded_mem_digest,
7891            "fixture: the foreign disk state must differ from the certified \
7892             memory outcome"
7893        );
7894
7895        // Cycle 3: the failure persists (the meta obstruction is still in
7896        // place). The tick is certified (memory unchanged), its rotations
7897        // durably advance a family only if due work exists - here none - and
7898        // the chained advance must never certify the foreign durable state.
7899        let deferred3 = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7900        assert!(!deferred3, "cycle 3 must defer again (same obstruction)");
7901        let after_foreign = read_deferred_predecessor(dir.path())
7902            .expect("the record must survive the third deferral");
7903        assert_eq!(
7904            after_foreign.committed_digest, commit_before.component_digest,
7905            "the committed predecessor must be preserved"
7906        );
7907        assert!(
7908            !after_foreign.states.contains(&foreign_reload),
7909            "the foreign durable state left by another writer must never be \
7910             certified into the record"
7911        );
7912        assert_eq!(
7913            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7914            pending_before,
7915            "deferred cycles must not touch the pending ledger"
7916        );
7917    }
7918
7919    #[tokio::test]
7920    async fn rotation_cycle_lock_serializes_writers() {
7921        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_CYCLE_LOCK_TEST";
7922        if std::env::var_os(ISOLATED).is_none() {
7923            let status = std::process::Command::new(std::env::current_exe().unwrap())
7924                .args([
7925                    "--exact",
7926                    "auth::key_rotation::tests::rotation_cycle_lock_serializes_writers",
7927                    "--nocapture",
7928                ])
7929                .env(ISOLATED, "1")
7930                .status()
7931                .unwrap();
7932            assert!(status.success());
7933            return;
7934        }
7935
7936        let dir = TempDir::new().unwrap();
7937        let config = test_config();
7938        let now = chrono::Utc::now().timestamp();
7939
7940        let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
7941        let active_slot = KeySlot::new(
7942            SigningKey::generate(&mut rand::rngs::OsRng),
7943            now - 14 * 86400,
7944            now + 6 * 3600,
7945        );
7946        persist_slot(dir.path(), "active", &active_slot).unwrap();
7947        let ed_store = SigningKeyStore::new(KeySlots {
7948            drain: None,
7949            active: Some(active_slot),
7950            lead: None,
7951        });
7952
7953        let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7954        initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7955            .await
7956            .expect("healthy starting authority must initialize composite");
7957        let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7958        let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7959
7960        // A competing writer holding the cycle lock makes this tick skip
7961        // entirely: no rotation, no record lifecycle, no publication.
7962        let cycle_lock = std::fs::OpenOptions::new()
7963            .create(true)
7964            .truncate(false)
7965            .read(true)
7966            .write(true)
7967            .open(cycle_lock_path(dir.path()))
7968            .unwrap();
7969        use nix::fcntl::{flock, FlockArg};
7970        use std::os::fd::AsRawFd as _;
7971        flock(cycle_lock.as_raw_fd(), FlockArg::LockExclusiveNonblock).unwrap();
7972
7973        let extra = RotationStores {
7974            es256: None,
7975            ml_dsa: Some(Arc::clone(&pq_store)),
7976            composite_ca_key: Arc::clone(&ca),
7977        };
7978        let skipped = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7979        assert!(
7980            skipped,
7981            "a lock-contended cycle must skip without reporting not-restored"
7982        );
7983        let (commit_skipped, _) = read_committed_composite_ledger(dir.path()).unwrap();
7984        assert_eq!(commit_skipped.version, commit_before.version);
7985        assert_eq!(
7986            std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7987            pending_before,
7988            "a skipped cycle must not touch the pending ledger"
7989        );
7990
7991        // Lock released: the next tick runs normally and may publish.
7992        drop(cycle_lock);
7993        let ran = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7994        assert!(ran, "an uncontended cycle must run and stay restored");
7995        let (commit_ran, _) = read_committed_composite_ledger(dir.path()).unwrap();
7996        assert!(
7997            commit_ran.version >= commit_before.version,
7998            "an uncontended cycle must not regress the committed authority"
7999        );
8000    }
8001
8002    #[tokio::test]
8003    async fn committed_marker_failures_never_reinitialize_from_mutable_authority() {
8004        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_FAIL_CLOSED_TEST";
8005        if std::env::var_os(ISOLATED).is_none() {
8006            let status = std::process::Command::new(std::env::current_exe().unwrap())
8007                .args([
8008                    "--exact",
8009                    "auth::key_rotation::tests::committed_marker_failures_never_reinitialize_from_mutable_authority",
8010                    "--nocapture",
8011                ])
8012                .env(ISOLATED, "1")
8013                .status()
8014                .unwrap();
8015            assert!(status.success());
8016            return;
8017        }
8018
8019        enum ImmutableFailure {
8020            Missing,
8021            Corrupt,
8022            Mismatched,
8023            Unavailable,
8024        }
8025
8026        for failure in [
8027            ImmutableFailure::Missing,
8028            ImmutableFailure::Corrupt,
8029            ImmutableFailure::Mismatched,
8030            ImmutableFailure::Unavailable,
8031        ] {
8032            let dir = TempDir::new().unwrap();
8033            let config = test_config();
8034            let ca = Arc::new(SigningKey::from_bytes(&[0x6a; 32]));
8035            let ed = load_or_init_key_store(dir.path(), &config);
8036            let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8037            initialize_composite_key_set(dir.path(), &ed, &pq, Arc::clone(&ca), 300)
8038                .await
8039                .unwrap();
8040
8041            let marker_bytes = std::fs::read(composite_committed_path(dir.path())).unwrap();
8042            let commit: CompositeCommit = serde_json::from_slice(&marker_bytes).unwrap();
8043            let immutable = composite_committed_ledger_path(dir.path(), &commit);
8044            let mut pending: CompositeLedger =
8045                serde_json::from_slice(&std::fs::read(composite_ledger_path(dir.path())).unwrap())
8046                    .unwrap();
8047            pending.version = pending.version.saturating_add(1);
8048            pending.component_digest = "staged-component-C".to_owned();
8049            let pending_bytes = serde_json::to_vec(&pending).unwrap();
8050            std::fs::write(composite_ledger_path(dir.path()), &pending_bytes).unwrap();
8051
8052            match failure {
8053                ImmutableFailure::Missing => std::fs::remove_file(&immutable).unwrap(),
8054                ImmutableFailure::Corrupt => std::fs::write(&immutable, b"{").unwrap(),
8055                ImmutableFailure::Mismatched => {
8056                    std::fs::write(&immutable, &pending_bytes).unwrap();
8057                }
8058                ImmutableFailure::Unavailable => {
8059                    std::fs::remove_file(&immutable).unwrap();
8060                    std::fs::create_dir(&immutable).unwrap();
8061                }
8062            }
8063
8064            let error = initialize_composite_key_set(dir.path(), &ed, &pq, Arc::clone(&ca), 300)
8065                .await
8066                .expect_err("marker-selected authority failure must fail closed");
8067            assert!(!error.to_string().is_empty());
8068            assert_eq!(
8069                std::fs::read(composite_committed_path(dir.path())).unwrap(),
8070                marker_bytes,
8071                "failure rewrote the committed marker"
8072            );
8073            assert_eq!(
8074                std::fs::read(composite_ledger_path(dir.path())).unwrap(),
8075                pending_bytes,
8076                "failure published over staged mutable C"
8077            );
8078            assert!(
8079                !composite_committed_ledger_path(
8080                    dir.path(),
8081                    &CompositeCommit {
8082                        version: pending.version,
8083                        component_digest: pending.component_digest.clone(),
8084                    },
8085                )
8086                .exists(),
8087                "failure materialized staged C as committed authority"
8088            );
8089        }
8090    }
8091
8092    #[tokio::test]
8093    async fn committed_verifier_generation_restores_local_signer_without_rotation() {
8094        const PHASE: &str = "HYPRSTREAM_COMPOSITE_SIGNER_ATTACH_TEST";
8095        const DIRECTORY: &str = "HYPRSTREAM_COMPOSITE_SIGNER_ATTACH_DIR";
8096        let phase = std::env::var(PHASE).unwrap_or_default();
8097        if phase.is_empty() {
8098            let dir = TempDir::new().unwrap();
8099            for phase in ["bootstrap", "restore"] {
8100                let status = std::process::Command::new(std::env::current_exe().unwrap())
8101                    .args(["--exact", "auth::key_rotation::tests::committed_verifier_generation_restores_local_signer_without_rotation", "--nocapture"])
8102                    .env(PHASE, phase).env(DIRECTORY, dir.path()).status().unwrap();
8103                assert!(status.success(), "{phase} subprocess failed");
8104            }
8105            return;
8106        }
8107        let dir = std::path::PathBuf::from(std::env::var_os(DIRECTORY).unwrap());
8108        let config = test_config();
8109        let ca = Arc::new(SigningKey::from_bytes(&[0x69; 32]));
8110        let ed = load_or_init_key_store(&dir, &config);
8111        let pq = load_or_init_ml_dsa_key_store(&dir, &config);
8112        if phase == "bootstrap" {
8113            initialize_composite_key_set(&dir, &ed, &pq, ca, 300)
8114                .await
8115                .unwrap();
8116            return;
8117        }
8118        let marker = std::fs::read(composite_committed_path(&dir)).unwrap();
8119        let (commit, _) = read_committed_composite_ledger(&dir).unwrap();
8120        let immutable_path = composite_committed_ledger_path(&dir, &commit);
8121        let immutable = std::fs::read(&immutable_path).unwrap();
8122        // Exact production order: main restores verifier-only N, then the
8123        // OAuth factory restores the local signing authority at that same N.
8124        restore_composite_verifying_key_set(&dir, &ed, &pq, ca.verifying_key())
8125            .await
8126            .unwrap();
8127        let keys = hyprstream_rpc::auth::global_composite_key_set();
8128        use hyprstream_rpc::auth::CompositePairRole;
8129        let before = keys.mint_snapshot().unwrap();
8130        assert!(before
8131            .active_signing_pair(CompositePairRole::OAuth)
8132            .is_none());
8133        for _ in 0..2 {
8134            initialize_composite_key_set(&dir, &ed, &pq, Arc::clone(&ca), 300)
8135                .await
8136                .unwrap();
8137            let after = keys.mint_snapshot().unwrap();
8138            assert_eq!(after.version(), before.version());
8139            assert_eq!(after.component_digest(), before.component_digest());
8140            for role in [CompositePairRole::OAuth, CompositePairRole::Policy] {
8141                assert!(after.active_signing_pair(role).is_some());
8142            }
8143        }
8144        // Matching file metadata is insufficient: missing local components
8145        // must still fail closed instead of manufacturing a signing identity.
8146        let absent = SigningKeyStore::new(KeySlots {
8147            drain: None,
8148            active: None,
8149            lead: None,
8150        });
8151        assert!(initialize_composite_key_set(&dir, &absent, &pq, ca, 300)
8152            .await
8153            .is_err());
8154        assert_eq!(
8155            std::fs::read(composite_committed_path(&dir)).unwrap(),
8156            marker
8157        );
8158        assert_eq!(std::fs::read(immutable_path).unwrap(), immutable);
8159    }
8160
8161    #[tokio::test]
8162    async fn first_boot_slot_lock_failure_never_creates_or_activates_keys() {
8163        let dir = TempDir::new().unwrap();
8164        std::fs::create_dir(dir.path().join("signing-slots.lock")).unwrap();
8165        let config = test_config();
8166        let ed = load_or_init_key_store(dir.path(), &config);
8167        let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8168        let es = load_or_init_es256_key_store(dir.path(), &config);
8169        assert!(ed.0.read().await.active.is_none());
8170        assert!(pq.active_key().await.is_none());
8171        assert!(es.0.read().active.is_none());
8172        for family in ["jwt", "ml-dsa", "es256"] {
8173            assert!(!dir
8174                .path()
8175                .join(format!("{family}-signing-key.active"))
8176                .exists());
8177        }
8178        assert!(
8179            initialize_composite_key_set(dir.path(), &ed, &pq, authority_ca_key(), 300)
8180                .await
8181                .is_err()
8182        );
8183        assert!(!composite_committed_path(dir.path()).exists());
8184    }
8185
8186    #[tokio::test]
8187    async fn first_boot_slot_lock_defers_rotation_without_mutation() {
8188        let dir = TempDir::new().unwrap();
8189        let config = test_config();
8190        let ed = load_or_init_key_store(dir.path(), &config);
8191        let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8192        let es = load_or_init_es256_key_store(dir.path(), &config);
8193        let ca = authority_ca_key();
8194        let before = component_state_digest(&ed, &pq, ca.verifying_key()).await;
8195        let es_before = es.0.read().active.as_ref().unwrap().kid();
8196        let lock = lock_signing_slots(dir.path(), false).unwrap();
8197        let now = chrono::Utc::now().timestamp();
8198        assert!(!rotate_jwt_keys(&config, dir.path(), &ed, now).await);
8199        assert!(!rotate_ml_dsa_keys(&config, dir.path(), &pq, now).await);
8200        assert!(!rotate_es256_keys(&config, dir.path(), &es, now).await);
8201        assert_eq!(
8202            component_state_digest(&ed, &pq, ca.verifying_key()).await,
8203            before
8204        );
8205        drop(lock);
8206        let reloaded_ed = load_or_init_key_store(dir.path(), &config);
8207        let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8208        let reloaded_es = load_or_init_es256_key_store(dir.path(), &config);
8209        assert_eq!(
8210            component_state_digest(&reloaded_ed, &reloaded_pq, ca.verifying_key()).await,
8211            before
8212        );
8213        assert_eq!(
8214            reloaded_es.0.read().active.as_ref().unwrap().kid(),
8215            es_before
8216        );
8217        assert!(rotate_jwt_keys(&config, dir.path(), &ed, now).await);
8218        assert!(rotate_ml_dsa_keys(&config, dir.path(), &pq, now).await);
8219    }
8220
8221    #[tokio::test]
8222    async fn first_boot_slot_creation_is_serialized_across_processes() {
8223        use nix::fcntl::{flock, FlockArg};
8224        use std::os::fd::AsRawFd;
8225
8226        const CHILD: &str = "HYPRSTREAM_FIRST_BOOT_SLOT_CHILD";
8227        const DIRECTORY: &str = "HYPRSTREAM_FIRST_BOOT_SLOT_DIR";
8228        const TEST: &str =
8229            "auth::key_rotation::tests::first_boot_slot_creation_is_serialized_across_processes";
8230        if let Ok(child) = std::env::var(CHILD) {
8231            let dir = PathBuf::from(std::env::var_os(DIRECTORY).unwrap());
8232            std::fs::write(dir.join(format!("{child}.ready")), b"ready").unwrap();
8233            let config = test_config();
8234            // Match service startup: PQ before Ed, then the factory's ES256.
8235            let pq = load_or_init_ml_dsa_key_store(&dir, &config);
8236            let ed = load_or_init_key_store(&dir, &config);
8237            let es = load_or_init_es256_key_store(&dir, &config);
8238            let ca = authority_ca_key();
8239            initialize_composite_key_set(&dir, &ed, &pq, Arc::clone(&ca), 300)
8240                .await
8241                .unwrap();
8242            // Only public identity digests leave this disposable test process.
8243            let digest = component_state_digest(&ed, &pq, ca.verifying_key()).await;
8244            let es_kid = es.0.read().active.as_ref().unwrap().kid();
8245            std::fs::write(
8246                dir.join(format!("{child}.done")),
8247                format!("{digest}:{es_kid}"),
8248            )
8249            .unwrap();
8250            return;
8251        }
8252
8253        let dir = TempDir::new().unwrap();
8254        let lock = std::fs::OpenOptions::new()
8255            .create(true)
8256            .truncate(false)
8257            .read(true)
8258            .write(true)
8259            .open(dir.path().join("signing-slots.lock"))
8260            .unwrap();
8261        flock(lock.as_raw_fd(), FlockArg::LockExclusive).unwrap();
8262        let spawn = |name: &str| {
8263            std::process::Command::new(std::env::current_exe().unwrap())
8264                .args(["--exact", TEST, "--nocapture"])
8265                .env(CHILD, name)
8266                .env(DIRECTORY, dir.path())
8267                .spawn()
8268                .unwrap()
8269        };
8270        let mut first = spawn("first");
8271        let mut second = spawn("second");
8272        wait_path(&dir.path().join("first.ready"));
8273        wait_path(&dir.path().join("second.ready"));
8274        // The old initializer ignores this held interprocess lock and writes
8275        // active slots. Check the causal boundary before releasing either child.
8276        std::thread::sleep(std::time::Duration::from_millis(300));
8277        let wrote_while_locked = ["jwt", "ml-dsa", "es256"].iter().any(|family| {
8278            dir.path()
8279                .join(format!("{family}-signing-key.active"))
8280                .exists()
8281        });
8282        drop(lock);
8283        assert!(first.wait().unwrap().success());
8284        assert!(second.wait().unwrap().success());
8285        assert!(
8286            !wrote_while_locked,
8287            "slot creation ignored the interprocess lock"
8288        );
8289        assert_eq!(
8290            std::fs::read(dir.path().join("first.done")).unwrap(),
8291            std::fs::read(dir.path().join("second.done")).unwrap(),
8292            "concurrent starters must adopt the same durable component identities"
8293        );
8294        let marker = std::fs::read(composite_committed_path(dir.path())).unwrap();
8295        let mut restarted = spawn("restarted");
8296        assert!(restarted.wait().unwrap().success());
8297        assert_eq!(
8298            std::fs::read(dir.path().join("first.done")).unwrap(),
8299            std::fs::read(dir.path().join("restarted.done")).unwrap()
8300        );
8301        assert_eq!(
8302            std::fs::read(composite_committed_path(dir.path())).unwrap(),
8303            marker
8304        );
8305    }
8306
8307    #[tokio::test]
8308    async fn first_bootstrap_and_locked_legacy_migration_are_distinct() {
8309        const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_LEGACY_MIGRATION_TEST";
8310        if std::env::var_os(ISOLATED).is_none() {
8311            let status = std::process::Command::new(std::env::current_exe().unwrap())
8312                .args([
8313                    "--exact",
8314                    "auth::key_rotation::tests::first_bootstrap_and_locked_legacy_migration_are_distinct",
8315                    "--nocapture",
8316                ])
8317                .env(ISOLATED, "1")
8318                .status()
8319                .unwrap();
8320            assert!(status.success());
8321            return;
8322        }
8323
8324        let dir = TempDir::new().unwrap();
8325        let config = test_config();
8326        let ca = Arc::new(SigningKey::from_bytes(&[0x6b; 32]));
8327        let ed = load_or_init_key_store(dir.path(), &config);
8328        let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8329
8330        assert!(!composite_committed_path(dir.path()).exists());
8331        initialize_composite_key_set(dir.path(), &ed, &pq, Arc::clone(&ca), 300)
8332            .await
8333            .unwrap();
8334        let commit: CompositeCommit =
8335            serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
8336                .unwrap();
8337        let immutable = composite_committed_ledger_path(dir.path(), &commit);
8338        let committed_bytes = std::fs::read(&immutable).unwrap();
8339
8340        // Recreate the round-three layout, then restart through the production
8341        // initializer. It must durably pin B before mutable C can be staged.
8342        std::fs::remove_file(&immutable).unwrap();
8343        initialize_composite_key_set(dir.path(), &ed, &pq, ca, 300)
8344            .await
8345            .unwrap();
8346        assert_eq!(std::fs::read(&immutable).unwrap(), committed_bytes);
8347
8348        let mut pending: CompositeLedger = serde_json::from_slice(&committed_bytes).unwrap();
8349        pending.version = pending.version.saturating_add(1);
8350        pending.component_digest = "staged-component-C".to_owned();
8351        std::fs::write(
8352            composite_ledger_path(dir.path()),
8353            serde_json::to_vec(&pending).unwrap(),
8354        )
8355        .unwrap();
8356        let (_, selected) = read_committed_composite_ledger(dir.path()).unwrap();
8357        assert_eq!(selected.version, commit.version);
8358        assert_eq!(selected.component_digest, commit.component_digest);
8359    }
8360
8361    #[tokio::test]
8362    async fn composite_ledger_survives_joint_rotation_restart_and_drain_expiry() {
8363        use super::ml_dsa_rotation::*;
8364
8365        let dir = TempDir::new().unwrap();
8366        let now = chrono::Utc::now().timestamp();
8367        let drain_secs = 600;
8368        let ca = Arc::new(SigningKey::from_bytes(&[91; 32]));
8369        let old_ed = KeySlot::new(SigningKey::from_bytes(&[92; 32]), now - 60, now + 60);
8370        let old_pq = generate_ml_dsa_slot(now - 60, now + 60);
8371        persist_slot(dir.path(), "active", &old_ed).unwrap();
8372        persist_ml_dsa_slot(dir.path(), "active", &old_pq).unwrap();
8373        let ed_store = SigningKeyStore::new(KeySlots {
8374            drain: None,
8375            active: Some(old_ed.clone()),
8376            lead: None,
8377        });
8378        let pq_store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
8379            drain: None,
8380            active: Some(old_pq.clone()),
8381            lead: None,
8382        });
8383        initialize_composite_key_set(
8384            dir.path(),
8385            &ed_store,
8386            &pq_store,
8387            Arc::clone(&ca),
8388            drain_secs,
8389        )
8390        .await
8391        .unwrap();
8392        let initial = hyprstream_rpc::auth::global_composite_key_set().snapshot();
8393        let initial_component_digest = initial.component_digest().to_owned();
8394        let old_pair = initial
8395            .active_signing_pair(hyprstream_rpc::auth::CompositePairRole::OAuth)
8396            .unwrap();
8397        let old_kid = old_pair.kid().to_owned();
8398        let (old_pq_signing, old_ed_signing) = old_pair.signing_keys().unwrap();
8399        let claims = hyprstream_rpc::auth::Claims::new("alice".to_owned(), now, now + 300)
8400            .with_issuer("https://local".to_owned())
8401            .with_audience(Some("https://resource".to_owned()));
8402        let token = crate::auth::jwt::encode_composite_ml_dsa_65_ed25519(
8403            &claims,
8404            &old_pq_signing,
8405            &old_ed_signing,
8406        );
8407
8408        // Deterministic joint promotion: old exact pair becomes drain and a
8409        // new exact pair becomes active in one publication.
8410        // Keep the active generation outside the configured lead window so a
8411        // restart reloads this exact persisted component generation.
8412        let new_ed = KeySlot::new(SigningKey::from_bytes(&[93; 32]), now, now + 15 * 86_400);
8413        let new_pq = generate_ml_dsa_slot(now, now + 15 * 86_400);
8414        persist_slot(dir.path(), "drain", &old_ed).unwrap();
8415        persist_slot(dir.path(), "active", &new_ed).unwrap();
8416        persist_ml_dsa_slot(dir.path(), "drain", &old_pq).unwrap();
8417        persist_ml_dsa_slot(dir.path(), "active", &new_pq).unwrap();
8418        let rotated_ed = SigningKeyStore::new(KeySlots {
8419            drain: Some(old_ed),
8420            active: Some(new_ed),
8421            lead: None,
8422        });
8423        let rotated_pq = MlDsaSigningKeyStore::new(MlDsaKeySlots {
8424            drain: Some(old_pq),
8425            active: Some(new_pq),
8426            lead: None,
8427        });
8428        refresh_composite_key_set(
8429            dir.path(),
8430            &rotated_ed,
8431            &rotated_pq,
8432            Arc::clone(&ca),
8433            drain_secs,
8434            &initial_component_digest,
8435        )
8436        .await
8437        .unwrap();
8438        let rotated = hyprstream_rpc::auth::global_composite_key_set().snapshot();
8439        let retained = rotated
8440            .pair(&old_kid)
8441            .expect("old exact pair retained as drain");
8442        let dispatch = hyprstream_rpc::auth::parse_composite_dispatch(&token, &["at+jwt"]).unwrap();
8443        hyprstream_rpc::auth::jwt::decode_composite(
8444            &token,
8445            retained.ml_dsa(),
8446            retained.ed25519(),
8447            Some("https://resource"),
8448            &dispatch,
8449        )
8450        .unwrap();
8451
8452        // Restart from component files plus the exact persisted association.
8453        let reloaded_ed = load_or_init_key_store(dir.path(), &test_config());
8454        let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &test_config());
8455        initialize_composite_key_set(
8456            dir.path(),
8457            &reloaded_ed,
8458            &reloaded_pq,
8459            Arc::clone(&ca),
8460            drain_secs,
8461        )
8462        .await
8463        .unwrap();
8464        let restarted = hyprstream_rpc::auth::global_composite_key_set().snapshot();
8465        assert!(restarted.pair(&old_kid).is_some());
8466        let jwks_kids: Vec<_> = restarted
8467            .pairs()
8468            .iter()
8469            .map(|pair| {
8470                crate::auth::jwt::composite_jwk(pair.ml_dsa(), pair.ed25519())["kid"]
8471                    .as_str()
8472                    .unwrap()
8473                    .to_owned()
8474            })
8475            .collect();
8476        assert!(jwks_kids.contains(&old_kid));
8477
8478        // Removing the expired drain from component authority requires a new
8479        // committed publication. A mutable-ledger edit alone is pending input
8480        // and must never revoke the marker-selected live snapshot.
8481        reloaded_ed.0.write().await.drain = None;
8482        reloaded_pq.0.write().await.drain = None;
8483        refresh_composite_key_set(
8484            dir.path(),
8485            &reloaded_ed,
8486            &reloaded_pq,
8487            ca,
8488            drain_secs,
8489            restarted.component_digest(),
8490        )
8491        .await
8492        .unwrap();
8493        assert!(hyprstream_rpc::auth::global_composite_key_set()
8494            .snapshot()
8495            .pair(&old_kid)
8496            .is_none());
8497    }
8498}