1use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
14use ed25519_dalek::SigningKey;
15use serde::{Deserialize, Serialize};
16use sha2::{Digest as _, Sha256};
17use std::path::{Path, PathBuf};
18use std::sync::atomic::{AtomicBool, Ordering};
19use std::sync::Arc;
20use tokio::sync::RwLock;
21use tokio::time::MissedTickBehavior;
22use tracing::{error, info, warn};
23
24use crate::config::OAuthConfig;
25
26#[allow(clippy::disallowed_types)]
34static ML_DSA_VERIFYING_KEYS: std::sync::OnceLock<
35 std::sync::Arc<std::sync::RwLock<Vec<hyprstream_rpc::crypto::pq::MlDsaVerifyingKey>>>,
36> = std::sync::OnceLock::new();
37
38#[allow(clippy::disallowed_types)]
40pub fn global_ml_dsa_verifying_keys(
41) -> std::sync::Arc<std::sync::RwLock<Vec<hyprstream_rpc::crypto::pq::MlDsaVerifyingKey>>> {
42 ML_DSA_VERIFYING_KEYS
43 .get_or_init(|| std::sync::Arc::new(std::sync::RwLock::new(Vec::new())))
44 .clone()
45}
46
47#[allow(clippy::disallowed_types)]
58pub type PublishedEd25519Keys = std::sync::Arc<std::sync::RwLock<Vec<ed25519_dalek::VerifyingKey>>>;
59
60static ED25519_VERIFYING_KEYS: std::sync::OnceLock<PublishedEd25519Keys> =
66 std::sync::OnceLock::new();
67
68#[allow(clippy::disallowed_types)]
73pub fn global_ed25519_verifying_keys() -> PublishedEd25519Keys {
74 ED25519_VERIFYING_KEYS
75 .get_or_init(|| std::sync::Arc::new(std::sync::RwLock::new(Vec::new())))
76 .clone()
77}
78
79pub async fn refresh_ed25519_verifying_keys(store: &SigningKeyStore) {
85 let vks: Vec<ed25519_dalek::VerifyingKey> = store
86 .all_slots_snapshot()
87 .await
88 .iter()
89 .map(|slot| slot.key.verifying_key())
90 .collect();
91 let shared = global_ed25519_verifying_keys();
92 let _ = shared.write().map(|mut guard| *guard = vks);
93}
94
95pub async fn refresh_ml_dsa_verifying_keys(store: &Arc<MlDsaSigningKeyStore>) {
97 let vks: Vec<hyprstream_rpc::crypto::pq::MlDsaVerifyingKey> = store
98 .all_slots_snapshot()
99 .await
100 .iter()
101 .map(ml_dsa_rotation::MlDsaKeySlot::verifying_key)
102 .collect();
103 let shared = global_ml_dsa_verifying_keys();
104 let _ = shared.write().map(|mut guard| *guard = vks);
105}
106
107#[derive(Clone, Serialize, Deserialize)]
108struct CompositeLedger {
109 version: u64,
110 #[serde(default)]
111 component_digest: String,
112 pairs: Vec<CompositeLedgerPair>,
113}
114
115#[derive(Clone, Serialize, Deserialize)]
116struct CompositeCommit {
117 version: u64,
118 component_digest: String,
119}
120
121#[derive(Clone, Serialize, Deserialize)]
122struct CompositeAcknowledgement {
123 version: u64,
124 component_digest: String,
125}
126
127#[derive(Clone, Serialize, Deserialize)]
128struct CompositeLedgerPair {
129 kid: String,
130 ml_dsa_public: String,
131 ed25519_public: String,
132 role: String,
133 state: String,
134 not_before: i64,
135 expires_at: i64,
136}
137
138fn composite_ledger_path(secrets_dir: &Path) -> PathBuf {
139 secrets_dir.join("jwt-composite-pairs.json")
140}
141
142fn composite_committed_path(dir: &Path) -> PathBuf {
143 dir.join("jwt-composite-pairs.committed")
144}
145
146fn has_committed_composite_marker(dir: &Path) -> bool {
153 !matches!(
154 std::fs::symlink_metadata(composite_committed_path(dir)),
155 Err(error) if error.kind() == std::io::ErrorKind::NotFound
156 )
157}
158fn composite_committed_ledger_path(dir: &Path, commit: &CompositeCommit) -> PathBuf {
159 dir.join(format!(
160 "jwt-composite-pairs.committed-{}-{}.json",
161 commit.version, commit.component_digest
162 ))
163}
164fn composite_ledger_lock_path(dir: &Path) -> PathBuf {
165 dir.join("jwt-composite-pairs.lock")
166}
167fn composite_writer_lock_path(dir: &Path) -> PathBuf {
168 dir.join("jwt-composite-pairs.writer.lock")
169}
170fn composite_subscribers_dir(dir: &Path) -> PathBuf {
171 dir.join("jwt-composite-subscribers")
172}
173
174static COMPOSITE_SUBSCRIPTION_STARTED: AtomicBool = AtomicBool::new(false);
175static COMPOSITE_PUBLISHING: AtomicBool = AtomicBool::new(false);
176static COMPOSITE_CA_SIGNING_KEY: std::sync::OnceLock<parking_lot::RwLock<Option<Arc<SigningKey>>>> =
177 std::sync::OnceLock::new();
178fn composite_ca_signing_key() -> &'static parking_lot::RwLock<Option<Arc<SigningKey>>> {
179 COMPOSITE_CA_SIGNING_KEY.get_or_init(|| parking_lot::RwLock::new(None))
180}
181fn configure_composite_authority(dir: &Path) {
182 hyprstream_rpc::auth::global_composite_key_set().configure_authority(
183 composite_ledger_path(dir),
184 composite_committed_path(dir),
185 dir.join("jwt-composite-pairs.committed"),
186 composite_ledger_lock_path(dir),
187 );
188}
189
190fn read_committed_composite_ledger(
194 dir: &Path,
195) -> anyhow::Result<(CompositeCommit, CompositeLedger)> {
196 let commit: CompositeCommit =
197 serde_json::from_slice(&std::fs::read(composite_committed_path(dir))?)?;
198 let ledger = read_composite_ledger_selected_by_commit(dir, &commit)?;
199 Ok((commit, ledger))
200}
201
202fn read_composite_ledger_selected_by_commit(
203 dir: &Path,
204 commit: &CompositeCommit,
205) -> anyhow::Result<CompositeLedger> {
206 let immutable = composite_committed_ledger_path(dir, commit);
207 let ledger: CompositeLedger = match std::fs::read(&immutable) {
208 Ok(bytes) => serde_json::from_slice(&bytes)?,
209 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
210 match std::fs::symlink_metadata(&immutable) {
211 Err(metadata_error) if metadata_error.kind() == std::io::ErrorKind::NotFound => {
212 serde_json::from_slice(&std::fs::read(composite_ledger_path(dir))?)?
213 }
214 Ok(_) => return Err(error.into()),
215 Err(metadata_error) => return Err(metadata_error.into()),
216 }
217 }
218 Err(error) => return Err(error.into()),
219 };
220 anyhow::ensure!(
221 ledger.version == commit.version
222 && !ledger.component_digest.is_empty()
223 && ledger.component_digest == commit.component_digest,
224 "committed composite ledger does not match its commit marker"
225 );
226 Ok(ledger)
227}
228
229fn load_or_migrate_committed_composite_ledger(
235 dir: &Path,
236) -> anyhow::Result<Option<CompositeLedger>> {
237 let marker_path = composite_committed_path(dir);
238 let marker_bytes = match std::fs::read(&marker_path) {
239 Ok(bytes) => bytes,
240 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
241 match std::fs::symlink_metadata(&marker_path) {
242 Err(metadata_error) if metadata_error.kind() == std::io::ErrorKind::NotFound => {
243 return Ok(None);
244 }
245 Ok(_) => return Err(error.into()),
246 Err(metadata_error) => return Err(metadata_error.into()),
247 }
248 }
249 Err(error) => return Err(error.into()),
250 };
251 let commit: CompositeCommit = serde_json::from_slice(&marker_bytes)?;
252 let immutable = composite_committed_ledger_path(dir, &commit);
253 let immutable_missing = match std::fs::symlink_metadata(&immutable) {
254 Ok(_) => false,
255 Err(error) if error.kind() == std::io::ErrorKind::NotFound => true,
256 Err(error) => return Err(error.into()),
257 };
258 let ledger = read_composite_ledger_selected_by_commit(dir, &commit)?;
259 let name = immutable
260 .file_name()
261 .and_then(|name| name.to_str())
262 .ok_or_else(|| anyhow::anyhow!("invalid committed composite ledger path"))?;
263 if immutable_missing {
264 super::identity_store::write_secret_durable(dir, name, &serde_json::to_vec(&ledger)?)?;
265 } else {
266 super::identity_store::sync_secret_durable(dir, name)?;
267 }
268 super::identity_store::sync_secret_durable(dir, "jwt-composite-pairs.committed")?;
273 Ok(Some(ledger))
274}
275
276fn digest_component_slot(
277 digest: &mut Sha256,
278 family: &[u8],
279 name: &[u8],
280 public: Option<&[u8]>,
281 validity: Option<(i64, i64)>,
282) {
283 digest.update((family.len() as u64).to_be_bytes());
284 digest.update(family);
285 digest.update((name.len() as u64).to_be_bytes());
286 digest.update(name);
287 match (public, validity) {
288 (Some(public), Some((nbf, exp))) => {
289 digest.update([1]);
290 digest.update((public.len() as u64).to_be_bytes());
291 digest.update(public);
292 digest.update(nbf.to_be_bytes());
293 digest.update(exp.to_be_bytes());
294 }
295 _ => digest.update([0]),
296 }
297}
298
299fn component_state_digest_from_slots(
300 ed: &KeySlots,
301 pq: &ml_dsa_rotation::MlDsaKeySlots,
302 ca_key: ed25519_dalek::VerifyingKey,
303) -> String {
304 let mut digest = Sha256::new();
305 digest.update(b"hyprstream-composite-component-state-v1");
306 let ed_drain = ed.drain.as_ref().map(KeySlot::verifying_key_bytes);
307 let ed_active = ed.active.as_ref().map(KeySlot::verifying_key_bytes);
308 let ed_lead = ed.lead.as_ref().map(KeySlot::verifying_key_bytes);
309 digest_component_slot(
310 &mut digest,
311 b"ed25519",
312 b"drain",
313 ed_drain.as_ref().map(<[u8; 32]>::as_slice),
314 ed.drain.as_ref().map(|slot| (slot.nbf, slot.exp)),
315 );
316 digest_component_slot(
317 &mut digest,
318 b"ed25519",
319 b"active",
320 ed_active.as_ref().map(<[u8; 32]>::as_slice),
321 ed.active.as_ref().map(|slot| (slot.nbf, slot.exp)),
322 );
323 digest_component_slot(
324 &mut digest,
325 b"ed25519",
326 b"lead",
327 ed_lead.as_ref().map(<[u8; 32]>::as_slice),
328 ed.lead.as_ref().map(|slot| (slot.nbf, slot.exp)),
329 );
330 for (name, slot) in [
331 (b"drain".as_slice(), pq.drain.as_ref()),
332 (b"active".as_slice(), pq.active.as_ref()),
333 (b"lead".as_slice(), pq.lead.as_ref()),
334 ] {
335 let public =
336 slot.map(|slot| hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&slot.verifying_key()));
337 digest_component_slot(
338 &mut digest,
339 b"ml-dsa-65",
340 name,
341 public.as_deref(),
342 slot.map(|slot| (slot.nbf, slot.exp)),
343 );
344 }
345 digest_component_slot(
346 &mut digest,
347 b"ed25519",
348 b"policy-ca",
349 Some(&ca_key.to_bytes()),
350 Some((i64::MIN, i64::MAX)),
351 );
352 URL_SAFE_NO_PAD.encode(digest.finalize())
353}
354
355fn fingerprint_component_slot_files(
360 secrets_dir: &Path,
361 fingerprint: &mut Sha256,
362) -> anyhow::Result<()> {
363 use std::os::unix::fs::MetadataExt;
364 let state_dir = rotation_state_dir(secrets_dir);
365 for dir in [secrets_dir, state_dir.as_path()] {
366 for family in ["jwt-signing-key", "ml-dsa-signing-key"] {
367 for slot in ["drain", "active", "lead"] {
368 for suffix in ["", ".meta"] {
369 let path = dir.join(format!("{family}.{slot}{suffix}"));
370 match std::fs::metadata(&path) {
371 Ok(metadata) => {
372 fingerprint.update([1]);
373 fingerprint.update(metadata.dev().to_be_bytes());
374 fingerprint.update(metadata.ino().to_be_bytes());
375 fingerprint.update(metadata.len().to_be_bytes());
376 fingerprint.update(metadata.mtime().to_be_bytes());
377 fingerprint.update(metadata.mtime_nsec().to_be_bytes());
378 }
379 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
380 fingerprint.update([0]);
381 }
382 Err(error) => return Err(error.into()),
383 }
384 }
385 }
386 }
387 }
388 Ok(())
389}
390
391async fn component_state_digest(
392 ed_store: &SigningKeyStore,
393 pq_store: &MlDsaSigningKeyStore,
394 ca_key: ed25519_dalek::VerifyingKey,
395) -> String {
396 let ed = ed_store.0.read().await;
397 let pq = pq_store.0.read().await;
398 component_state_digest_from_slots(&ed, &pq, ca_key)
399}
400
401pub async fn restore_composite_verifying_key_set(
403 secrets_dir: &Path,
404 _ed_store: &SigningKeyStore,
405 _ml_dsa_store: &MlDsaSigningKeyStore,
406 _ca_key: ed25519_dalek::VerifyingKey,
407) -> anyhow::Result<()> {
408 use hyprstream_rpc::auth::{CompositeKeyPair, CompositePairRole, CompositePairState};
409
410 configure_composite_authority(secrets_dir);
411 #[cfg(not(test))]
412 start_composite_authority_subscription(secrets_dir.to_path_buf(), _ca_key)?;
413 use nix::fcntl::{flock, FlockArg};
414 use std::os::fd::AsRawFd;
415 let lock = std::fs::OpenOptions::new()
416 .create(true)
417 .truncate(false)
418 .read(true)
419 .write(true)
420 .open(composite_ledger_lock_path(secrets_dir))?;
421 flock(lock.as_raw_fd(), FlockArg::LockShared)?;
422 let (_, ledger) = read_committed_composite_ledger(secrets_dir)?;
423 let now = chrono::Utc::now().timestamp();
424 let mut pairs = Vec::new();
425 for record in ledger.pairs {
426 if record.expires_at <= now {
427 continue;
428 }
429 let pq_bytes = URL_SAFE_NO_PAD.decode(&record.ml_dsa_public)?;
430 let pq = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&pq_bytes)?;
431 let ed_bytes: [u8; 32] = URL_SAFE_NO_PAD
432 .decode(&record.ed25519_public)?
433 .try_into()
434 .map_err(|_| anyhow::anyhow!("invalid committed Ed25519 key length"))?;
435 let ed = ed25519_dalek::VerifyingKey::from_bytes(&ed_bytes)?;
436 pairs.push(CompositeKeyPair::verifying(
437 record.kid,
438 pq,
439 ed,
440 if record.role == "policy" {
441 CompositePairRole::Policy
442 } else {
443 CompositePairRole::OAuth
444 },
445 if record.state == "active" {
446 CompositePairState::Active
447 } else {
448 CompositePairState::Drain
449 },
450 record.not_before,
451 record.expires_at,
452 ));
453 }
454 let key_set = hyprstream_rpc::auth::global_composite_key_set();
455 if ledger.version > key_set.snapshot().version() {
456 key_set.publish(ledger.version, ledger.component_digest, pairs)?;
457 }
458 Ok(())
459}
460
461fn start_composite_authority_subscription(
462 secrets_dir: PathBuf,
463 ca_key: ed25519_dalek::VerifyingKey,
464) -> anyhow::Result<()> {
465 if COMPOSITE_SUBSCRIPTION_STARTED.swap(true, Ordering::AcqRel) {
466 return Ok(());
467 }
468 configure_composite_authority(&secrets_dir);
469 let subscribers = composite_subscribers_dir(&secrets_dir);
470 std::fs::create_dir_all(&subscribers)?;
471 let name = format!("{}-{}", std::process::id(), uuid::Uuid::new_v4());
472 super::identity_store::write_secret(&subscribers, &name, b"0")?;
473 std::thread::Builder::new()
474 .name("composite-authority".into())
475 .spawn(move || {
476 use nix::fcntl::{flock, FlockArg};
477 use std::os::fd::AsRawFd;
478 let mut last_error: Option<String> = None;
479 let mut rejected_pending: Option<([u8; 32], std::time::Instant)> = None;
480 loop {
481 if COMPOSITE_PUBLISHING.load(Ordering::Acquire) {
482 std::thread::sleep(std::time::Duration::from_millis(10));
483 continue;
484 }
485 let load = (|| -> anyhow::Result<()> {
486 let lock = std::fs::OpenOptions::new()
487 .create(true)
488 .truncate(false)
489 .read(true)
490 .write(true)
491 .open(composite_ledger_lock_path(&secrets_dir))?;
492 flock(lock.as_raw_fd(), FlockArg::LockShared)?;
493 let pending_bytes = std::fs::read(composite_ledger_path(&secrets_dir))?;
494 let marker_bytes = std::fs::read(composite_committed_path(&secrets_dir))?;
495 let mut fingerprint = Sha256::new();
496 fingerprint.update(&pending_bytes);
497 fingerprint.update(&marker_bytes);
498 fingerprint_component_slot_files(&secrets_dir, &mut fingerprint)?;
499 let fingerprint: [u8; 32] = fingerprint.finalize().into();
500 if rejected_pending.as_ref().is_some_and(|(previous, at)| {
501 *previous == fingerprint
502 && at.elapsed() < std::time::Duration::from_secs(60)
503 }) {
504 return Ok(());
508 }
509 let pending: CompositeLedger = serde_json::from_slice(&pending_bytes)?;
510 let ed = load_or_init_key_store(&secrets_dir, &OAuthConfig::default());
511 let pq = load_or_init_ml_dsa_key_store(&secrets_dir, &OAuthConfig::default());
512 let local_digest = {
516 let ed_slots = ed.0.blocking_read();
517 let pq_slots = pq.0.blocking_read();
518 component_state_digest_from_slots(&ed_slots, &pq_slots, ca_key)
519 };
520 if pending.component_digest != local_digest {
521 rejected_pending = Some((fingerprint, std::time::Instant::now()));
522 anyhow::bail!(
523 "pending composite ledger does not match local component authority"
524 );
525 }
526 rejected_pending = None;
527 let _staged = ledger_pairs_from_local_keys(&pending, &ed, &pq, ca_key, true)?;
528 super::identity_store::write_secret(
529 &subscribers,
530 &name,
531 &serde_json::to_vec(&CompositeAcknowledgement {
532 version: pending.version,
533 component_digest: pending.component_digest,
534 })?,
535 )?;
536 let (_, committed) = read_committed_composite_ledger(&secrets_dir)?;
537 let key_set = hyprstream_rpc::auth::global_composite_key_set();
538 if committed.version > key_set.snapshot().version()
539 || (committed.version == key_set.snapshot().version()
540 && committed.component_digest != key_set.snapshot().component_digest())
541 {
542 key_set.publish(
543 committed.version,
544 committed.component_digest.clone(),
545 ledger_pairs_from_local_keys(&committed, &ed, &pq, ca_key, false)?,
546 )?;
547 }
548 Ok(())
549 })();
550 match load {
551 Ok(()) => {
552 last_error = None;
553 std::thread::sleep(std::time::Duration::from_millis(25));
554 }
555 Err(error) => {
556 let message = format!("{error:#}");
557 if last_error.as_deref() != Some(&message) {
558 tracing::warn!("composite authority reload failed closed: {message}");
559 last_error = Some(message);
560 }
561 std::thread::sleep(std::time::Duration::from_secs(1));
566 }
567 }
568 }
569 })?;
570 Ok(())
571}
572
573fn ledger_pairs_from_local_keys(
574 ledger: &CompositeLedger,
575 ed_store: &SigningKeyStore,
576 pq_store: &MlDsaSigningKeyStore,
577 ca_key: ed25519_dalek::VerifyingKey,
578 require_local_pairs: bool,
579) -> anyhow::Result<Vec<hyprstream_rpc::auth::CompositeKeyPair>> {
580 use hyprstream_rpc::auth::{CompositeKeyPair, CompositePairRole, CompositePairState};
581 let ed_slots = ed_store.0.blocking_read();
582 let pq_slots = pq_store.0.blocking_read();
583 let ca_signing = composite_ca_signing_key().read().clone();
584 let now = chrono::Utc::now().timestamp();
585 ledger
586 .pairs
587 .iter()
588 .filter(|r| r.expires_at > now)
589 .map(|r| {
590 let pq_bytes = URL_SAFE_NO_PAD.decode(&r.ml_dsa_public)?;
591 let pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&pq_bytes)?;
592 let ed_bytes: [u8; 32] = URL_SAFE_NO_PAD
593 .decode(&r.ed25519_public)?
594 .try_into()
595 .map_err(|_| anyhow::anyhow!("invalid Ed25519 key length"))?;
596 let ed_vk = ed25519_dalek::VerifyingKey::from_bytes(&ed_bytes)?;
597 let role = if r.role == "policy" {
598 CompositePairRole::Policy
599 } else {
600 CompositePairRole::OAuth
601 };
602 let state = if r.state == "active" {
603 CompositePairState::Active
604 } else {
605 CompositePairState::Drain
606 };
607 let pq_signing = pq_slots
608 .all()
609 .into_iter()
610 .find(|s| {
611 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&s.verifying_key()) == pq_bytes
612 })
613 .map(|s| Arc::clone(&s.key));
614 let ed_signing = if ca_key == ed_vk {
615 ca_signing.clone()
616 } else {
617 ed_slots
618 .all()
619 .into_iter()
620 .find(|s| s.key.verifying_key() == ed_vk)
621 .map(|s| Arc::clone(&s.key))
622 };
623 if require_local_pairs {
624 anyhow::ensure!(
625 pq_signing.is_some(),
626 "composite PQ key is absent from the pending component generation"
627 );
628 anyhow::ensure!(
629 ca_key == ed_vk || ed_signing.is_some(),
630 "composite Ed25519 key is absent from the pending component generation"
631 );
632 }
633 Ok(match (pq_signing, ed_signing) {
634 (Some(pq), Some(ed)) => CompositeKeyPair::signing(
635 r.kid.clone(),
636 pq,
637 ed,
638 role,
639 state,
640 r.not_before,
641 r.expires_at,
642 ),
643 _ => CompositeKeyPair::verifying(
644 r.kid.clone(),
645 pq_vk,
646 ed_vk,
647 role,
648 state,
649 r.not_before,
650 r.expires_at,
651 ),
652 })
653 })
654 .collect()
655}
656
657pub async fn initialize_composite_key_set(
661 secrets_dir: &Path,
662 ed_store: &SigningKeyStore,
663 ml_dsa_store: &MlDsaSigningKeyStore,
664 ca_key: Arc<SigningKey>,
665 drain_secs: i64,
666) -> anyhow::Result<()> {
667 publish_composite_key_set(
668 secrets_dir,
669 ed_store,
670 ml_dsa_store,
671 ca_key,
672 drain_secs,
673 true,
674 None,
675 )
676 .await
677}
678
679pub async fn refresh_composite_key_set(
681 secrets_dir: &Path,
682 ed_store: &SigningKeyStore,
683 ml_dsa_store: &MlDsaSigningKeyStore,
684 ca_key: Arc<SigningKey>,
685 drain_secs: i64,
686 expected_component_digest: &str,
687) -> anyhow::Result<()> {
688 publish_composite_key_set(
689 secrets_dir,
690 ed_store,
691 ml_dsa_store,
692 ca_key,
693 drain_secs,
694 false,
695 Some(expected_component_digest.to_owned()),
696 )
697 .await
698}
699
700async fn publish_composite_key_set(
701 secrets_dir: &Path,
702 ed_store: &SigningKeyStore,
703 ml_dsa_store: &MlDsaSigningKeyStore,
704 ca_key: Arc<SigningKey>,
705 drain_secs: i64,
706 restore: bool,
707 expected_component_digest: Option<String>,
708) -> anyhow::Result<()> {
709 use hyprstream_rpc::auth::{CompositeKeyPair, CompositePairRole, CompositePairState};
710 use nix::fcntl::{flock, FlockArg};
711 use std::os::fd::AsRawFd;
712 struct Guard;
713 impl Drop for Guard {
714 fn drop(&mut self) {
715 COMPOSITE_PUBLISHING.store(false, Ordering::Release);
716 }
717 }
718 configure_composite_authority(secrets_dir);
719 *composite_ca_signing_key().write() = Some(Arc::clone(&ca_key));
720 let writer = std::fs::OpenOptions::new()
721 .create(true)
722 .truncate(false)
723 .read(true)
724 .write(true)
725 .open(composite_writer_lock_path(secrets_dir))?;
726 flock(writer.as_raw_fd(), FlockArg::LockExclusive)?;
727 COMPOSITE_PUBLISHING.store(true, Ordering::Release);
728 let _guard = Guard;
729
730 let now = chrono::Utc::now().timestamp();
731 let ed_component_slots = ed_store.0.read().await.clone();
732 let pq_component_slots = ml_dsa_store.0.read().await.clone();
733 let ed_slots: Vec<_> = ed_component_slots.all().into_iter().cloned().collect();
734 let pq_slots: Vec<_> = pq_component_slots.all().into_iter().cloned().collect();
735 let proposed_component_digest = component_state_digest_from_slots(
736 &ed_component_slots,
737 &pq_component_slots,
738 ca_key.verifying_key(),
739 );
740 let key_set = hyprstream_rpc::auth::global_composite_key_set();
741 let current = key_set.snapshot();
742 let ledger_lock = std::fs::OpenOptions::new()
743 .create(true)
744 .truncate(false)
745 .read(true)
746 .write(true)
747 .open(composite_ledger_lock_path(secrets_dir))?;
748 flock(ledger_lock.as_raw_fd(), FlockArg::LockExclusive)?;
749 let persisted = load_or_migrate_committed_composite_ledger(secrets_dir)?;
750 let restoring_committed = restore && persisted.is_some();
751 let persisted_version = persisted.as_ref().map_or(0, |ledger| ledger.version);
752 let persisted_digest = persisted
753 .as_ref()
754 .map(|ledger| ledger.component_digest.clone());
755 if persisted.is_none() {
756 let ed_active = ed_component_slots.active.is_some();
766 let pq_active = pq_component_slots.active.is_some();
767 anyhow::ensure!(
768 ed_active && pq_active,
769 "refusing to initialize composite authority from an incomplete \
770 local signer set (ed25519_active={} ml_dsa_active={}): resolve \
771 first-boot persistence and retry",
772 ed_active,
773 pq_active
774 );
775 }
776 match (&persisted, expected_component_digest.as_deref()) {
777 (Some(ledger), Some(expected)) => anyhow::ensure!(
778 !ledger.component_digest.is_empty() && ledger.component_digest == expected,
779 "stale composite component authority: expected {expected}, authoritative digest is {}",
780 ledger.component_digest
781 ),
782 (Some(ledger), None) => anyhow::ensure!(
783 restore && !ledger.component_digest.is_empty(),
784 "persisted composite authority is not restorable"
785 ),
786 (None, Some(_)) => {
787 anyhow::bail!("stale composite component authority: expected generation has no ledger")
788 }
789 (None, None) => anyhow::ensure!(restore, "composite authority is not initialized"),
790 }
791 let mut pairs = Vec::new();
792
793 for record in persisted.into_iter().flat_map(|ledger| ledger.pairs) {
794 if record.expires_at <= now {
795 continue;
796 }
797 let pq_bytes = URL_SAFE_NO_PAD.decode(&record.ml_dsa_public)?;
798 let pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&pq_bytes)?;
799 let pq_signing = pq_slots.iter().find(|slot| {
800 URL_SAFE_NO_PAD.encode(hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
801 &slot.verifying_key(),
802 )) == record.ml_dsa_public
803 });
804 let ed_bytes: [u8; 32] = URL_SAFE_NO_PAD
805 .decode(&record.ed25519_public)?
806 .try_into()
807 .map_err(|_| anyhow::anyhow!("invalid persisted Ed25519 key length"))?;
808 let ed_vk = ed25519_dalek::VerifyingKey::from_bytes(&ed_bytes)?;
809 let policy_ca_ed_matched =
810 URL_SAFE_NO_PAD.encode(ca_key.verifying_key().to_bytes()) == record.ed25519_public;
811 let ed_signing = if policy_ca_ed_matched {
812 Some(Arc::clone(&ca_key))
813 } else {
814 ed_slots
815 .iter()
816 .find(|slot| {
817 URL_SAFE_NO_PAD.encode(slot.key.verifying_key().to_bytes())
818 == record.ed25519_public
819 })
820 .map(|slot| Arc::clone(&slot.key))
821 };
822 let role = if record.role == "policy" {
823 CompositePairRole::Policy
824 } else {
825 CompositePairRole::OAuth
826 };
827 let state = if restoring_committed && record.state == "active" {
828 CompositePairState::Active
829 } else {
830 CompositePairState::Drain
831 };
832 if !restoring_committed && (pq_signing.is_none() || ed_signing.is_none()) {
836 continue;
837 }
838 if restoring_committed && state == CompositePairState::Active {
839 anyhow::ensure!(
840 pq_signing.is_some() && ed_signing.is_some(),
841 "committed active composite signing pair is unavailable after restart: \
842 kid={:?} role={:?} ed_component_available={} pq_component_available={} \
843 policy_ca_ed_matched={}",
844 record.kid,
845 record.role,
846 ed_signing.is_some(),
847 pq_signing.is_some(),
848 policy_ca_ed_matched,
849 );
850 }
851 pairs.push(match (pq_signing, ed_signing) {
852 (Some(pq), Some(ed)) => CompositeKeyPair::signing(
853 record.kid,
854 Arc::clone(&pq.key),
855 ed,
856 role,
857 state,
858 record.not_before,
859 record.expires_at,
860 ),
861 _ => CompositeKeyPair::verifying(
862 record.kid,
863 pq_vk,
864 ed_vk,
865 role,
866 state,
867 record.not_before,
868 record.expires_at,
869 ),
870 });
871 }
872
873 if restoring_committed {
874 anyhow::ensure!(
875 [CompositePairRole::OAuth, CompositePairRole::Policy]
876 .into_iter()
877 .all(|role| pairs.iter().any(|pair| {
878 pair.role() == role
879 && pair.state() == CompositePairState::Active
880 && pair.signing_keys().is_some()
881 })),
882 "committed active composite signing authority is incomplete"
883 );
884 let digest = persisted_digest.ok_or_else(|| {
885 anyhow::anyhow!("committed composite digest disappeared during restore")
886 })?;
887 if persisted_version > current.version() {
888 key_set.publish(persisted_version, digest, pairs)?;
889 } else {
890 drop(ledger_lock);
896 key_set.attach_committed_signing_keys(persisted_version, &digest, pairs)?;
897 }
898 #[cfg(not(test))]
899 start_composite_authority_subscription(secrets_dir.to_path_buf(), ca_key.verifying_key())?;
900 return Ok(());
901 }
902
903 let active_pq_key = ml_dsa_store.active_key().await;
904 let active_pq = active_pq_key.as_ref().and_then(|active| {
905 let active_vk = ml_dsa::Keypair::verifying_key(&**active);
906 pq_slots
907 .iter()
908 .find(|slot| {
909 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&active_vk)
910 == hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&slot.verifying_key())
911 })
912 .cloned()
913 });
914 let active_ed = ed_store.active_key().await;
915 if let Some(pq) = active_pq {
916 if let Some(ed) = active_ed {
917 upsert_active_pair(
918 &mut pairs,
919 Arc::clone(&pq.key),
920 ed,
921 CompositePairRole::OAuth,
922 pq.nbf,
923 pq.exp + drain_secs,
924 );
925 }
926 upsert_active_pair(
927 &mut pairs,
928 pq.key,
929 Arc::clone(&ca_key),
930 CompositePairRole::Policy,
931 pq.nbf,
932 pq.exp + drain_secs,
933 );
934 }
935
936 let version = current.version().max(persisted_version).saturating_add(1);
937 let ledger = CompositeLedger {
938 version,
939 component_digest: proposed_component_digest.clone(),
940 pairs: pairs.iter().map(ledger_record).collect(),
941 };
942 super::identity_store::write_secret(
943 secrets_dir,
944 "jwt-composite-pairs.json",
945 &serde_json::to_vec(&ledger)?,
946 )?;
947 #[cfg(test)]
948 if let Some(marker) = std::env::var_os("HYPRSTREAM_COMPOSITE_STAGE_MARKER") {
949 super::identity_store::write_secret(
950 Path::new(&marker).parent().ok_or_else(|| {
951 anyhow::anyhow!("composite stage marker must have a parent directory")
952 })?,
953 Path::new(&marker)
954 .file_name()
955 .and_then(|name| name.to_str())
956 .ok_or_else(|| anyhow::anyhow!("invalid composite stage marker"))?,
957 b"1",
958 )?;
959 }
960 #[cfg(test)]
961 if std::env::var_os("HYPRSTREAM_COMPOSITE_CRASH_AFTER_STAGE").is_some() {
962 std::process::exit(86);
963 }
964 drop(ledger_lock);
965 wait_for_composite_acknowledgements(secrets_dir, version, &proposed_component_digest)?;
966 let ledger_lock = std::fs::OpenOptions::new()
967 .create(true)
968 .truncate(false)
969 .read(true)
970 .write(true)
971 .open(composite_ledger_lock_path(secrets_dir))?;
972 flock(ledger_lock.as_raw_fd(), FlockArg::LockExclusive)?;
973 let pending: CompositeLedger =
974 serde_json::from_slice(&std::fs::read(composite_ledger_path(secrets_dir))?)?;
975 anyhow::ensure!(
976 pending.version == version && pending.component_digest == proposed_component_digest,
977 "composite publication changed before commit"
978 );
979 persist_committed_composite_ledger(secrets_dir, &pending)?;
980 if version > key_set.snapshot().version() {
981 key_set.publish(version, proposed_component_digest, pairs)?;
982 }
983 #[cfg(not(test))]
984 start_composite_authority_subscription(secrets_dir.to_path_buf(), ca_key.verifying_key())?;
985 Ok(())
986}
987
988fn persist_committed_composite_ledger(dir: &Path, ledger: &CompositeLedger) -> anyhow::Result<()> {
989 let commit = CompositeCommit {
990 version: ledger.version,
991 component_digest: ledger.component_digest.clone(),
992 };
993 let committed_name = composite_committed_ledger_path(dir, &commit)
997 .file_name()
998 .and_then(|name| name.to_str())
999 .ok_or_else(|| anyhow::anyhow!("invalid committed composite ledger path"))?
1000 .to_owned();
1001 super::identity_store::write_secret_durable(
1005 dir,
1006 &committed_name,
1007 &serde_json::to_vec(ledger)?,
1008 )?;
1009 super::identity_store::write_secret_durable(
1010 dir,
1011 "jwt-composite-pairs.committed",
1012 &serde_json::to_vec(&commit)?,
1013 )?;
1014 Ok(())
1015}
1016
1017fn wait_for_composite_acknowledgements(
1018 dir: &Path,
1019 version: u64,
1020 component_digest: &str,
1021) -> anyhow::Result<()> {
1022 let subscribers = composite_subscribers_dir(dir);
1023 std::fs::create_dir_all(&subscribers)?;
1024 let own = format!("{}-", std::process::id());
1025 let required: Vec<PathBuf> = std::fs::read_dir(&subscribers)?
1026 .filter_map(Result::ok)
1027 .map(|e| e.path())
1028 .filter(|path| {
1029 let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
1030 return false;
1031 };
1032 if name.starts_with(&own) {
1033 return false;
1034 }
1035 let pid = name
1036 .split_once('-')
1037 .and_then(|(p, _)| p.parse::<u32>().ok());
1038 if pid.is_some_and(|p| !composite_subscriber_alive(p)) {
1039 let _ = std::fs::remove_file(path);
1040 return false;
1041 }
1042 true
1043 })
1044 .collect();
1045 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
1046 loop {
1047 if required.iter().all(|p| {
1048 std::fs::read(p)
1049 .ok()
1050 .and_then(|bytes| serde_json::from_slice::<CompositeAcknowledgement>(&bytes).ok())
1051 .is_some_and(|ack| {
1052 ack.version == version && ack.component_digest == component_digest
1053 })
1054 }) {
1055 return Ok(());
1056 }
1057 anyhow::ensure!(
1058 std::time::Instant::now() < deadline,
1059 "composite generation {version} was not acknowledged by all live service processes"
1060 );
1061 std::thread::sleep(std::time::Duration::from_millis(10));
1062 }
1063}
1064
1065fn composite_subscriber_alive(pid: u32) -> bool {
1066 use nix::errno::Errno;
1067 use nix::sys::signal::kill;
1068 use nix::unistd::Pid;
1069 !matches!(kill(Pid::from_raw(pid as i32), None), Err(Errno::ESRCH))
1070}
1071
1072fn upsert_active_pair(
1073 pairs: &mut Vec<hyprstream_rpc::auth::CompositeKeyPair>,
1074 ml_dsa: Arc<hyprstream_rpc::crypto::pq::MlDsaSigningKey>,
1075 ed25519: Arc<SigningKey>,
1076 role: hyprstream_rpc::auth::CompositePairRole,
1077 not_before: i64,
1078 expires_at: i64,
1079) {
1080 let ml_vk = ml_dsa::Keypair::verifying_key(&*ml_dsa).clone();
1081 let kid = crate::auth::jwt::composite_kid(&ml_vk, &ed25519.verifying_key());
1082 pairs.retain(|pair| {
1083 pair.kid() != kid
1084 && !(pair.role() == role
1085 && pair.state() == hyprstream_rpc::auth::CompositePairState::Active)
1086 });
1087 pairs.push(hyprstream_rpc::auth::CompositeKeyPair::signing(
1088 kid,
1089 ml_dsa,
1090 ed25519,
1091 role,
1092 hyprstream_rpc::auth::CompositePairState::Active,
1093 not_before,
1094 expires_at,
1095 ));
1096}
1097
1098fn ledger_record(pair: &hyprstream_rpc::auth::CompositeKeyPair) -> CompositeLedgerPair {
1099 CompositeLedgerPair {
1100 kid: pair.kid().to_owned(),
1101 ml_dsa_public: URL_SAFE_NO_PAD
1102 .encode(hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(pair.ml_dsa())),
1103 ed25519_public: URL_SAFE_NO_PAD.encode(pair.ed25519().to_bytes()),
1104 role: if pair.role() == hyprstream_rpc::auth::CompositePairRole::Policy {
1105 "policy"
1106 } else {
1107 "oauth"
1108 }
1109 .to_owned(),
1110 state: if pair.state() == hyprstream_rpc::auth::CompositePairState::Active {
1111 "active"
1112 } else {
1113 "drain"
1114 }
1115 .to_owned(),
1116 not_before: pair.not_before(),
1117 expires_at: pair.expires_at(),
1118 }
1119}
1120
1121static ML_DSA_SIGNING_STORE: std::sync::OnceLock<Arc<MlDsaSigningKeyStore>> =
1126 std::sync::OnceLock::new();
1127static ED25519_SIGNING_STORE: std::sync::OnceLock<Arc<SigningKeyStore>> =
1128 std::sync::OnceLock::new();
1129
1130pub fn global_ed25519_key_store(secrets_dir: &Path, config: &OAuthConfig) -> Arc<SigningKeyStore> {
1132 ED25519_SIGNING_STORE
1133 .get_or_init(|| Arc::new(load_or_init_key_store(secrets_dir, config)))
1134 .clone()
1135}
1136
1137pub fn global_ml_dsa_key_store(
1141 secrets_dir: &Path,
1142 config: &OAuthConfig,
1143) -> Arc<MlDsaSigningKeyStore> {
1144 ML_DSA_SIGNING_STORE
1145 .get_or_init(|| Arc::new(load_or_init_ml_dsa_key_store(secrets_dir, config)))
1146 .clone()
1147}
1148
1149static ES256_SIGNING_STORE: std::sync::OnceLock<Arc<Es256SigningKeyStore>> =
1151 std::sync::OnceLock::new();
1152
1153pub fn global_es256_key_store(
1155 secrets_dir: &Path,
1156 config: &OAuthConfig,
1157) -> Arc<Es256SigningKeyStore> {
1158 let store = ES256_SIGNING_STORE
1159 .get_or_init(|| Arc::new(load_or_init_es256_key_store(secrets_dir, config)))
1160 .clone();
1161 store.bind_secrets_dir(rotation_state_dir(secrets_dir));
1162 store
1163}
1164
1165#[derive(Clone)]
1168pub struct KeySlot {
1169 pub key: Arc<SigningKey>,
1170 pub nbf: i64,
1172 pub exp: i64,
1174}
1175
1176impl KeySlot {
1177 pub fn new(key: SigningKey, nbf: i64, exp: i64) -> Self {
1178 Self {
1179 key: Arc::new(key),
1180 nbf,
1181 exp,
1182 }
1183 }
1184
1185 pub fn verifying_key_bytes(&self) -> [u8; 32] {
1186 self.key.verifying_key().to_bytes()
1187 }
1188
1189 pub fn kid(&self) -> String {
1190 crate::services::oauth::jwks::compute_kid(&self.verifying_key_bytes())
1191 }
1192}
1193
1194#[derive(Clone, Default)]
1197pub struct KeySlots {
1198 pub drain: Option<KeySlot>,
1199 pub active: Option<KeySlot>,
1200 pub lead: Option<KeySlot>,
1201}
1202
1203impl KeySlots {
1204 pub fn all(&self) -> Vec<&KeySlot> {
1206 [&self.drain, &self.active, &self.lead]
1207 .into_iter()
1208 .flatten()
1209 .collect()
1210 }
1211}
1212
1213#[derive(Clone)]
1216pub struct SigningKeyStore(pub Arc<RwLock<KeySlots>>);
1217
1218impl SigningKeyStore {
1219 pub fn new(slots: KeySlots) -> Self {
1220 Self(Arc::new(RwLock::new(slots)))
1221 }
1222
1223 pub async fn active_key(&self) -> Option<Arc<SigningKey>> {
1224 let slots = self.0.read().await;
1225 slots.active.as_ref().map(|s| Arc::clone(&s.key))
1226 }
1227
1228 pub async fn active_verifying_key_bytes(&self) -> Option<[u8; 32]> {
1229 let slots = self.0.read().await;
1230 slots.active.as_ref().map(KeySlot::verifying_key_bytes)
1231 }
1232
1233 pub async fn all_slots_snapshot(&self) -> Vec<KeySlot> {
1234 let slots = self.0.read().await;
1235 slots.all().into_iter().cloned().collect()
1236 }
1237
1238 pub async fn slots_snapshot(&self) -> KeySlots {
1240 self.0.read().await.clone()
1241 }
1242
1243 pub async fn verifying_key_for_kid(&self, kid: &str) -> Option<ed25519_dalek::VerifyingKey> {
1245 let slots = self.0.read().await;
1246 for slot in slots.all() {
1247 if slot.kid() == kid {
1248 return Some(slot.key.verifying_key());
1249 }
1250 }
1251 None
1252 }
1253}
1254
1255pub fn rotation_state_dir(secrets_dir: &Path) -> PathBuf {
1268 if super::identity_store::is_writable(secrets_dir) {
1269 return secrets_dir.to_path_buf();
1270 }
1271 let systemd_state = std::env::var("STATE_DIRECTORY").ok().and_then(|v| {
1275 let first = v.split(':').next().unwrap_or_default();
1276 if first.is_empty() {
1277 None
1278 } else {
1279 Some(PathBuf::from(first))
1280 }
1281 });
1282 let base = match systemd_state {
1283 Some(dir) => match std::env::var("HYPRSTREAM_INSTANCE") {
1285 Ok(inst) if !inst.is_empty() => Some(dir.join("instances").join(inst)),
1286 _ => Some(dir),
1287 },
1288 None => crate::storage::paths::StoragePaths::new()
1291 .and_then(|s| s.state_dir())
1292 .ok(),
1293 };
1294 match base {
1295 Some(dir) => dir.join("credentials"),
1296 None => {
1297 error!(
1298 "secrets dir '{}' is read-only and no writable state directory \
1299 could be resolved; JWT rotation state will NOT survive restart \
1300 (all issued tokens are invalidated on every restart)",
1301 secrets_dir.display()
1302 );
1303 secrets_dir.to_path_buf()
1304 }
1305 }
1306}
1307
1308#[derive(Serialize, Deserialize)]
1309struct SlotMeta {
1310 nbf: i64,
1311 exp: i64,
1312}
1313
1314fn slot_paths(secrets_dir: &Path, name: &str) -> (PathBuf, PathBuf) {
1315 (
1316 secrets_dir.join(format!("jwt-signing-key.{name}")),
1317 secrets_dir.join(format!("jwt-signing-key.{name}.meta")),
1318 )
1319}
1320
1321fn load_slot(secrets_dir: &Path, name: &str) -> Option<KeySlot> {
1322 let (key_path, meta_path) = slot_paths(secrets_dir, name);
1323 let seed = std::fs::read(&key_path).ok()?;
1324 if seed.len() != 32 {
1325 warn!(
1326 "JWT key slot '{name}': unexpected seed length {}",
1327 seed.len()
1328 );
1329 return None;
1330 }
1331 let meta_bytes = std::fs::read(&meta_path).ok()?;
1332 let meta: SlotMeta = serde_json::from_slice(&meta_bytes).ok()?;
1333 let mut seed_arr = [0u8; 32];
1334 seed_arr.copy_from_slice(&seed);
1335 Some(KeySlot::new(
1336 SigningKey::from_bytes(&seed_arr),
1337 meta.nbf,
1338 meta.exp,
1339 ))
1340}
1341
1342fn persist_slot(secrets_dir: &Path, name: &str, slot: &KeySlot) -> anyhow::Result<()> {
1343 super::identity_store::write_secret(
1346 secrets_dir,
1347 &format!("jwt-signing-key.{name}"),
1348 &slot.key.to_bytes(),
1349 )?;
1350 let meta = SlotMeta {
1351 nbf: slot.nbf,
1352 exp: slot.exp,
1353 };
1354 super::identity_store::write_secret(
1355 secrets_dir,
1356 &format!("jwt-signing-key.{name}.meta"),
1357 &serde_json::to_vec(&meta)?,
1358 )?;
1359 Ok(())
1360}
1361
1362fn delete_slot(secrets_dir: &Path, name: &str) {
1363 let (key_path, meta_path) = slot_paths(secrets_dir, name);
1364 let _ = std::fs::remove_file(&key_path);
1365 let _ = std::fs::remove_file(&meta_path);
1366}
1367
1368fn generate_slot(nbf: i64, exp: i64) -> KeySlot {
1369 let key = SigningKey::generate(&mut rand::rngs::OsRng);
1370 KeySlot::new(key, nbf, exp)
1371}
1372
1373fn lock_signing_slots(state_dir: &Path, nonblocking: bool) -> std::io::Result<std::fs::File> {
1384 use nix::fcntl::{flock, FlockArg};
1385 use std::os::fd::AsRawFd;
1386 use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
1387
1388 std::fs::DirBuilder::new()
1389 .recursive(true)
1390 .mode(0o700)
1391 .create(state_dir)?;
1392 let file = std::fs::OpenOptions::new()
1393 .create(true)
1394 .truncate(false)
1395 .read(true)
1396 .write(true)
1397 .mode(0o600)
1398 .open(state_dir.join("signing-slots.lock"))?;
1399 flock(
1400 file.as_raw_fd(),
1401 if nonblocking {
1402 FlockArg::LockExclusiveNonblock
1403 } else {
1404 FlockArg::LockExclusive
1405 },
1406 )?;
1407 Ok(file)
1408}
1409
1410pub fn load_or_init_key_store(secrets_dir: &Path, config: &OAuthConfig) -> SigningKeyStore {
1417 let now = chrono::Utc::now().timestamp();
1418 let active_secs = config.active_secs();
1419 let lead_secs = config.lead_secs();
1420
1421 let state_dir = rotation_state_dir(secrets_dir);
1424 let _slot_lock = match lock_signing_slots(&state_dir, false) {
1425 Ok(lock) => lock,
1426 Err(error) => {
1427 error!("Could not lock JWT signing slots: {error}; refusing initialization");
1428 return SigningKeyStore::new(KeySlots::default());
1429 }
1430 };
1431 let has_committed_authority = has_committed_composite_marker(secrets_dir);
1432 let drain = load_slot(&state_dir, "drain").or_else(|| load_slot(secrets_dir, "drain"));
1433 let mut active = load_slot(&state_dir, "active").or_else(|| load_slot(secrets_dir, "active"));
1434 let lead = load_slot(&state_dir, "lead").or_else(|| load_slot(secrets_dir, "lead"));
1435
1436 if active.is_none() {
1437 if has_committed_authority {
1438 error!(
1439 "Committed composite authority exists but no active JWT signing slot is \
1440 available; refusing implicit key generation"
1441 );
1442 } else {
1443 info!("No active JWT signing key found — generating on first boot");
1444 let slot = generate_slot(now, now + active_secs);
1445 match persist_slot(&state_dir, "active", &slot) {
1450 Ok(()) => {
1451 info!("Active JWT key generated (kid={})", slot.kid());
1452 active = load_slot(&state_dir, "active");
1453 }
1454 Err(e) => error!(
1455 "Could not persist active JWT key to '{}': {e}. Refusing to \
1456 activate the unpersisted key; no active signer is available \
1457 until persistence succeeds on a restart.",
1458 state_dir.display()
1459 ),
1460 }
1461 }
1462 }
1463
1464 let should_gen_lead = !has_committed_authority
1466 && lead.is_none()
1467 && active.as_ref().is_some_and(|a| a.exp - now < lead_secs);
1468 if should_gen_lead {
1469 let lead_nbf = active.as_ref().map(|a| a.exp).unwrap_or(now) - lead_secs;
1470 let lead_exp = lead_nbf + active_secs;
1471 let slot = generate_slot(lead_nbf, lead_exp);
1472 if let Err(e) = persist_slot(&state_dir, "lead", &slot) {
1473 error!(
1474 "Could not persist lead JWT key to '{}': {e}. Rotation state will \
1475 not survive restart.",
1476 state_dir.display()
1477 );
1478 } else {
1479 info!("Lead JWT key pre-generated at startup (kid={})", slot.kid());
1480 }
1481 }
1482
1483 let lead = if should_gen_lead {
1485 load_slot(&state_dir, "lead")
1486 } else {
1487 lead
1488 };
1489
1490 SigningKeyStore::new(KeySlots {
1491 drain,
1492 active,
1493 lead,
1494 })
1495}
1496
1497pub fn load_or_init_root_identity_key_store(
1504 secrets_dir: &Path,
1505 config: &OAuthConfig,
1506 initial_key: SigningKey,
1507) -> SigningKeyStore {
1508 let now = chrono::Utc::now().timestamp();
1509 let active_secs = config.active_secs();
1510 let lead_secs = config.lead_secs();
1511
1512 let drain = load_slot(secrets_dir, "drain");
1513 let mut active = load_slot(secrets_dir, "active");
1514 let lead = load_slot(secrets_dir, "lead");
1515
1516 if active.is_none() {
1522 let slot = KeySlot::new(initial_key, now, now + active_secs);
1523 if let Err(error) = persist_slot(secrets_dir, "active", &slot) {
1524 warn!("Could not persist root-DID active identity key: {error}");
1525 } else {
1526 info!("Seeded root-DID active identity key (kid={})", slot.kid());
1527 }
1528 active = Some(slot);
1529 }
1530
1531 let should_generate_lead = lead.is_none()
1532 && active
1533 .as_ref()
1534 .is_some_and(|slot| slot.exp - now < lead_secs);
1535 if should_generate_lead {
1536 let lead_nbf = active.as_ref().map_or(now, |slot| slot.exp - lead_secs);
1537 let slot = generate_slot(lead_nbf, lead_nbf + active_secs);
1538 if let Err(error) = persist_slot(secrets_dir, "lead", &slot) {
1539 warn!("Could not persist root-DID lead identity key: {error}");
1540 }
1541 }
1542 let lead = if should_generate_lead {
1543 load_slot(secrets_dir, "lead")
1544 } else {
1545 lead
1546 };
1547
1548 SigningKeyStore::new(KeySlots {
1549 drain,
1550 active,
1551 lead,
1552 })
1553}
1554
1555pub async fn rotate_jwt_keys(
1563 config: &OAuthConfig,
1564 secrets_dir: &Path,
1565 store: &SigningKeyStore,
1566 now: i64,
1567) -> bool {
1568 let state_dir = rotation_state_dir(secrets_dir);
1569 let _slot_lock = match lock_signing_slots(&state_dir, true) {
1570 Ok(lock) => lock,
1571 Err(error) => {
1572 warn!("Could not lock JWT signing slots; deferring rotation: {error}");
1573 return false;
1574 }
1575 };
1576 rotate_jwt_keys_in_state_dir(config, &state_dir, store, now).await
1577}
1578
1579async fn rotate_jwt_keys_in_state_dir(
1583 config: &OAuthConfig,
1584 state_dir: &Path,
1585 store: &SigningKeyStore,
1586 now: i64,
1587) -> bool {
1588 let mut slots = store.0.write().await;
1589
1590 let active_secs = config.active_secs();
1591 let lead_secs = config.lead_secs();
1592 let drain_secs = config.drain_secs();
1593
1594 if let Some(new_lead) = slots.lead.as_ref().filter(|l| l.nbf <= now).cloned() {
1605 let old_active = slots.active.clone();
1606 let old_drain = slots.drain.clone();
1607
1608 info!("Promoting lead JWT key (kid={}) to active", new_lead.kid());
1609
1610 if let Some(ref old) = old_active {
1615 if let Err(e) = persist_slot(state_dir, "drain", old) {
1616 let drain_restored = if let Some(ref prev) = old_drain {
1622 persist_slot(state_dir, "drain", prev)
1623 } else {
1624 delete_slot(state_dir, "drain");
1625 let (drain_key, drain_meta) = slot_paths(state_dir, "drain");
1629 let removed = [&drain_key, &drain_meta].iter().all(|path| {
1630 matches!(
1631 std::fs::symlink_metadata(path),
1632 Err(error) if error.kind() == std::io::ErrorKind::NotFound
1633 )
1634 });
1635 if !removed {
1636 Err(anyhow::anyhow!(
1637 "partial replacement drain slot still present after removal"
1638 ))
1639 } else {
1640 Ok(())
1641 }
1642 };
1643 if let Err(drain_error) = drain_restored {
1644 error!(
1645 "Could not restore previous drain JWT key to '{}': \
1646 {drain_error}. The durable drain slot is not fully \
1647 restored; retaining the old active in memory and \
1648 retrying next tick.",
1649 state_dir.display()
1650 );
1651 return false;
1652 }
1653 error!(
1654 "Could not persist drain JWT key to '{}': {e}. Retaining the \
1655 old active in memory; promotion will be retried next tick.",
1656 state_dir.display()
1657 );
1658 return true;
1659 }
1660 }
1661
1662 if let Err(e) = persist_slot(state_dir, "active", &new_lead) {
1663 let restored = match old_active {
1669 Some(ref old) => persist_slot(state_dir, "active", old),
1670 None => {
1671 delete_slot(state_dir, "active");
1672 Ok(())
1673 }
1674 };
1675 if let Err(restore_error) = restored {
1676 error!(
1677 "Could not restore active JWT key to '{}': {restore_error}. The \
1678 failed promotion left the old active signer only in its drain \
1679 slot; that recoverable copy is kept untouched and promotion \
1680 will be retried next tick.",
1681 state_dir.display()
1682 );
1683 return false;
1684 }
1685 if old_active.is_some() {
1686 if let Some(ref prev) = old_drain {
1687 if let Err(drain_error) = persist_slot(state_dir, "drain", prev) {
1688 error!(
1689 "Could not restore previous drain JWT key to '{}': \
1690 {drain_error}. The old active signer is restored in the \
1691 active slot, but the durable drain slot may hold a \
1692 partial write; promotion will be retried next tick.",
1693 state_dir.display()
1694 );
1695 return false;
1696 }
1697 } else {
1698 delete_slot(state_dir, "drain");
1699 }
1700 }
1701 error!(
1702 "Could not persist active JWT key to '{}': {e}. Retaining the old \
1703 active in memory; promotion will be retried next tick.",
1704 state_dir.display()
1705 );
1706 return true;
1707 }
1708
1709 if let Some(ref prev) = old_drain {
1712 info!(
1713 "Evicting previous drain JWT key (kid={}) during promotion",
1714 prev.kid()
1715 );
1716 }
1717 delete_slot(state_dir, "lead");
1718 if let Some(old) = old_active {
1719 slots.drain = Some(old);
1720 }
1721 slots.active = Some(new_lead);
1722 slots.lead = None;
1723 }
1724
1725 if slots
1727 .drain
1728 .as_ref()
1729 .is_some_and(|d| now >= d.exp + drain_secs)
1730 {
1731 let kid = slots.drain.as_ref().map(KeySlot::kid).unwrap_or_default();
1732 info!("Removing expired drain JWT key (kid={kid})");
1733 delete_slot(state_dir, "drain");
1734 slots.drain = None;
1735 }
1736
1737 if slots.lead.is_none() {
1739 if let Some(active) = &slots.active {
1740 if active.exp - now < lead_secs {
1741 let lead_nbf = active.exp - lead_secs;
1742 let lead_exp = lead_nbf + active_secs;
1743 let new_lead = generate_slot(lead_nbf, lead_exp);
1744 info!(
1745 "Generated new lead JWT key (kid={}, nbf={})",
1746 new_lead.kid(),
1747 new_lead.nbf
1748 );
1749 if let Err(e) = persist_slot(state_dir, "lead", &new_lead) {
1750 error!(
1751 "Could not persist lead JWT key to '{}': {e}. Rotation state \
1752 will not survive restart.",
1753 state_dir.display()
1754 );
1755 }
1756 slots.lead = Some(new_lead);
1757 }
1758 }
1759 }
1760 true
1761}
1762
1763pub fn spawn_root_identity_rotation_task(
1769 config: Arc<OAuthConfig>,
1770 secrets_dir: PathBuf,
1771 store: Arc<SigningKeyStore>,
1772) {
1773 tokio::task::spawn_local(async move {
1774 let mut interval = tokio::time::interval(
1775 config
1776 .rotation_check_interval()
1777 .min(std::time::Duration::from_secs(60)),
1778 );
1779 interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
1780 interval.tick().await;
1781 loop {
1782 interval.tick().await;
1783 rotate_jwt_keys(
1784 &config,
1785 &secrets_dir,
1786 &store,
1787 chrono::Utc::now().timestamp(),
1788 )
1789 .await;
1790 }
1791 });
1792}
1793
1794pub struct RotationStores {
1798 pub es256: Option<Arc<Es256SigningKeyStore>>,
1799 pub ml_dsa: Option<Arc<MlDsaSigningKeyStore>>,
1800 pub composite_ca_key: Arc<SigningKey>,
1801}
1802
1803#[derive(Clone, Serialize, Deserialize)]
1814struct DeferredPredecessorRecord {
1815 committed_digest: String,
1816 states: Vec<String>,
1817}
1818
1819static DEFERRED_PREDECESSOR_FALLBACK: parking_lot::RwLock<Option<DeferredPredecessorRecord>> =
1820 parking_lot::RwLock::new(None);
1821
1822fn deferred_predecessor_path(secrets_dir: &Path) -> PathBuf {
1823 rotation_state_dir(secrets_dir).join("jwt-composite-deferred-predecessor")
1824}
1825
1826fn cycle_lock_path(secrets_dir: &Path) -> PathBuf {
1833 rotation_state_dir(secrets_dir).join("jwt-composite-rotation-cycle.lock")
1834}
1835
1836fn read_deferred_predecessor(secrets_dir: &Path) -> Option<DeferredPredecessorRecord> {
1841 if let Some(record) = DEFERRED_PREDECESSOR_FALLBACK.read().clone() {
1842 return Some(record);
1843 }
1844 let value = std::fs::read_to_string(deferred_predecessor_path(secrets_dir)).ok()?;
1845 serde_json::from_str::<DeferredPredecessorRecord>(value.trim()).ok()
1846}
1847
1848fn write_deferred_predecessor(secrets_dir: &Path, record: &DeferredPredecessorRecord) -> bool {
1853 let state_dir = rotation_state_dir(secrets_dir);
1854 match serde_json::to_vec(record) {
1855 Ok(bytes) => super::identity_store::write_secret(
1856 &state_dir,
1857 "jwt-composite-deferred-predecessor",
1858 &bytes,
1859 )
1860 .is_ok(),
1861 Err(error) => {
1862 warn!("could not serialize the deferred predecessor record: {error}");
1863 false
1864 }
1865 }
1866}
1867
1868fn clear_deferred_predecessor(secrets_dir: &Path) {
1869 *DEFERRED_PREDECESSOR_FALLBACK.write() = None;
1870 let _ = std::fs::remove_file(deferred_predecessor_path(secrets_dir));
1871}
1872
1873async fn run_rotation_cycle(
1904 config: &OAuthConfig,
1905 secrets_dir: &Path,
1906 store: &SigningKeyStore,
1907 extra: &RotationStores,
1908 now: i64,
1909) -> bool {
1910 let cycle_lock = std::fs::OpenOptions::new()
1918 .create(true)
1919 .truncate(false)
1920 .read(true)
1921 .write(true)
1922 .open(cycle_lock_path(secrets_dir))
1923 .map_err(|error| {
1924 warn!("could not open the rotation cycle lock: {error}");
1925 });
1926 let cycle_lock = match cycle_lock {
1927 Ok(file) => file,
1928 Err(()) => return true,
1929 };
1930 use nix::fcntl::{flock, FlockArg};
1931 use std::os::fd::AsRawFd as _;
1932 if let Err(error) = flock(cycle_lock.as_raw_fd(), FlockArg::LockExclusiveNonblock) {
1933 warn!(
1934 "another rotation cycle holds the cycle lock; skipping this tick \
1935 ({error})"
1936 );
1937 return true;
1938 }
1939
1940 let pre_rotation_digest = if let Some(ref ml_dsa) = extra.ml_dsa {
1941 Some(component_state_digest(store, ml_dsa, extra.composite_ca_key.verifying_key()).await)
1942 } else {
1943 None
1944 };
1945 let mut all_restored = rotate_jwt_keys(config, secrets_dir, store, now).await;
1946 if !all_restored {
1947 warn!(
1948 "Ed25519 rotation ended not fully restored: a failed promotion \
1949 rollback left the signer slots only partially restored; the old \
1950 active signer remains active in memory and publication is \
1951 deferred until a retry fully restores it"
1952 );
1953 }
1954 refresh_ed25519_verifying_keys(store).await;
1959 if let Some(ref es256) = extra.es256 {
1960 rotate_es256_keys(config, secrets_dir, es256, now).await;
1961 if let Err(error) = super::op_log::advance_sealed_head(secrets_dir, es256).await {
1968 error!(
1969 "failed to re-seal op-log head after ES256 rotation; \
1970 cross-process readers will retain the prior generation: {error}"
1971 );
1972 }
1973 }
1974 if let (Some(ref ml_dsa), Some(pre_rotation)) = (&extra.ml_dsa, pre_rotation_digest.as_deref())
1975 {
1976 let pq_restored = rotate_ml_dsa_keys(config, secrets_dir, ml_dsa, now).await;
1977 all_restored &= pq_restored;
1978 if !pq_restored {
1979 warn!(
1980 "ML-DSA rotation ended not fully restored: a failed promotion \
1981 rollback left the signer slots only partially restored; the old \
1982 active signer remains active in memory and publication is \
1983 deferred until a retry fully restores it"
1984 );
1985 }
1986 refresh_ml_dsa_verifying_keys(ml_dsa).await;
1987 let post_memory =
1988 component_state_digest(store, ml_dsa, extra.composite_ca_key.verifying_key()).await;
1989 let retained = read_deferred_predecessor(secrets_dir);
1994 let certified = retained
1995 .as_ref()
1996 .map(|record| record.states.iter().any(|state| state == pre_rotation))
1997 .unwrap_or(false);
1998 let anchor = match retained.as_ref() {
1999 Some(record) if certified => {
2000 info!(
2001 "anchoring deferred composite publication on the retained \
2002 committed predecessor"
2003 );
2004 Some(record.committed_digest.clone())
2005 }
2006 _ => None,
2007 };
2008 if all_restored {
2012 let expected = anchor.as_deref().unwrap_or(pre_rotation);
2013 match refresh_composite_key_set(
2014 secrets_dir,
2015 store,
2016 ml_dsa,
2017 Arc::clone(&extra.composite_ca_key),
2018 config.drain_secs(),
2019 expected,
2020 )
2021 .await
2022 {
2023 Ok(()) => {
2024 if retained.is_some() {
2025 clear_deferred_predecessor(secrets_dir);
2026 }
2027 }
2028 Err(error) => {
2029 let superseded = retained.as_ref().map(|record| {
2038 read_committed_composite_ledger(secrets_dir)
2039 .map(|(_, ledger)| ledger.component_digest != record.committed_digest)
2040 .unwrap_or(false)
2041 });
2042 match (retained.as_ref(), superseded) {
2043 (Some(_record), Some(true)) => {
2044 clear_deferred_predecessor(secrets_dir);
2047 warn!(
2048 "deferred composite predecessor superseded by a \
2049 competing publication; retrying once anchored on \
2050 current component memory"
2051 );
2052 if let Err(retry_error) = refresh_composite_key_set(
2053 secrets_dir,
2054 store,
2055 ml_dsa,
2056 Arc::clone(&extra.composite_ca_key),
2057 config.drain_secs(),
2058 pre_rotation,
2059 )
2060 .await
2061 {
2062 warn!(
2063 "composite key-set publication failed; retaining \
2064 prior authority: {retry_error}"
2065 );
2066 }
2067 }
2068 (Some(record), _) if certified => {
2069 let advanced = DeferredPredecessorRecord {
2080 committed_digest: record.committed_digest.clone(),
2081 states: vec![post_memory.clone()],
2082 };
2083 if write_deferred_predecessor(secrets_dir, &advanced) {
2084 *DEFERRED_PREDECESSOR_FALLBACK.write() = None;
2085 } else {
2086 *DEFERRED_PREDECESSOR_FALLBACK.write() = Some(advanced);
2087 }
2088 warn!(
2089 "composite key-set publication failed; the certified \
2090 recovery state advanced with this tick's rotations \
2091 and publication will be retried"
2092 );
2093 }
2094 _ => {
2095 warn!(
2101 "composite key-set publication failed; retaining prior \
2102 authority: {error}"
2103 );
2104 }
2105 }
2106 }
2107 }
2108 } else {
2109 match retained.as_ref() {
2129 None => {
2130 if let Ok((_, ledger)) = read_committed_composite_ledger(secrets_dir) {
2131 if ledger.component_digest == pre_rotation {
2132 let record = DeferredPredecessorRecord {
2133 committed_digest: pre_rotation.to_owned(),
2134 states: vec![post_memory.clone()],
2135 };
2136 if write_deferred_predecessor(secrets_dir, &record) {
2137 info!(
2138 "composite publication deferred; committed predecessor \
2139 and certified deferred states recorded for recovery"
2140 );
2141 } else {
2142 *DEFERRED_PREDECESSOR_FALLBACK.write() = Some(record);
2143 warn!(
2144 "could not persist the deferred predecessor record; \
2145 retaining it in memory for this process"
2146 );
2147 }
2148 }
2149 }
2150 }
2151 Some(record) if certified => {
2152 let states = if post_memory == *pre_rotation {
2153 record.states.clone()
2156 } else {
2157 vec![post_memory.clone()]
2160 };
2161 let advanced = DeferredPredecessorRecord {
2162 committed_digest: record.committed_digest.clone(),
2163 states,
2164 };
2165 if write_deferred_predecessor(secrets_dir, &advanced) {
2166 *DEFERRED_PREDECESSOR_FALLBACK.write() = None;
2167 } else {
2168 *DEFERRED_PREDECESSOR_FALLBACK.write() = Some(advanced);
2169 }
2170 }
2171 Some(_) => {
2172 if !deferred_predecessor_path(secrets_dir).exists() {
2178 if let Some(record) = DEFERRED_PREDECESSOR_FALLBACK.read().clone() {
2179 write_deferred_predecessor(secrets_dir, &record);
2180 }
2181 }
2182 }
2183 }
2184 warn!(
2185 "rotation cycle ended not fully restored; deferring composite \
2186 key-set publication so the cycle cannot publish as successfully \
2187 restored"
2188 );
2189 }
2190 }
2191 all_restored
2192}
2193
2194pub fn spawn_rotation_task(
2195 config: Arc<OAuthConfig>,
2196 secrets_dir: PathBuf,
2197 store: Arc<SigningKeyStore>,
2198 extra: RotationStores,
2199) {
2200 tokio::task::spawn_local(async move {
2201 let mut interval = tokio::time::interval(
2205 config
2206 .rotation_check_interval()
2207 .min(std::time::Duration::from_secs(60)),
2208 );
2209 interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
2210 interval.tick().await;
2212 loop {
2213 interval.tick().await;
2214 let now = chrono::Utc::now().timestamp();
2215 run_rotation_cycle(&config, &secrets_dir, &store, &extra, now).await;
2216 }
2217 });
2218}
2219
2220use p256::ecdsa::SigningKey as Es256SigningKey;
2225
2226#[derive(Clone)]
2227pub struct Es256KeySlot {
2228 pub key: Arc<Es256SigningKey>,
2229 pub nbf: i64,
2230 pub exp: i64,
2231}
2232
2233impl Es256KeySlot {
2234 pub fn new(key: Es256SigningKey, nbf: i64, exp: i64) -> Self {
2235 Self {
2236 key: Arc::new(key),
2237 nbf,
2238 exp,
2239 }
2240 }
2241
2242 pub fn kid(&self) -> String {
2243 crate::auth::jwt::es256_kid(&self.key)
2244 }
2245}
2246
2247#[derive(Default, Clone)]
2248pub struct Es256KeySlots {
2249 pub drain: Option<Es256KeySlot>,
2250 pub active: Option<Es256KeySlot>,
2251 pub lead: Option<Es256KeySlot>,
2252}
2253
2254impl Es256KeySlots {
2255 pub fn all(&self) -> Vec<&Es256KeySlot> {
2256 [&self.drain, &self.active, &self.lead]
2257 .into_iter()
2258 .flatten()
2259 .collect()
2260 }
2261}
2262
2263type Es256PromotionHook = dyn Fn(Arc<Es256SigningKey>) -> anyhow::Result<()> + Send + Sync;
2264
2265#[derive(Clone)]
2266pub struct Es256SigningKeyStore(
2267 pub Arc<parking_lot::RwLock<Es256KeySlots>>,
2268 Arc<parking_lot::RwLock<Option<Arc<Es256PromotionHook>>>>,
2269 Arc<parking_lot::RwLock<Option<PathBuf>>>,
2270);
2271
2272impl Es256SigningKeyStore {
2273 pub fn new(slots: Es256KeySlots) -> Self {
2274 Self(
2275 Arc::new(parking_lot::RwLock::new(slots)),
2276 Arc::new(parking_lot::RwLock::new(None)),
2277 Arc::new(parking_lot::RwLock::new(None)),
2278 )
2279 }
2280
2281 pub fn bind_secrets_dir(&self, secrets_dir: PathBuf) {
2285 *self.2.write() = Some(secrets_dir);
2286 }
2287
2288 pub fn refresh_from_disk(&self) {
2289 let Some(secrets_dir) = self.2.read().clone() else {
2290 return;
2291 };
2292 let active = load_es256_slot(&secrets_dir, "active");
2293 let Some(active) = active else {
2296 warn!("ES256: durable active slot unavailable; retaining cached authority");
2297 return;
2298 };
2299 *self.0.write() = Es256KeySlots {
2300 drain: load_es256_slot(&secrets_dir, "drain"),
2301 active: Some(active),
2302 lead: load_es256_slot(&secrets_dir, "lead"),
2303 };
2304 }
2305
2306 pub(crate) fn set_promotion_hook(&self, hook: Arc<Es256PromotionHook>) {
2312 *self.1.write() = Some(hook);
2313 }
2314
2315 fn notify_promotion(&self, key: Arc<Es256SigningKey>) -> anyhow::Result<()> {
2316 let hook = self.1.read().clone();
2317 if let Some(hook) = hook {
2318 hook(key)?;
2319 }
2320 Ok(())
2321 }
2322
2323 pub fn active_key(&self) -> Option<Arc<Es256SigningKey>> {
2328 self.refresh_from_disk();
2329 self.0.read().active.as_ref().map(|s| Arc::clone(&s.key))
2330 }
2331
2332 pub fn slots_snapshot(&self) -> Es256KeySlots {
2335 self.refresh_from_disk();
2336 self.0.read().clone()
2337 }
2338
2339 pub fn active_slot(&self) -> Option<Es256KeySlot> {
2341 self.0.read().active.clone()
2342 }
2343
2344 pub fn drain_slot(&self) -> Option<Es256KeySlot> {
2346 self.0.read().drain.clone()
2347 }
2348
2349 pub fn lead_slot(&self) -> Option<Es256KeySlot> {
2352 self.0.read().lead.clone()
2353 }
2354
2355 pub fn all_slots_snapshot(&self) -> Vec<Es256KeySlot> {
2356 self.0.read().all().into_iter().cloned().collect()
2357 }
2358}
2359
2360pub(crate) fn es256_signing_key_for_kid(secrets_dir: &Path, kid: &str) -> Option<Es256SigningKey> {
2367 let state_dir = rotation_state_dir(secrets_dir);
2368 for name in ["active", "drain", "lead"] {
2369 if let Some(slot) =
2370 load_es256_slot(&state_dir, name).or_else(|| load_es256_slot(secrets_dir, name))
2371 {
2372 if slot.kid() == kid {
2373 return Some((*slot.key).clone());
2374 }
2375 }
2376 }
2377 None
2378}
2379
2380fn es256_slot_paths(secrets_dir: &Path, name: &str) -> (PathBuf, PathBuf) {
2383 (
2384 secrets_dir.join(format!("es256-signing-key.{name}")),
2385 secrets_dir.join(format!("es256-signing-key.{name}.meta")),
2386 )
2387}
2388
2389fn load_es256_slot(secrets_dir: &Path, name: &str) -> Option<Es256KeySlot> {
2390 let (key_path, meta_path) = es256_slot_paths(secrets_dir, name);
2391 let seed = std::fs::read(&key_path).ok()?;
2392 if seed.len() != 32 {
2393 warn!(
2394 "ES256 key slot '{name}': unexpected seed length {}",
2395 seed.len()
2396 );
2397 return None;
2398 }
2399 let meta_bytes = std::fs::read(&meta_path).ok()?;
2400 let meta: SlotMeta = serde_json::from_slice(&meta_bytes).ok()?;
2401 let key = Es256SigningKey::from_bytes(seed.as_slice().into()).ok()?;
2402 Some(Es256KeySlot::new(key, meta.nbf, meta.exp))
2403}
2404
2405pub(crate) fn persist_es256_slot(
2406 secrets_dir: &Path,
2407 name: &str,
2408 slot: &Es256KeySlot,
2409) -> anyhow::Result<()> {
2410 super::identity_store::write_secret(
2412 secrets_dir,
2413 &format!("es256-signing-key.{name}"),
2414 &slot.key.to_bytes(),
2415 )?;
2416 let meta = SlotMeta {
2417 nbf: slot.nbf,
2418 exp: slot.exp,
2419 };
2420 super::identity_store::write_secret(
2421 secrets_dir,
2422 &format!("es256-signing-key.{name}.meta"),
2423 &serde_json::to_vec(&meta)?,
2424 )?;
2425 Ok(())
2426}
2427
2428fn delete_es256_slot(secrets_dir: &Path, name: &str) {
2429 let (key_path, meta_path) = es256_slot_paths(secrets_dir, name);
2430 let _ = std::fs::remove_file(&key_path);
2431 let _ = std::fs::remove_file(&meta_path);
2432}
2433
2434fn generate_es256_slot(nbf: i64, exp: i64) -> Es256KeySlot {
2435 let key = Es256SigningKey::random(&mut rand::rngs::OsRng);
2436 Es256KeySlot::new(key, nbf, exp)
2437}
2438
2439pub fn load_or_init_es256_key_store(
2440 secrets_dir: &Path,
2441 config: &OAuthConfig,
2442) -> Es256SigningKeyStore {
2443 match super::op_log::resolve_oplog_state_dir(secrets_dir) {
2444 Ok(state_dir) => {
2445 load_or_init_es256_key_store_with_oplog_state_dir(secrets_dir, config, &state_dir)
2446 }
2447 Err(error) => {
2448 error!(
2449 "Could not resolve durable ES256 operation-log authority: {error}; refusing implicit key generation"
2450 );
2451 load_or_init_es256_key_store_with_authority(secrets_dir, config, || true)
2452 }
2453 }
2454}
2455
2456fn load_or_init_es256_key_store_with_oplog_state_dir(
2457 secrets_dir: &Path,
2458 config: &OAuthConfig,
2459 oplog_state_dir: &Path,
2460) -> Es256SigningKeyStore {
2461 load_or_init_es256_key_store_with_authority(secrets_dir, config, || {
2462 match has_es256_oplog_authority(oplog_state_dir) {
2463 Ok(present) => present,
2464 Err(error) => {
2465 error!(
2466 "Could not inspect durable ES256 operation-log authority: {error}; refusing implicit key generation"
2467 );
2468 true
2469 }
2470 }
2471 })
2472}
2473
2474fn has_es256_oplog_authority(state_dir: &Path) -> std::io::Result<bool> {
2475 use std::io::ErrorKind;
2476
2477 for name in [
2478 super::op_log::SEALED_HEAD_FILENAME,
2479 super::op_log::SEALED_HEAD_MAX_SEQ_FILENAME,
2480 super::op_log::HEAD_VERIFYING_KEY_FILENAME,
2481 ] {
2482 match std::fs::symlink_metadata(state_dir.join(name)) {
2483 Ok(_) => return Ok(true),
2484 Err(error) if error.kind() == ErrorKind::NotFound => {}
2485 Err(error) => return Err(error),
2486 }
2487 }
2488 Ok(false)
2489}
2490
2491fn load_or_init_es256_key_store_with_authority(
2492 secrets_dir: &Path,
2493 config: &OAuthConfig,
2494 has_durable_authority: impl FnOnce() -> bool,
2495) -> Es256SigningKeyStore {
2496 let now = chrono::Utc::now().timestamp();
2497 let active_secs = config.active_secs();
2498 let lead_secs = config.lead_secs();
2499
2500 let state_dir = rotation_state_dir(secrets_dir);
2501 let _slot_lock = match lock_signing_slots(&state_dir, false) {
2502 Ok(lock) => lock,
2503 Err(error) => {
2504 error!("Could not lock ES256 signing slots: {error}; refusing initialization");
2505 return Es256SigningKeyStore::new(Es256KeySlots::default());
2506 }
2507 };
2508 let has_durable_authority = has_durable_authority();
2509 let mut drain =
2510 load_es256_slot(&state_dir, "drain").or_else(|| load_es256_slot(secrets_dir, "drain"));
2511 let mut active =
2512 load_es256_slot(&state_dir, "active").or_else(|| load_es256_slot(secrets_dir, "active"));
2513 let mut lead =
2514 load_es256_slot(&state_dir, "lead").or_else(|| load_es256_slot(secrets_dir, "lead"));
2515
2516 if drain.as_ref().is_some_and(|slot| now >= slot.exp) {
2519 delete_es256_slot(&state_dir, "drain");
2520 drain = None;
2521 }
2522 if lead.as_ref().is_some_and(|slot| now >= slot.exp) {
2523 delete_es256_slot(&state_dir, "lead");
2524 lead = None;
2525 }
2526
2527 if active.is_none() {
2528 if has_durable_authority {
2529 error!(
2530 "Durable ES256 operation-log authority exists but no active signing slot is \
2531 available; refusing implicit key generation"
2532 );
2533 } else {
2534 info!("No active ES256 signing key found — generating on first boot");
2535 let slot = generate_es256_slot(now, now + active_secs);
2536 match persist_es256_slot(&state_dir, "active", &slot) {
2537 Ok(()) => {
2538 info!("Active ES256 key generated (kid={})", slot.kid());
2539 active = load_es256_slot(&state_dir, "active");
2540 }
2541 Err(error) => error!(
2542 "Could not persist active ES256 key to '{}': {error}. Refusing \
2543 to activate the unpersisted key; no active signer is available \
2544 until persistence succeeds on a restart.",
2545 state_dir.display()
2546 ),
2547 }
2548 }
2549 }
2550
2551 let should_gen_lead =
2552 lead.is_none() && active.as_ref().is_some_and(|a| a.exp - now < lead_secs);
2553 if should_gen_lead {
2554 let lead_nbf = active.as_ref().map(|a| a.exp).unwrap_or(now) - lead_secs;
2555 let lead_exp = lead_nbf + active_secs;
2556 let slot = generate_es256_slot(lead_nbf, lead_exp);
2557 if let Err(e) = persist_es256_slot(&state_dir, "lead", &slot) {
2558 error!(
2559 "Could not persist lead ES256 key to '{}': {e}. Rotation state will \
2560 not survive restart.",
2561 state_dir.display()
2562 );
2563 }
2564 }
2565
2566 let lead = if should_gen_lead {
2567 load_es256_slot(&state_dir, "lead")
2568 } else {
2569 lead
2570 };
2571 Es256SigningKeyStore::new(Es256KeySlots {
2572 drain,
2573 active,
2574 lead,
2575 })
2576}
2577
2578pub async fn rotate_es256_keys(
2587 config: &OAuthConfig,
2588 secrets_dir: &Path,
2589 store: &Es256SigningKeyStore,
2590 now: i64,
2591) -> bool {
2592 let state_dir = rotation_state_dir(secrets_dir);
2593 let _slot_lock = match lock_signing_slots(&state_dir, true) {
2594 Ok(lock) => lock,
2595 Err(error) => {
2596 warn!("Could not lock ES256 signing slots; deferring rotation: {error}");
2597 return false;
2598 }
2599 };
2600 rotate_es256_keys_in_state_dir(config, &state_dir, store, now).await
2601}
2602
2603async fn rotate_es256_keys_in_state_dir(
2604 config: &OAuthConfig,
2605 state_dir: &Path,
2606 store: &Es256SigningKeyStore,
2607 now: i64,
2608) -> bool {
2609 let mut slots = store.0.write();
2610 let active_secs = config.active_secs();
2611 let lead_secs = config.lead_secs();
2612 let drain_secs = config.drain_secs();
2613 let mut promoted = false;
2614
2615 if let Some(new_active) = slots.lead.as_ref().filter(|lead| lead.nbf <= now).cloned() {
2617 let old_active = slots.active.clone();
2618 let old_drain = slots.drain.clone();
2619 let new_drain = old_active.as_ref().map(|old_active| {
2620 Es256KeySlot::new(
2624 old_active.key.as_ref().clone(),
2625 old_active.nbf,
2626 now.saturating_add(drain_secs),
2627 )
2628 });
2629
2630 if let Some(ref drain) = new_drain {
2633 if let Err(error) = persist_es256_slot(state_dir, "drain", drain) {
2634 if let Some(ref old_drain) = old_drain {
2635 let _ = persist_es256_slot(state_dir, "drain", old_drain);
2636 } else {
2637 delete_es256_slot(state_dir, "drain");
2638 }
2639 error!(
2640 "ES256: failed to persist drain slot to '{}'; retaining old active: {error}",
2641 state_dir.display()
2642 );
2643 return false;
2644 }
2645 }
2646
2647 if let Err(error) = persist_es256_slot(state_dir, "active", &new_active) {
2648 if let Some(ref old_active) = old_active {
2649 let _ = persist_es256_slot(state_dir, "active", old_active);
2650 } else {
2651 delete_es256_slot(state_dir, "active");
2652 }
2653 if let Some(ref old_drain) = old_drain {
2654 let _ = persist_es256_slot(state_dir, "drain", old_drain);
2655 } else {
2656 delete_es256_slot(state_dir, "drain");
2657 }
2658 error!(
2659 "ES256: failed to persist promoted active to '{}'; retaining old active: {error}",
2660 state_dir.display()
2661 );
2662 return false;
2663 }
2664
2665 if let Err(error) = store.notify_promotion(Arc::clone(&new_active.key)) {
2666 if let Some(ref old_active) = old_active {
2670 if let Err(rollback_error) = store.notify_promotion(Arc::clone(&old_active.key)) {
2671 warn!(
2672 "ES256: failed to restore PDS head after promotion failure: {rollback_error}"
2673 );
2674 }
2675 if let Err(rollback_error) = persist_es256_slot(state_dir, "active", old_active) {
2676 warn!(
2677 "ES256: failed to restore persisted active after promotion failure: {rollback_error}"
2678 );
2679 }
2680 } else {
2681 delete_es256_slot(state_dir, "active");
2682 }
2683 if let Some(ref old_drain) = old_drain {
2684 if let Err(rollback_error) = persist_es256_slot(state_dir, "drain", old_drain) {
2685 warn!(
2686 "ES256: failed to restore persisted drain after promotion failure: {rollback_error}"
2687 );
2688 }
2689 } else {
2690 delete_es256_slot(state_dir, "drain");
2691 }
2692 warn!(
2693 "ES256: failed to re-sign PDS head; retaining old active and retrying next tick: {error}"
2694 );
2695 return false;
2696 }
2697
2698 delete_es256_slot(state_dir, "lead");
2699 slots.drain = new_drain.or(old_drain);
2700 slots.active = Some(new_active);
2701 slots.lead = None;
2702 promoted = true;
2703 info!("ES256: promoted lead → active");
2704 }
2705
2706 if let Some(ref drain) = slots.drain {
2708 if now >= drain.exp {
2709 delete_es256_slot(state_dir, "drain");
2710 slots.drain = None;
2711 info!("ES256: removed expired drain slot");
2712 }
2713 }
2714
2715 if slots.lead.as_ref().is_some_and(|lead| now >= lead.exp) {
2718 delete_es256_slot(state_dir, "lead");
2719 slots.lead = None;
2720 info!("ES256: removed expired lead slot");
2721 }
2722
2723 if let Some(ref active) = slots.active {
2725 if slots.lead.is_none() && active.exp - now < lead_secs {
2726 let lead_nbf = active.exp - lead_secs;
2727 let lead_exp = lead_nbf + active_secs;
2728 let new_lead = generate_es256_slot(lead_nbf, lead_exp);
2729 if let Err(e) = persist_es256_slot(state_dir, "lead", &new_lead) {
2730 error!(
2731 "ES256: failed to persist new lead to '{}': {e}. Rotation state \
2732 will not survive restart.",
2733 state_dir.display()
2734 );
2735 } else {
2736 info!("ES256: generated new lead key (kid={})", new_lead.kid());
2737 }
2738 slots.lead = Some(new_lead);
2739 }
2740 }
2741 promoted
2742}
2743
2744mod ml_dsa_rotation {
2749 use super::*;
2750 use hyprstream_rpc::crypto::pq::{
2751 ml_dsa_generate_keypair, ml_dsa_sk_from_seed, ml_dsa_sk_to_seed, MlDsaSigningKey,
2752 };
2753
2754 #[derive(Clone)]
2755 pub struct MlDsaKeySlot {
2756 pub key: Arc<MlDsaSigningKey>,
2757 pub nbf: i64,
2758 pub exp: i64,
2759 }
2760
2761 impl MlDsaKeySlot {
2762 pub fn new(key: MlDsaSigningKey, nbf: i64, exp: i64) -> Self {
2763 Self {
2764 key: Arc::new(key),
2765 nbf,
2766 exp,
2767 }
2768 }
2769
2770 pub fn verifying_key(&self) -> hyprstream_rpc::crypto::pq::MlDsaVerifyingKey {
2771 ml_dsa::Keypair::verifying_key(&*self.key).clone()
2772 }
2773 }
2774
2775 #[derive(Clone, Default)]
2776 pub struct MlDsaKeySlots {
2777 pub drain: Option<MlDsaKeySlot>,
2778 pub active: Option<MlDsaKeySlot>,
2779 pub lead: Option<MlDsaKeySlot>,
2780 }
2781
2782 impl MlDsaKeySlots {
2783 pub fn all(&self) -> Vec<&MlDsaKeySlot> {
2784 [&self.drain, &self.active, &self.lead]
2785 .into_iter()
2786 .flatten()
2787 .collect()
2788 }
2789 }
2790
2791 #[derive(Clone)]
2792 pub struct MlDsaSigningKeyStore(pub Arc<RwLock<MlDsaKeySlots>>);
2793
2794 impl MlDsaSigningKeyStore {
2795 pub fn new(slots: MlDsaKeySlots) -> Self {
2796 Self(Arc::new(RwLock::new(slots)))
2797 }
2798
2799 pub async fn active_key(&self) -> Option<Arc<MlDsaSigningKey>> {
2800 self.0
2801 .read()
2802 .await
2803 .active
2804 .as_ref()
2805 .map(|s| Arc::clone(&s.key))
2806 }
2807
2808 pub async fn all_slots_snapshot(&self) -> Vec<MlDsaKeySlot> {
2809 self.0.read().await.all().into_iter().cloned().collect()
2810 }
2811 }
2812
2813 fn ml_dsa_slot_paths(secrets_dir: &Path, name: &str) -> (PathBuf, PathBuf) {
2816 (
2817 secrets_dir.join(format!("ml-dsa-signing-key.{name}")),
2818 secrets_dir.join(format!("ml-dsa-signing-key.{name}.meta")),
2819 )
2820 }
2821
2822 pub(super) fn load_ml_dsa_slot(secrets_dir: &Path, name: &str) -> Option<MlDsaKeySlot> {
2823 let (key_path, meta_path) = ml_dsa_slot_paths(secrets_dir, name);
2824 let seed_bytes = std::fs::read(&key_path).ok()?;
2825 if seed_bytes.len() != 32 {
2826 warn!(
2827 "ML-DSA key slot '{name}': unexpected seed length {}",
2828 seed_bytes.len()
2829 );
2830 return None;
2831 }
2832 let meta_bytes = std::fs::read(&meta_path).ok()?;
2833 let meta: SlotMeta = serde_json::from_slice(&meta_bytes).ok()?;
2834 let mut seed = [0u8; 32];
2835 seed.copy_from_slice(&seed_bytes);
2836 let key = ml_dsa_sk_from_seed(&seed);
2837 Some(MlDsaKeySlot::new(key, meta.nbf, meta.exp))
2838 }
2839
2840 pub(super) fn persist_ml_dsa_slot(
2841 secrets_dir: &Path,
2842 name: &str,
2843 slot: &MlDsaKeySlot,
2844 ) -> anyhow::Result<()> {
2845 let seed = ml_dsa_sk_to_seed(&slot.key);
2847 super::super::identity_store::write_secret(
2848 secrets_dir,
2849 &format!("ml-dsa-signing-key.{name}"),
2850 &seed,
2851 )?;
2852 let meta = SlotMeta {
2853 nbf: slot.nbf,
2854 exp: slot.exp,
2855 };
2856 super::super::identity_store::write_secret(
2857 secrets_dir,
2858 &format!("ml-dsa-signing-key.{name}.meta"),
2859 &serde_json::to_vec(&meta)?,
2860 )?;
2861 Ok(())
2862 }
2863
2864 fn delete_ml_dsa_slot(secrets_dir: &Path, name: &str) {
2865 let (key_path, meta_path) = ml_dsa_slot_paths(secrets_dir, name);
2866 let _ = std::fs::remove_file(&key_path);
2867 let _ = std::fs::remove_file(&meta_path);
2868 }
2869
2870 pub(super) fn generate_ml_dsa_slot(nbf: i64, exp: i64) -> MlDsaKeySlot {
2871 let (key, _vk) = ml_dsa_generate_keypair();
2872 MlDsaKeySlot::new(key, nbf, exp)
2873 }
2874
2875 pub fn load_or_init_ml_dsa_key_store(
2876 secrets_dir: &Path,
2877 config: &OAuthConfig,
2878 ) -> MlDsaSigningKeyStore {
2879 let now = chrono::Utc::now().timestamp();
2880 let active_secs = config.active_secs();
2881 let lead_secs = config.lead_secs();
2882
2883 let state_dir = rotation_state_dir(secrets_dir);
2884 let _slot_lock = match lock_signing_slots(&state_dir, false) {
2885 Ok(lock) => lock,
2886 Err(error) => {
2887 error!("Could not lock ML-DSA signing slots: {error}; refusing initialization");
2888 return MlDsaSigningKeyStore::new(MlDsaKeySlots::default());
2889 }
2890 };
2891 let has_committed_authority = has_committed_composite_marker(secrets_dir);
2892 let drain = load_ml_dsa_slot(&state_dir, "drain")
2893 .or_else(|| load_ml_dsa_slot(secrets_dir, "drain"));
2894 let mut active = load_ml_dsa_slot(&state_dir, "active")
2895 .or_else(|| load_ml_dsa_slot(secrets_dir, "active"));
2896 let lead =
2897 load_ml_dsa_slot(&state_dir, "lead").or_else(|| load_ml_dsa_slot(secrets_dir, "lead"));
2898
2899 if active.is_none() {
2900 if has_committed_authority {
2901 error!(
2902 "Committed composite authority exists but no active ML-DSA signing slot is \
2903 available; refusing implicit key generation"
2904 );
2905 } else {
2906 info!("No active ML-DSA-65 signing key found — generating on first boot");
2907 let slot = generate_ml_dsa_slot(now, now + active_secs);
2908 match persist_ml_dsa_slot(&state_dir, "active", &slot) {
2911 Ok(()) => {
2912 info!("Active ML-DSA-65 key generated");
2913 active = load_ml_dsa_slot(&state_dir, "active");
2914 }
2915 Err(e) => error!(
2916 "Could not persist active ML-DSA key to '{}': {e}. Refusing \
2917 to activate the unpersisted key; no active signer is \
2918 available until persistence succeeds on a restart.",
2919 state_dir.display()
2920 ),
2921 }
2922 }
2923 }
2924
2925 let should_gen_lead = !has_committed_authority
2926 && lead.is_none()
2927 && active.as_ref().is_some_and(|a| a.exp - now < lead_secs);
2928 if should_gen_lead {
2929 let lead_nbf = active.as_ref().map(|a| a.exp).unwrap_or(now) - lead_secs;
2930 let lead_exp = lead_nbf + active_secs;
2931 let slot = generate_ml_dsa_slot(lead_nbf, lead_exp);
2932 if let Err(e) = persist_ml_dsa_slot(&state_dir, "lead", &slot) {
2933 error!(
2934 "Could not persist lead ML-DSA key to '{}': {e}. Rotation state \
2935 will not survive restart.",
2936 state_dir.display()
2937 );
2938 }
2939 }
2940
2941 let lead = if should_gen_lead {
2942 load_ml_dsa_slot(&state_dir, "lead")
2943 } else {
2944 lead
2945 };
2946 MlDsaSigningKeyStore::new(MlDsaKeySlots {
2947 drain,
2948 active,
2949 lead,
2950 })
2951 }
2952
2953 pub async fn rotate_ml_dsa_keys(
2959 config: &OAuthConfig,
2960 secrets_dir: &Path,
2961 store: &MlDsaSigningKeyStore,
2962 now: i64,
2963 ) -> bool {
2964 let state_dir = rotation_state_dir(secrets_dir);
2965 let _slot_lock = match lock_signing_slots(&state_dir, true) {
2966 Ok(lock) => lock,
2967 Err(error) => {
2968 warn!("Could not lock ML-DSA signing slots; deferring rotation: {error}");
2969 return false;
2970 }
2971 };
2972 rotate_ml_dsa_keys_in_state_dir(config, &state_dir, store, now).await
2973 }
2974
2975 pub(super) async fn rotate_ml_dsa_keys_in_state_dir(
2979 config: &OAuthConfig,
2980 state_dir: &Path,
2981 store: &MlDsaSigningKeyStore,
2982 now: i64,
2983 ) -> bool {
2984 let mut slots = store.0.write().await;
2985 let active_secs = config.active_secs();
2986 let lead_secs = config.lead_secs();
2987 let drain_secs = config.drain_secs();
2988
2989 if let Some(new_active) = slots.lead.as_ref().filter(|lead| lead.nbf <= now).cloned() {
2998 let old_active = slots.active.clone();
2999 let old_drain = slots.drain.clone();
3000
3001 info!("ML-DSA: promoting lead to active");
3002
3003 if let Some(ref old) = old_active {
3004 if let Err(e) = persist_ml_dsa_slot(state_dir, "drain", old) {
3005 let drain_restored = if let Some(ref prev) = old_drain {
3009 persist_ml_dsa_slot(state_dir, "drain", prev)
3010 } else {
3011 delete_ml_dsa_slot(state_dir, "drain");
3012 let (drain_key, drain_meta) = ml_dsa_slot_paths(state_dir, "drain");
3014 let removed = [&drain_key, &drain_meta].iter().all(|path| {
3015 matches!(
3016 std::fs::symlink_metadata(path),
3017 Err(error) if error.kind() == std::io::ErrorKind::NotFound
3018 )
3019 });
3020 if !removed {
3021 Err(anyhow::anyhow!(
3022 "partial replacement drain slot still present after removal"
3023 ))
3024 } else {
3025 Ok(())
3026 }
3027 };
3028 if let Err(drain_error) = drain_restored {
3029 error!(
3030 "ML-DSA: could not restore previous drain slot to '{}': \
3031 {drain_error}. The durable drain slot is not fully \
3032 restored; retaining the old active in memory and \
3033 retrying next tick.",
3034 state_dir.display()
3035 );
3036 return false;
3037 }
3038 error!(
3039 "ML-DSA: failed to persist drain slot to '{}': {e}. Retaining \
3040 the old active in memory; promotion will be retried next \
3041 tick.",
3042 state_dir.display()
3043 );
3044 return true;
3045 }
3046 }
3047
3048 if let Err(e) = persist_ml_dsa_slot(state_dir, "active", &new_active) {
3049 let restored = match old_active {
3056 Some(ref old) => persist_ml_dsa_slot(state_dir, "active", old),
3057 None => {
3058 delete_ml_dsa_slot(state_dir, "active");
3059 Ok(())
3060 }
3061 };
3062 if let Err(restore_error) = restored {
3063 error!(
3064 "ML-DSA: could not restore active key to '{}': {restore_error}. \
3065 The failed promotion left the old active signer only in its \
3066 drain slot; that recoverable copy is kept untouched and \
3067 promotion will be retried next tick.",
3068 state_dir.display()
3069 );
3070 return false;
3071 }
3072 if old_active.is_some() {
3073 if let Some(ref prev) = old_drain {
3074 if let Err(drain_error) = persist_ml_dsa_slot(state_dir, "drain", prev) {
3075 error!(
3076 "ML-DSA: could not restore previous drain slot to '{}': \
3077 {drain_error}. The old active signer is restored in the \
3078 active slot, but the durable drain slot may hold a \
3079 partial write; promotion will be retried next tick.",
3080 state_dir.display()
3081 );
3082 return false;
3083 }
3084 } else {
3085 delete_ml_dsa_slot(state_dir, "drain");
3086 }
3087 }
3088 error!(
3089 "ML-DSA: failed to persist promoted active to '{}': {e}. Retaining \
3090 the old active in memory; promotion will be retried next tick.",
3091 state_dir.display()
3092 );
3093 return true;
3094 }
3095
3096 delete_ml_dsa_slot(state_dir, "lead");
3097 if let Some(old) = old_active {
3098 slots.drain = Some(old);
3099 }
3100 slots.active = Some(new_active);
3101 slots.lead = None;
3102 info!("ML-DSA: promoted lead → active");
3103 }
3104
3105 if let Some(ref drain) = slots.drain {
3107 if now >= drain.exp + drain_secs {
3108 delete_ml_dsa_slot(state_dir, "drain");
3109 slots.drain = None;
3110 info!("ML-DSA: removed expired drain slot");
3111 }
3112 }
3113
3114 if let Some(ref active) = slots.active {
3116 if slots.lead.is_none() && active.exp - now < lead_secs {
3117 let lead_nbf = active.exp - lead_secs;
3118 let lead_exp = lead_nbf + active_secs;
3119 let new_lead = generate_ml_dsa_slot(lead_nbf, lead_exp);
3120 if let Err(e) = persist_ml_dsa_slot(state_dir, "lead", &new_lead) {
3121 error!(
3122 "ML-DSA: failed to persist new lead to '{}': {e}. Rotation state \
3123 will not survive restart.",
3124 state_dir.display()
3125 );
3126 } else {
3127 info!("ML-DSA: generated new lead key");
3128 }
3129 slots.lead = Some(new_lead);
3130 }
3131 }
3132 true
3133 }
3134}
3135
3136pub use ml_dsa_rotation::{
3137 load_or_init_ml_dsa_key_store, rotate_ml_dsa_keys, MlDsaKeySlot, MlDsaKeySlots,
3138 MlDsaSigningKeyStore,
3139};
3140
3141#[cfg(test)]
3144#[allow(clippy::unwrap_used, clippy::expect_used)]
3145mod tests {
3146 use super::*;
3147 use std::io::Write as _;
3148 use tempfile::TempDir;
3149
3150 fn durability_ledger(version: u64) -> CompositeLedger {
3151 CompositeLedger {
3154 version,
3155 component_digest: format!("test-components-{version}"),
3156 pairs: Vec::new(),
3157 }
3158 }
3159
3160 fn durability_ledger_name(dir: &Path, ledger: &CompositeLedger) -> String {
3161 composite_committed_ledger_path(
3162 dir,
3163 &CompositeCommit {
3164 version: ledger.version,
3165 component_digest: ledger.component_digest.clone(),
3166 },
3167 )
3168 .file_name()
3169 .unwrap()
3170 .to_str()
3171 .unwrap()
3172 .to_owned()
3173 }
3174
3175 #[test]
3176 fn composite_durable_commit_orders_generation_before_marker() {
3177 use super::super::identity_store::durable_write_test::{self, Stage};
3178 let dir = TempDir::new().unwrap();
3179 let ledger = durability_ledger(1);
3180 let name = durability_ledger_name(dir.path(), &ledger);
3181 let _scope = durable_write_test::install(None);
3182 persist_committed_composite_ledger(dir.path(), &ledger).unwrap();
3183 assert_eq!(
3184 durable_write_test::events(),
3185 vec![
3186 (name.clone(), Stage::FileSync),
3187 (name, Stage::DirectorySync),
3188 ("jwt-composite-pairs.committed".into(), Stage::FileSync),
3189 ("jwt-composite-pairs.committed".into(), Stage::DirectorySync),
3190 ]
3191 );
3192 let (commit, selected) = read_committed_composite_ledger(dir.path()).unwrap();
3193 assert_eq!(commit.version, 1);
3194 assert_eq!(selected.component_digest, ledger.component_digest);
3195 }
3196
3197 #[test]
3198 fn composite_durable_commit_sync_failure_preserves_selected_authority() {
3199 use super::super::identity_store::durable_write_test::{self, Stage};
3200 for (marker_fault, stage) in [
3201 (false, Stage::FileSync),
3202 (false, Stage::DirectorySync),
3203 (true, Stage::FileSync),
3204 ] {
3205 let dir = TempDir::new().unwrap();
3206 let old = durability_ledger(1);
3207 persist_committed_composite_ledger(dir.path(), &old).unwrap();
3208 let old_marker = std::fs::read(composite_committed_path(dir.path())).unwrap();
3209 let old_snapshot = std::fs::read(composite_committed_ledger_path(
3210 dir.path(),
3211 &CompositeCommit {
3212 version: 1,
3213 component_digest: old.component_digest.clone(),
3214 },
3215 ))
3216 .unwrap();
3217 let next = durability_ledger(2);
3218 let name = if marker_fault {
3219 "jwt-composite-pairs.committed".to_owned()
3220 } else {
3221 durability_ledger_name(dir.path(), &next)
3222 };
3223 let _scope = durable_write_test::install(Some((name, stage)));
3224 assert!(persist_committed_composite_ledger(dir.path(), &next).is_err());
3225 let (commit, selected) = read_committed_composite_ledger(dir.path()).unwrap();
3226 assert_eq!(
3227 commit.version, 1,
3228 "a failed sync selected pending authority"
3229 );
3230 assert_eq!(selected.component_digest, old.component_digest);
3231 assert_eq!(
3232 std::fs::read(composite_committed_path(dir.path())).unwrap(),
3233 old_marker
3234 );
3235 assert_eq!(
3236 std::fs::read(composite_committed_ledger_path(dir.path(), &commit)).unwrap(),
3237 old_snapshot
3238 );
3239 if !marker_fault {
3240 assert!(durable_write_test::events()
3241 .iter()
3242 .all(|(file, _)| file != "jwt-composite-pairs.committed"));
3243 }
3244 }
3245 }
3246
3247 #[test]
3248 fn composite_durable_commit_marker_directory_failure_never_selects_partial_ledger() {
3249 use super::super::identity_store::durable_write_test::{self, Stage};
3250 let dir = TempDir::new().unwrap();
3251 let old = durability_ledger(1);
3252 persist_committed_composite_ledger(dir.path(), &old).unwrap();
3253 let old_marker = std::fs::read(composite_committed_path(dir.path())).unwrap();
3254 let next = durability_ledger(2);
3255 let _scope = durable_write_test::install(Some((
3256 "jwt-composite-pairs.committed".into(),
3257 Stage::DirectorySync,
3258 )));
3259 assert!(persist_committed_composite_ledger(dir.path(), &next).is_err());
3260 let (commit, selected) = read_committed_composite_ledger(dir.path()).unwrap();
3263 assert_eq!(commit.version, 2);
3264 assert_eq!(selected.component_digest, next.component_digest);
3265 std::fs::write(composite_committed_path(dir.path()), old_marker).unwrap();
3268 let (commit, selected) = read_committed_composite_ledger(dir.path()).unwrap();
3269 assert_eq!(commit.version, 1);
3270 assert_eq!(selected.component_digest, old.component_digest);
3271 }
3272
3273 #[test]
3274 fn composite_durable_commit_legacy_migration_sync_failure_propagates() {
3275 use super::super::identity_store::durable_write_test::{self, Stage};
3276 let dir = TempDir::new().unwrap();
3277 let ledger = durability_ledger(1);
3278 let commit = CompositeCommit {
3279 version: ledger.version,
3280 component_digest: ledger.component_digest.clone(),
3281 };
3282 std::fs::write(
3283 composite_ledger_path(dir.path()),
3284 serde_json::to_vec(&ledger).unwrap(),
3285 )
3286 .unwrap();
3287 std::fs::write(
3288 composite_committed_path(dir.path()),
3289 serde_json::to_vec(&commit).unwrap(),
3290 )
3291 .unwrap();
3292 let marker = std::fs::read(composite_committed_path(dir.path())).unwrap();
3293 let _scope = durable_write_test::install(Some((
3294 durability_ledger_name(dir.path(), &ledger),
3295 Stage::DirectorySync,
3296 )));
3297 assert!(load_or_migrate_committed_composite_ledger(dir.path()).is_err());
3298 assert_eq!(
3299 std::fs::read(composite_committed_path(dir.path())).unwrap(),
3300 marker
3301 );
3302 }
3303
3304 #[tokio::test]
3305 async fn composite_durable_commit_initializer_retry_reestablishes_both_barriers() {
3306 use super::super::identity_store::durable_write_test::{self, Stage};
3307 const MODE: &str = "HYPRSTREAM_COMPOSITE_DURABILITY_RETRY_MODE";
3308 const DIRECTORY: &str = "HYPRSTREAM_COMPOSITE_DURABILITY_RETRY_DIR";
3309 let mode = std::env::var(MODE).unwrap_or_default();
3310 if mode.is_empty() {
3311 for retry_mode in ["migration", "marker"] {
3312 let dir = TempDir::new().unwrap();
3313 let phases: &[&str] = if retry_mode == "migration" {
3314 &["bootstrap", "migration"]
3315 } else {
3316 &["marker"]
3317 };
3318 for phase in phases {
3319 let status = std::process::Command::new(std::env::current_exe().unwrap())
3320 .args([
3321 "--exact",
3322 "auth::key_rotation::tests::composite_durable_commit_initializer_retry_reestablishes_both_barriers",
3323 "--nocapture",
3324 ])
3325 .env(MODE, phase)
3326 .env(DIRECTORY, dir.path())
3327 .status()
3328 .unwrap();
3329 assert!(status.success(), "{phase} retry subprocess failed");
3330 }
3331 }
3332 return;
3333 }
3334 let dir = PathBuf::from(std::env::var_os(DIRECTORY).unwrap());
3335 let config = test_config();
3336 let ca = Arc::new(SigningKey::from_bytes(&[0x6b; 32]));
3337 let ed = load_or_init_key_store(&dir, &config);
3338 let pq = load_or_init_ml_dsa_key_store(&dir, &config);
3339 if mode == "bootstrap" {
3340 initialize_composite_key_set(&dir, &ed, &pq, ca, 300)
3341 .await
3342 .unwrap();
3343 return;
3344 }
3345 let fault_name = if mode == "migration" {
3346 let (commit, ledger) = read_committed_composite_ledger(&dir).unwrap();
3347 let name = durability_ledger_name(&dir, &ledger);
3348 std::fs::remove_file(composite_committed_ledger_path(&dir, &commit)).unwrap();
3349 name
3350 } else {
3351 "jwt-composite-pairs.committed".to_owned()
3352 };
3353 let key_set = hyprstream_rpc::auth::global_composite_key_set();
3354 assert_eq!(key_set.snapshot().version(), 0);
3355 let failed = durable_write_test::install(Some((fault_name, Stage::DirectorySync)));
3356 for _ in 0..2 {
3360 let mutable_before = std::fs::read(composite_ledger_path(&dir)).ok();
3361 assert!(
3362 initialize_composite_key_set(&dir, &ed, &pq, Arc::clone(&ca), 300)
3363 .await
3364 .is_err()
3365 );
3366 assert_eq!(key_set.snapshot().version(), 0);
3367 if let Some(before) = mutable_before {
3368 assert_eq!(std::fs::read(composite_ledger_path(&dir)).unwrap(), before);
3369 }
3370 }
3371 let (commit, ledger) = read_committed_composite_ledger(&dir).unwrap();
3372 let name = durability_ledger_name(&dir, &ledger);
3373 drop(failed);
3374 let _success = durable_write_test::install(None);
3375 initialize_composite_key_set(&dir, &ed, &pq, ca, 300)
3376 .await
3377 .unwrap();
3378 assert_eq!(key_set.snapshot().version(), commit.version);
3379 assert_eq!(
3380 durable_write_test::events(),
3381 vec![
3382 (name.clone(), Stage::FileSync),
3383 (name, Stage::DirectorySync),
3384 ("jwt-composite-pairs.committed".into(), Stage::FileSync),
3385 ("jwt-composite-pairs.committed".into(), Stage::DirectorySync),
3386 ]
3387 );
3388 }
3389
3390 #[tokio::test]
3391 async fn committed_active_pair_failure_reports_safe_component_availability() {
3392 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_DIAGNOSTIC_TEST";
3393 if std::env::var_os(ISOLATED).is_none() {
3394 let status = std::process::Command::new(std::env::current_exe().unwrap())
3395 .args([
3396 "--exact",
3397 "auth::key_rotation::tests::committed_active_pair_failure_reports_safe_component_availability",
3398 "--nocapture",
3399 ])
3400 .env(ISOLATED, "1")
3401 .status()
3402 .unwrap();
3403 assert!(status.success());
3404 return;
3405 }
3406
3407 let dir = TempDir::new().unwrap();
3408 let now = chrono::Utc::now().timestamp();
3409 let ca = Arc::new(SigningKey::from_bytes(&[0x61; 32]));
3410 let ed = SigningKeyStore::new(KeySlots {
3411 active: Some(KeySlot::new(
3412 SigningKey::from_bytes(&[0x62; 32]),
3413 now - 60,
3414 now + 3600,
3415 )),
3416 ..KeySlots::default()
3417 });
3418 let pq = MlDsaSigningKeyStore::new(MlDsaKeySlots {
3419 active: Some(ml_dsa_rotation::generate_ml_dsa_slot(now - 60, now + 3600)),
3420 ..MlDsaKeySlots::default()
3421 });
3422 initialize_composite_key_set(dir.path(), &ed, &pq, Arc::clone(&ca), 300)
3423 .await
3424 .unwrap();
3425 let commit: CompositeCommit =
3426 serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
3427 .unwrap();
3428 let immutable = composite_committed_ledger_path(dir.path(), &commit);
3429 let mut ledger: CompositeLedger =
3430 serde_json::from_slice(&std::fs::read(&immutable).unwrap()).unwrap();
3431 let oauth = ledger
3432 .pairs
3433 .iter()
3434 .find(|pair| pair.role == "oauth" && pair.state == "active")
3435 .unwrap();
3436
3437 let missing_ed = SigningKeyStore::new(KeySlots::default());
3438 let error =
3439 initialize_composite_key_set(dir.path(), &missing_ed, &pq, Arc::clone(&ca), 300)
3440 .await
3441 .expect_err("missing committed OAuth Ed component must fail closed");
3442 assert_eq!(
3443 error.to_string(),
3444 format!(
3445 "committed active composite signing pair is unavailable after restart: \
3446 kid={:?} role={:?} ed_component_available=false pq_component_available=true \
3447 policy_ca_ed_matched=false",
3448 oauth.kid, oauth.role,
3449 )
3450 );
3451 assert!(!error.to_string().contains(&oauth.ed25519_public));
3452 assert!(!error.to_string().contains(&oauth.ml_dsa_public));
3453
3454 ledger.pairs.sort_by_key(|pair| pair.role != "policy");
3457 std::fs::write(&immutable, serde_json::to_vec(&ledger).unwrap()).unwrap();
3458 let policy = ledger
3459 .pairs
3460 .iter()
3461 .find(|pair| pair.role == "policy" && pair.state == "active")
3462 .unwrap();
3463 let missing_pq = MlDsaSigningKeyStore::new(MlDsaKeySlots::default());
3464 let error = initialize_composite_key_set(dir.path(), &ed, &missing_pq, ca, 300)
3465 .await
3466 .expect_err("missing committed Policy PQ component must fail closed");
3467 assert_eq!(
3468 error.to_string(),
3469 format!(
3470 "committed active composite signing pair is unavailable after restart: \
3471 kid={:?} role={:?} ed_component_available=true pq_component_available=false \
3472 policy_ca_ed_matched=true",
3473 policy.kid, policy.role,
3474 )
3475 );
3476 assert!(!error.to_string().contains(&policy.ed25519_public));
3477 assert!(!error.to_string().contains(&policy.ml_dsa_public));
3478 }
3479
3480 #[test]
3481 fn rejected_composite_proposal_rechecks_replaced_component_slot() {
3482 let dir = TempDir::new().unwrap();
3483 let fingerprint = || {
3484 let mut digest = Sha256::new();
3485 fingerprint_component_slot_files(dir.path(), &mut digest).unwrap();
3486 digest.finalize().to_vec()
3487 };
3488 let missing = fingerprint();
3489 super::super::identity_store::write_secret(
3490 dir.path(),
3491 "jwt-signing-key.active",
3492 b"first-component",
3493 )
3494 .unwrap();
3495 let first = fingerprint();
3496 assert_ne!(missing, first);
3497 super::super::identity_store::write_secret(
3498 dir.path(),
3499 "jwt-signing-key.active",
3500 b"second-component",
3501 )
3502 .unwrap();
3503 assert_ne!(first, fingerprint());
3504 }
3505
3506 struct PermitFixtureAccountReads;
3507
3508 impl hyprstream_pds_service::AccountRecordReadAuthorizer for PermitFixtureAccountReads {
3509 fn check_read(
3510 &self,
3511 _subject: &hyprstream_rpc::Subject,
3512 _verified_tenant: Option<&str>,
3513 _security_context: Option<&hyprstream_rpc::auth::mac::SecurityContext>,
3514 _object_id: &str,
3515 ) -> hyprstream_rpc::auth::mac::MacDecision {
3516 hyprstream_rpc::auth::mac::MacDecision::Permit
3517 }
3518 }
3519
3520 fn hosted_account_store(
3521 label: &str,
3522 zone: &str,
3523 ) -> anyhow::Result<Arc<hyprstream_pds_service::AccountRecordStore>> {
3524 let ed = ed25519_dalek::SigningKey::generate(&mut rand::rngs::OsRng);
3525 let (pq, pq_vk) = hyprstream_crypto::pq::ml_dsa_generate_keypair();
3526 let hybrid = hyprstream_pds::did_op::HybridRotationKey::new(
3527 ed.verifying_key().to_bytes(),
3528 hyprstream_crypto::pq::ml_dsa_vk_bytes(&pq_vk),
3529 )?;
3530 let rotations = hyprstream_pds::did_op::GenesisRotationKeys::new(
3531 hyprstream_pds::did_op::UserRotationKey::new(hybrid),
3532 hyprstream_pds::did_op::RecoveryKeyEnrollment::Declined,
3533 hyprstream_pds::did_op::HostKeyEnrollment::Absent,
3534 )?;
3535 let name =
3536 hyprstream_pds::AllocatedAccountName::new(label, format!("did:web:{label}.{zone}"))?;
3537 let mint = hyprstream_pds::HostedAccountMint::begin(name, rotations)?;
3538 let document = mint.seal_did_document(&format!("https://{zone}"))?;
3539 let pending =
3540 mint.prepare_genesis(document, hyprstream_pds::did_op::GenesisRepoHead::EmptyRepo)?;
3541 let signature = hyprstream_pds::did_op::sign_genesis(pending.unsigned_genesis(), &ed, &pq)?;
3542 let record = pending.seal(signature)?.record_bytes().to_vec();
3543 let root = hyprstream_vfs::SyntheticNode::dir().with_child(
3544 zone,
3545 hyprstream_vfs::SyntheticNode::dir().with_child(
3546 "accounts",
3547 hyprstream_vfs::SyntheticNode::dir().with_child(
3548 label,
3549 hyprstream_vfs::SyntheticNode::dir().with_child(
3550 "account-record.cbor",
3551 hyprstream_vfs::SyntheticNode::file(record),
3552 ),
3553 ),
3554 ),
3555 );
3556 Ok(Arc::new(hyprstream_pds_service::AccountRecordStore::new(
3557 Arc::new(hyprstream_vfs::SyntheticMount::new(root)),
3558 Arc::new(PermitFixtureAccountReads),
3559 )))
3560 }
3561
3562 fn test_config() -> OAuthConfig {
3563 OAuthConfig {
3564 jwt_key_active_days: 14,
3565 jwt_key_lead_days: 7,
3566 jwt_key_drain_days: 30,
3567 ..OAuthConfig::default()
3568 }
3569 }
3570
3571 struct ProductionAuthorityVerifier {
3572 transport: hyprstream_rpc::transport::TransportConfig,
3573 signing_key: hyprstream_rpc::prelude::SigningKey,
3574 key_source: Arc<hyprstream_rpc::auth::ClusterKeySource>,
3575 accepted: Arc<std::sync::atomic::AtomicBool>,
3576 error: Arc<parking_lot::Mutex<Option<String>>>,
3577 }
3578
3579 #[async_trait::async_trait(?Send)]
3580 impl crate::services::RequestService for ProductionAuthorityVerifier {
3581 fn decode_request_body(
3582 &self,
3583 signed_body: &[u8],
3584 ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
3585 Ok(hyprstream_rpc::service::DecodedRequestBody::opaque(
3586 signed_body.to_vec(),
3587 ))
3588 }
3589
3590 async fn handle_request(
3591 &self,
3592 _ctx: &crate::services::EnvelopeContext,
3593 _body: &hyprstream_rpc::service::DecodedRequestBody,
3594 ) -> anyhow::Result<(Vec<u8>, Option<crate::services::Continuation>)> {
3595 self.accepted.store(true, Ordering::Release);
3596 Ok((Vec::new(), None))
3597 }
3598
3599 fn name(&self) -> &str {
3600 "multiprocess-authority-verifier"
3601 }
3602
3603 fn transport(&self) -> &hyprstream_rpc::transport::TransportConfig {
3604 &self.transport
3605 }
3606
3607 fn signing_key(&self) -> hyprstream_rpc::prelude::SigningKey {
3608 self.signing_key.clone()
3609 }
3610
3611 fn jwt_key_source(&self) -> Option<Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
3612 Some(self.key_source.clone())
3613 }
3614
3615 fn expected_audience(&self) -> Option<&str> {
3616 Some("multiprocess")
3617 }
3618
3619 fn require_cnf_binding(&self) -> bool {
3620 false
3621 }
3622
3623 fn jwt_verify_policy(&self) -> hyprstream_rpc::crypto::CryptoPolicy {
3624 hyprstream_rpc::crypto::CryptoPolicy::Hybrid
3625 }
3626
3627 fn build_error_payload(&self, _request_id: u64, error: &str) -> Vec<u8> {
3628 *self.error.lock() = Some(error.to_owned());
3629 Vec::new()
3630 }
3631 }
3632
3633 async fn verify_through_rpc_endpoint(dir: &Path, token: String) -> anyhow::Result<()> {
3634 use hyprstream_rpc::rpc_client::RpcClientImpl;
3635 use hyprstream_rpc::signer::LocalSigner;
3636 use hyprstream_rpc::transport::lazy_uds::LazyUdsTransport;
3637 let rpc = RpcClientImpl::new(
3638 LocalSigner::new(SigningKey::from_bytes(&[0x75; 32])),
3639 LazyUdsTransport::new(dir.join("authority-rpc.sock")),
3640 Some(SigningKey::from_bytes(&[0x74; 32]).verifying_key()),
3641 )
3642 .with_response_verify_policy(hyprstream_rpc::crypto::CryptoPolicy::Classical)
3643 .with_default_jwt(token);
3644 rpc.call(b"verify-composite-authority".to_vec()).await?;
3645 Ok(())
3646 }
3647
3648 fn policy_client_for_socket(
3649 dir: &Path,
3650 ) -> anyhow::Result<hyprstream_rpc_std::policy_client::PolicyClient> {
3651 policy_client_for_named_socket(dir, "authority-policy.sock")
3652 }
3653
3654 fn policy_client_for_named_socket(
3655 dir: &Path,
3656 socket: &str,
3657 ) -> anyhow::Result<hyprstream_rpc_std::policy_client::PolicyClient> {
3658 use hyprstream_rpc::rpc_client::RpcClientImpl;
3659 use hyprstream_rpc::signer::LocalSigner;
3660 use hyprstream_rpc::transport::lazy_uds::LazyUdsTransport;
3661 let rpc = RpcClientImpl::new(
3662 LocalSigner::new(SigningKey::from_bytes(&[0x76; 32])),
3663 LazyUdsTransport::new(dir.join(socket)),
3664 Some(SigningKey::from_bytes(&[0x73; 32]).verifying_key()),
3665 )
3666 .with_response_verify_policy(hyprstream_rpc::crypto::CryptoPolicy::Classical);
3667 Ok(hyprstream_rpc_std::policy_client::PolicyClient::new(
3668 Arc::new(rpc),
3669 ))
3670 }
3671
3672 fn authority_process_dir() -> Option<PathBuf> {
3673 let dir = std::env::var_os("HYPRSTREAM_COMPOSITE_PRODUCTION_TEST_DIR")?;
3674 crate::mac::install_explicit_test_dispatch_pep();
3677 Some(PathBuf::from(dir))
3678 }
3679
3680 fn authority_ca_key() -> Arc<SigningKey> {
3681 Arc::new(SigningKey::from_bytes(&[0x5a; 32]))
3682 }
3683
3684 fn install_hybrid_request_anchor(client_key: &SigningKey) -> anyhow::Result<()> {
3685 let mut store = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
3686 let pq_key = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(client_key);
3687 store.bind(
3688 client_key.verifying_key().to_bytes(),
3689 &ml_dsa::Keypair::verifying_key(&pq_key),
3690 );
3691 hyprstream_rpc::envelope::install_verify_config(
3692 hyprstream_rpc::envelope::EnvelopeVerifyConfig {
3693 policy: hyprstream_rpc::crypto::CryptoPolicy::Hybrid,
3694 pq_store: Some(Arc::new(store)),
3695 },
3696 )
3697 }
3698
3699 async fn install_signing_authority(
3700 dir: &Path,
3701 subscribe: bool,
3702 ) -> anyhow::Result<(Arc<SigningKeyStore>, Arc<MlDsaSigningKeyStore>)> {
3703 let config = test_config();
3704 let ca = authority_ca_key();
3705 let ed = Arc::new(load_or_init_key_store(dir, &config));
3706 let pq = Arc::new(load_or_init_ml_dsa_key_store(dir, &config));
3707 *composite_ca_signing_key().write() = Some(Arc::clone(&ca));
3708 configure_composite_authority(dir);
3709 initialize_composite_key_set(dir, &ed, &pq, ca, 300).await?;
3710 if subscribe {
3711 start_composite_authority_subscription(
3712 dir.to_path_buf(),
3713 authority_ca_key().verifying_key(),
3714 )?;
3715 }
3716 Ok((ed, pq))
3717 }
3718
3719 async fn install_verifying_authority(dir: &Path) -> anyhow::Result<()> {
3720 let config = test_config();
3721 let ca = authority_ca_key();
3722 let ed = load_or_init_key_store(dir, &config);
3723 let pq = load_or_init_ml_dsa_key_store(dir, &config);
3724 restore_composite_verifying_key_set(dir, &ed, &pq, ca.verifying_key()).await?;
3725 start_composite_authority_subscription(dir.to_path_buf(), ca.verifying_key())?;
3726 Ok(())
3727 }
3728
3729 const PRODUCTION_AUTHORITY_BARRIER_TIMEOUT: std::time::Duration =
3730 std::time::Duration::from_secs(120);
3731
3732 fn wait_path(path: &Path) {
3733 let deadline = std::time::Instant::now() + PRODUCTION_AUTHORITY_BARRIER_TIMEOUT;
3734 while !path.exists() {
3735 assert!(
3736 std::time::Instant::now() < deadline,
3737 "timeout: {}",
3738 path.display()
3739 );
3740 std::thread::sleep(std::time::Duration::from_millis(10));
3741 }
3742 }
3743
3744 fn wait_for_target_authority(dir: &Path) -> CompositeCommit {
3745 let target_path = dir.join("target-authority.json");
3746 wait_path(&target_path);
3747 let target: CompositeCommit =
3748 serde_json::from_slice(&std::fs::read(target_path).unwrap()).unwrap();
3749 let deadline = std::time::Instant::now() + PRODUCTION_AUTHORITY_BARRIER_TIMEOUT;
3750 loop {
3751 let snapshot = hyprstream_rpc::auth::global_composite_key_set().snapshot();
3752 if snapshot.version() == target.version
3753 && snapshot.component_digest() == target.component_digest
3754 {
3755 return target;
3756 }
3757 assert!(
3758 std::time::Instant::now() < deadline,
3759 "process did not converge to {} {}",
3760 target.version,
3761 target.component_digest
3762 );
3763 std::thread::sleep(std::time::Duration::from_millis(10));
3764 }
3765 }
3766
3767 async fn verify_returned_tokens(dir: &Path, process: &str) -> anyhow::Result<()> {
3768 wait_path(&dir.join("tokens-ready"));
3769 for token_file in ["oauth-token", "policy-token"] {
3770 verify_through_rpc_endpoint(dir, std::fs::read_to_string(dir.join(token_file))?)
3771 .await?;
3772 }
3773 std::fs::write(dir.join(format!("verified-{process}")), b"1")?;
3774 Ok(())
3775 }
3776
3777 fn wait_for_done(dir: &Path) {
3778 wait_path(&dir.join("done"));
3779 }
3780
3781 fn publish_target_authority(dir: &Path, target: &CompositeCommit) {
3788 let bytes = serde_json::to_vec(target).unwrap();
3789 let mut temporary = tempfile::NamedTempFile::new_in(dir).unwrap();
3790 temporary.write_all(&bytes).unwrap();
3791 temporary.flush().unwrap();
3792 temporary
3793 .persist(dir.join("target-authority.json"))
3794 .unwrap();
3795 }
3796
3797 async fn mutate_authority_for_failure(
3798 ed: &SigningKeyStore,
3799 pq: &MlDsaSigningKeyStore,
3800 persist_dir: Option<&Path>,
3801 ) {
3802 let now = chrono::Utc::now().timestamp();
3803 let new_ed = KeySlot::new(
3804 SigningKey::generate(&mut rand::rngs::OsRng),
3805 now,
3806 now + 1200,
3807 );
3808 let mut ed_slots = ed.0.write().await;
3809 ed_slots.drain = ed_slots.active.take();
3810 ed_slots.active = Some(new_ed.clone());
3811 if let Some(dir) = persist_dir {
3812 if let Some(drain) = &ed_slots.drain {
3813 persist_slot(dir, "drain", drain).unwrap();
3814 }
3815 persist_slot(dir, "active", &new_ed).unwrap();
3816 }
3817 drop(ed_slots);
3818
3819 let (pq_key, _) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
3820 let new_pq = MlDsaKeySlot::new(pq_key, now, now + 1200);
3821 let mut pq_slots = pq.0.write().await;
3822 pq_slots.drain = pq_slots.active.take();
3823 pq_slots.active = Some(new_pq.clone());
3824 if let Some(dir) = persist_dir {
3825 if let Some(drain) = &pq_slots.drain {
3826 ml_dsa_rotation::persist_ml_dsa_slot(dir, "drain", drain).unwrap();
3827 }
3828 ml_dsa_rotation::persist_ml_dsa_slot(dir, "active", &new_pq).unwrap();
3829 }
3830 }
3831
3832 fn test_primary_key() -> [u8; 32] {
3836 [0x66; 32]
3837 }
3838 fn test_user_pub_key_b64() -> String {
3839 use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
3840 URL_SAFE_NO_PAD.encode(test_primary_key())
3841 }
3842 fn test_permissive_primary_resolver(
3846 ) -> Arc<dyn crate::services::policy::PrimaryEnrollmentResolver> {
3847 struct R;
3848 impl crate::services::policy::PrimaryEnrollmentResolver for R {
3849 fn primary_group(
3850 &self,
3851 _principal: &str,
3852 _tenant: &str,
3853 ) -> Option<crate::services::policy::PrimaryGroup> {
3854 Some(crate::services::policy::PrimaryGroup {
3855 suite_id: hyprstream_rpc::auth::SUITE_CLASSICAL_ED25519.to_owned(),
3856 ordered_component_keys: vec![test_primary_key().to_vec()],
3857 })
3858 }
3859 }
3860 Arc::new(R)
3861 }
3862
3863 #[test]
3864 fn composite_oauth_production_process() {
3865 let Some(dir) = authority_process_dir() else {
3866 return;
3867 };
3868 let runtime = tokio::runtime::Runtime::new().unwrap();
3869 runtime
3870 .block_on(async {
3871 install_signing_authority(&dir, true).await?;
3872 wait_path(&dir.join("authority-policy.sock"));
3873 let _ = hyprstream_rpc::auth::set_global_session_registry(Arc::new(
3886 crate::services::revocation::PolicyAuthoritySessionRegistry::new(
3887 policy_client_for_socket(&dir)?,
3888 ),
3889 ));
3890 let _ = hyprstream_rpc::auth::set_global_credential_revocation_store(Arc::new(
3891 crate::services::revocation::PolicyAuthorityRevocationStore::new(
3892 policy_client_for_socket(&dir)?,
3893 ),
3894 ));
3895 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
3896 let address = listener.local_addr()?;
3897 let mut config = test_config();
3898 config.external_url = Some(format!("http://{address}"));
3899 config.token_ttl_seconds = 60;
3900 let unused: Arc<dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor> =
3901 Arc::new(hyprstream_rpc::transport::rpc_session::from_fn(|_| async {
3902 Ok(bytes::Bytes::new())
3903 }));
3904 hyprstream_rpc::dial::register_inproc("multiprocess-unused-discovery", &unused);
3905 let discovery = hyprstream_rpc_std::discovery_client::DiscoveryClient::for_local_endpoint_bootstrap(
3906 "inproc://multiprocess-unused-discovery",
3907 SigningKey::from_bytes(&[0x76; 32]),
3908 SigningKey::from_bytes(&[0x77; 32]).verifying_key(),
3909 None,
3910 )?;
3911 let user_store = Arc::new(crate::auth::RocksDbUserStore::open(
3912 &dir.join("oauth-users"),
3913 )?);
3914 crate::auth::UserStore::register(user_store.as_ref(), "multiprocess-oauth").await?;
3915 crate::auth::UserStore::set_profile(
3916 user_store.as_ref(),
3917 "multiprocess-oauth",
3918 crate::auth::UserProfilePatch {
3919 atproto_did: Some(Some(
3920 "did:web:multiprocess-oauth.example.test".to_owned(),
3921 )),
3922 ..Default::default()
3923 },
3924 )
3925 .await?;
3926 let hosted_store = hosted_account_store("multiprocess-oauth", "example.test")?;
3927 hosted_store
3932 .refresh_hosted_did_index(&hyprstream_rpc::Subject::new(
3933 hyprstream_pds_service::OAUTH_ACCOUNT_RESOLVER_SUBJECT,
3934 ))
3935 .await?;
3936 let state = Arc::new(
3937 crate::services::oauth::state::OAuthState::new(
3938 &config,
3939 policy_client_for_socket(&dir)?,
3940 discovery,
3941 authority_ca_key().verifying_key().to_bytes(),
3942 )
3943 .with_user_store(crate::auth::ProductionUserStore::for_test(user_store))
3944 .with_hosted_account_zone(crate::account::AccountZone::new(
3945 "example.test",
3946 )?)
3947 .with_hosted_account_store(hosted_store),
3948 );
3949 let verifier = "multiprocess-pkce-verifier";
3950 let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes()));
3951 for code in [
3952 "multiprocess-code",
3953 "post-failure-code",
3954 "timeout-code",
3955 "post-crash-code",
3956 ] {
3957 state.pending_codes.write().await.insert(
3958 code.to_owned(),
3959 crate::services::oauth::state::PendingAuthCode {
3960 code: code.to_owned(),
3961 client_id: "multiprocess-client".to_owned(),
3962 redirect_uri: "https://client.test/callback".to_owned(),
3963 code_challenge: challenge.clone(),
3964 scopes: vec!["read".to_owned()],
3965 resource: Some("multiprocess".to_owned()),
3966 oidc_nonce: None,
3967 created_at: std::time::Instant::now(),
3968 expires_at: std::time::Instant::now()
3969 + std::time::Duration::from_secs(60),
3970 username: "multiprocess-oauth".to_owned(),
3971 verifying_key: None,
3972 dpop_jkt: None,
3973 client_assertion_jkt: None,
3974 },
3975 );
3976 }
3977 let app = crate::services::oauth::create_app(
3978 state,
3979 &crate::config::CorsConfig::default(),
3980 );
3981 let server = tokio::spawn(async move { axum::serve(listener, app).await });
3982 std::fs::write(dir.join("oauth-http-url"), format!("http://{address}"))?;
3983 std::fs::write(dir.join("ready-oauth"), b"1")?;
3984 wait_for_target_authority(&dir);
3985 std::fs::write(dir.join("converged-oauth"), b"1")?;
3986 verify_returned_tokens(&dir, "oauth").await?;
3987 wait_for_done(&dir);
3988 server.abort();
3989 anyhow::Ok(())
3990 })
3991 .unwrap();
3992 }
3993
3994 #[test]
3995 fn composite_policy_production_process() {
3996 let Some(dir) = authority_process_dir() else {
3997 return;
3998 };
3999 let runtime = tokio::runtime::Runtime::new().unwrap();
4000 runtime
4001 .block_on(async {
4002 install_signing_authority(&dir, true).await?;
4003 let _ = hyprstream_rpc::auth::set_global_session_registry(Arc::new(
4012 hyprstream_rpc::auth::InMemorySessionRegistry::new(),
4013 ));
4014 let _ = hyprstream_rpc::auth::set_global_credential_revocation_store(Arc::new(
4015 hyprstream_rpc::auth::InMemoryCredentialRevocationStore::new(),
4016 ));
4017 let db = dir.join(format!("policy-db-{}", std::process::id()));
4018 std::fs::create_dir_all(&db)?;
4019 let policy_manager = Arc::new(
4020 crate::auth::PolicyManager::new(&db.join(".registry/policies")).await?,
4021 );
4022 let git2db = Arc::new(tokio::sync::RwLock::new(git2db::Git2DB::open(&db).await?));
4023 let client_key = SigningKey::from_bytes(&[0x76; 32]);
4024 install_hybrid_request_anchor(&client_key)?;
4025 hyprstream_service::global_trust_store().insert(
4026 client_key.verifying_key(),
4027 hyprstream_service::Attestation {
4028 scopes: std::iter::once("oauth".to_owned()).collect(),
4029 subject: None,
4030 jwt: None,
4031 expires_at: chrono::Utc::now().timestamp() + 300,
4032 attested_by: None,
4033 },
4034 );
4035 let service = crate::services::PolicyService::new(
4036 policy_manager,
4037 Arc::new(SigningKey::from_bytes(&[0x73; 32])),
4038 crate::config::TokenConfig::default(),
4039 git2db,
4040 hyprstream_rpc::transport::TransportConfig::ipc(
4041 dir.join("authority-policy.sock"),
4042 ),
4043 )
4044 .with_default_audience("multiprocess".to_owned())
4045 .with_jwt_key_source(Arc::new(hyprstream_rpc::auth::ClusterKeySource::new(
4050 SigningKey::from_bytes(&[0x73; 32]).verifying_key(),
4051 "multiprocess".to_owned(),
4052 )))
4053 .with_primary_enrollment_resolver(test_permissive_primary_resolver());
4054 let shutdown = Arc::new(tokio::sync::Notify::new());
4055 let shutdown_server = Arc::clone(&shutdown);
4056 std::thread::spawn(move || {
4057 use hyprstream_rpc::service::Spawnable as _;
4058 Box::new(service).run(shutdown_server, None).unwrap();
4059 });
4060 wait_path(&dir.join("authority-policy.sock"));
4061 std::fs::write(dir.join("ready-policy"), b"1")?;
4062 wait_for_target_authority(&dir);
4063 std::fs::write(dir.join("converged-policy"), b"1")?;
4064 let token = policy_client_for_socket(&dir)?
4065 .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4066 requested_scopes: Some(vec!["read".to_owned()]),
4067 ttl: Some(60),
4068 audience: Some("multiprocess".to_owned()),
4069 subject: Some("multiprocess-policy".to_owned()),
4070 user_pub_key: Some(test_user_pub_key_b64()),
4071 dpop_jkt: None,
4072 issuer: None,
4073 tenant: None,
4074 require_clearance: false,
4075 session_id: None,
4076 issuance_profile:
4077 hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4078 client_id: Some("hyprstream-oauth-client-1".to_owned()),
4079 })
4080 .await?
4081 .token;
4082 std::fs::write(dir.join("policy-token"), token)?;
4083 verify_returned_tokens(&dir, "policy").await?;
4084 wait_for_done(&dir);
4085 shutdown.notify_waiters();
4086 anyhow::Ok(())
4087 })
4088 .unwrap();
4089 }
4090
4091 #[test]
4092 fn composite_jwks_production_process() {
4093 let Some(dir) = authority_process_dir() else {
4094 return;
4095 };
4096 let runtime = tokio::runtime::Runtime::new().unwrap();
4097 runtime
4098 .block_on(async {
4099 install_verifying_authority(&dir).await?;
4100 std::fs::write(dir.join("ready-jwks"), b"1")?;
4101 wait_for_target_authority(&dir);
4102 std::fs::write(dir.join("converged-jwks"), b"1")?;
4103 wait_path(&dir.join("jwks-response.json"));
4104 verify_returned_tokens(&dir, "jwks").await?;
4105 anyhow::Ok(())
4106 })
4107 .unwrap();
4108 wait_for_done(&dir);
4109 }
4110
4111 #[test]
4112 fn composite_rpc_production_process() {
4113 let Some(dir) = authority_process_dir() else {
4114 return;
4115 };
4116 let runtime = tokio::runtime::Runtime::new().unwrap();
4117 runtime
4118 .block_on(async {
4119 install_verifying_authority(&dir).await?;
4120 install_hybrid_request_anchor(&SigningKey::from_bytes(&[0x75; 32]))?;
4121 let accepted = Arc::new(std::sync::atomic::AtomicBool::new(false));
4122 let service = ProductionAuthorityVerifier {
4123 transport: hyprstream_rpc::transport::TransportConfig::ipc(
4124 dir.join("authority-rpc.sock"),
4125 ),
4126 signing_key: SigningKey::from_bytes(&[0x74; 32]),
4127 key_source: Arc::new(hyprstream_rpc::auth::ClusterKeySource::new(
4128 authority_ca_key().verifying_key(),
4129 "https://multiprocess.test".to_owned(),
4130 )),
4131 accepted,
4132 error: Arc::new(parking_lot::Mutex::new(None)),
4133 };
4134 let shutdown = Arc::new(tokio::sync::Notify::new());
4135 let shutdown_server = Arc::clone(&shutdown);
4136 std::thread::spawn(move || {
4137 use hyprstream_rpc::service::Spawnable as _;
4138 Box::new(service).run(shutdown_server, None).unwrap();
4139 });
4140 wait_path(&dir.join("authority-rpc.sock"));
4141 std::fs::write(dir.join("ready-rpc"), b"1")?;
4142 wait_for_target_authority(&dir);
4143 std::fs::write(dir.join("converged-rpc"), b"1")?;
4144 verify_returned_tokens(&dir, "rpc").await?;
4145 wait_for_done(&dir);
4146 shutdown.notify_waiters();
4147 anyhow::Ok(())
4148 })
4149 .unwrap();
4150 }
4151
4152 #[test]
4153 fn composite_stale_policy_production_process() {
4154 let Some(dir) = authority_process_dir() else {
4155 return;
4156 };
4157 let runtime = tokio::runtime::Runtime::new().unwrap();
4158 runtime
4159 .block_on(async {
4160 install_signing_authority(&dir, false).await?;
4161 std::fs::write(dir.join("ready-stale-policy"), b"1")?;
4162 wait_path(&dir.join("stale-attempt-gate"));
4163 let db = dir.join(format!("stale-policy-db-{}", std::process::id()));
4164 std::fs::create_dir_all(&db)?;
4165 let policy_manager = Arc::new(
4166 crate::auth::PolicyManager::new(&db.join(".registry/policies")).await?,
4167 );
4168 let git2db = Arc::new(tokio::sync::RwLock::new(git2db::Git2DB::open(&db).await?));
4169 let client_key = SigningKey::from_bytes(&[0x76; 32]);
4170 install_hybrid_request_anchor(&client_key)?;
4171 hyprstream_service::global_trust_store().insert(
4172 client_key.verifying_key(),
4173 hyprstream_service::Attestation {
4174 scopes: std::iter::once("oauth".to_owned()).collect(),
4175 subject: None,
4176 jwt: None,
4177 expires_at: chrono::Utc::now().timestamp() + 300,
4178 attested_by: None,
4179 },
4180 );
4181 let service = crate::services::PolicyService::new(
4182 policy_manager,
4183 Arc::new(SigningKey::from_bytes(&[0x73; 32])),
4184 crate::config::TokenConfig::default(),
4185 git2db,
4186 hyprstream_rpc::transport::TransportConfig::ipc(dir.join("stale-policy.sock")),
4187 )
4188 .with_default_audience("multiprocess".to_owned())
4189 .with_jwt_key_source(Arc::new(hyprstream_rpc::auth::ClusterKeySource::new(
4190 SigningKey::from_bytes(&[0x73; 32]).verifying_key(),
4191 "multiprocess".to_owned(),
4192 )))
4193 .with_primary_enrollment_resolver(test_permissive_primary_resolver());
4194 let shutdown = Arc::new(tokio::sync::Notify::new());
4195 let shutdown_server = Arc::clone(&shutdown);
4196 std::thread::spawn(move || {
4197 use hyprstream_rpc::service::Spawnable as _;
4198 Box::new(service).run(shutdown_server, None).unwrap();
4199 });
4200 wait_path(&dir.join("stale-policy.sock"));
4201 let result = policy_client_for_named_socket(&dir, "stale-policy.sock")?
4202 .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4203 requested_scopes: Some(vec!["read".to_owned()]),
4204 ttl: Some(60),
4205 audience: Some("multiprocess".to_owned()),
4206 subject: Some("stale-policy".to_owned()),
4207 user_pub_key: Some(test_user_pub_key_b64()),
4208 dpop_jkt: None,
4209 issuer: None,
4210 tenant: None,
4211 require_clearance: false,
4212 session_id: None,
4213 issuance_profile:
4214 hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4215 client_id: Some("hyprstream-oauth-client-1".to_owned()),
4216 })
4217 .await;
4218 anyhow::ensure!(result.is_err(), "stale PolicyService minted a token");
4219 shutdown.notify_waiters();
4220 std::fs::write(dir.join("stale-policy-refused"), b"1")?;
4221 anyhow::Ok(())
4222 })
4223 .unwrap();
4224 }
4225
4226 #[test]
4227 fn composite_stale_writer_process() {
4228 let Some(dir) = authority_process_dir() else {
4229 return;
4230 };
4231 let runtime = tokio::runtime::Runtime::new().unwrap();
4232 runtime
4233 .block_on(async {
4234 let (ed, pq) = install_signing_authority(&dir, false).await?;
4235 let expected = hyprstream_rpc::auth::global_composite_key_set()
4236 .snapshot()
4237 .component_digest()
4238 .to_owned();
4239 std::fs::write(dir.join("ready-stale-writer"), b"1")?;
4240 wait_path(&dir.join("stale-attempt-gate"));
4241 let result =
4242 refresh_composite_key_set(&dir, &ed, &pq, authority_ca_key(), 300, &expected)
4243 .await;
4244 let error = result.expect_err("stale writer publication unexpectedly succeeded");
4245 anyhow::ensure!(
4246 error
4247 .to_string()
4248 .contains("stale composite component authority"),
4249 "unexpected stale writer error: {error:#}"
4250 );
4251 std::fs::write(dir.join("stale-writer-refused"), error.to_string())?;
4252 anyhow::Ok(())
4253 })
4254 .unwrap();
4255 }
4256
4257 #[test]
4258 fn composite_crashing_writer_process() {
4259 let Some(dir) = authority_process_dir() else {
4260 return;
4261 };
4262 let runtime = tokio::runtime::Runtime::new().unwrap();
4263 runtime
4264 .block_on(async {
4265 let config = test_config();
4266 let ed = load_or_init_key_store(&dir, &config);
4267 let pq = load_or_init_ml_dsa_key_store(&dir, &config);
4268 mutate_authority_for_failure(&ed, &pq, Some(&dir)).await;
4269 let expected = std::fs::read_to_string(dir.join("crash-expected-digest"))?;
4270 refresh_composite_key_set(&dir, &ed, &pq, authority_ca_key(), 300, &expected).await
4271 })
4272 .unwrap();
4273 panic!("crash mutation returned instead of exiting after stage");
4274 }
4275
4276 #[test]
4277 fn composite_timing_out_writer_process() {
4278 let Some(dir) = authority_process_dir() else {
4279 return;
4280 };
4281 let runtime = tokio::runtime::Runtime::new().unwrap();
4282 runtime
4283 .block_on(async {
4284 let config = test_config();
4285 let ed = load_or_init_key_store(&dir, &config);
4286 let pq = load_or_init_ml_dsa_key_store(&dir, &config);
4287 mutate_authority_for_failure(&ed, &pq, None).await;
4288 let expected = std::fs::read_to_string(dir.join("timeout-expected-digest"))?;
4289 let error =
4290 refresh_composite_key_set(&dir, &ed, &pq, authority_ca_key(), 300, &expected)
4291 .await
4292 .expect_err("unacknowledged composite generation committed");
4293 anyhow::ensure!(
4294 error.to_string().contains("was not acknowledged"),
4295 "unexpected acknowledgement-timeout error: {error:#}"
4296 );
4297 std::fs::write(dir.join("timeout-writer-refused"), error.to_string())?;
4298 anyhow::Ok(())
4299 })
4300 .unwrap();
4301 }
4302
4303 #[test]
4304 fn composite_restart_rpc_production_process() {
4305 let Some(dir) = authority_process_dir() else {
4306 return;
4307 };
4308 let runtime = tokio::runtime::Runtime::new().unwrap();
4309 runtime
4310 .block_on(async {
4311 install_verifying_authority(&dir).await?;
4312 verify_through_rpc_endpoint(
4313 &dir,
4314 std::fs::read_to_string(dir.join("old-oauth-token"))?,
4315 )
4316 .await?;
4317 std::fs::write(dir.join("restart-drain-verified"), b"1")?;
4318 anyhow::Ok(())
4319 })
4320 .unwrap();
4321 }
4322
4323 #[test]
4324 fn composite_restart_policy_production_process() {
4325 let Some(dir) = authority_process_dir() else {
4326 return;
4327 };
4328 let runtime = tokio::runtime::Runtime::new().unwrap();
4329 runtime
4330 .block_on(async {
4331 install_signing_authority(&dir, false).await?;
4332 let db = dir.join(format!("restart-policy-db-{}", std::process::id()));
4333 std::fs::create_dir_all(&db)?;
4334 let policy_manager = Arc::new(
4335 crate::auth::PolicyManager::new(&db.join(".registry/policies")).await?,
4336 );
4337 let git2db = Arc::new(tokio::sync::RwLock::new(git2db::Git2DB::open(&db).await?));
4338 let client_key = SigningKey::from_bytes(&[0x76; 32]);
4339 install_hybrid_request_anchor(&client_key)?;
4340 hyprstream_service::global_trust_store().insert(
4341 client_key.verifying_key(),
4342 hyprstream_service::Attestation {
4343 scopes: std::iter::once("oauth".to_owned()).collect(),
4344 subject: None,
4345 jwt: None,
4346 expires_at: chrono::Utc::now().timestamp() + 300,
4347 attested_by: None,
4348 },
4349 );
4350 let socket = "restart-policy.sock";
4351 let service = crate::services::PolicyService::new(
4352 policy_manager,
4353 Arc::new(SigningKey::from_bytes(&[0x73; 32])),
4354 crate::config::TokenConfig::default(),
4355 git2db,
4356 hyprstream_rpc::transport::TransportConfig::ipc(dir.join(socket)),
4357 )
4358 .with_default_audience("multiprocess".to_owned())
4359 .with_jwt_key_source(Arc::new(hyprstream_rpc::auth::ClusterKeySource::new(
4360 SigningKey::from_bytes(&[0x73; 32]).verifying_key(),
4361 "multiprocess".to_owned(),
4362 )))
4363 .with_primary_enrollment_resolver(test_permissive_primary_resolver());
4364 let shutdown = Arc::new(tokio::sync::Notify::new());
4365 let shutdown_server = Arc::clone(&shutdown);
4366 std::thread::spawn(move || {
4367 use hyprstream_rpc::service::Spawnable as _;
4368 Box::new(service).run(shutdown_server, None).unwrap();
4369 });
4370 wait_path(&dir.join(socket));
4371 let token = policy_client_for_named_socket(&dir, socket)?
4372 .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4373 requested_scopes: Some(vec!["read".to_owned()]),
4374 ttl: Some(60),
4375 audience: Some("multiprocess".to_owned()),
4376 subject: Some("restart-policy".to_owned()),
4377 user_pub_key: Some(test_user_pub_key_b64()),
4378 dpop_jkt: None,
4379 issuer: None,
4380 tenant: None,
4381 require_clearance: false,
4382 session_id: None,
4383 issuance_profile:
4384 hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4385 client_id: Some("hyprstream-oauth-client-1".to_owned()),
4386 })
4387 .await?
4388 .token;
4389 verify_through_rpc_endpoint(&dir, token).await?;
4390 std::fs::write(dir.join("restart-policy-minted"), b"1")?;
4391 shutdown.notify_waiters();
4392 anyhow::Ok(())
4393 })
4394 .unwrap();
4395 }
4396
4397 fn spawn_production_authority_process(dir: &Path, test_name: &str) -> std::process::Child {
4398 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
4399 command
4400 .args(["--exact", test_name, "--nocapture"])
4401 .env("HYPRSTREAM_COMPOSITE_PRODUCTION_TEST_DIR", dir);
4402 if test_name.ends_with("composite_crashing_writer_process") {
4403 command.env("HYPRSTREAM_COMPOSITE_CRASH_AFTER_STAGE", "1");
4404 }
4405 if test_name.ends_with("composite_timing_out_writer_process") {
4406 command.env(
4407 "HYPRSTREAM_COMPOSITE_STAGE_MARKER",
4408 dir.join("timeout-staged"),
4409 );
4410 }
4411 command.spawn().unwrap()
4412 }
4413
4414 #[test]
4415 fn composite_authority_production_paths_reject_semantic_rollback() {
4416 const ORCHESTRATOR_DIR: &str = "HYPRSTREAM_COMPOSITE_ORCHESTRATOR_TEST_DIR";
4417 if std::env::var_os(ORCHESTRATOR_DIR).is_none() {
4418 let dir = TempDir::new().unwrap();
4421 let status = std::process::Command::new(std::env::current_exe().unwrap())
4422 .args([
4423 "--exact",
4424 "auth::key_rotation::tests::composite_authority_production_paths_reject_semantic_rollback",
4425 "--nocapture",
4426 ])
4427 .env(ORCHESTRATOR_DIR, dir.path())
4428 .status()
4429 .unwrap();
4430 assert!(status.success());
4431 return;
4432 }
4433
4434 struct AuthorityTestDir(PathBuf);
4435 impl AuthorityTestDir {
4436 fn path(&self) -> &Path {
4437 &self.0
4438 }
4439 }
4440 let dir = AuthorityTestDir(PathBuf::from(std::env::var_os(ORCHESTRATOR_DIR).unwrap()));
4441 let config = test_config();
4442 let now = chrono::Utc::now().timestamp();
4443 let old_ed = KeySlot::new(SigningKey::from_bytes(&[0x11; 32]), now - 10, now + 600);
4444 let (old_pq_key, _) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
4445 let old_pq = MlDsaKeySlot::new(old_pq_key, now - 10, now + 600);
4446 persist_slot(dir.path(), "active", &old_ed).unwrap();
4447 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &old_pq).unwrap();
4448 let ca = authority_ca_key();
4449 let runtime = tokio::runtime::Runtime::new().unwrap();
4450 let old_ed_store = load_or_init_key_store(dir.path(), &config);
4451 let old_pq_store = load_or_init_ml_dsa_key_store(dir.path(), &config);
4452 runtime
4453 .block_on(initialize_composite_key_set(
4454 dir.path(),
4455 &old_ed_store,
4456 &old_pq_store,
4457 Arc::clone(&ca),
4458 300,
4459 ))
4460 .unwrap();
4461 let old_digest = hyprstream_rpc::auth::global_composite_key_set()
4462 .snapshot()
4463 .component_digest()
4464 .to_owned();
4465
4466 let process_tests = [
4467 (
4468 "oauth",
4469 "auth::key_rotation::tests::composite_oauth_production_process",
4470 ),
4471 (
4472 "policy",
4473 "auth::key_rotation::tests::composite_policy_production_process",
4474 ),
4475 (
4476 "jwks",
4477 "auth::key_rotation::tests::composite_jwks_production_process",
4478 ),
4479 (
4480 "rpc",
4481 "auth::key_rotation::tests::composite_rpc_production_process",
4482 ),
4483 ];
4484 let mut services: Vec<_> = process_tests
4485 .iter()
4486 .map(|(_, test)| spawn_production_authority_process(dir.path(), test))
4487 .collect();
4488 let mut stale_policy = spawn_production_authority_process(
4489 dir.path(),
4490 "auth::key_rotation::tests::composite_stale_policy_production_process",
4491 );
4492 let mut stale_writer = spawn_production_authority_process(
4493 dir.path(),
4494 "auth::key_rotation::tests::composite_stale_writer_process",
4495 );
4496 for process in [
4497 "oauth",
4498 "policy",
4499 "jwks",
4500 "rpc",
4501 "stale-policy",
4502 "stale-writer",
4503 ] {
4504 wait_path(&dir.path().join(format!("ready-{process}")));
4505 }
4506
4507 let oauth_url = std::fs::read_to_string(dir.path().join("oauth-http-url")).unwrap();
4510 let old_token = runtime
4511 .block_on(async {
4512 policy_client_for_socket(dir.path())?
4513 .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4514 requested_scopes: Some(vec!["read".to_owned()]),
4515 ttl: Some(60),
4516 audience: Some("multiprocess".to_owned()),
4517 subject: Some("pre-rotation-policy".to_owned()),
4518 user_pub_key: Some(test_user_pub_key_b64()),
4519 dpop_jkt: None,
4520 issuer: None,
4521 tenant: None,
4522 require_clearance: false,
4523 session_id: None,
4524 issuance_profile:
4525 hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4526 client_id: Some("hyprstream-oauth-client-1".to_owned()),
4527 })
4528 .await?;
4529 let client = reqwest::Client::new();
4530 let authorization_code_form = [
4531 ("grant_type", "authorization_code"),
4532 ("client_id", "multiprocess-client"),
4533 ("code", "multiprocess-code"),
4534 ("redirect_uri", "https://client.test/callback"),
4535 ("code_verifier", "multiprocess-pkce-verifier"),
4536 ];
4537 let response = client
4538 .post(format!("{oauth_url}/oauth/token"))
4539 .form(&authorization_code_form)
4540 .send()
4541 .await?;
4542 if !response.status().is_success() {
4543 let status = response.status();
4544 let body = response.text().await.unwrap_or_default();
4545 anyhow::bail!("pre-rotation OAuth endpoint returned {status}: {body}");
4546 }
4547 let body: serde_json::Value = response.json().await?;
4548 let token = body
4549 .get("access_token")
4550 .and_then(serde_json::Value::as_str)
4551 .map(str::to_owned)
4552 .ok_or_else(|| {
4553 anyhow::anyhow!("pre-rotation OAuth response omitted access_token")
4554 })?;
4555 let replay = client
4556 .post(format!("{oauth_url}/oauth/token"))
4557 .form(&authorization_code_form)
4558 .send()
4559 .await?;
4560 anyhow::ensure!(
4561 replay.status() == reqwest::StatusCode::BAD_REQUEST,
4562 "authorization-code replay returned {}, expected 400",
4563 replay.status()
4564 );
4565 let replay_body: serde_json::Value = replay.json().await?;
4566 anyhow::ensure!(
4567 replay_body.get("error").and_then(serde_json::Value::as_str)
4568 == Some("invalid_grant"),
4569 "authorization-code replay was not rejected as invalid_grant: {replay_body}"
4570 );
4571 anyhow::Ok(token)
4572 })
4573 .unwrap();
4574 std::fs::write(dir.path().join("old-oauth-token"), old_token).unwrap();
4575
4576 persist_slot(dir.path(), "drain", &old_ed).unwrap();
4577 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "drain", &old_pq).unwrap();
4578 let new_ed = KeySlot::new(SigningKey::from_bytes(&[0x22; 32]), now, now + 1200);
4579 let (new_pq_key, _) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
4580 let new_pq = MlDsaKeySlot::new(new_pq_key, now, now + 1200);
4581 persist_slot(dir.path(), "active", &new_ed).unwrap();
4582 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &new_pq).unwrap();
4583 let new_ed_store = load_or_init_key_store(dir.path(), &config);
4584 let new_pq_store = load_or_init_ml_dsa_key_store(dir.path(), &config);
4585 runtime
4586 .block_on(refresh_composite_key_set(
4587 dir.path(),
4588 &new_ed_store,
4589 &new_pq_store,
4590 Arc::clone(&ca),
4591 300,
4592 &old_digest,
4593 ))
4594 .unwrap();
4595 let committed: CompositeCommit =
4596 serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
4597 .unwrap();
4598 let ledger_after_rotation: CompositeLedger =
4599 serde_json::from_slice(&std::fs::read(composite_ledger_path(dir.path())).unwrap())
4600 .unwrap();
4601 assert_eq!(committed.version, ledger_after_rotation.version);
4602 assert_eq!(
4603 committed.component_digest,
4604 ledger_after_rotation.component_digest
4605 );
4606 assert_ne!(committed.component_digest, old_digest);
4607
4608 let committed_immutable = composite_committed_ledger_path(dir.path(), &committed);
4613 std::fs::remove_file(&committed_immutable).unwrap();
4614 runtime
4615 .block_on(initialize_composite_key_set(
4616 dir.path(),
4617 &new_ed_store,
4618 &new_pq_store,
4619 Arc::clone(&ca),
4620 300,
4621 ))
4622 .unwrap();
4623 let migrated: CompositeLedger =
4624 serde_json::from_slice(&std::fs::read(&committed_immutable).unwrap()).unwrap();
4625 assert_eq!(migrated.version, committed.version);
4626 assert_eq!(migrated.component_digest, committed.component_digest);
4627 assert_eq!(
4628 std::fs::read(composite_ledger_path(dir.path())).unwrap(),
4629 serde_json::to_vec(&ledger_after_rotation).unwrap(),
4630 "legacy migration changed mutable B before pending C was staged"
4631 );
4632 publish_target_authority(dir.path(), &committed);
4633 for process in ["oauth", "policy", "jwks", "rpc"] {
4634 wait_path(&dir.path().join(format!("converged-{process}")));
4635 }
4636
4637 let oauth_url = std::fs::read_to_string(dir.path().join("oauth-http-url")).unwrap();
4638 let (oauth_token, jwks) = runtime
4639 .block_on(async {
4640 let client = reqwest::Client::new();
4641 let response = client
4642 .post(format!("{oauth_url}/oauth/token"))
4643 .form(&[
4644 ("grant_type", "authorization_code"),
4645 ("client_id", "multiprocess-client"),
4646 ("code", "post-failure-code"),
4647 ("redirect_uri", "https://client.test/callback"),
4648 ("code_verifier", "multiprocess-pkce-verifier"),
4649 ])
4650 .send()
4651 .await?;
4652 anyhow::ensure!(
4653 response.status().is_success(),
4654 "ordinary OAuth token endpoint returned {}: {}",
4655 response.status(),
4656 response.text().await?
4657 );
4658 let body: serde_json::Value = response.json().await?;
4659 let token = body
4660 .get("access_token")
4661 .and_then(serde_json::Value::as_str)
4662 .ok_or_else(|| anyhow::anyhow!("OAuth response omitted access_token"))?
4663 .to_owned();
4664 let jwks_response = client.get(format!("{oauth_url}/oauth/jwks")).send().await?;
4665 anyhow::ensure!(
4666 jwks_response.status().is_success(),
4667 "HTTP JWKS endpoint returned {}",
4668 jwks_response.status()
4669 );
4670 let jwks = jwks_response.json::<serde_json::Value>().await?;
4671 anyhow::Ok((token, jwks))
4672 })
4673 .unwrap();
4674 std::fs::write(dir.path().join("oauth-token"), oauth_token).unwrap();
4675 std::fs::write(
4676 dir.path().join("jwks-response.json"),
4677 serde_json::to_vec(&jwks).unwrap(),
4678 )
4679 .unwrap();
4680
4681 wait_path(&dir.path().join("oauth-token"));
4682 wait_path(&dir.path().join("policy-token"));
4683 std::fs::write(dir.path().join("tokens-ready"), b"1").unwrap();
4684 for process in ["oauth", "policy", "jwks", "rpc"] {
4685 wait_path(&dir.path().join(format!("verified-{process}")));
4686 }
4687 wait_path(&dir.path().join("jwks-response.json"));
4688 let jwks: serde_json::Value =
4689 serde_json::from_slice(&std::fs::read(dir.path().join("jwks-response.json")).unwrap())
4690 .unwrap();
4691 assert_eq!(jwks["composite_version"].as_u64(), Some(committed.version));
4692 assert_eq!(
4693 jwks["composite_component_digest"].as_str(),
4694 Some(committed.component_digest.as_str())
4695 );
4696 let active_kids: Vec<&str> = ledger_after_rotation
4697 .pairs
4698 .iter()
4699 .filter(|pair| pair.state == "active")
4700 .map(|pair| pair.kid.as_str())
4701 .collect();
4702 for kid in active_kids {
4703 assert!(jwks["keys"]
4704 .as_array()
4705 .unwrap()
4706 .iter()
4707 .any(|key| { key.get("kid").and_then(serde_json::Value::as_str) == Some(kid) }));
4708 }
4709
4710 std::fs::write(
4714 dir.path().join("timeout-expected-digest"),
4715 &committed.component_digest,
4716 )
4717 .unwrap();
4718 let mut timing_out_writer = spawn_production_authority_process(
4719 dir.path(),
4720 "auth::key_rotation::tests::composite_timing_out_writer_process",
4721 );
4722 wait_path(&dir.path().join("timeout-staged"));
4723 let (timeout_oauth_token, timeout_policy_token, timeout_jwks) = runtime
4724 .block_on(async {
4725 let client = reqwest::Client::new();
4726 let response = client
4727 .post(format!("{oauth_url}/oauth/token"))
4728 .form(&[
4729 ("grant_type", "authorization_code"),
4730 ("client_id", "multiprocess-client"),
4731 ("code", "timeout-code"),
4732 ("redirect_uri", "https://client.test/callback"),
4733 ("code_verifier", "multiprocess-pkce-verifier"),
4734 ])
4735 .send()
4736 .await?;
4737 anyhow::ensure!(
4738 response.status().is_success(),
4739 "OAuth mint failed while publication was pending: {}",
4740 response.text().await?
4741 );
4742 let oauth = response.json::<serde_json::Value>().await?["access_token"]
4743 .as_str()
4744 .ok_or_else(|| anyhow::anyhow!("pending-timeout OAuth token missing"))?
4745 .to_owned();
4746 let policy = policy_client_for_socket(dir.path())?
4747 .issue_token(&hyprstream_rpc_std::policy_client::IssueToken {
4748 requested_scopes: Some(vec!["read".to_owned()]),
4749 ttl: Some(60),
4750 audience: Some("multiprocess".to_owned()),
4751 subject: Some("timeout-policy".to_owned()),
4752 user_pub_key: Some(test_user_pub_key_b64()),
4753 dpop_jkt: None,
4754 issuer: None,
4755 tenant: None,
4756 require_clearance: false,
4757 session_id: None,
4758 issuance_profile:
4759 hyprstream_rpc_std::policy_client::IssueTokenProfile::Rfc8693,
4760 client_id: Some("hyprstream-oauth-client-1".to_owned()),
4761 })
4762 .await?
4763 .token;
4764 let jwks = client
4765 .get(format!("{oauth_url}/oauth/jwks"))
4766 .send()
4767 .await?
4768 .json::<serde_json::Value>()
4769 .await?;
4770 verify_through_rpc_endpoint(dir.path(), oauth.clone()).await?;
4771 verify_through_rpc_endpoint(dir.path(), policy.clone()).await?;
4772 anyhow::Ok((oauth, policy, jwks))
4773 })
4774 .unwrap();
4775 assert_eq!(
4776 timeout_jwks["composite_version"].as_u64(),
4777 Some(committed.version)
4778 );
4779 assert_eq!(
4780 timeout_jwks["composite_component_digest"].as_str(),
4781 Some(committed.component_digest.as_str())
4782 );
4783 wait_path(&dir.path().join("timeout-writer-refused"));
4784 assert!(timing_out_writer.wait().unwrap().success());
4785 let committed_after_timeout: CompositeCommit =
4786 serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
4787 .unwrap();
4788 assert_eq!(committed_after_timeout.version, committed.version);
4789 assert_eq!(
4790 committed_after_timeout.component_digest,
4791 committed.component_digest
4792 );
4793
4794 std::fs::write(
4798 dir.path().join("crash-expected-digest"),
4799 &committed.component_digest,
4800 )
4801 .unwrap();
4802 let mut crashing_writer = spawn_production_authority_process(
4803 dir.path(),
4804 "auth::key_rotation::tests::composite_crashing_writer_process",
4805 );
4806 assert_eq!(crashing_writer.wait().unwrap().code(), Some(86));
4807 let pending_after_crash: CompositeLedger =
4808 serde_json::from_slice(&std::fs::read(composite_ledger_path(dir.path())).unwrap())
4809 .unwrap();
4810 assert_ne!(
4811 pending_after_crash.component_digest,
4812 committed.component_digest
4813 );
4814 let (post_crash_token, post_crash_jwks) = runtime
4815 .block_on(async {
4816 let client = reqwest::Client::new();
4817 let response = client
4818 .post(format!("{oauth_url}/oauth/token"))
4819 .form(&[
4820 ("grant_type", "authorization_code"),
4821 ("client_id", "multiprocess-client"),
4822 ("code", "post-crash-code"),
4823 ("redirect_uri", "https://client.test/callback"),
4824 ("code_verifier", "multiprocess-pkce-verifier"),
4825 ])
4826 .send()
4827 .await?;
4828 anyhow::ensure!(
4829 response.status().is_success(),
4830 "OAuth mint failed after writer crash: {}",
4831 response.text().await?
4832 );
4833 let token = response.json::<serde_json::Value>().await?["access_token"]
4834 .as_str()
4835 .ok_or_else(|| anyhow::anyhow!("post-crash OAuth token missing"))?
4836 .to_owned();
4837 let jwks = client
4838 .get(format!("{oauth_url}/oauth/jwks"))
4839 .send()
4840 .await?
4841 .json::<serde_json::Value>()
4842 .await?;
4843 verify_through_rpc_endpoint(dir.path(), token.clone()).await?;
4844 anyhow::Ok((token, jwks))
4845 })
4846 .unwrap();
4847 assert_eq!(
4848 post_crash_jwks["composite_version"].as_u64(),
4849 Some(committed.version)
4850 );
4851 assert_eq!(
4852 post_crash_jwks["composite_component_digest"].as_str(),
4853 Some(committed.component_digest.as_str())
4854 );
4855 for pending_only in pending_after_crash.pairs.iter().filter(|candidate| {
4856 !ledger_after_rotation
4857 .pairs
4858 .iter()
4859 .any(|committed_pair| committed_pair.kid == candidate.kid)
4860 }) {
4861 assert!(!post_crash_jwks["keys"]
4862 .as_array()
4863 .unwrap()
4864 .iter()
4865 .any(|key| key["kid"].as_str() == Some(pending_only.kid.as_str())));
4866 }
4867 let _ = (timeout_oauth_token, timeout_policy_token, post_crash_token);
4868
4869 let mut restart = spawn_production_authority_process(
4870 dir.path(),
4871 "auth::key_rotation::tests::composite_restart_rpc_production_process",
4872 );
4873 wait_path(&dir.path().join("restart-drain-verified"));
4874 assert!(restart.wait().unwrap().success());
4875
4876 let mut restart_policy = spawn_production_authority_process(
4877 dir.path(),
4878 "auth::key_rotation::tests::composite_restart_policy_production_process",
4879 );
4880 wait_path(&dir.path().join("restart-policy-minted"));
4881 assert!(restart_policy.wait().unwrap().success());
4882
4883 std::fs::write(dir.path().join("stale-attempt-gate"), b"1").unwrap();
4884 wait_path(&dir.path().join("stale-policy-refused"));
4885 wait_path(&dir.path().join("stale-writer-refused"));
4886 assert!(stale_policy.wait().unwrap().success());
4887 assert!(stale_writer.wait().unwrap().success());
4888 let ledger_after_stale: CompositeLedger =
4889 serde_json::from_slice(&std::fs::read(composite_ledger_path(dir.path())).unwrap())
4890 .unwrap();
4891 assert_eq!(ledger_after_stale.version, pending_after_crash.version);
4892 assert_eq!(
4893 ledger_after_stale.component_digest,
4894 pending_after_crash.component_digest
4895 );
4896 assert_eq!(
4897 ledger_after_stale.pairs.len(),
4898 pending_after_crash.pairs.len()
4899 );
4900 for pair in &pending_after_crash.pairs {
4901 assert!(ledger_after_stale.pairs.iter().any(|candidate| {
4902 candidate.kid == pair.kid
4903 && candidate.role == pair.role
4904 && candidate.state == pair.state
4905 }));
4906 }
4907 let committed_after_stale: CompositeCommit =
4908 serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
4909 .unwrap();
4910 assert_eq!(committed_after_stale.version, committed.version);
4911 assert_eq!(
4912 committed_after_stale.component_digest,
4913 committed.component_digest
4914 );
4915 runtime
4916 .block_on(async {
4917 for token_file in ["oauth-token", "policy-token"] {
4918 verify_through_rpc_endpoint(
4919 dir.path(),
4920 std::fs::read_to_string(dir.path().join(token_file))?,
4921 )
4922 .await?;
4923 }
4924 anyhow::Ok(())
4925 })
4926 .unwrap();
4927
4928 std::fs::write(dir.path().join("done"), b"1").unwrap();
4929 for service in &mut services {
4930 assert!(service.wait().unwrap().success());
4931 }
4932 }
4933
4934 #[test]
4935 fn load_or_init_creates_active_key_on_first_boot() {
4936 let dir = TempDir::new().unwrap();
4937 let config = test_config();
4938 let store = load_or_init_key_store(dir.path(), &config);
4939 let rt = tokio::runtime::Runtime::new().unwrap();
4940 let vk = rt.block_on(store.active_verifying_key_bytes());
4941 assert!(
4942 vk.is_some(),
4943 "active key should be present after first boot"
4944 );
4945 }
4946
4947 #[test]
4948 fn committed_marker_with_missing_slots_does_not_bootstrap_replacement_keys() {
4949 let dir = TempDir::new().unwrap();
4950 let config = test_config();
4951 let marker = composite_committed_path(dir.path());
4952 std::fs::write(&marker, b"committed but unreadable").unwrap();
4956
4957 let ed = load_or_init_key_store(dir.path(), &config);
4958 let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
4959 let rt = tokio::runtime::Runtime::new().unwrap();
4960 assert!(rt.block_on(ed.active_verifying_key_bytes()).is_none());
4961 assert!(rt.block_on(pq.active_key()).is_none());
4962 assert_eq!(std::fs::read(&marker).unwrap(), b"committed but unreadable");
4963 for name in [
4964 "jwt-signing-key.active",
4965 "jwt-signing-key.active.meta",
4966 "jwt-signing-key.lead",
4967 "jwt-signing-key.lead.meta",
4968 "ml-dsa-signing-key.active",
4969 "ml-dsa-signing-key.active.meta",
4970 "ml-dsa-signing-key.lead",
4971 "ml-dsa-signing-key.lead.meta",
4972 ] {
4973 assert!(
4974 !dir.path().join(name).exists(),
4975 "committed store unexpectedly created {name}"
4976 );
4977 }
4978 }
4979
4980 #[tokio::test]
4981 async fn committed_marker_does_not_generate_missing_lead_slots() {
4982 let dir = TempDir::new().unwrap();
4983 let config = test_config();
4984 let now = chrono::Utc::now().timestamp();
4985 let marker = composite_committed_path(dir.path());
4986 std::fs::write(&marker, b"committed").unwrap();
4987 persist_slot(
4988 dir.path(),
4989 "active",
4990 &KeySlot::new(SigningKey::from_bytes(&[0x71; 32]), now - 60, now + 1),
4991 )
4992 .unwrap();
4993 ml_dsa_rotation::persist_ml_dsa_slot(
4994 dir.path(),
4995 "active",
4996 &ml_dsa_rotation::generate_ml_dsa_slot(now - 60, now + 1),
4997 )
4998 .unwrap();
4999
5000 let ed = load_or_init_key_store(dir.path(), &config);
5001 let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
5002 assert!(ed.0.read().await.active.is_some());
5003 assert!(ed.0.read().await.lead.is_none());
5004 assert!(pq.0.read().await.active.is_some());
5005 assert!(pq.0.read().await.lead.is_none());
5006 for name in [
5007 "jwt-signing-key.lead",
5008 "jwt-signing-key.lead.meta",
5009 "ml-dsa-signing-key.lead",
5010 "ml-dsa-signing-key.lead.meta",
5011 ] {
5012 assert!(
5013 !dir.path().join(name).exists(),
5014 "committed store unexpectedly created {name}"
5015 );
5016 }
5017 }
5018
5019 #[test]
5020 fn persist_and_reload_slot() {
5021 let dir = TempDir::new().unwrap();
5022 let config = test_config();
5023 {
5024 let store = load_or_init_key_store(dir.path(), &config);
5025 let rt = tokio::runtime::Runtime::new().unwrap();
5026 let vk1 = rt.block_on(store.active_verifying_key_bytes()).unwrap();
5027 let store2 = load_or_init_key_store(dir.path(), &config);
5029 let vk2 = rt.block_on(store2.active_verifying_key_bytes()).unwrap();
5030 assert_eq!(vk1, vk2, "persisted key must reload identically");
5031 }
5032 }
5033
5034 #[tokio::test]
5035 async fn rotate_promotes_lead_to_active() {
5036 let dir = TempDir::new().unwrap();
5037 let config = test_config();
5038 let now = chrono::Utc::now().timestamp();
5039
5040 let active_slot = KeySlot::new(
5042 SigningKey::generate(&mut rand::rngs::OsRng),
5043 now - 14 * 86400,
5044 now + 1,
5045 );
5046 let lead_slot = KeySlot::new(
5047 SigningKey::generate(&mut rand::rngs::OsRng),
5048 now - 1, now + 14 * 86400,
5050 );
5051 persist_slot(dir.path(), "active", &active_slot).unwrap();
5052 persist_slot(dir.path(), "lead", &lead_slot).unwrap();
5053 let lead_vk = lead_slot.verifying_key_bytes();
5054
5055 let store = SigningKeyStore::new(KeySlots {
5056 drain: None,
5057 active: Some(active_slot),
5058 lead: Some(lead_slot),
5059 });
5060
5061 rotate_jwt_keys(&config, dir.path(), &store, now).await;
5062
5063 let new_active_vk = store.active_verifying_key_bytes().await.unwrap();
5064 assert_eq!(
5065 new_active_vk, lead_vk,
5066 "lead must become active after promotion"
5067 );
5068
5069 let slots = store.0.read().await;
5071 assert!(slots.drain.is_some(), "old active must become drain");
5072 assert!(
5073 slots.lead.is_none(),
5074 "lead slot must be cleared after promotion"
5075 );
5076 }
5077
5078 #[tokio::test]
5079 async fn drain_is_removed_after_drain_window() {
5080 let dir = TempDir::new().unwrap();
5081 let config = test_config();
5082 let now = chrono::Utc::now().timestamp();
5083
5084 let drain_slot = KeySlot::new(
5085 SigningKey::generate(&mut rand::rngs::OsRng),
5086 now - 60 * 86400,
5087 now - 31 * 86400, );
5089 let active_slot = KeySlot::new(
5090 SigningKey::generate(&mut rand::rngs::OsRng),
5091 now - 86400,
5092 now + 13 * 86400,
5093 );
5094 persist_slot(dir.path(), "drain", &drain_slot).unwrap();
5095 persist_slot(dir.path(), "active", &active_slot).unwrap();
5096
5097 let store = SigningKeyStore::new(KeySlots {
5098 drain: Some(drain_slot),
5099 active: Some(active_slot),
5100 lead: None,
5101 });
5102
5103 rotate_jwt_keys(&config, dir.path(), &store, now).await;
5104
5105 let slots = store.0.read().await;
5106 assert!(
5107 slots.drain.is_none(),
5108 "drain must be removed after drain window closes"
5109 );
5110 }
5111
5112 #[tokio::test]
5113 async fn lead_is_generated_when_active_near_expiry() {
5114 let dir = TempDir::new().unwrap();
5115 let config = test_config();
5116 let now = chrono::Utc::now().timestamp();
5117
5118 let active_slot = KeySlot::new(
5120 SigningKey::generate(&mut rand::rngs::OsRng),
5121 now - 8 * 86400,
5122 now + 6 * 86400,
5123 );
5124 persist_slot(dir.path(), "active", &active_slot).unwrap();
5125
5126 let store = SigningKeyStore::new(KeySlots {
5127 drain: None,
5128 active: Some(active_slot),
5129 lead: None,
5130 });
5131
5132 rotate_jwt_keys(&config, dir.path(), &store, now).await;
5133
5134 let slots = store.0.read().await;
5135 assert!(
5136 slots.lead.is_some(),
5137 "lead must be generated when active is within lead window"
5138 );
5139 }
5140
5141 #[tokio::test]
5150 async fn first_boot_persistence_failure_leaves_no_active_ed25519_key() {
5151 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_FIRST_BOOT_ED_TEST";
5156 if std::env::var_os(ISOLATED).is_none() {
5157 let status = std::process::Command::new(std::env::current_exe().unwrap())
5158 .args([
5159 "--exact",
5160 "auth::key_rotation::tests::first_boot_persistence_failure_leaves_no_active_ed25519_key",
5161 "--nocapture",
5162 ])
5163 .env(ISOLATED, "1")
5164 .status()
5165 .unwrap();
5166 assert!(status.success());
5167 return;
5168 }
5169 let dir = TempDir::new().unwrap();
5170 let config = test_config();
5171 std::fs::create_dir(dir.path().join("jwt-signing-key.active")).unwrap();
5174
5175 let store = load_or_init_key_store(dir.path(), &config);
5176 assert!(
5177 store.active_verifying_key_bytes().await.is_none(),
5178 "an unpersisted generated key must not be exposed as active"
5179 );
5180 assert!(
5181 !dir.path().join("jwt-signing-key.active.meta").exists(),
5182 "no slot metadata may exist when the key bytes failed to persist"
5183 );
5184
5185 let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
5189 let error = initialize_composite_key_set(
5190 dir.path(),
5191 &store,
5192 &pq,
5193 Arc::new(SigningKey::from_bytes(&[0x5A; 32])),
5194 30,
5195 )
5196 .await
5197 .expect_err("incomplete first-boot store must not create composite authority");
5198 assert!(
5199 error.to_string().contains("incomplete local signer set"),
5200 "unexpected initialization error: {error:#}"
5201 );
5202 assert!(
5203 !dir.path().join("jwt-composite-pairs.json").exists(),
5204 "no pending ledger may be created from an incomplete store"
5205 );
5206 assert!(
5207 !dir.path().join("jwt-composite-pairs.committed").exists(),
5208 "no commit marker may be created from an incomplete store"
5209 );
5210 let snapshots = std::fs::read_dir(dir.path())
5211 .unwrap()
5212 .filter_map(Result::ok)
5213 .filter(|entry| {
5214 entry
5215 .file_name()
5216 .to_string_lossy()
5217 .starts_with("jwt-composite-pairs.committed-")
5218 })
5219 .count();
5220 assert_eq!(
5221 snapshots, 0,
5222 "no immutable committed snapshot may be created from an incomplete store"
5223 );
5224
5225 std::fs::remove_dir(dir.path().join("jwt-signing-key.active")).unwrap();
5228 let fresh_ed = load_or_init_key_store(dir.path(), &config);
5229 let fresh_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
5230 initialize_composite_key_set(
5231 dir.path(),
5232 &fresh_ed,
5233 &fresh_pq,
5234 Arc::new(SigningKey::from_bytes(&[0x5A; 32])),
5235 30,
5236 )
5237 .await
5238 .expect("normal initialization must succeed once persistence recovers");
5239 assert!(dir.path().join("jwt-composite-pairs.json").exists());
5240 assert!(dir.path().join("jwt-composite-pairs.committed").exists());
5241 }
5242
5243 #[tokio::test]
5244 async fn ed25519_rotation_keeps_active_when_drain_persist_fails() {
5245 let dir = TempDir::new().unwrap();
5246 let config = test_config();
5247 let now = chrono::Utc::now().timestamp();
5248
5249 let active_slot = KeySlot::new(
5250 SigningKey::generate(&mut rand::rngs::OsRng),
5251 now - 14 * 86400,
5252 now + 1,
5253 );
5254 let lead_slot = KeySlot::new(
5255 SigningKey::generate(&mut rand::rngs::OsRng),
5256 now - 1, now + 14 * 86400,
5258 );
5259 let drain_slot = KeySlot::new(
5260 SigningKey::generate(&mut rand::rngs::OsRng),
5261 now - 45 * 86400,
5262 now + 5 * 86400,
5263 );
5264 let active_vk = active_slot.verifying_key_bytes();
5265 let lead_vk = lead_slot.verifying_key_bytes();
5266 let drain_vk = drain_slot.verifying_key_bytes();
5267
5268 let store = SigningKeyStore::new(KeySlots {
5269 drain: Some(drain_slot),
5270 active: Some(active_slot),
5271 lead: Some(lead_slot),
5272 });
5273
5274 let broken_state = dir.path().join("not-a-directory");
5277 std::fs::write(&broken_state, b"file").unwrap();
5278 rotate_jwt_keys_in_state_dir(&config, &broken_state, &store, now).await;
5279
5280 let slots = store.0.read().await;
5281 assert_eq!(
5282 slots.active.as_ref().map(KeySlot::verifying_key_bytes),
5283 Some(active_vk),
5284 "the old active must remain the in-memory active signer"
5285 );
5286 assert_eq!(
5287 slots.lead.as_ref().map(KeySlot::verifying_key_bytes),
5288 Some(lead_vk),
5289 "the queued lead must stay in memory for the next tick"
5290 );
5291 assert_eq!(
5292 slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5293 Some(drain_vk),
5294 "a failed promotion must not evict the in-memory drain"
5295 );
5296 drop(slots);
5297
5298 rotate_jwt_keys_in_state_dir(&config, dir.path(), &store, now).await;
5301 assert_eq!(
5302 store.active_verifying_key_bytes().await,
5303 Some(lead_vk),
5304 "the queued lead must promote once persistence succeeds"
5305 );
5306 let persisted = load_slot(dir.path(), "active").expect("promoted active must be persisted");
5307 assert_eq!(
5308 persisted.verifying_key_bytes(),
5309 lead_vk,
5310 "memory promotion must follow the durable active write"
5311 );
5312 let slots = store.0.read().await;
5313 assert_eq!(
5314 slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5315 Some(active_vk),
5316 "the old active becomes drain only after its persist succeeded"
5317 );
5318 assert!(slots.lead.is_none());
5319 assert!(!dir.path().join("jwt-signing-key.lead").exists());
5320 }
5321
5322 #[tokio::test]
5323 async fn ed25519_rotation_keeps_active_when_active_persist_fails() {
5324 let dir = TempDir::new().unwrap();
5325 let config = test_config();
5326 let now = chrono::Utc::now().timestamp();
5327
5328 let active_slot = KeySlot::new(
5329 SigningKey::generate(&mut rand::rngs::OsRng),
5330 now - 14 * 86400,
5331 now + 1,
5332 );
5333 let lead_slot = KeySlot::new(
5334 SigningKey::generate(&mut rand::rngs::OsRng),
5335 now - 1,
5336 now + 14 * 86400,
5337 );
5338 let active_vk = active_slot.verifying_key_bytes();
5339 let lead_vk = lead_slot.verifying_key_bytes();
5340 persist_slot(dir.path(), "lead", &lead_slot).unwrap();
5341
5342 let store = SigningKeyStore::new(KeySlots {
5343 drain: None,
5344 active: Some(active_slot),
5345 lead: Some(lead_slot),
5346 });
5347
5348 std::fs::create_dir(dir.path().join("jwt-signing-key.active")).unwrap();
5354 let restored = rotate_jwt_keys_in_state_dir(&config, dir.path(), &store, now).await;
5355 assert!(
5356 !restored,
5357 "a failed rollback must report the stores as not fully restored"
5358 );
5359
5360 assert_eq!(
5361 store.active_verifying_key_bytes().await,
5362 Some(active_vk),
5363 "the failed new key must not be exposed as active in memory"
5364 );
5365 {
5366 let slots = store.0.read().await;
5367 assert!(slots.lead.is_some(), "the lead stays queued for retry");
5368 assert!(
5369 slots.drain.is_none(),
5370 "in-memory drain must not change before every write succeeded"
5371 );
5372 }
5373 assert_eq!(
5374 load_slot(dir.path(), "drain")
5375 .expect("the drain copy of the old active must survive a failed rollback")
5376 .verifying_key_bytes(),
5377 active_vk,
5378 "a failed rollback must not erase the recoverable old active signer"
5379 );
5380 assert!(
5381 dir.path().join("jwt-signing-key.active").is_dir()
5382 && load_slot(dir.path(), "active").is_none(),
5383 "the active slot must not be durably replaced by the failed key; until the retry lands, the old active survives in memory and in drain"
5384 );
5385
5386 std::fs::remove_dir(dir.path().join("jwt-signing-key.active")).unwrap();
5388 rotate_jwt_keys_in_state_dir(&config, dir.path(), &store, now).await;
5389 assert_eq!(store.active_verifying_key_bytes().await, Some(lead_vk));
5390 assert_eq!(
5391 load_slot(dir.path(), "active")
5392 .expect("promoted active must be persisted")
5393 .verifying_key_bytes(),
5394 lead_vk,
5395 "memory promotion must be preceded by the durable active write"
5396 );
5397 let slots = store.0.read().await;
5398 assert_eq!(
5399 slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5400 Some(active_vk)
5401 );
5402 assert!(slots.lead.is_none());
5403 }
5404
5405 #[tokio::test]
5406 async fn ed25519_failed_rollback_preserves_recoverable_active_in_drain() {
5407 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_ROLLBACK_RECOVERY_TEST";
5412 if std::env::var_os(ISOLATED).is_none() {
5413 let status = std::process::Command::new(std::env::current_exe().unwrap())
5414 .args([
5415 "--exact",
5416 "auth::key_rotation::tests::ed25519_failed_rollback_preserves_recoverable_active_in_drain",
5417 "--nocapture",
5418 ])
5419 .env(ISOLATED, "1")
5420 .status()
5421 .unwrap();
5422 assert!(status.success());
5423 return;
5424 }
5425 for prev_drain_present in [false, true] {
5426 let dir = TempDir::new().unwrap();
5427 let config = test_config();
5428 let now = chrono::Utc::now().timestamp();
5429
5430 let active_slot = KeySlot::new(
5431 SigningKey::generate(&mut rand::rngs::OsRng),
5432 now - 14 * 86400,
5433 now + 6 * 3600,
5434 );
5435 let lead_slot = KeySlot::new(
5436 SigningKey::generate(&mut rand::rngs::OsRng),
5437 now - 1, now + 14 * 86400,
5439 );
5440 let active_seed = active_slot.key.to_bytes();
5441 let active_vk = active_slot.verifying_key_bytes();
5442 let lead_vk = lead_slot.verifying_key_bytes();
5443 persist_slot(dir.path(), "active", &active_slot).unwrap();
5444 persist_slot(dir.path(), "lead", &lead_slot).unwrap();
5445 let prev_drain = if prev_drain_present {
5446 let drain = KeySlot::new(
5447 SigningKey::generate(&mut rand::rngs::OsRng),
5448 now - 60 * 86400,
5449 now - 45 * 86400,
5450 );
5451 persist_slot(dir.path(), "drain", &drain).unwrap();
5452 Some(drain)
5453 } else {
5454 None
5455 };
5456
5457 let store = SigningKeyStore::new(KeySlots {
5458 drain: prev_drain.clone(),
5459 active: Some(active_slot),
5460 lead: Some(lead_slot),
5461 });
5462
5463 let pq_store = load_or_init_ml_dsa_key_store(dir.path(), &config);
5466 let ca_key = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
5467 initialize_composite_key_set(dir.path(), &store, &pq_store, Arc::clone(&ca_key), 30)
5468 .await
5469 .expect("healthy starting authority must initialize composite");
5470 let (commit_before, ledger_before) =
5471 read_committed_composite_ledger(dir.path()).unwrap();
5472 let oauth_active_ed = ledger_before
5473 .pairs
5474 .iter()
5475 .find(|record| record.role == "oauth" && record.state == "active")
5476 .expect("initialized authority must carry an active OAuth pair");
5477 assert_eq!(
5478 oauth_active_ed.ed25519_public,
5479 URL_SAFE_NO_PAD.encode(active_vk),
5480 "prior_drain={prev_drain_present}: the committed active pair must \
5481 name the old active Ed25519 signer"
5482 );
5483
5484 let meta_path = dir.path().join("jwt-signing-key.active.meta");
5491 std::fs::remove_file(&meta_path).unwrap();
5492 std::fs::create_dir(&meta_path).unwrap();
5493
5494 let restored = rotate_jwt_keys_in_state_dir(&config, dir.path(), &store, now).await;
5495 assert!(
5496 !restored,
5497 "prior_drain={prev_drain_present}: a failed rollback must report \
5498 the stores as not fully restored"
5499 );
5500
5501 let slots = store.0.read().await;
5503 assert_eq!(
5504 slots.active.as_ref().map(KeySlot::verifying_key_bytes),
5505 Some(active_vk),
5506 "prior_drain={prev_drain_present}: the old active must remain the \
5507 in-memory active signer"
5508 );
5509 assert_eq!(
5510 slots.lead.as_ref().map(KeySlot::verifying_key_bytes),
5511 Some(lead_vk),
5512 "prior_drain={prev_drain_present}: the lead must stay queued"
5513 );
5514 assert_eq!(
5515 slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5516 prev_drain.as_ref().map(KeySlot::verifying_key_bytes),
5517 "prior_drain={prev_drain_present}: memory drain must not change \
5518 before the rollback is confirmed"
5519 );
5520 drop(slots);
5521
5522 let recovered = load_slot(dir.path(), "drain").expect(
5526 "prior_drain={prev_drain_present}: drain copy of the old active \
5527 must survive a failed rollback",
5528 );
5529 assert_eq!(
5530 recovered.verifying_key_bytes(),
5531 active_vk,
5532 "prior_drain={prev_drain_present}: failed rollback must not erase \
5533 the recoverable old active signer"
5534 );
5535 assert_eq!(
5536 std::fs::read(dir.path().join("jwt-signing-key.active")).unwrap(),
5537 active_seed,
5538 "prior_drain={prev_drain_present}: the rollback must have restored \
5539 the old active seed before failing on metadata"
5540 );
5541 assert!(
5542 load_slot(dir.path(), "active").is_none(),
5543 "prior_drain={prev_drain_present}: the active slot is incomplete \
5544 without its metadata and must not load"
5545 );
5546
5547 let fresh = load_or_init_key_store(dir.path(), &config);
5553 {
5554 let fresh_slots = fresh.0.read().await;
5555 assert_eq!(
5556 fresh_slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5557 Some(active_vk),
5558 "prior_drain={prev_drain_present}: fresh reload must still have \
5559 the committed old active signer recoverable in drain"
5560 );
5561 assert!(
5562 fresh_slots.active.is_none(),
5563 "prior_drain={prev_drain_present}: the torn active slot must not \
5564 load as a usable signer"
5565 );
5566 }
5567
5568 let fresh_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
5575 initialize_composite_key_set(dir.path(), &fresh, &fresh_pq, Arc::clone(&ca_key), 30)
5576 .await
5577 .expect(
5578 "prior_drain={prev_drain_present}: committed authority must restore \
5579 from the recovered drain slot after a failed rollback",
5580 );
5581 let (commit_after, ledger_after) = read_committed_composite_ledger(dir.path()).unwrap();
5582 assert_eq!(
5583 commit_after.version, commit_before.version,
5584 "prior_drain={prev_drain_present}: recovery must not need a new \
5585 committed generation"
5586 );
5587 assert_eq!(
5588 commit_after.component_digest,
5589 commit_before.component_digest
5590 );
5591 let oauth_active_ed_after = ledger_after
5592 .pairs
5593 .iter()
5594 .find(|record| record.role == "oauth" && record.state == "active")
5595 .expect("restored authority must still carry an active OAuth pair");
5596 assert_eq!(
5597 oauth_active_ed_after.ed25519_public,
5598 URL_SAFE_NO_PAD.encode(active_vk),
5599 "prior_drain={prev_drain_present}: restored committed authority \
5600 must still name the recovered old active signer"
5601 );
5602
5603 std::fs::remove_dir(&meta_path).unwrap();
5607 let retried = rotate_jwt_keys_in_state_dir(&config, dir.path(), &fresh, now).await;
5608 assert!(
5609 retried,
5610 "prior_drain={prev_drain_present}: the retry must fully restore and \
5611 promote"
5612 );
5613 assert_eq!(
5614 fresh.active_verifying_key_bytes().await,
5615 Some(lead_vk),
5616 "prior_drain={prev_drain_present}: the queued lead promotes after \
5617 recovery"
5618 );
5619 let slots = fresh.0.read().await;
5620 assert_eq!(
5621 slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
5622 Some(active_vk),
5623 "prior_drain={prev_drain_present}: the recovered old active must \
5624 survive the retry in drain"
5625 );
5626 assert!(slots.lead.is_none());
5627 }
5628 }
5629
5630 #[test]
5633 fn es256_load_or_init_creates_active_key() {
5634 let dir = TempDir::new().unwrap();
5635 let config = test_config();
5636 let store = load_or_init_es256_key_store(dir.path(), &config);
5637 let key = store.active_key();
5638 assert!(key.is_some());
5639 }
5640
5641 #[test]
5642 fn es256_missing_active_with_durable_oplog_authority_does_not_regenerate() {
5643 for marker in [
5644 super::super::op_log::SEALED_HEAD_FILENAME,
5645 super::super::op_log::SEALED_HEAD_MAX_SEQ_FILENAME,
5646 super::super::op_log::HEAD_VERIFYING_KEY_FILENAME,
5647 ] {
5648 let root = TempDir::new().unwrap();
5649 let secrets_dir = root.path().join("credentials");
5650 let oplog_dir = root.path().join("oplog-state");
5651 std::fs::create_dir(&secrets_dir).unwrap();
5652 std::fs::create_dir(&oplog_dir).unwrap();
5653 std::fs::write(oplog_dir.join(marker), b"existing authority marker").unwrap();
5654
5655 let store = load_or_init_es256_key_store_with_oplog_state_dir(
5656 &secrets_dir,
5657 &test_config(),
5658 &oplog_dir,
5659 );
5660 assert!(store.active_key().is_none(), "marker={marker}");
5661 assert!(
5662 load_es256_slot(&secrets_dir, "active").is_none(),
5663 "marker={marker}"
5664 );
5665 assert!(
5666 load_es256_slot(&secrets_dir, "drain").is_none(),
5667 "marker={marker}"
5668 );
5669 assert!(
5670 load_es256_slot(&secrets_dir, "lead").is_none(),
5671 "marker={marker}"
5672 );
5673 }
5674 }
5675
5676 #[tokio::test]
5677 async fn es256_failed_first_boot_persistence_can_recover_without_published_marker() {
5678 let root = TempDir::new().unwrap();
5679 let secrets_dir = root.path().join("credentials");
5680 let oplog_dir = root.path().join("oplog-state");
5681 std::fs::create_dir(&secrets_dir).unwrap();
5682 std::fs::create_dir(&oplog_dir).unwrap();
5683
5684 let blocked_metadata = secrets_dir.join("es256-signing-key.active.meta");
5687 std::fs::create_dir(&blocked_metadata).unwrap();
5688 let failed_boot = load_or_init_es256_key_store_with_oplog_state_dir(
5689 &secrets_dir,
5690 &test_config(),
5691 &oplog_dir,
5692 );
5693 assert!(failed_boot.active_key().is_none());
5694
5695 assert!(super::super::op_log::advance_sealed_head_with_state_dir(
5696 &secrets_dir,
5697 &oplog_dir,
5698 &failed_boot,
5699 )
5700 .await
5701 .is_err());
5702 assert!(!oplog_dir
5703 .join(super::super::op_log::HEAD_VERIFYING_KEY_FILENAME)
5704 .exists());
5705 assert!(!oplog_dir
5706 .join(super::super::op_log::SEALED_HEAD_FILENAME)
5707 .exists());
5708
5709 std::fs::remove_dir(&blocked_metadata).unwrap();
5712 let recovered = load_or_init_es256_key_store_with_oplog_state_dir(
5713 &secrets_dir,
5714 &test_config(),
5715 &oplog_dir,
5716 );
5717 assert!(recovered.active_key().is_some());
5718 super::super::op_log::advance_sealed_head_with_state_dir(
5719 &secrets_dir,
5720 &oplog_dir,
5721 &recovered,
5722 )
5723 .await
5724 .unwrap();
5725 assert!(oplog_dir
5726 .join(super::super::op_log::HEAD_VERIFYING_KEY_FILENAME)
5727 .exists());
5728 assert!(oplog_dir
5729 .join(super::super::op_log::SEALED_HEAD_FILENAME)
5730 .exists());
5731 }
5732
5733 #[tokio::test]
5734 async fn advance_oplog_does_not_reload_unpromoted_es256_disk_candidate() {
5735 let root = TempDir::new().unwrap();
5736 let secrets_dir = root.path().join("credentials");
5737 let oplog_dir = root.path().join("oplog-state");
5738 std::fs::create_dir(&secrets_dir).unwrap();
5739 std::fs::create_dir(&oplog_dir).unwrap();
5740
5741 let now = chrono::Utc::now().timestamp();
5742 let retained = generate_es256_slot(now - 120, now + 86_400);
5743 let rejected_candidate = generate_es256_slot(now, now + 172_800);
5744 let retained_kid = retained.kid();
5745 persist_es256_slot(&secrets_dir, "active", &rejected_candidate).unwrap();
5746 persist_es256_slot(&secrets_dir, "drain", &retained).unwrap();
5747
5748 let store = Es256SigningKeyStore::new(Es256KeySlots {
5751 drain: None,
5752 active: Some(retained),
5753 lead: Some(rejected_candidate),
5754 });
5755 store.bind_secrets_dir(secrets_dir.clone());
5756 assert_eq!(store.active_slot().unwrap().kid(), retained_kid);
5757
5758 super::super::op_log::advance_sealed_head_with_state_dir(&secrets_dir, &oplog_dir, &store)
5759 .await
5760 .unwrap();
5761
5762 assert_eq!(
5763 store.active_slot().unwrap().kid(),
5764 retained_kid,
5765 "op-log advancement must not reload an unpromoted disk candidate"
5766 );
5767 let head: super::super::op_log::SealedOpLogHead = serde_json::from_slice(
5768 &std::fs::read(oplog_dir.join(super::super::op_log::SEALED_HEAD_FILENAME)).unwrap(),
5769 )
5770 .unwrap();
5771 assert_eq!(head.active_kid, retained_kid);
5772 }
5773
5774 #[test]
5775 fn es256_persist_and_reload() {
5776 let dir = TempDir::new().unwrap();
5777 let config = test_config();
5778 let store1 = load_or_init_es256_key_store(dir.path(), &config);
5779 let kid1 = {
5780 let slots = store1.all_slots_snapshot();
5781 slots[0].kid()
5782 };
5783 let store2 = load_or_init_es256_key_store(dir.path(), &config);
5784 let kid2 = {
5785 let slots = store2.all_slots_snapshot();
5786 slots[0].kid()
5787 };
5788 assert_eq!(kid1, kid2, "ES256 key must survive reload from disk");
5789 }
5790
5791 #[test]
5792 fn es256_startup_removes_expired_bounded_slots() {
5793 let dir = TempDir::new().unwrap();
5794 let config = test_config();
5795 let now = chrono::Utc::now().timestamp();
5796 let active = generate_es256_slot(now - 60, now + 86_400);
5797 let expired_drain = generate_es256_slot(now - 120, now);
5798 persist_es256_slot(dir.path(), "active", &active).unwrap();
5799 persist_es256_slot(dir.path(), "drain", &expired_drain).unwrap();
5800
5801 let store = load_or_init_es256_key_store(dir.path(), &config);
5802 assert!(store.drain_slot().is_none());
5803 assert!(load_es256_slot(dir.path(), "drain").is_none());
5804 }
5805
5806 #[tokio::test]
5807 async fn es256_rotate_promotes_lead() {
5808 let dir = TempDir::new().unwrap();
5809 let config = test_config();
5810 let now = chrono::Utc::now().timestamp();
5811
5812 let active = generate_es256_slot(now - 14 * 86400, now + 1);
5813 let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5814 persist_es256_slot(dir.path(), "active", &active).unwrap();
5815 persist_es256_slot(dir.path(), "lead", &lead).unwrap();
5816 let lead_kid = lead.kid();
5817
5818 let store = Es256SigningKeyStore::new(Es256KeySlots {
5819 drain: None,
5820 active: Some(active),
5821 lead: Some(lead),
5822 });
5823 store.set_promotion_hook(Arc::new(|_| Ok(())));
5824
5825 rotate_es256_keys(&config, dir.path(), &store, now).await;
5826
5827 let slots = store.0.read();
5828 assert_eq!(slots.active.as_ref().unwrap().kid(), lead_kid);
5829 assert!(slots.drain.is_some());
5830 assert!(slots.lead.is_none());
5831 }
5832
5833 #[tokio::test]
5834 async fn es256_drain_is_removed_at_its_published_expiry() {
5835 let dir = TempDir::new().unwrap();
5836 let config = test_config();
5837 let now = chrono::Utc::now().timestamp();
5838 let active = generate_es256_slot(now - 14 * 86400, now + 1);
5839 let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5840 let store = Es256SigningKeyStore::new(Es256KeySlots {
5841 drain: None,
5842 active: Some(active),
5843 lead: Some(lead),
5844 });
5845 store.set_promotion_hook(Arc::new(|_| Ok(())));
5846
5847 assert!(rotate_es256_keys(&config, dir.path(), &store, now).await);
5848 let published_expiry = store.drain_slot().expect("drain after promotion").exp;
5849 assert!(
5850 rotate_es256_keys(&config, dir.path(), &store, published_expiry).await
5851 || store.active_slot().is_some(),
5852 "cleanup tick remains operational"
5853 );
5854 assert!(
5855 store.drain_slot().is_none(),
5856 "the drain slot must leave the publication source at its expiry"
5857 );
5858 }
5859
5860 #[tokio::test]
5861 async fn es256_rotate_without_hook_publishes_drain_for_ipc_workers() {
5862 let dir = TempDir::new().unwrap();
5863 let config = test_config();
5864 let now = chrono::Utc::now().timestamp();
5865
5866 let active = generate_es256_slot(now - 14 * 86400, now + 1);
5867 let active_kid = active.kid();
5868 let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5869 let lead_kid = lead.kid();
5870 persist_es256_slot(dir.path(), "active", &active).unwrap();
5871 persist_es256_slot(dir.path(), "lead", &lead).unwrap();
5872 let store = Es256SigningKeyStore::new(Es256KeySlots {
5873 drain: None,
5874 active: Some(active),
5875 lead: Some(lead),
5876 });
5877
5878 assert!(rotate_es256_keys(&config, dir.path(), &store, now).await);
5879 let slots = store.0.read();
5880 assert_eq!(slots.active.as_ref().unwrap().kid(), lead_kid);
5881 assert!(slots.lead.is_none());
5882 assert_eq!(slots.drain.as_ref().unwrap().kid(), active_kid);
5883 assert_eq!(
5884 load_es256_slot(dir.path(), "active").unwrap().kid(),
5885 lead_kid
5886 );
5887 }
5888
5889 #[tokio::test]
5890 async fn es256_rotation_retries_after_promotion_hook_failure() {
5891 let dir = TempDir::new().unwrap();
5892 let config = test_config();
5893 let now = chrono::Utc::now().timestamp();
5894
5895 let active = generate_es256_slot(now - 14 * 86400, now + 1);
5896 let active_kid = active.kid();
5897 let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5898 let lead_kid = lead.kid();
5899 persist_es256_slot(dir.path(), "active", &active).unwrap();
5900 persist_es256_slot(dir.path(), "lead", &lead).unwrap();
5901
5902 let store = Es256SigningKeyStore::new(Es256KeySlots {
5903 drain: None,
5904 active: Some(active),
5905 lead: Some(lead),
5906 });
5907 let fail_once = Arc::new(std::sync::atomic::AtomicBool::new(true));
5908 let injected_failure = Arc::clone(&fail_once);
5909 store.set_promotion_hook(Arc::new(move |_| {
5910 if injected_failure.swap(false, Ordering::AcqRel) {
5911 anyhow::bail!("injected head re-sign failure");
5912 }
5913 Ok(())
5914 }));
5915
5916 assert!(
5917 !rotate_es256_keys(&config, dir.path(), &store, now).await,
5918 "a failed promotion hook must leave the candidate queued"
5919 );
5920 {
5921 let slots = store.0.read();
5922 assert_eq!(slots.active.as_ref().unwrap().kid(), active_kid);
5923 assert_eq!(slots.lead.as_ref().unwrap().kid(), lead_kid);
5924 assert!(slots.drain.is_none());
5925 }
5926 assert_eq!(
5927 load_es256_slot(dir.path(), "active").unwrap().kid(),
5928 active_kid,
5929 "failed promotion must restore the durable active key"
5930 );
5931
5932 assert!(rotate_es256_keys(&config, dir.path(), &store, now).await);
5933 let slots = store.0.read();
5934 assert_eq!(slots.active.as_ref().unwrap().kid(), lead_kid);
5935 assert!(slots.lead.is_none());
5936 }
5937
5938 #[tokio::test]
5939 async fn es256_rotate_retries_after_active_persistence_failure() {
5940 let dir = TempDir::new().unwrap();
5941 let invalid_secrets_dir = dir.path().join("not-a-directory");
5942 std::fs::write(&invalid_secrets_dir, b"file").unwrap();
5943 let valid_secrets_dir = dir.path().join("secrets");
5944 let config = test_config();
5945 let now = chrono::Utc::now().timestamp();
5946
5947 let active = generate_es256_slot(now - 14 * 86400, now + 1);
5948 let active_kid = active.kid();
5949 let lead = generate_es256_slot(now - 1, now + 14 * 86400);
5950 let lead_kid = lead.kid();
5951 let store = Es256SigningKeyStore::new(Es256KeySlots {
5952 drain: None,
5953 active: Some(active),
5954 lead: Some(lead),
5955 });
5956 assert!(
5957 !rotate_es256_keys_in_state_dir(&config, &invalid_secrets_dir, &store, now).await,
5958 "failed active persistence must not report promotion"
5959 );
5960 {
5961 let slots = store.0.read();
5962 assert_eq!(slots.active.as_ref().unwrap().kid(), active_kid);
5963 assert_eq!(slots.lead.as_ref().unwrap().kid(), lead_kid);
5964 }
5965 assert!(
5966 rotate_es256_keys(&config, &valid_secrets_dir, &store, now).await,
5967 "queued lead must be retried after persistence recovers"
5968 );
5969 assert_eq!(store.active_slot().unwrap().kid(), lead_kid);
5970 }
5971
5972 #[test]
5975 fn ml_dsa_load_or_init_creates_active_key() {
5976 let dir = TempDir::new().unwrap();
5977 let config = test_config();
5978 let store = load_or_init_ml_dsa_key_store(dir.path(), &config);
5979 let rt = tokio::runtime::Runtime::new().unwrap();
5980 let key = rt.block_on(store.active_key());
5981 assert!(key.is_some());
5982 }
5983
5984 #[test]
5985 fn ml_dsa_persist_and_reload() {
5986 let dir = TempDir::new().unwrap();
5987 let config = test_config();
5988 let store1 = load_or_init_ml_dsa_key_store(dir.path(), &config);
5989 let rt = tokio::runtime::Runtime::new().unwrap();
5990 let vk1 = {
5991 let key = rt.block_on(store1.active_key()).unwrap();
5992 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
5993 &ml_dsa::Keypair::verifying_key(&*key).clone(),
5994 )
5995 };
5996 let store2 = load_or_init_ml_dsa_key_store(dir.path(), &config);
5997 let vk2 = {
5998 let key = rt.block_on(store2.active_key()).unwrap();
5999 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
6000 &ml_dsa::Keypair::verifying_key(&*key).clone(),
6001 )
6002 };
6003 assert_eq!(vk1, vk2, "ML-DSA key must survive reload from disk");
6004 }
6005
6006 #[tokio::test]
6007 async fn ml_dsa_rotate_promotes_lead() {
6008 use super::ml_dsa_rotation::*;
6009 let dir = TempDir::new().unwrap();
6010 let config = test_config();
6011 let now = chrono::Utc::now().timestamp();
6012
6013 let active = generate_ml_dsa_slot(now - 14 * 86400, now + 1);
6014 let lead = generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6015 persist_ml_dsa_slot(dir.path(), "active", &active).unwrap();
6016 persist_ml_dsa_slot(dir.path(), "lead", &lead).unwrap();
6017 let lead_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
6018 &ml_dsa::Keypair::verifying_key(&*lead.key).clone(),
6019 );
6020
6021 let store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
6022 drain: None,
6023 active: Some(active),
6024 lead: Some(lead),
6025 });
6026
6027 rotate_ml_dsa_keys(&config, dir.path(), &store, now).await;
6028
6029 let slots = store.0.read().await;
6030 let active_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(
6031 &ml_dsa::Keypair::verifying_key(&*slots.active.as_ref().unwrap().key).clone(),
6032 );
6033 assert_eq!(
6034 active_vk, lead_vk,
6035 "lead must become active after promotion"
6036 );
6037 assert!(slots.drain.is_some());
6038 assert!(slots.lead.is_none());
6039 }
6040
6041 fn ml_dsa_slot_vk_bytes(slot: &ml_dsa_rotation::MlDsaKeySlot) -> Vec<u8> {
6042 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&slot.verifying_key())
6043 }
6044
6045 fn ml_dsa_key_vk_bytes(key: &hyprstream_rpc::crypto::pq::MlDsaSigningKey) -> Vec<u8> {
6046 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&ml_dsa::Keypair::verifying_key(key).clone())
6047 }
6048
6049 #[tokio::test]
6050 async fn ml_dsa_first_boot_persistence_failure_leaves_no_active_key() {
6051 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_FIRST_BOOT_ML_DSA_TEST";
6056 if std::env::var_os(ISOLATED).is_none() {
6057 let status = std::process::Command::new(std::env::current_exe().unwrap())
6058 .args([
6059 "--exact",
6060 "auth::key_rotation::tests::ml_dsa_first_boot_persistence_failure_leaves_no_active_key",
6061 "--nocapture",
6062 ])
6063 .env(ISOLATED, "1")
6064 .status()
6065 .unwrap();
6066 assert!(status.success());
6067 return;
6068 }
6069 let dir = TempDir::new().unwrap();
6070 let config = test_config();
6071 std::fs::create_dir(dir.path().join("ml-dsa-signing-key.active")).unwrap();
6074
6075 let store = load_or_init_ml_dsa_key_store(dir.path(), &config);
6076 assert!(
6077 store.active_key().await.is_none(),
6078 "an unpersisted generated ML-DSA key must not be exposed as active"
6079 );
6080 assert!(
6081 !dir.path().join("ml-dsa-signing-key.active.meta").exists(),
6082 "no slot metadata may exist when the key bytes failed to persist"
6083 );
6084
6085 let ed = load_or_init_key_store(dir.path(), &config);
6090 let error = initialize_composite_key_set(
6091 dir.path(),
6092 &ed,
6093 &store,
6094 Arc::new(SigningKey::from_bytes(&[0x5A; 32])),
6095 30,
6096 )
6097 .await
6098 .expect_err("incomplete first-boot store must not create composite authority");
6099 assert!(
6100 error.to_string().contains("incomplete local signer set"),
6101 "unexpected initialization error: {error:#}"
6102 );
6103 assert!(
6104 !dir.path().join("jwt-composite-pairs.json").exists(),
6105 "no pending ledger may be created from an incomplete store"
6106 );
6107 assert!(
6108 !dir.path().join("jwt-composite-pairs.committed").exists(),
6109 "no commit marker may be created from an incomplete store"
6110 );
6111 let snapshots = std::fs::read_dir(dir.path())
6112 .unwrap()
6113 .filter_map(Result::ok)
6114 .filter(|entry| {
6115 entry
6116 .file_name()
6117 .to_string_lossy()
6118 .starts_with("jwt-composite-pairs.committed-")
6119 })
6120 .count();
6121 assert_eq!(
6122 snapshots, 0,
6123 "no immutable committed snapshot may be created from an incomplete store"
6124 );
6125
6126 std::fs::remove_dir(dir.path().join("ml-dsa-signing-key.active")).unwrap();
6129 let fresh_ed = load_or_init_key_store(dir.path(), &config);
6130 let fresh_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
6131 initialize_composite_key_set(
6132 dir.path(),
6133 &fresh_ed,
6134 &fresh_pq,
6135 Arc::new(SigningKey::from_bytes(&[0x5A; 32])),
6136 30,
6137 )
6138 .await
6139 .expect("normal initialization must succeed once persistence recovers");
6140 assert!(dir.path().join("jwt-composite-pairs.json").exists());
6141 assert!(dir.path().join("jwt-composite-pairs.committed").exists());
6142 }
6143
6144 #[tokio::test]
6145 async fn ml_dsa_rotation_keeps_active_when_drain_persist_fails() {
6146 let dir = TempDir::new().unwrap();
6147 let config = test_config();
6148 let now = chrono::Utc::now().timestamp();
6149
6150 let active = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 1);
6151 let lead = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6152 let drain = ml_dsa_rotation::generate_ml_dsa_slot(now - 45 * 86400, now + 5 * 86400);
6153 let active_vk = ml_dsa_slot_vk_bytes(&active);
6154 let lead_vk = ml_dsa_slot_vk_bytes(&lead);
6155 let drain_vk = ml_dsa_slot_vk_bytes(&drain);
6156
6157 let store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
6158 drain: Some(drain),
6159 active: Some(active),
6160 lead: Some(lead),
6161 });
6162
6163 let broken_state = dir.path().join("not-a-directory");
6166 std::fs::write(&broken_state, b"file").unwrap();
6167 ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, &broken_state, &store, now).await;
6168
6169 let slots = store.0.read().await;
6170 assert_eq!(
6171 slots.active.as_ref().map(ml_dsa_slot_vk_bytes),
6172 Some(active_vk.clone()),
6173 "the old active must remain the in-memory active signer"
6174 );
6175 assert_eq!(
6176 slots.lead.as_ref().map(ml_dsa_slot_vk_bytes),
6177 Some(lead_vk.clone()),
6178 "the queued lead must stay in memory for the next tick"
6179 );
6180 assert_eq!(
6181 slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6182 Some(drain_vk.clone()),
6183 "a failed promotion must not evict the in-memory drain"
6184 );
6185 drop(slots);
6186
6187 ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &store, now).await;
6190 let persisted = ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "active")
6191 .expect("promoted active must be persisted");
6192 assert_eq!(
6193 ml_dsa_slot_vk_bytes(&persisted),
6194 lead_vk,
6195 "memory promotion must follow the durable active write"
6196 );
6197 let slots = store.0.read().await;
6198 assert_eq!(
6199 slots.active.as_ref().map(ml_dsa_slot_vk_bytes),
6200 Some(lead_vk),
6201 "the queued lead must promote once persistence succeeds"
6202 );
6203 assert_eq!(
6204 slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6205 Some(active_vk),
6206 "the old active becomes drain only after its persist succeeded"
6207 );
6208 assert!(slots.lead.is_none());
6209 assert!(!dir.path().join("ml-dsa-signing-key.lead").exists());
6210 }
6211
6212 #[tokio::test]
6213 async fn ml_dsa_rotation_keeps_active_when_active_persist_fails() {
6214 let dir = TempDir::new().unwrap();
6215 let config = test_config();
6216 let now = chrono::Utc::now().timestamp();
6217
6218 let active = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 1);
6219 let lead = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6220 let active_vk = ml_dsa_slot_vk_bytes(&active);
6221 let lead_vk = ml_dsa_slot_vk_bytes(&lead);
6222 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &lead).unwrap();
6223
6224 let store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
6225 drain: None,
6226 active: Some(active),
6227 lead: Some(lead),
6228 });
6229
6230 std::fs::create_dir(dir.path().join("ml-dsa-signing-key.active")).unwrap();
6235 let restored =
6236 ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &store, now)
6237 .await;
6238 assert!(
6239 !restored,
6240 "a failed rollback must report the stores as not fully restored"
6241 );
6242
6243 assert_eq!(
6244 store.active_key().await.as_deref().map(ml_dsa_key_vk_bytes),
6245 Some(active_vk.clone()),
6246 "the failed new key must not be exposed as active in memory"
6247 );
6248 {
6249 let slots = store.0.read().await;
6250 assert!(slots.lead.is_some(), "the lead stays queued for retry");
6251 assert!(
6252 slots.drain.is_none(),
6253 "in-memory drain must not change before every write succeeded"
6254 );
6255 }
6256 assert_eq!(
6257 ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "drain")
6258 .map(|slot| ml_dsa_slot_vk_bytes(&slot)),
6259 Some(active_vk.clone()),
6260 "a failed rollback must not erase the drain copy of the old active \
6261 signer"
6262 );
6263 assert!(
6264 dir.path().join("ml-dsa-signing-key.active").is_dir()
6265 && ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "active").is_none(),
6266 "the active slot must not be durably replaced by the failed key; \
6267 until the retry lands, the old active survives in memory and in \
6268 drain"
6269 );
6270
6271 std::fs::remove_dir(dir.path().join("ml-dsa-signing-key.active")).unwrap();
6273 ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &store, now).await;
6274 assert_eq!(
6275 store.active_key().await.as_deref().map(ml_dsa_key_vk_bytes),
6276 Some(lead_vk.clone()),
6277 "the queued lead must promote once persistence succeeds"
6278 );
6279 let persisted = ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "active")
6280 .expect("promoted active must be persisted");
6281 assert_eq!(
6282 ml_dsa_key_vk_bytes(persisted.key.as_ref()),
6283 lead_vk,
6284 "memory promotion must be preceded by the durable active write"
6285 );
6286 let slots = store.0.read().await;
6287 assert_eq!(
6288 slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6289 Some(active_vk),
6290 "the old active becomes drain only after its persist succeeded"
6291 );
6292 assert!(slots.lead.is_none());
6293 }
6294
6295 #[tokio::test]
6296 async fn ml_dsa_failed_rollback_preserves_recoverable_active_in_drain() {
6297 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ML_DSA_ROLLBACK_RECOVERY_TEST";
6302 if std::env::var_os(ISOLATED).is_none() {
6303 let status = std::process::Command::new(std::env::current_exe().unwrap())
6304 .args([
6305 "--exact",
6306 "auth::key_rotation::tests::ml_dsa_failed_rollback_preserves_recoverable_active_in_drain",
6307 "--nocapture",
6308 ])
6309 .env(ISOLATED, "1")
6310 .status()
6311 .unwrap();
6312 assert!(status.success());
6313 return;
6314 }
6315 for prev_drain_present in [false, true] {
6316 let dir = TempDir::new().unwrap();
6317 let config = test_config();
6318 let now = chrono::Utc::now().timestamp();
6319
6320 let active = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 6 * 3600);
6324 let lead = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6325 let active_vk = ml_dsa_slot_vk_bytes(&active);
6326 let lead_vk = ml_dsa_slot_vk_bytes(&lead);
6327 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &active).unwrap();
6328 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &lead).unwrap();
6329 let prev_drain = if prev_drain_present {
6330 let drain =
6331 ml_dsa_rotation::generate_ml_dsa_slot(now - 60 * 86400, now - 45 * 86400);
6332 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "drain", &drain).unwrap();
6333 Some(drain)
6334 } else {
6335 None
6336 };
6337
6338 let store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
6339 drain: prev_drain.clone(),
6340 active: Some(active),
6341 lead: Some(lead),
6342 });
6343
6344 let ed_store = load_or_init_key_store(dir.path(), &config);
6347 let ca_key = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
6348 initialize_composite_key_set(dir.path(), &ed_store, &store, Arc::clone(&ca_key), 30)
6349 .await
6350 .expect("healthy starting authority must initialize composite");
6351 let (commit_before, ledger_before) =
6352 read_committed_composite_ledger(dir.path()).unwrap();
6353 let oauth_active_pq = ledger_before
6354 .pairs
6355 .iter()
6356 .find(|record| record.role == "oauth" && record.state == "active")
6357 .expect("initialized authority must carry an active OAuth pair");
6358 assert_eq!(
6359 oauth_active_pq.ml_dsa_public,
6360 URL_SAFE_NO_PAD.encode(&active_vk),
6361 "prior_drain={prev_drain_present}: the committed active pair must \
6362 name the old active ML-DSA signer"
6363 );
6364
6365 let meta_path = dir.path().join("ml-dsa-signing-key.active.meta");
6370 std::fs::remove_file(&meta_path).unwrap();
6371 std::fs::create_dir(&meta_path).unwrap();
6372
6373 let restored =
6374 ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &store, now)
6375 .await;
6376 assert!(
6377 !restored,
6378 "prior_drain={prev_drain_present}: a failed rollback must report \
6379 the stores as not fully restored"
6380 );
6381
6382 let slots = store.0.read().await;
6383 assert_eq!(
6384 slots.active.as_ref().map(ml_dsa_slot_vk_bytes),
6385 Some(active_vk.clone()),
6386 "prior_drain={prev_drain_present}: the old active must remain the \
6387 in-memory active signer"
6388 );
6389 assert_eq!(
6390 slots.lead.as_ref().map(ml_dsa_slot_vk_bytes),
6391 Some(lead_vk.clone()),
6392 "prior_drain={prev_drain_present}: the lead must stay queued"
6393 );
6394 assert_eq!(
6395 slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6396 prev_drain.as_ref().map(ml_dsa_slot_vk_bytes),
6397 "prior_drain={prev_drain_present}: memory drain must not change \
6398 before the rollback is confirmed"
6399 );
6400 drop(slots);
6401
6402 let recovered = ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "drain").expect(
6403 "prior_drain={prev_drain_present}: drain copy of the old active \
6404 must survive a failed rollback",
6405 );
6406 assert_eq!(
6407 ml_dsa_slot_vk_bytes(&recovered),
6408 active_vk,
6409 "prior_drain={prev_drain_present}: failed rollback must not erase \
6410 the recoverable old active signer"
6411 );
6412 assert!(
6413 ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "active").is_none(),
6414 "prior_drain={prev_drain_present}: the active slot is incomplete \
6415 without its metadata and must not load"
6416 );
6417
6418 let fresh = load_or_init_ml_dsa_key_store(dir.path(), &config);
6421 {
6422 let fresh_slots = fresh.0.read().await;
6423 assert_eq!(
6424 fresh_slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6425 Some(active_vk.clone()),
6426 "prior_drain={prev_drain_present}: fresh reload must still have \
6427 the committed old active signer recoverable in drain"
6428 );
6429 assert!(
6430 fresh_slots.active.is_none(),
6431 "prior_drain={prev_drain_present}: the torn active slot must not \
6432 load as a usable signer"
6433 );
6434 }
6435
6436 let fresh_ed = load_or_init_key_store(dir.path(), &config);
6441 initialize_composite_key_set(dir.path(), &fresh_ed, &fresh, Arc::clone(&ca_key), 30)
6442 .await
6443 .expect(
6444 "prior_drain={prev_drain_present}: committed authority must restore \
6445 from the recovered drain slot after a failed rollback",
6446 );
6447 let (commit_after, ledger_after) = read_committed_composite_ledger(dir.path()).unwrap();
6448 assert_eq!(
6449 commit_after.version, commit_before.version,
6450 "prior_drain={prev_drain_present}: recovery must not need a new \
6451 committed generation"
6452 );
6453 assert_eq!(
6454 commit_after.component_digest,
6455 commit_before.component_digest
6456 );
6457 let oauth_active_pq_after = ledger_after
6458 .pairs
6459 .iter()
6460 .find(|record| record.role == "oauth" && record.state == "active")
6461 .expect("restored authority must still carry an active OAuth pair");
6462 assert_eq!(
6463 oauth_active_pq_after.ml_dsa_public,
6464 URL_SAFE_NO_PAD.encode(&active_vk),
6465 "prior_drain={prev_drain_present}: restored committed authority \
6466 must still name the recovered old active signer"
6467 );
6468
6469 std::fs::remove_dir(&meta_path).unwrap();
6471 let retried =
6472 ml_dsa_rotation::rotate_ml_dsa_keys_in_state_dir(&config, dir.path(), &fresh, now)
6473 .await;
6474 assert!(
6475 retried,
6476 "prior_drain={prev_drain_present}: the retry must fully restore and \
6477 promote"
6478 );
6479 assert_eq!(
6480 fresh.active_key().await.as_deref().map(ml_dsa_key_vk_bytes),
6481 Some(lead_vk),
6482 "prior_drain={prev_drain_present}: the queued lead promotes after \
6483 recovery"
6484 );
6485 let slots = fresh.0.read().await;
6486 assert_eq!(
6487 slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
6488 Some(active_vk),
6489 "prior_drain={prev_drain_present}: the recovered old active must \
6490 survive the retry in drain"
6491 );
6492 assert!(slots.lead.is_none());
6493 }
6494 }
6495
6496 #[tokio::test]
6497 async fn failed_rollback_defers_composite_publication() {
6498 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_PUBLICATION_GATE_TEST";
6503 if std::env::var_os(ISOLATED).is_none() {
6504 let status = std::process::Command::new(std::env::current_exe().unwrap())
6505 .args([
6506 "--exact",
6507 "auth::key_rotation::tests::failed_rollback_defers_composite_publication",
6508 "--nocapture",
6509 ])
6510 .env(ISOLATED, "1")
6511 .status()
6512 .unwrap();
6513 assert!(status.success());
6514 return;
6515 }
6516 let dir = TempDir::new().unwrap();
6517 let config = test_config();
6518 let now = chrono::Utc::now().timestamp();
6519
6520 let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6523 let active_slot = KeySlot::new(
6524 SigningKey::generate(&mut rand::rngs::OsRng),
6525 now - 14 * 86400,
6526 now + 1,
6527 );
6528 let lead_slot = KeySlot::new(
6529 SigningKey::generate(&mut rand::rngs::OsRng),
6530 now - 1,
6531 now + 14 * 86400,
6532 );
6533 persist_slot(dir.path(), "active", &active_slot).unwrap();
6534 persist_slot(dir.path(), "lead", &lead_slot).unwrap();
6535 let ed_store = Arc::new(SigningKeyStore::new(KeySlots {
6536 drain: None,
6537 active: Some(active_slot),
6538 lead: Some(lead_slot),
6539 }));
6540
6541 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
6542 initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
6543 .await
6544 .expect("a fully available signer set must initialize composite authority");
6545 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
6546 let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
6547 let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
6548 .snapshot()
6549 .version();
6550
6551 let meta_path = dir.path().join("jwt-signing-key.active.meta");
6552 std::fs::remove_file(&meta_path).unwrap();
6553 std::fs::create_dir(&meta_path).unwrap();
6554
6555 let extra = RotationStores {
6556 es256: None,
6557 ml_dsa: Some(Arc::clone(&pq_store)),
6558 composite_ca_key: Arc::clone(&ca),
6559 };
6560 run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
6561
6562 let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
6566 assert_eq!(
6567 commit_after.version, commit_before.version,
6568 "a not-fully-restored cycle must not advance the commit marker"
6569 );
6570 assert_eq!(
6571 commit_after.component_digest, commit_before.component_digest,
6572 "a not-fully-restored cycle must not change the committed digest"
6573 );
6574 assert_eq!(
6575 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6576 pending_before,
6577 "a not-fully-restored cycle must not restate the pending ledger"
6578 );
6579 assert_eq!(
6580 hyprstream_rpc::auth::global_composite_key_set()
6581 .snapshot()
6582 .version(),
6583 snapshot_version_before,
6584 "a not-fully-restored cycle must not publish to the live key set"
6585 );
6586 assert_eq!(
6588 read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
6589 Some(commit_before.component_digest.clone()),
6590 "a deferring cycle must retain the committed predecessor digest"
6591 );
6592 }
6593
6594 #[tokio::test]
6595 async fn ed25519_restart_mismatch_fails_closed() {
6596 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_RESTART_MISMATCH_TEST";
6597 if std::env::var_os(ISOLATED).is_none() {
6598 let status = std::process::Command::new(std::env::current_exe().unwrap())
6599 .args([
6600 "--exact",
6601 "auth::key_rotation::tests::ed25519_restart_mismatch_fails_closed",
6602 "--nocapture",
6603 ])
6604 .env(ISOLATED, "1")
6605 .status()
6606 .unwrap();
6607 assert!(status.success());
6608 return;
6609 }
6610
6611 let dir = TempDir::new().unwrap();
6612 let config = test_config();
6613 let now = chrono::Utc::now().timestamp();
6614
6615 let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6618 let new_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6619 let pq_lead = ml_dsa_rotation::MlDsaSigningKeyStore::new(MlDsaKeySlots {
6620 drain: None,
6621 active: pq_store.active_key().await.map(|key| {
6622 ml_dsa_rotation::MlDsaKeySlot::new((*key).clone(), now - 3600, now + 14 * 86400)
6623 }),
6624 lead: Some(new_pq.clone()),
6625 });
6626 let old_ed = KeySlot::new(
6627 SigningKey::generate(&mut rand::rngs::OsRng),
6628 now - 14 * 86400,
6629 now + 6 * 3600,
6630 );
6631 let new_ed = KeySlot::new(
6632 SigningKey::generate(&mut rand::rngs::OsRng),
6633 now - 1,
6634 now + 14 * 86400,
6635 );
6636 persist_slot(dir.path(), "active", &old_ed).unwrap();
6637 persist_slot(dir.path(), "lead", &new_ed).unwrap();
6638 let ed_store = Arc::new(SigningKeyStore::new(KeySlots {
6639 drain: None,
6640 active: Some(old_ed.clone()),
6641 lead: Some(new_ed.clone()),
6642 }));
6643
6644 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
6645 initialize_composite_key_set(dir.path(), &ed_store, &pq_lead, Arc::clone(&ca), 30)
6649 .await
6650 .expect("healthy starting authority must initialize composite");
6651 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
6652 let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
6653 let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
6654 .snapshot()
6655 .version();
6656
6657 let meta_path = dir.path().join("jwt-signing-key.active.meta");
6661 std::fs::remove_file(&meta_path).unwrap();
6662 std::fs::create_dir(&meta_path).unwrap();
6663 let extra = RotationStores {
6664 es256: None,
6665 ml_dsa: Some(Arc::new(pq_lead)),
6666 composite_ca_key: Arc::clone(&ca),
6667 };
6668 let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
6669 assert!(
6670 !all_restored,
6671 "the mixed cycle must report not fully restored"
6672 );
6673 let (commit_deferred, _) = read_committed_composite_ledger(dir.path()).unwrap();
6674 assert_eq!(commit_deferred.version, commit_before.version);
6675 assert_eq!(
6676 commit_deferred.component_digest,
6677 commit_before.component_digest
6678 );
6679 assert_eq!(
6680 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6681 pending_before,
6682 "a deferred cycle must not restate the pending ledger"
6683 );
6684 assert_eq!(
6685 hyprstream_rpc::auth::global_composite_key_set()
6686 .snapshot()
6687 .version(),
6688 snapshot_version_before,
6689 "a deferred cycle must not publish to the live key set"
6690 );
6691 assert_eq!(
6692 read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
6693 Some(commit_before.component_digest.clone()),
6694 "the committed predecessor digest must be retained across the \
6695 deferred cycle"
6696 );
6697
6698 std::fs::remove_dir(&meta_path).unwrap();
6704 let fresh_ed = Arc::new(load_or_init_key_store(dir.path(), &config));
6705 let fresh_pq = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6706 let extra_fresh = RotationStores {
6707 es256: None,
6708 ml_dsa: Some(Arc::clone(&fresh_pq)),
6709 composite_ca_key: Arc::clone(&ca),
6710 };
6711 let initial_record = read_deferred_predecessor(dir.path()).unwrap();
6712 let reloaded_digest =
6713 component_state_digest(&fresh_ed, &fresh_pq, ca.verifying_key()).await;
6714 assert!(
6715 !initial_record.states.contains(&reloaded_digest),
6716 "a restart reload not certified by this process must remain untrusted"
6717 );
6718 let marker_path = dir.path().join("jwt-composite-pairs.committed");
6719 let marker_bytes = std::fs::read(&marker_path).unwrap();
6720 std::fs::write(&marker_path, b"corrupt").unwrap();
6721 let _failed_cycle =
6724 run_rotation_cycle(&config, dir.path(), &fresh_ed, &extra_fresh, now).await;
6725 std::fs::write(&marker_path, &marker_bytes).unwrap();
6729 let (commit_failed, _) = read_committed_composite_ledger(dir.path()).unwrap();
6730 assert_eq!(
6731 commit_failed.version, commit_before.version,
6732 "a failed recovery publication must not advance the committed \
6733 generation"
6734 );
6735 let record_after_failure = read_deferred_predecessor(dir.path())
6736 .expect("a failed recovery publication must retain the record");
6737 assert_eq!(
6738 record_after_failure.committed_digest, commit_before.component_digest,
6739 "the retained record must keep the committed predecessor"
6740 );
6741 assert_eq!(
6742 record_after_failure.states, initial_record.states,
6743 "an untrusted restart state must not be added to the recovery record"
6744 );
6745 assert_eq!(
6746 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6747 pending_before,
6748 "the failed recovery publication must not restate the pending ledger"
6749 );
6750
6751 std::fs::write(&marker_path, &marker_bytes).unwrap();
6755 let recovered = run_rotation_cycle(&config, dir.path(), &fresh_ed, &extra_fresh, now).await;
6756 assert!(
6757 recovered,
6758 "the tick after a failed recovery publication must publish"
6759 );
6760 let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
6761 assert_eq!(commit_after.version, commit_before.version);
6762 assert_eq!(
6763 commit_after.component_digest,
6764 commit_before.component_digest
6765 );
6766 assert_eq!(
6767 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6768 pending_before
6769 );
6770 assert!(read_deferred_predecessor(dir.path()).is_some());
6771 assert_eq!(
6772 hyprstream_rpc::auth::global_composite_key_set()
6773 .snapshot()
6774 .version(),
6775 snapshot_version_before
6776 );
6777 }
6778
6779 #[tokio::test]
6780 async fn ml_dsa_restart_mismatch_fails_closed() {
6781 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ML_DSA_RESTART_MISMATCH_TEST";
6782 if std::env::var_os(ISOLATED).is_none() {
6783 let status = std::process::Command::new(std::env::current_exe().unwrap())
6784 .args([
6785 "--exact",
6786 "auth::key_rotation::tests::ml_dsa_restart_mismatch_fails_closed",
6787 "--nocapture",
6788 ])
6789 .env(ISOLATED, "1")
6790 .status()
6791 .unwrap();
6792 assert!(status.success());
6793 return;
6794 }
6795
6796 let dir = TempDir::new().unwrap();
6797 let config = test_config();
6798 let now = chrono::Utc::now().timestamp();
6799
6800 let ed_store = Arc::new(load_or_init_key_store(dir.path(), &config));
6803 let new_ed = KeySlot::new(
6804 SigningKey::generate(&mut rand::rngs::OsRng),
6805 now - 1,
6806 now + 14 * 86400,
6807 );
6808 let ed_lead = SigningKeyStore::new(KeySlots {
6809 drain: None,
6810 active: ed_store
6811 .active_key()
6812 .await
6813 .map(|key| KeySlot::new((*key).clone(), now - 3600, now + 14 * 86400)),
6814 lead: Some(new_ed.clone()),
6815 });
6816 let old_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 6 * 3600);
6817 let new_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
6818 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &old_pq).unwrap();
6819 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &new_pq).unwrap();
6820 let pq_store = Arc::new(MlDsaSigningKeyStore::new(MlDsaKeySlots {
6821 drain: None,
6822 active: Some(old_pq.clone()),
6823 lead: Some(new_pq.clone()),
6824 }));
6825
6826 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
6827 initialize_composite_key_set(dir.path(), &ed_lead, &pq_store, Arc::clone(&ca), 30)
6831 .await
6832 .expect("healthy starting authority must initialize composite");
6833 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
6834 let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
6835 let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
6836 .snapshot()
6837 .version();
6838
6839 let meta_path = dir.path().join("ml-dsa-signing-key.active.meta");
6842 std::fs::remove_file(&meta_path).unwrap();
6843 std::fs::create_dir(&meta_path).unwrap();
6844 let extra = RotationStores {
6845 es256: None,
6846 ml_dsa: Some(Arc::clone(&pq_store)),
6847 composite_ca_key: Arc::clone(&ca),
6848 };
6849 let all_restored = run_rotation_cycle(&config, dir.path(), &ed_lead, &extra, now).await;
6850 assert!(
6851 !all_restored,
6852 "the mixed cycle must report not fully restored"
6853 );
6854 let (commit_deferred, _) = read_committed_composite_ledger(dir.path()).unwrap();
6855 assert_eq!(commit_deferred.version, commit_before.version);
6856 assert_eq!(
6857 commit_deferred.component_digest,
6858 commit_before.component_digest
6859 );
6860 assert_eq!(
6861 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6862 pending_before,
6863 "a deferred cycle must not restate the pending ledger"
6864 );
6865 assert_eq!(
6866 hyprstream_rpc::auth::global_composite_key_set()
6867 .snapshot()
6868 .version(),
6869 snapshot_version_before,
6870 "a deferred cycle must not publish to the live key set"
6871 );
6872 assert_eq!(
6873 read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
6874 Some(commit_before.component_digest.clone()),
6875 "the committed predecessor digest must be retained across the \
6876 deferred cycle"
6877 );
6878
6879 std::fs::remove_dir(&meta_path).unwrap();
6884 let fresh_ed = Arc::new(load_or_init_key_store(dir.path(), &config));
6885 let fresh_pq = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6886 let extra_fresh = RotationStores {
6887 es256: None,
6888 ml_dsa: Some(Arc::clone(&fresh_pq)),
6889 composite_ca_key: Arc::clone(&ca),
6890 };
6891 let initial_record = read_deferred_predecessor(dir.path()).unwrap();
6892 let reloaded_digest =
6893 component_state_digest(&fresh_ed, &fresh_pq, ca.verifying_key()).await;
6894 assert!(
6895 !initial_record.states.contains(&reloaded_digest),
6896 "a restart reload not certified by this process must remain untrusted"
6897 );
6898 let marker_path = dir.path().join("jwt-composite-pairs.committed");
6899 let marker_bytes = std::fs::read(&marker_path).unwrap();
6900 std::fs::write(&marker_path, b"corrupt").unwrap();
6901 let _failed_cycle =
6904 run_rotation_cycle(&config, dir.path(), &fresh_ed, &extra_fresh, now).await;
6905 std::fs::write(&marker_path, &marker_bytes).unwrap();
6909 let (commit_failed, _) = read_committed_composite_ledger(dir.path()).unwrap();
6910 assert_eq!(
6911 commit_failed.version, commit_before.version,
6912 "a failed recovery publication must not advance the committed \
6913 generation"
6914 );
6915 let record_after_failure = read_deferred_predecessor(dir.path())
6916 .expect("a failed recovery publication must retain the record");
6917 assert_eq!(
6918 record_after_failure.committed_digest, commit_before.component_digest,
6919 "the retained record must keep the committed predecessor"
6920 );
6921 assert_eq!(
6922 record_after_failure.states, initial_record.states,
6923 "an untrusted restart state must not be added to the recovery record"
6924 );
6925 assert_eq!(
6926 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6927 pending_before,
6928 "the failed recovery publication must not restate the pending ledger"
6929 );
6930
6931 std::fs::write(&marker_path, &marker_bytes).unwrap();
6935 let recovered = run_rotation_cycle(&config, dir.path(), &fresh_ed, &extra_fresh, now).await;
6936 assert!(
6937 recovered,
6938 "the rotation operation itself should remain usable"
6939 );
6940 let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
6941 assert_eq!(commit_after.version, commit_before.version);
6942 assert_eq!(
6943 commit_after.component_digest,
6944 commit_before.component_digest
6945 );
6946 assert_eq!(
6947 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
6948 pending_before
6949 );
6950 assert!(read_deferred_predecessor(dir.path()).is_some());
6951 assert_eq!(
6952 hyprstream_rpc::auth::global_composite_key_set()
6953 .snapshot()
6954 .version(),
6955 snapshot_version_before,
6956 "the restart mismatch must not publish new signer authority"
6957 );
6958 }
6959
6960 #[tokio::test]
6961 async fn ed25519_failed_drain_rollback_defers_publication() {
6962 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_DRAIN_ROLLBACK_TEST";
6963 if std::env::var_os(ISOLATED).is_none() {
6964 let status = std::process::Command::new(std::env::current_exe().unwrap())
6965 .args([
6966 "--exact",
6967 "auth::key_rotation::tests::ed25519_failed_drain_rollback_defers_publication",
6968 "--nocapture",
6969 ])
6970 .env(ISOLATED, "1")
6971 .status()
6972 .unwrap();
6973 assert!(status.success());
6974 return;
6975 }
6976
6977 for prev_drain_present in [false, true] {
6978 let dir = TempDir::new().unwrap();
6979 let config = test_config();
6980 let now = chrono::Utc::now().timestamp();
6981
6982 let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
6983 let active_slot = KeySlot::new(
6984 SigningKey::generate(&mut rand::rngs::OsRng),
6985 now - 14 * 86400,
6986 now + 6 * 3600,
6987 );
6988 let lead_slot = KeySlot::new(
6989 SigningKey::generate(&mut rand::rngs::OsRng),
6990 now - 1,
6991 now + 14 * 86400,
6992 );
6993 let active_vk = active_slot.verifying_key_bytes();
6994 let lead_vk = lead_slot.verifying_key_bytes();
6995 let prev_drain = if prev_drain_present {
6996 let drain = KeySlot::new(
6997 SigningKey::generate(&mut rand::rngs::OsRng),
6998 now - 60 * 86400,
6999 now - 45 * 86400,
7000 );
7001 persist_slot(dir.path(), "drain", &drain).unwrap();
7002 Some(drain)
7003 } else {
7004 None
7005 };
7006 persist_slot(dir.path(), "active", &active_slot).unwrap();
7007 persist_slot(dir.path(), "lead", &lead_slot).unwrap();
7008
7009 let ed_store = SigningKeyStore::new(KeySlots {
7010 drain: prev_drain.clone(),
7011 active: Some(active_slot),
7012 lead: Some(lead_slot),
7013 });
7014
7015 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7016 initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7017 .await
7018 .expect("healthy starting authority must initialize composite");
7019 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7020 let pending_before =
7021 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7022 let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
7023 .snapshot()
7024 .version();
7025
7026 let meta_path = dir.path().join("jwt-signing-key.drain.meta");
7032 if meta_path.exists() {
7033 std::fs::remove_file(&meta_path).unwrap();
7034 }
7035 std::fs::create_dir(&meta_path).unwrap();
7036
7037 let extra = RotationStores {
7038 es256: None,
7039 ml_dsa: Some(Arc::clone(&pq_store)),
7040 composite_ca_key: Arc::clone(&ca),
7041 };
7042 let all_restored =
7043 run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7044 assert!(
7045 !all_restored,
7046 "prior_drain={prev_drain_present}: an unconfirmed previous-drain \
7047 restoration must report the stores as not fully restored"
7048 );
7049
7050 let slots = ed_store.0.read().await;
7051 assert_eq!(
7052 slots.active.as_ref().map(KeySlot::verifying_key_bytes),
7053 Some(active_vk),
7054 "prior_drain={prev_drain_present}: the old active must remain the \
7055 in-memory active signer"
7056 );
7057 assert_eq!(
7058 slots.lead.as_ref().map(KeySlot::verifying_key_bytes),
7059 Some(lead_vk),
7060 "prior_drain={prev_drain_present}: the lead must stay queued"
7061 );
7062 assert_eq!(
7063 slots.drain.as_ref().map(KeySlot::verifying_key_bytes),
7064 prev_drain.as_ref().map(KeySlot::verifying_key_bytes),
7065 "prior_drain={prev_drain_present}: memory drain must not change"
7066 );
7067 drop(slots);
7068
7069 assert!(
7070 load_slot(dir.path(), "drain").is_none(),
7071 "prior_drain={prev_drain_present}: the torn drain slot must not load"
7072 );
7073
7074 let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
7078 assert_eq!(commit_after.version, commit_before.version);
7079 assert_eq!(
7080 commit_after.component_digest,
7081 commit_before.component_digest
7082 );
7083 assert_eq!(
7084 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7085 pending_before,
7086 "a not-fully-restored cycle must not restate the pending ledger"
7087 );
7088 assert_eq!(
7089 hyprstream_rpc::auth::global_composite_key_set()
7090 .snapshot()
7091 .version(),
7092 snapshot_version_before,
7093 "a not-fully-restored cycle must not publish to the live key set"
7094 );
7095 assert_eq!(
7096 read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
7097 Some(commit_before.component_digest.clone()),
7098 "a deferring cycle must retain the committed predecessor digest"
7099 );
7100 }
7101 }
7102
7103 #[tokio::test]
7104 async fn ml_dsa_failed_drain_rollback_defers_publication() {
7105 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ML_DSA_DRAIN_ROLLBACK_TEST";
7106 if std::env::var_os(ISOLATED).is_none() {
7107 let status = std::process::Command::new(std::env::current_exe().unwrap())
7108 .args([
7109 "--exact",
7110 "auth::key_rotation::tests::ml_dsa_failed_drain_rollback_defers_publication",
7111 "--nocapture",
7112 ])
7113 .env(ISOLATED, "1")
7114 .status()
7115 .unwrap();
7116 assert!(status.success());
7117 return;
7118 }
7119
7120 for prev_drain_present in [false, true] {
7121 let dir = TempDir::new().unwrap();
7122 let config = test_config();
7123 let now = chrono::Utc::now().timestamp();
7124
7125 let ed_store = Arc::new(load_or_init_key_store(dir.path(), &config));
7126 let active = ml_dsa_rotation::generate_ml_dsa_slot(now - 14 * 86400, now + 6 * 3600);
7127 let lead = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7128 let active_vk = ml_dsa_slot_vk_bytes(&active);
7129 let lead_vk = ml_dsa_slot_vk_bytes(&lead);
7130 let prev_drain = if prev_drain_present {
7131 let drain =
7132 ml_dsa_rotation::generate_ml_dsa_slot(now - 60 * 86400, now - 45 * 86400);
7133 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "drain", &drain).unwrap();
7134 Some(drain)
7135 } else {
7136 None
7137 };
7138 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &active).unwrap();
7139 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &lead).unwrap();
7140
7141 let pq_store = Arc::new(MlDsaSigningKeyStore::new(MlDsaKeySlots {
7142 drain: prev_drain.clone(),
7143 active: Some(active),
7144 lead: Some(lead),
7145 }));
7146
7147 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7148 initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7149 .await
7150 .expect("healthy starting authority must initialize composite");
7151 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7152 let pending_before =
7153 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7154 let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
7155 .snapshot()
7156 .version();
7157
7158 let meta_path = dir.path().join("ml-dsa-signing-key.drain.meta");
7160 if meta_path.exists() {
7161 std::fs::remove_file(&meta_path).unwrap();
7162 }
7163 std::fs::create_dir(&meta_path).unwrap();
7164
7165 let extra = RotationStores {
7166 es256: None,
7167 ml_dsa: Some(Arc::clone(&pq_store)),
7168 composite_ca_key: Arc::clone(&ca),
7169 };
7170 let all_restored =
7171 run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7172 assert!(
7173 !all_restored,
7174 "prior_drain={prev_drain_present}: an unconfirmed previous-drain \
7175 restoration must report the stores as not fully restored"
7176 );
7177
7178 let slots = pq_store.0.read().await;
7179 assert_eq!(
7180 slots.active.as_ref().map(ml_dsa_slot_vk_bytes),
7181 Some(active_vk.clone()),
7182 "prior_drain={prev_drain_present}: the old active must remain the \
7183 in-memory active signer"
7184 );
7185 assert_eq!(
7186 slots.lead.as_ref().map(ml_dsa_slot_vk_bytes),
7187 Some(lead_vk.clone()),
7188 "prior_drain={prev_drain_present}: the lead must stay queued"
7189 );
7190 assert_eq!(
7191 slots.drain.as_ref().map(ml_dsa_slot_vk_bytes),
7192 prev_drain.as_ref().map(ml_dsa_slot_vk_bytes),
7193 "prior_drain={prev_drain_present}: memory drain must not change"
7194 );
7195 drop(slots);
7196
7197 assert!(
7198 ml_dsa_rotation::load_ml_dsa_slot(dir.path(), "drain").is_none(),
7199 "prior_drain={prev_drain_present}: the torn drain slot must not load"
7200 );
7201
7202 let (commit_after, _) = read_committed_composite_ledger(dir.path()).unwrap();
7203 assert_eq!(commit_after.version, commit_before.version);
7204 assert_eq!(
7205 commit_after.component_digest,
7206 commit_before.component_digest
7207 );
7208 assert_eq!(
7209 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7210 pending_before,
7211 "a not-fully-restored cycle must not restate the pending ledger"
7212 );
7213 assert_eq!(
7214 hyprstream_rpc::auth::global_composite_key_set()
7215 .snapshot()
7216 .version(),
7217 snapshot_version_before,
7218 "a not-fully-restored cycle must not publish to the live key set"
7219 );
7220 assert_eq!(
7221 read_deferred_predecessor(dir.path()).map(|record| record.committed_digest.clone()),
7222 Some(commit_before.component_digest.clone()),
7223 "a deferring cycle must retain the committed predecessor digest"
7224 );
7225 }
7226 }
7227
7228 #[tokio::test]
7229 async fn ed25519_stale_anchor_does_not_authorize_publication() {
7230 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_STALE_ANCHOR_TEST";
7231 if std::env::var_os(ISOLATED).is_none() {
7232 let status = std::process::Command::new(std::env::current_exe().unwrap())
7233 .args([
7234 "--exact",
7235 "auth::key_rotation::tests::ed25519_stale_anchor_does_not_authorize_publication",
7236 "--nocapture",
7237 ])
7238 .env(ISOLATED, "1")
7239 .status()
7240 .unwrap();
7241 assert!(status.success());
7242 return;
7243 }
7244
7245 let dir = TempDir::new().unwrap();
7246 let config = test_config();
7247 let now = chrono::Utc::now().timestamp();
7248
7249 let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
7250 let active_slot = KeySlot::new(
7254 SigningKey::generate(&mut rand::rngs::OsRng),
7255 now - 14 * 86400,
7256 now + 8 * 86400,
7257 );
7258 persist_slot(dir.path(), "active", &active_slot).unwrap();
7259 let ed_store = SigningKeyStore::new(KeySlots {
7260 drain: None,
7261 active: Some(active_slot),
7262 lead: None,
7263 });
7264
7265 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7266 initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7267 .await
7268 .expect("healthy starting authority must initialize composite");
7269 let (commit_before, ledger_before) = read_committed_composite_ledger(dir.path()).unwrap();
7270 let oauth_before = ledger_before
7271 .pairs
7272 .iter()
7273 .find(|record| record.role == "oauth" && record.state == "active")
7274 .expect("initialized authority must carry an active OAuth pair")
7275 .clone();
7276
7277 let foreign_ed = load_or_init_key_store(&dir.path().join("foreign-ed"), &config);
7284 let foreign_pq = load_or_init_ml_dsa_key_store(&dir.path().join("foreign-pq"), &config);
7285 let foreign_state =
7286 component_state_digest(&foreign_ed, &foreign_pq, ca.verifying_key()).await;
7287 let record = DeferredPredecessorRecord {
7288 committed_digest: commit_before.component_digest.clone(),
7289 states: vec![foreign_state],
7290 };
7291 assert!(
7292 write_deferred_predecessor(dir.path(), &record),
7293 "the test fixture must be able to record a foreign anchor"
7294 );
7295
7296 let extra = RotationStores {
7297 es256: None,
7298 ml_dsa: Some(Arc::clone(&pq_store)),
7299 composite_ca_key: Arc::clone(&ca),
7300 };
7301 let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7302 assert!(all_restored);
7303
7304 let (commit_after, ledger_after) = read_committed_composite_ledger(dir.path()).unwrap();
7308 let oauth_active_after = ledger_after
7309 .pairs
7310 .iter()
7311 .find(|record| record.role == "oauth" && record.state == "active")
7312 .expect("authority must still carry an active OAuth pair");
7313 assert_eq!(
7314 oauth_active_after.ed25519_public, oauth_before.ed25519_public,
7315 "a foreign certified state must not replace the committed OAuth \
7316 signer"
7317 );
7318 assert_ne!(
7319 oauth_active_after.ed25519_public,
7320 URL_SAFE_NO_PAD.encode(foreign_ed.active_verifying_key_bytes().await.unwrap()),
7321 "the foreign stale component state must never be published"
7322 );
7323 assert_eq!(
7324 commit_after.component_digest, commit_before.component_digest,
7325 "an ignored record must not change the committed digest"
7326 );
7327 assert!(
7328 read_deferred_predecessor(dir.path()).is_none(),
7329 "the successful publication clears the ignored record"
7330 );
7331 }
7332
7333 #[tokio::test]
7334 async fn ed25519_stale_writer_ticks_do_not_certify_or_publish() {
7335 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_STALE_WRITER_TEST";
7336 if std::env::var_os(ISOLATED).is_none() {
7337 let status = std::process::Command::new(std::env::current_exe().unwrap())
7338 .args([
7339 "--exact",
7340 "auth::key_rotation::tests::ed25519_stale_writer_ticks_do_not_certify_or_publish",
7341 "--nocapture",
7342 ])
7343 .env(ISOLATED, "1")
7344 .status()
7345 .unwrap();
7346 assert!(status.success());
7347 return;
7348 }
7349
7350 let dir = TempDir::new().unwrap();
7351 let config = test_config();
7352 let now = chrono::Utc::now().timestamp();
7353
7354 let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
7358 let old_ed = KeySlot::new(
7359 SigningKey::generate(&mut rand::rngs::OsRng),
7360 now - 14 * 86400,
7361 now + 6 * 3600,
7362 );
7363 let new_ed = KeySlot::new(
7364 SigningKey::generate(&mut rand::rngs::OsRng),
7365 now - 1,
7366 now + 14 * 86400,
7367 );
7368 let new_ed_vk = new_ed.verifying_key_bytes();
7369 persist_slot(dir.path(), "active", &old_ed).unwrap();
7370 persist_slot(dir.path(), "lead", &new_ed).unwrap();
7371 let ed_store = SigningKeyStore::new(KeySlots {
7372 drain: None,
7373 active: Some(old_ed),
7374 lead: Some(new_ed),
7375 });
7376
7377 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7378 initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7379 .await
7380 .expect("healthy starting authority must initialize composite");
7381 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7382 let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7383 let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
7384 .snapshot()
7385 .version();
7386
7387 let foreign_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 30 * 86400, now + 8 * 86400);
7392 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &foreign_pq).unwrap();
7393
7394 let meta_path = dir.path().join("jwt-signing-key.active.meta");
7395 std::fs::remove_file(&meta_path).unwrap();
7396 std::fs::create_dir(&meta_path).unwrap();
7397 let extra = RotationStores {
7398 es256: None,
7399 ml_dsa: Some(Arc::clone(&pq_store)),
7400 composite_ca_key: Arc::clone(&ca),
7401 };
7402 let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7403 assert!(
7404 !all_restored,
7405 "the mixed cycle must defer and record its certified states"
7406 );
7407 let seeded = read_deferred_predecessor(dir.path())
7408 .expect("the deferring cycle must retain a recovery record");
7409 assert_eq!(
7410 seeded.committed_digest, commit_before.component_digest,
7411 "the record must anchor on the current committed generation"
7412 );
7413 assert_eq!(
7414 seeded.states.len(),
7415 1,
7416 "only this process's memory state is recorded"
7417 );
7418 let foreign_reload = {
7419 let reloaded_ed = load_or_init_key_store(dir.path(), &config);
7420 let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
7421 component_state_digest(&reloaded_ed, &reloaded_pq, ca.verifying_key()).await
7422 };
7423 assert_ne!(
7424 foreign_reload, seeded.states[0],
7425 "the mutable slot state predated this record and is not trusted"
7426 );
7427 assert!(
7428 !seeded.states.contains(&foreign_reload),
7429 "a foreign reload must never be certified into the first record"
7430 );
7431 let certified_states = seeded.states.clone();
7432
7433 let stale_ed = KeySlot::new(
7437 SigningKey::generate(&mut rand::rngs::OsRng),
7438 now - 30 * 86400,
7439 now + 8 * 86400,
7440 );
7441 let stale_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 30 * 86400, now + 8 * 86400);
7442 let stale_ed_store = SigningKeyStore::new(KeySlots {
7443 drain: None,
7444 active: Some(stale_ed),
7445 lead: None,
7446 });
7447 let stale_pq_store = Arc::new(MlDsaSigningKeyStore::new(MlDsaKeySlots {
7448 drain: None,
7449 active: Some(stale_pq),
7450 lead: None,
7451 }));
7452 let stale_extra = RotationStores {
7453 es256: None,
7454 ml_dsa: Some(Arc::clone(&stale_pq_store)),
7455 composite_ca_key: Arc::clone(&ca),
7456 };
7457
7458 for tick in 1..=2 {
7462 let all_restored =
7463 run_rotation_cycle(&config, dir.path(), &stale_ed_store, &stale_extra, now).await;
7464 assert!(
7465 all_restored,
7466 "stale tick {tick}: the no-op writer is restored"
7467 );
7468 let (commit_after_tick, _ledger_after_tick) =
7469 read_committed_composite_ledger(dir.path()).unwrap();
7470 assert_eq!(
7471 commit_after_tick.version, commit_before.version,
7472 "stale tick {tick}: a rejected stale writer must not advance the \
7473 committed generation"
7474 );
7475 assert_eq!(
7476 commit_after_tick.component_digest, commit_before.component_digest,
7477 "stale tick {tick}: the committed digest must not change"
7478 );
7479 assert_eq!(
7480 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7481 pending_before,
7482 "stale tick {tick}: the pending ledger must be untouched"
7483 );
7484 assert_eq!(
7485 hyprstream_rpc::auth::global_composite_key_set()
7486 .snapshot()
7487 .version(),
7488 snapshot_version_before,
7489 "stale tick {tick}: a rejected stale writer must not publish to \
7490 the live key set"
7491 );
7492 let record = read_deferred_predecessor(dir.path())
7493 .expect("stale tick {tick}: the recovery record must survive");
7494 assert_eq!(
7495 record.committed_digest, seeded.committed_digest,
7496 "stale tick {tick}: the recorded committed predecessor must be \
7497 preserved"
7498 );
7499 assert_eq!(
7500 record.states, certified_states,
7501 "stale tick {tick}: the stale writer's outcome must never be \
7502 certified into the record"
7503 );
7504 }
7505
7506 std::fs::remove_dir(&meta_path).unwrap();
7509 let recovered = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7510 assert!(recovered, "the certified writer must recover and publish");
7511 let (commit_recovered, ledger_recovered) =
7512 read_committed_composite_ledger(dir.path()).unwrap();
7513 assert!(
7514 commit_recovered.version > commit_before.version,
7515 "the certified recovery must publish a new committed generation"
7516 );
7517 let oauth_active = ledger_recovered
7518 .pairs
7519 .iter()
7520 .find(|record| record.role == "oauth" && record.state == "active")
7521 .expect("the published generation must carry an active OAuth pair");
7522 assert_eq!(
7523 oauth_active.ed25519_public,
7524 URL_SAFE_NO_PAD.encode(new_ed_vk),
7525 "the published generation must carry the recovered Ed25519 signer"
7526 );
7527 assert!(read_deferred_predecessor(dir.path()).is_none());
7528 }
7529
7530 #[tokio::test]
7531 async fn ed25519_foreign_disk_state_never_certified() {
7532 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_FOREIGN_DISK_STATE_TEST";
7533 if std::env::var_os(ISOLATED).is_none() {
7534 let status = std::process::Command::new(std::env::current_exe().unwrap())
7535 .args([
7536 "--exact",
7537 "auth::key_rotation::tests::ed25519_foreign_disk_state_never_certified",
7538 "--nocapture",
7539 ])
7540 .env(ISOLATED, "1")
7541 .status()
7542 .unwrap();
7543 assert!(status.success());
7544 return;
7545 }
7546
7547 let dir = TempDir::new().unwrap();
7548 let config = test_config();
7549 let now = chrono::Utc::now().timestamp();
7550
7551 let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
7554 let new_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7555 let pq_lead = Arc::new(MlDsaSigningKeyStore::new(MlDsaKeySlots {
7556 drain: None,
7557 active: pq_store.active_key().await.map(|key| {
7558 ml_dsa_rotation::MlDsaKeySlot::new((*key).clone(), now - 3600, now + 14 * 86400)
7559 }),
7560 lead: Some(new_pq.clone()),
7561 }));
7562 let old_ed = KeySlot::new(
7563 SigningKey::generate(&mut rand::rngs::OsRng),
7564 now - 14 * 86400,
7565 now + 6 * 3600,
7566 );
7567 let new_ed = KeySlot::new(
7568 SigningKey::generate(&mut rand::rngs::OsRng),
7569 now - 1,
7570 now + 14 * 86400,
7571 );
7572 let new_ed_vk = new_ed.verifying_key_bytes();
7573 let new_pq_vk = ml_dsa_slot_vk_bytes(&new_pq);
7574 persist_slot(dir.path(), "active", &old_ed).unwrap();
7575 persist_slot(dir.path(), "lead", &new_ed).unwrap();
7576 let ed_store = SigningKeyStore::new(KeySlots {
7577 drain: None,
7578 active: Some(old_ed),
7579 lead: Some(new_ed),
7580 });
7581
7582 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7583 initialize_composite_key_set(dir.path(), &ed_store, &pq_lead, Arc::clone(&ca), 30)
7587 .await
7588 .expect("healthy starting authority must initialize composite");
7589 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7590 let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7591 let snapshot_version_before = hyprstream_rpc::auth::global_composite_key_set()
7592 .snapshot()
7593 .version();
7594
7595 let meta_path = dir.path().join("jwt-signing-key.active.meta");
7598 std::fs::remove_file(&meta_path).unwrap();
7599 std::fs::create_dir(&meta_path).unwrap();
7600 let extra = RotationStores {
7601 es256: None,
7602 ml_dsa: Some(Arc::clone(&pq_lead)),
7603 composite_ca_key: Arc::clone(&ca),
7604 };
7605 let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7606 assert!(!all_restored, "the mixed cycle must defer");
7607 let seeded = read_deferred_predecessor(dir.path())
7608 .expect("the deferring cycle must retain a recovery record");
7609 assert_eq!(
7610 seeded.committed_digest, commit_before.component_digest,
7611 "the record must anchor on the current committed generation"
7612 );
7613 assert_eq!(
7614 seeded.states.len(),
7615 1,
7616 "only process-computed state recorded"
7617 );
7618
7619 std::fs::remove_dir(&meta_path).unwrap();
7621
7622 let new_ed_b = KeySlot::new(
7628 SigningKey::generate(&mut rand::rngs::OsRng),
7629 now - 1,
7630 now + 14 * 86400,
7631 );
7632 let new_pq_b = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7633 let new_pq_b_vk = ml_dsa_slot_vk_bytes(&new_pq_b);
7634 let ed_b = SigningKeyStore::new(KeySlots {
7635 drain: None,
7636 active: ed_store
7637 .active_key()
7638 .await
7639 .map(|key| KeySlot::new((*key).clone(), now - 3600, now + 14 * 86400)),
7640 lead: Some(new_ed_b.clone()),
7641 });
7642 let pq_b = MlDsaSigningKeyStore::new(MlDsaKeySlots {
7643 drain: None,
7644 active: Some(ml_dsa_rotation::generate_ml_dsa_slot(
7645 now - 3600,
7646 now + 14 * 86400,
7647 )),
7648 lead: Some(new_pq_b.clone()),
7649 });
7650 let extra_b = RotationStores {
7651 es256: None,
7652 ml_dsa: Some(Arc::new(pq_b)),
7653 composite_ca_key: Arc::clone(&ca),
7654 };
7655 let b_restored = run_rotation_cycle(&config, dir.path(), &ed_b, &extra_b, now).await;
7656 assert!(b_restored, "writer B's own rotations must succeed");
7657
7658 let marker_path = dir.path().join("jwt-composite-pairs.committed");
7664 let marker_bytes = std::fs::read(&marker_path).unwrap();
7665 std::fs::write(&marker_path, b"corrupt").unwrap();
7666 let all_restored = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7667 assert!(
7668 all_restored,
7669 "writer A's rotation is restored; only its publication fails"
7670 );
7671 let record = read_deferred_predecessor(dir.path())
7672 .expect("the recovery record must survive the failed publication");
7673 assert_eq!(
7674 record.committed_digest, commit_before.component_digest,
7675 "the record must keep the committed predecessor"
7676 );
7677 let post_memory = component_state_digest(&ed_store, &pq_lead, ca.verifying_key()).await;
7680 assert_eq!(
7681 record.states,
7682 vec![post_memory.clone()],
7683 "the advance must certify only writer A's own memory outcome"
7684 );
7685 let foreign_reload = {
7686 let reloaded_ed = load_or_init_key_store(dir.path(), &config);
7687 let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
7688 component_state_digest(&reloaded_ed, &reloaded_pq, ca.verifying_key()).await
7689 };
7690 assert_ne!(
7691 foreign_reload, post_memory,
7692 "writer B's disk slots must still be on disk for this regression"
7693 );
7694 assert!(
7695 !record.states.contains(&foreign_reload),
7696 "the durable state left by the rejected writer must never be \
7697 certified into the record"
7698 );
7699 assert_eq!(
7700 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7701 pending_before,
7702 "the failed publication must not restate the pending ledger"
7703 );
7704 assert_eq!(
7705 hyprstream_rpc::auth::global_composite_key_set()
7706 .snapshot()
7707 .version(),
7708 snapshot_version_before,
7709 "the failed publication must not publish to the live key set"
7710 );
7711
7712 std::fs::write(&marker_path, &marker_bytes).unwrap();
7715 let (commit_recovered_check, _) = read_committed_composite_ledger(dir.path()).unwrap();
7716 assert_eq!(
7717 commit_recovered_check.version, commit_before.version,
7718 "the failed publication must not advance the committed generation"
7719 );
7720
7721 let b_retry = run_rotation_cycle(&config, dir.path(), &ed_b, &extra_b, now).await;
7725 assert!(b_retry, "writer B's no-op tick is restored");
7726 let (commit_b_retry, ledger_b_retry) = read_committed_composite_ledger(dir.path()).unwrap();
7727 assert_eq!(commit_b_retry.version, commit_before.version);
7728 let oauth_b = ledger_b_retry
7729 .pairs
7730 .iter()
7731 .find(|record| record.role == "oauth" && record.state == "active")
7732 .expect("authority must still carry an active OAuth pair");
7733 assert_ne!(
7734 oauth_b.ml_dsa_public,
7735 URL_SAFE_NO_PAD.encode(&new_pq_b_vk),
7736 "writer B's stale ML-DSA outcome must never become committed \
7737 authority"
7738 );
7739 assert_eq!(
7740 read_deferred_predecessor(dir.path())
7741 .expect("the record must survive writer B's retry")
7742 .states,
7743 vec![post_memory.clone()],
7744 "writer B's retry must not certify its outcome into the record"
7745 );
7746
7747 std::fs::write(&marker_path, &marker_bytes).unwrap();
7750 let recovered = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7751 assert!(
7752 recovered,
7753 "writer A's certified tick must recover and publish"
7754 );
7755 let (commit_recovered, ledger_recovered) =
7756 read_committed_composite_ledger(dir.path()).unwrap();
7757 assert!(
7758 commit_recovered.version > commit_before.version,
7759 "the certified recovery must publish a new committed generation"
7760 );
7761 let oauth_recovered = ledger_recovered
7762 .pairs
7763 .iter()
7764 .find(|record| record.role == "oauth" && record.state == "active")
7765 .expect("the published generation must carry an active OAuth pair");
7766 assert_eq!(
7767 oauth_recovered.ed25519_public,
7768 URL_SAFE_NO_PAD.encode(new_ed_vk),
7769 "the published generation must carry the recovered Ed25519 signer"
7770 );
7771 assert_eq!(
7772 oauth_recovered.ml_dsa_public,
7773 URL_SAFE_NO_PAD.encode(&new_pq_vk),
7774 "the published generation must carry the promoted ML-DSA signer"
7775 );
7776 assert!(read_deferred_predecessor(dir.path()).is_none());
7777 }
7778
7779 #[tokio::test]
7780 async fn ed25519_chained_deferral_never_certifies_reload() {
7781 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_ED_CHAINED_DEFER_TEST";
7782 if std::env::var_os(ISOLATED).is_none() {
7783 let status = std::process::Command::new(std::env::current_exe().unwrap())
7784 .args([
7785 "--exact",
7786 "auth::key_rotation::tests::ed25519_chained_deferral_never_certifies_reload",
7787 "--nocapture",
7788 ])
7789 .env(ISOLATED, "1")
7790 .status()
7791 .unwrap();
7792 assert!(status.success());
7793 return;
7794 }
7795
7796 let dir = TempDir::new().unwrap();
7797 let config = test_config();
7798 let now = chrono::Utc::now().timestamp();
7799
7800 load_or_init_ml_dsa_key_store(dir.path(), &config);
7803 let old_ed = KeySlot::new(
7804 SigningKey::generate(&mut rand::rngs::OsRng),
7805 now - 14 * 86400,
7806 now + 6 * 3600,
7807 );
7808 let new_ed = KeySlot::new(
7809 SigningKey::generate(&mut rand::rngs::OsRng),
7810 now - 1,
7811 now + 14 * 86400,
7812 );
7813 let old_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 3600, now + 14 * 86400);
7814 let new_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7815 persist_slot(dir.path(), "active", &old_ed).unwrap();
7816 persist_slot(dir.path(), "lead", &new_ed).unwrap();
7817 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &old_pq).unwrap();
7818 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "lead", &new_pq).unwrap();
7819 let ed_store = SigningKeyStore::new(KeySlots {
7820 drain: None,
7821 active: Some(old_ed),
7822 lead: Some(new_ed),
7823 });
7824 let pq_store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
7825 drain: None,
7826 active: Some(old_pq),
7827 lead: Some(new_pq),
7828 });
7829
7830 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7831 initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7832 .await
7833 .expect("healthy starting authority must initialize composite");
7834 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7835 let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7836
7837 let meta_path = dir.path().join("jwt-signing-key.active.meta");
7838 std::fs::remove_file(&meta_path).unwrap();
7839 std::fs::create_dir(&meta_path).unwrap();
7840 let pq_store = Arc::new(pq_store);
7841 let extra = RotationStores {
7842 es256: None,
7843 ml_dsa: Some(Arc::clone(&pq_store)),
7844 composite_ca_key: Arc::clone(&ca),
7845 };
7846
7847 let deferred1 = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7850 assert!(!deferred1, "cycle 1 must defer");
7851 let seeded =
7852 read_deferred_predecessor(dir.path()).expect("cycle 1 must retain a recovery record");
7853 assert_eq!(seeded.states.len(), 1, "seed records process memory only");
7854 let seeded_mem_digest =
7855 component_state_digest(&ed_store, &pq_store, ca.verifying_key()).await;
7856 assert_eq!(
7857 seeded.states,
7858 vec![seeded_mem_digest.clone()],
7859 "the seed must match only this process's computed memory"
7860 );
7861
7862 let deferred2 = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7866 assert!(!deferred2, "cycle 2 must defer again (same obstruction)");
7867 let advanced = read_deferred_predecessor(dir.path())
7868 .expect("the record must survive the chained deferral");
7869 assert_eq!(
7870 advanced.committed_digest, commit_before.component_digest,
7871 "the chained deferral must keep the committed predecessor"
7872 );
7873 assert_eq!(
7874 advanced.states, seeded.states,
7875 "an unchanged chained deferral must keep the record as-is"
7876 );
7877
7878 let foreign_pq = ml_dsa_rotation::generate_ml_dsa_slot(now - 1, now + 14 * 86400);
7883 ml_dsa_rotation::persist_ml_dsa_slot(dir.path(), "active", &foreign_pq).unwrap();
7884 let foreign_reload = {
7885 let reloaded_ed = load_or_init_key_store(dir.path(), &config);
7886 let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
7887 component_state_digest(&reloaded_ed, &reloaded_pq, ca.verifying_key()).await
7888 };
7889 assert_ne!(
7890 foreign_reload, seeded_mem_digest,
7891 "fixture: the foreign disk state must differ from the certified \
7892 memory outcome"
7893 );
7894
7895 let deferred3 = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7900 assert!(!deferred3, "cycle 3 must defer again (same obstruction)");
7901 let after_foreign = read_deferred_predecessor(dir.path())
7902 .expect("the record must survive the third deferral");
7903 assert_eq!(
7904 after_foreign.committed_digest, commit_before.component_digest,
7905 "the committed predecessor must be preserved"
7906 );
7907 assert!(
7908 !after_foreign.states.contains(&foreign_reload),
7909 "the foreign durable state left by another writer must never be \
7910 certified into the record"
7911 );
7912 assert_eq!(
7913 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7914 pending_before,
7915 "deferred cycles must not touch the pending ledger"
7916 );
7917 }
7918
7919 #[tokio::test]
7920 async fn rotation_cycle_lock_serializes_writers() {
7921 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_CYCLE_LOCK_TEST";
7922 if std::env::var_os(ISOLATED).is_none() {
7923 let status = std::process::Command::new(std::env::current_exe().unwrap())
7924 .args([
7925 "--exact",
7926 "auth::key_rotation::tests::rotation_cycle_lock_serializes_writers",
7927 "--nocapture",
7928 ])
7929 .env(ISOLATED, "1")
7930 .status()
7931 .unwrap();
7932 assert!(status.success());
7933 return;
7934 }
7935
7936 let dir = TempDir::new().unwrap();
7937 let config = test_config();
7938 let now = chrono::Utc::now().timestamp();
7939
7940 let pq_store = Arc::new(load_or_init_ml_dsa_key_store(dir.path(), &config));
7941 let active_slot = KeySlot::new(
7942 SigningKey::generate(&mut rand::rngs::OsRng),
7943 now - 14 * 86400,
7944 now + 6 * 3600,
7945 );
7946 persist_slot(dir.path(), "active", &active_slot).unwrap();
7947 let ed_store = SigningKeyStore::new(KeySlots {
7948 drain: None,
7949 active: Some(active_slot),
7950 lead: None,
7951 });
7952
7953 let ca: Arc<SigningKey> = Arc::new(SigningKey::from_bytes(&[0x5A; 32]));
7954 initialize_composite_key_set(dir.path(), &ed_store, &pq_store, Arc::clone(&ca), 30)
7955 .await
7956 .expect("healthy starting authority must initialize composite");
7957 let (commit_before, _) = read_committed_composite_ledger(dir.path()).unwrap();
7958 let pending_before = std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap();
7959
7960 let cycle_lock = std::fs::OpenOptions::new()
7963 .create(true)
7964 .truncate(false)
7965 .read(true)
7966 .write(true)
7967 .open(cycle_lock_path(dir.path()))
7968 .unwrap();
7969 use nix::fcntl::{flock, FlockArg};
7970 use std::os::fd::AsRawFd as _;
7971 flock(cycle_lock.as_raw_fd(), FlockArg::LockExclusiveNonblock).unwrap();
7972
7973 let extra = RotationStores {
7974 es256: None,
7975 ml_dsa: Some(Arc::clone(&pq_store)),
7976 composite_ca_key: Arc::clone(&ca),
7977 };
7978 let skipped = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7979 assert!(
7980 skipped,
7981 "a lock-contended cycle must skip without reporting not-restored"
7982 );
7983 let (commit_skipped, _) = read_committed_composite_ledger(dir.path()).unwrap();
7984 assert_eq!(commit_skipped.version, commit_before.version);
7985 assert_eq!(
7986 std::fs::read(dir.path().join("jwt-composite-pairs.json")).unwrap(),
7987 pending_before,
7988 "a skipped cycle must not touch the pending ledger"
7989 );
7990
7991 drop(cycle_lock);
7993 let ran = run_rotation_cycle(&config, dir.path(), &ed_store, &extra, now).await;
7994 assert!(ran, "an uncontended cycle must run and stay restored");
7995 let (commit_ran, _) = read_committed_composite_ledger(dir.path()).unwrap();
7996 assert!(
7997 commit_ran.version >= commit_before.version,
7998 "an uncontended cycle must not regress the committed authority"
7999 );
8000 }
8001
8002 #[tokio::test]
8003 async fn committed_marker_failures_never_reinitialize_from_mutable_authority() {
8004 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_FAIL_CLOSED_TEST";
8005 if std::env::var_os(ISOLATED).is_none() {
8006 let status = std::process::Command::new(std::env::current_exe().unwrap())
8007 .args([
8008 "--exact",
8009 "auth::key_rotation::tests::committed_marker_failures_never_reinitialize_from_mutable_authority",
8010 "--nocapture",
8011 ])
8012 .env(ISOLATED, "1")
8013 .status()
8014 .unwrap();
8015 assert!(status.success());
8016 return;
8017 }
8018
8019 enum ImmutableFailure {
8020 Missing,
8021 Corrupt,
8022 Mismatched,
8023 Unavailable,
8024 }
8025
8026 for failure in [
8027 ImmutableFailure::Missing,
8028 ImmutableFailure::Corrupt,
8029 ImmutableFailure::Mismatched,
8030 ImmutableFailure::Unavailable,
8031 ] {
8032 let dir = TempDir::new().unwrap();
8033 let config = test_config();
8034 let ca = Arc::new(SigningKey::from_bytes(&[0x6a; 32]));
8035 let ed = load_or_init_key_store(dir.path(), &config);
8036 let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8037 initialize_composite_key_set(dir.path(), &ed, &pq, Arc::clone(&ca), 300)
8038 .await
8039 .unwrap();
8040
8041 let marker_bytes = std::fs::read(composite_committed_path(dir.path())).unwrap();
8042 let commit: CompositeCommit = serde_json::from_slice(&marker_bytes).unwrap();
8043 let immutable = composite_committed_ledger_path(dir.path(), &commit);
8044 let mut pending: CompositeLedger =
8045 serde_json::from_slice(&std::fs::read(composite_ledger_path(dir.path())).unwrap())
8046 .unwrap();
8047 pending.version = pending.version.saturating_add(1);
8048 pending.component_digest = "staged-component-C".to_owned();
8049 let pending_bytes = serde_json::to_vec(&pending).unwrap();
8050 std::fs::write(composite_ledger_path(dir.path()), &pending_bytes).unwrap();
8051
8052 match failure {
8053 ImmutableFailure::Missing => std::fs::remove_file(&immutable).unwrap(),
8054 ImmutableFailure::Corrupt => std::fs::write(&immutable, b"{").unwrap(),
8055 ImmutableFailure::Mismatched => {
8056 std::fs::write(&immutable, &pending_bytes).unwrap();
8057 }
8058 ImmutableFailure::Unavailable => {
8059 std::fs::remove_file(&immutable).unwrap();
8060 std::fs::create_dir(&immutable).unwrap();
8061 }
8062 }
8063
8064 let error = initialize_composite_key_set(dir.path(), &ed, &pq, Arc::clone(&ca), 300)
8065 .await
8066 .expect_err("marker-selected authority failure must fail closed");
8067 assert!(!error.to_string().is_empty());
8068 assert_eq!(
8069 std::fs::read(composite_committed_path(dir.path())).unwrap(),
8070 marker_bytes,
8071 "failure rewrote the committed marker"
8072 );
8073 assert_eq!(
8074 std::fs::read(composite_ledger_path(dir.path())).unwrap(),
8075 pending_bytes,
8076 "failure published over staged mutable C"
8077 );
8078 assert!(
8079 !composite_committed_ledger_path(
8080 dir.path(),
8081 &CompositeCommit {
8082 version: pending.version,
8083 component_digest: pending.component_digest.clone(),
8084 },
8085 )
8086 .exists(),
8087 "failure materialized staged C as committed authority"
8088 );
8089 }
8090 }
8091
8092 #[tokio::test]
8093 async fn committed_verifier_generation_restores_local_signer_without_rotation() {
8094 const PHASE: &str = "HYPRSTREAM_COMPOSITE_SIGNER_ATTACH_TEST";
8095 const DIRECTORY: &str = "HYPRSTREAM_COMPOSITE_SIGNER_ATTACH_DIR";
8096 let phase = std::env::var(PHASE).unwrap_or_default();
8097 if phase.is_empty() {
8098 let dir = TempDir::new().unwrap();
8099 for phase in ["bootstrap", "restore"] {
8100 let status = std::process::Command::new(std::env::current_exe().unwrap())
8101 .args(["--exact", "auth::key_rotation::tests::committed_verifier_generation_restores_local_signer_without_rotation", "--nocapture"])
8102 .env(PHASE, phase).env(DIRECTORY, dir.path()).status().unwrap();
8103 assert!(status.success(), "{phase} subprocess failed");
8104 }
8105 return;
8106 }
8107 let dir = std::path::PathBuf::from(std::env::var_os(DIRECTORY).unwrap());
8108 let config = test_config();
8109 let ca = Arc::new(SigningKey::from_bytes(&[0x69; 32]));
8110 let ed = load_or_init_key_store(&dir, &config);
8111 let pq = load_or_init_ml_dsa_key_store(&dir, &config);
8112 if phase == "bootstrap" {
8113 initialize_composite_key_set(&dir, &ed, &pq, ca, 300)
8114 .await
8115 .unwrap();
8116 return;
8117 }
8118 let marker = std::fs::read(composite_committed_path(&dir)).unwrap();
8119 let (commit, _) = read_committed_composite_ledger(&dir).unwrap();
8120 let immutable_path = composite_committed_ledger_path(&dir, &commit);
8121 let immutable = std::fs::read(&immutable_path).unwrap();
8122 restore_composite_verifying_key_set(&dir, &ed, &pq, ca.verifying_key())
8125 .await
8126 .unwrap();
8127 let keys = hyprstream_rpc::auth::global_composite_key_set();
8128 use hyprstream_rpc::auth::CompositePairRole;
8129 let before = keys.mint_snapshot().unwrap();
8130 assert!(before
8131 .active_signing_pair(CompositePairRole::OAuth)
8132 .is_none());
8133 for _ in 0..2 {
8134 initialize_composite_key_set(&dir, &ed, &pq, Arc::clone(&ca), 300)
8135 .await
8136 .unwrap();
8137 let after = keys.mint_snapshot().unwrap();
8138 assert_eq!(after.version(), before.version());
8139 assert_eq!(after.component_digest(), before.component_digest());
8140 for role in [CompositePairRole::OAuth, CompositePairRole::Policy] {
8141 assert!(after.active_signing_pair(role).is_some());
8142 }
8143 }
8144 let absent = SigningKeyStore::new(KeySlots {
8147 drain: None,
8148 active: None,
8149 lead: None,
8150 });
8151 assert!(initialize_composite_key_set(&dir, &absent, &pq, ca, 300)
8152 .await
8153 .is_err());
8154 assert_eq!(
8155 std::fs::read(composite_committed_path(&dir)).unwrap(),
8156 marker
8157 );
8158 assert_eq!(std::fs::read(immutable_path).unwrap(), immutable);
8159 }
8160
8161 #[tokio::test]
8162 async fn first_boot_slot_lock_failure_never_creates_or_activates_keys() {
8163 let dir = TempDir::new().unwrap();
8164 std::fs::create_dir(dir.path().join("signing-slots.lock")).unwrap();
8165 let config = test_config();
8166 let ed = load_or_init_key_store(dir.path(), &config);
8167 let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8168 let es = load_or_init_es256_key_store(dir.path(), &config);
8169 assert!(ed.0.read().await.active.is_none());
8170 assert!(pq.active_key().await.is_none());
8171 assert!(es.0.read().active.is_none());
8172 for family in ["jwt", "ml-dsa", "es256"] {
8173 assert!(!dir
8174 .path()
8175 .join(format!("{family}-signing-key.active"))
8176 .exists());
8177 }
8178 assert!(
8179 initialize_composite_key_set(dir.path(), &ed, &pq, authority_ca_key(), 300)
8180 .await
8181 .is_err()
8182 );
8183 assert!(!composite_committed_path(dir.path()).exists());
8184 }
8185
8186 #[tokio::test]
8187 async fn first_boot_slot_lock_defers_rotation_without_mutation() {
8188 let dir = TempDir::new().unwrap();
8189 let config = test_config();
8190 let ed = load_or_init_key_store(dir.path(), &config);
8191 let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8192 let es = load_or_init_es256_key_store(dir.path(), &config);
8193 let ca = authority_ca_key();
8194 let before = component_state_digest(&ed, &pq, ca.verifying_key()).await;
8195 let es_before = es.0.read().active.as_ref().unwrap().kid();
8196 let lock = lock_signing_slots(dir.path(), false).unwrap();
8197 let now = chrono::Utc::now().timestamp();
8198 assert!(!rotate_jwt_keys(&config, dir.path(), &ed, now).await);
8199 assert!(!rotate_ml_dsa_keys(&config, dir.path(), &pq, now).await);
8200 assert!(!rotate_es256_keys(&config, dir.path(), &es, now).await);
8201 assert_eq!(
8202 component_state_digest(&ed, &pq, ca.verifying_key()).await,
8203 before
8204 );
8205 drop(lock);
8206 let reloaded_ed = load_or_init_key_store(dir.path(), &config);
8207 let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8208 let reloaded_es = load_or_init_es256_key_store(dir.path(), &config);
8209 assert_eq!(
8210 component_state_digest(&reloaded_ed, &reloaded_pq, ca.verifying_key()).await,
8211 before
8212 );
8213 assert_eq!(
8214 reloaded_es.0.read().active.as_ref().unwrap().kid(),
8215 es_before
8216 );
8217 assert!(rotate_jwt_keys(&config, dir.path(), &ed, now).await);
8218 assert!(rotate_ml_dsa_keys(&config, dir.path(), &pq, now).await);
8219 }
8220
8221 #[tokio::test]
8222 async fn first_boot_slot_creation_is_serialized_across_processes() {
8223 use nix::fcntl::{flock, FlockArg};
8224 use std::os::fd::AsRawFd;
8225
8226 const CHILD: &str = "HYPRSTREAM_FIRST_BOOT_SLOT_CHILD";
8227 const DIRECTORY: &str = "HYPRSTREAM_FIRST_BOOT_SLOT_DIR";
8228 const TEST: &str =
8229 "auth::key_rotation::tests::first_boot_slot_creation_is_serialized_across_processes";
8230 if let Ok(child) = std::env::var(CHILD) {
8231 let dir = PathBuf::from(std::env::var_os(DIRECTORY).unwrap());
8232 std::fs::write(dir.join(format!("{child}.ready")), b"ready").unwrap();
8233 let config = test_config();
8234 let pq = load_or_init_ml_dsa_key_store(&dir, &config);
8236 let ed = load_or_init_key_store(&dir, &config);
8237 let es = load_or_init_es256_key_store(&dir, &config);
8238 let ca = authority_ca_key();
8239 initialize_composite_key_set(&dir, &ed, &pq, Arc::clone(&ca), 300)
8240 .await
8241 .unwrap();
8242 let digest = component_state_digest(&ed, &pq, ca.verifying_key()).await;
8244 let es_kid = es.0.read().active.as_ref().unwrap().kid();
8245 std::fs::write(
8246 dir.join(format!("{child}.done")),
8247 format!("{digest}:{es_kid}"),
8248 )
8249 .unwrap();
8250 return;
8251 }
8252
8253 let dir = TempDir::new().unwrap();
8254 let lock = std::fs::OpenOptions::new()
8255 .create(true)
8256 .truncate(false)
8257 .read(true)
8258 .write(true)
8259 .open(dir.path().join("signing-slots.lock"))
8260 .unwrap();
8261 flock(lock.as_raw_fd(), FlockArg::LockExclusive).unwrap();
8262 let spawn = |name: &str| {
8263 std::process::Command::new(std::env::current_exe().unwrap())
8264 .args(["--exact", TEST, "--nocapture"])
8265 .env(CHILD, name)
8266 .env(DIRECTORY, dir.path())
8267 .spawn()
8268 .unwrap()
8269 };
8270 let mut first = spawn("first");
8271 let mut second = spawn("second");
8272 wait_path(&dir.path().join("first.ready"));
8273 wait_path(&dir.path().join("second.ready"));
8274 std::thread::sleep(std::time::Duration::from_millis(300));
8277 let wrote_while_locked = ["jwt", "ml-dsa", "es256"].iter().any(|family| {
8278 dir.path()
8279 .join(format!("{family}-signing-key.active"))
8280 .exists()
8281 });
8282 drop(lock);
8283 assert!(first.wait().unwrap().success());
8284 assert!(second.wait().unwrap().success());
8285 assert!(
8286 !wrote_while_locked,
8287 "slot creation ignored the interprocess lock"
8288 );
8289 assert_eq!(
8290 std::fs::read(dir.path().join("first.done")).unwrap(),
8291 std::fs::read(dir.path().join("second.done")).unwrap(),
8292 "concurrent starters must adopt the same durable component identities"
8293 );
8294 let marker = std::fs::read(composite_committed_path(dir.path())).unwrap();
8295 let mut restarted = spawn("restarted");
8296 assert!(restarted.wait().unwrap().success());
8297 assert_eq!(
8298 std::fs::read(dir.path().join("first.done")).unwrap(),
8299 std::fs::read(dir.path().join("restarted.done")).unwrap()
8300 );
8301 assert_eq!(
8302 std::fs::read(composite_committed_path(dir.path())).unwrap(),
8303 marker
8304 );
8305 }
8306
8307 #[tokio::test]
8308 async fn first_bootstrap_and_locked_legacy_migration_are_distinct() {
8309 const ISOLATED: &str = "HYPRSTREAM_COMPOSITE_LEGACY_MIGRATION_TEST";
8310 if std::env::var_os(ISOLATED).is_none() {
8311 let status = std::process::Command::new(std::env::current_exe().unwrap())
8312 .args([
8313 "--exact",
8314 "auth::key_rotation::tests::first_bootstrap_and_locked_legacy_migration_are_distinct",
8315 "--nocapture",
8316 ])
8317 .env(ISOLATED, "1")
8318 .status()
8319 .unwrap();
8320 assert!(status.success());
8321 return;
8322 }
8323
8324 let dir = TempDir::new().unwrap();
8325 let config = test_config();
8326 let ca = Arc::new(SigningKey::from_bytes(&[0x6b; 32]));
8327 let ed = load_or_init_key_store(dir.path(), &config);
8328 let pq = load_or_init_ml_dsa_key_store(dir.path(), &config);
8329
8330 assert!(!composite_committed_path(dir.path()).exists());
8331 initialize_composite_key_set(dir.path(), &ed, &pq, Arc::clone(&ca), 300)
8332 .await
8333 .unwrap();
8334 let commit: CompositeCommit =
8335 serde_json::from_slice(&std::fs::read(composite_committed_path(dir.path())).unwrap())
8336 .unwrap();
8337 let immutable = composite_committed_ledger_path(dir.path(), &commit);
8338 let committed_bytes = std::fs::read(&immutable).unwrap();
8339
8340 std::fs::remove_file(&immutable).unwrap();
8343 initialize_composite_key_set(dir.path(), &ed, &pq, ca, 300)
8344 .await
8345 .unwrap();
8346 assert_eq!(std::fs::read(&immutable).unwrap(), committed_bytes);
8347
8348 let mut pending: CompositeLedger = serde_json::from_slice(&committed_bytes).unwrap();
8349 pending.version = pending.version.saturating_add(1);
8350 pending.component_digest = "staged-component-C".to_owned();
8351 std::fs::write(
8352 composite_ledger_path(dir.path()),
8353 serde_json::to_vec(&pending).unwrap(),
8354 )
8355 .unwrap();
8356 let (_, selected) = read_committed_composite_ledger(dir.path()).unwrap();
8357 assert_eq!(selected.version, commit.version);
8358 assert_eq!(selected.component_digest, commit.component_digest);
8359 }
8360
8361 #[tokio::test]
8362 async fn composite_ledger_survives_joint_rotation_restart_and_drain_expiry() {
8363 use super::ml_dsa_rotation::*;
8364
8365 let dir = TempDir::new().unwrap();
8366 let now = chrono::Utc::now().timestamp();
8367 let drain_secs = 600;
8368 let ca = Arc::new(SigningKey::from_bytes(&[91; 32]));
8369 let old_ed = KeySlot::new(SigningKey::from_bytes(&[92; 32]), now - 60, now + 60);
8370 let old_pq = generate_ml_dsa_slot(now - 60, now + 60);
8371 persist_slot(dir.path(), "active", &old_ed).unwrap();
8372 persist_ml_dsa_slot(dir.path(), "active", &old_pq).unwrap();
8373 let ed_store = SigningKeyStore::new(KeySlots {
8374 drain: None,
8375 active: Some(old_ed.clone()),
8376 lead: None,
8377 });
8378 let pq_store = MlDsaSigningKeyStore::new(MlDsaKeySlots {
8379 drain: None,
8380 active: Some(old_pq.clone()),
8381 lead: None,
8382 });
8383 initialize_composite_key_set(
8384 dir.path(),
8385 &ed_store,
8386 &pq_store,
8387 Arc::clone(&ca),
8388 drain_secs,
8389 )
8390 .await
8391 .unwrap();
8392 let initial = hyprstream_rpc::auth::global_composite_key_set().snapshot();
8393 let initial_component_digest = initial.component_digest().to_owned();
8394 let old_pair = initial
8395 .active_signing_pair(hyprstream_rpc::auth::CompositePairRole::OAuth)
8396 .unwrap();
8397 let old_kid = old_pair.kid().to_owned();
8398 let (old_pq_signing, old_ed_signing) = old_pair.signing_keys().unwrap();
8399 let claims = hyprstream_rpc::auth::Claims::new("alice".to_owned(), now, now + 300)
8400 .with_issuer("https://local".to_owned())
8401 .with_audience(Some("https://resource".to_owned()));
8402 let token = crate::auth::jwt::encode_composite_ml_dsa_65_ed25519(
8403 &claims,
8404 &old_pq_signing,
8405 &old_ed_signing,
8406 );
8407
8408 let new_ed = KeySlot::new(SigningKey::from_bytes(&[93; 32]), now, now + 15 * 86_400);
8413 let new_pq = generate_ml_dsa_slot(now, now + 15 * 86_400);
8414 persist_slot(dir.path(), "drain", &old_ed).unwrap();
8415 persist_slot(dir.path(), "active", &new_ed).unwrap();
8416 persist_ml_dsa_slot(dir.path(), "drain", &old_pq).unwrap();
8417 persist_ml_dsa_slot(dir.path(), "active", &new_pq).unwrap();
8418 let rotated_ed = SigningKeyStore::new(KeySlots {
8419 drain: Some(old_ed),
8420 active: Some(new_ed),
8421 lead: None,
8422 });
8423 let rotated_pq = MlDsaSigningKeyStore::new(MlDsaKeySlots {
8424 drain: Some(old_pq),
8425 active: Some(new_pq),
8426 lead: None,
8427 });
8428 refresh_composite_key_set(
8429 dir.path(),
8430 &rotated_ed,
8431 &rotated_pq,
8432 Arc::clone(&ca),
8433 drain_secs,
8434 &initial_component_digest,
8435 )
8436 .await
8437 .unwrap();
8438 let rotated = hyprstream_rpc::auth::global_composite_key_set().snapshot();
8439 let retained = rotated
8440 .pair(&old_kid)
8441 .expect("old exact pair retained as drain");
8442 let dispatch = hyprstream_rpc::auth::parse_composite_dispatch(&token, &["at+jwt"]).unwrap();
8443 hyprstream_rpc::auth::jwt::decode_composite(
8444 &token,
8445 retained.ml_dsa(),
8446 retained.ed25519(),
8447 Some("https://resource"),
8448 &dispatch,
8449 )
8450 .unwrap();
8451
8452 let reloaded_ed = load_or_init_key_store(dir.path(), &test_config());
8454 let reloaded_pq = load_or_init_ml_dsa_key_store(dir.path(), &test_config());
8455 initialize_composite_key_set(
8456 dir.path(),
8457 &reloaded_ed,
8458 &reloaded_pq,
8459 Arc::clone(&ca),
8460 drain_secs,
8461 )
8462 .await
8463 .unwrap();
8464 let restarted = hyprstream_rpc::auth::global_composite_key_set().snapshot();
8465 assert!(restarted.pair(&old_kid).is_some());
8466 let jwks_kids: Vec<_> = restarted
8467 .pairs()
8468 .iter()
8469 .map(|pair| {
8470 crate::auth::jwt::composite_jwk(pair.ml_dsa(), pair.ed25519())["kid"]
8471 .as_str()
8472 .unwrap()
8473 .to_owned()
8474 })
8475 .collect();
8476 assert!(jwks_kids.contains(&old_kid));
8477
8478 reloaded_ed.0.write().await.drain = None;
8482 reloaded_pq.0.write().await.drain = None;
8483 refresh_composite_key_set(
8484 dir.path(),
8485 &reloaded_ed,
8486 &reloaded_pq,
8487 ca,
8488 drain_secs,
8489 restarted.component_digest(),
8490 )
8491 .await
8492 .unwrap();
8493 assert!(hyprstream_rpc::auth::global_composite_key_set()
8494 .snapshot()
8495 .pair(&old_kid)
8496 .is_none());
8497 }
8498}