Skip to main content

hyprstream_core/auth/
op_log.rs

1//! C4 (#1170): the sealed op-log head — cross-process source of truth for the
2//! active `#atproto` ES256 generation. **Closes #1123.**
3//!
4//! Plan t1124 §3.4: under `--ipc`, discovery/registry/oauth agree on the active
5//! generation with **no event-delivery mechanism** — every process reads ONE
6//! sealed head that the rotator (OAuth) writes after each promotion. The
7//! process-local `OnceLock` stays the rotator's own working state; it is no
8//! longer the source of truth for *other* processes.
9//!
10//! ## Credentials (split, least-privilege)
11//!
12//! The head is signed by a **dedicated** Ed25519 keypair — NOT a key derived
13//! from the CA root. The private key (`oplog-head-key`) is a secret held only
14//! by the rotator; the public verifying key (`atproto-oplog-head-pubkey`) is
15//! published to the shared state dir and loaded by readers. A process that can
16//! *verify* a head therefore **cannot forge** one (F3). Readers never touch
17//! `ca-key` (which is PolicyService-only under systemd IPC).
18//!
19//! ## Head state directory (F4)
20//!
21//! The head is a signed, public artifact — it carries no secrecy, so it does
22//! NOT live in the read-only per-unit credentials directory. It lives in a
23//! dedicated **shared writable state dir** ([`resolve_oplog_state_dir`]),
24//! env-overridable so a systemd `--ipc` deployment can point it at a shared
25//! `StateDirectory=`. Where that shared storage is provisioned is deployment
26//! plumbing (#808); this module takes the path, never assumes `secrets_dir`.
27//!
28//! ## Consistency guarantee (implemented)
29//!
30//! A reader is NOT guaranteed to observe a rotation instantaneously (bounded
31//! by one rotation tick + filesystem visibility). The fail-closed contract:
32//! the reader verifies the head's node signature, **rejects a `seq`
33//! regression** against both the highest generation it has observed in memory
34//! and the rotator's durable high-water mark, and resolves the signing key
35//! **by kid** from the key-material store. A retired key is never presented as
36//! active; replaying only an older-but-valid head is rejected even after a
37//! reader restart. As with any node-local checkpoint, coordinated rollback of
38//! the head and its high-water mark is outside this file-level guarantee.
39//!
40//! ## Dependency honesty — C2 (#1168) is not built
41//!
42//! [`seal_op_log_head`] is rename-atomic at the file level. C2 must harden
43//! the full rotation ordering — *re-sign repo head → durably write head →
44//! seal DidOp → publish* — as one crash-atomic unit, so no crash leaves a
45//! head naming a key whose repo head was not durably re-signed. Full
46//! crash-injection at every boundary is C2's job; the writer verifies every
47//! prior head before carrying its `seq`/`prev` forward (F5). This is NOT the
48//! DidOp format (C1 #1167) and does not encode rotation-key custody (epic
49//! #1158 Q2 / one-way door #4); the projection carries its own `version`.
50
51use anyhow::{anyhow, Context as _, Result};
52use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
53use ed25519_dalek::{
54    Signature, Signer, SigningKey as Ed25519SigningKey, Verifier,
55    VerifyingKey as Ed25519VerifyingKey,
56};
57use p256::ecdsa::{SigningKey as Es256SigningKey, VerifyingKey as Es256VerifyingKey};
58use serde::{Deserialize, Serialize};
59use sha2::{Digest as _, Sha256};
60use std::path::{Path, PathBuf};
61use std::sync::atomic::{AtomicU64, Ordering};
62
63use super::jwt::es256_kid;
64use super::key_rotation::Es256SigningKeyStore;
65
66/// Projection format version (NOT the DidOp format — C1 #1167 owns that).
67pub const PROJECTION_VERSION: u16 = 1;
68
69/// Secret seed for the head-signing Ed25519 key (rotator only).
70pub const HEAD_SIGNING_KEY_NAME: &str = "oplog-head-key";
71/// Public verifying key for the head signature (all readers).
72pub const HEAD_VERIFYING_KEY_FILENAME: &str = "atproto-oplog-head-pubkey";
73/// The sealed head file.
74pub const SEALED_HEAD_FILENAME: &str = "atproto-oplog-head.json";
75/// Durable sequence high-water mark, written by the rotator after each head.
76pub const SEALED_HEAD_MAX_SEQ_FILENAME: &str = "atproto-oplog-head-max-seq";
77/// Env override for the shared head state dir (#808 systemd seam).
78pub const OPLOG_STATE_DIR_ENV: &str = "HYPRSTREAM_OPLOG_STATE_DIR";
79
80/// Resolve the shared, writable head state dir.
81///
82/// Default: a dedicated `oplog-state/` sibling of the secrets dir (shared and
83/// writable in the single-process daemon). A systemd `--ipc` deployment sets
84/// `HYPRSTREAM_OPLOG_STATE_DIR` to a shared `StateDirectory=` path —
85/// provisioning that shared storage is #808.
86pub fn resolve_oplog_state_dir(secrets_dir: &Path) -> Result<PathBuf> {
87    if let Some(from_env) = std::env::var_os(OPLOG_STATE_DIR_ENV) {
88        let path = PathBuf::from(from_env);
89        std::fs::create_dir_all(&path)
90            .with_context(|| format!("creating oplog state dir {}", path.display()))?;
91        return Ok(path);
92    }
93    let parent = secrets_dir
94        .parent()
95        .ok_or_else(|| anyhow!("secrets dir has no parent for oplog state"))?;
96    let path = parent.join("oplog-state");
97    std::fs::create_dir_all(&path)
98        .with_context(|| format!("creating oplog state dir {}", path.display()))?;
99    Ok(path)
100}
101
102/// Load or initialize the dedicated head-signing Ed25519 key (rotator only).
103pub fn load_or_init_head_signing_key(secrets_dir: &Path) -> Result<Ed25519SigningKey> {
104    if let Some(bytes) = super::identity_store::read_secret(secrets_dir, HEAD_SIGNING_KEY_NAME)? {
105        let mut seed = [0u8; 32];
106        let v: Vec<u8> = bytes;
107        seed.copy_from_slice(&v);
108        return Ok(Ed25519SigningKey::from_bytes(&seed));
109    }
110    let key = Ed25519SigningKey::generate(&mut rand::rngs::OsRng);
111    super::identity_store::write_secret(secrets_dir, HEAD_SIGNING_KEY_NAME, &key.to_bytes())?;
112    Ok(key)
113}
114
115/// Publish the head-signing *verifying* key to the shared state dir (public).
116pub fn publish_head_verifying_key(state_dir: &Path, head_sk: &Ed25519SigningKey) -> Result<()> {
117    let vk = head_sk.verifying_key();
118    atomic_write_public(&state_dir.join(HEAD_VERIFYING_KEY_FILENAME), &vk.to_bytes())
119}
120
121/// Atomic (rename) write with world-readable perms (0644). The head and its
122/// verifying key are signed/public artifacts shared across services, which may
123/// run as distinct users under systemd `--ipc` — so unlike secrets (0600)
124/// they must be readable by every reader process.
125fn atomic_write_public(path: &Path, bytes: &[u8]) -> Result<()> {
126    use std::io::Write as _;
127    let dir = path
128        .parent()
129        .ok_or_else(|| anyhow!("head path has no parent directory"))?;
130    let mut tmp = tempfile::NamedTempFile::new_in(dir)
131        .with_context(|| format!("creating temp file in {}", dir.display()))?;
132    tmp.write_all(bytes)?;
133    #[cfg(unix)]
134    {
135        use std::os::unix::fs::PermissionsExt;
136        std::fs::set_permissions(tmp.path(), std::fs::Permissions::from_mode(0o644))?;
137    }
138    tmp.persist(path)
139        .with_context(|| format!("persisting {}", path.display()))?;
140    Ok(())
141}
142
143/// Load the head-signing verifying key from the shared state dir (readers).
144pub fn load_head_verifying_key(state_dir: &Path) -> Result<Ed25519VerifyingKey> {
145    let bytes = std::fs::read(state_dir.join(HEAD_VERIFYING_KEY_FILENAME))
146        .context("head verifying key is not published (rotator has not run)")?;
147    let arr: [u8; 32] = bytes
148        .as_slice()
149        .try_into()
150        .map_err(|_| anyhow!("head verifying key must be 32 bytes"))?;
151    Ed25519VerifyingKey::from_bytes(&arr).map_err(|e| anyhow!("invalid head verifying key: {e}"))
152}
153
154/// The on-disk sealed head: an authenticated, ordered projection of "which
155/// `#atproto` ES256 generation is active right now."
156#[derive(Clone, Debug, Serialize, Deserialize)]
157pub struct SealedOpLogHead {
158    pub version: u16,
159    /// Monotonic generation; strictly advances. Readers reject a regression.
160    pub seq: u64,
161    pub active_kid: String,
162    /// Active P-256 verifying key, SEC1 compressed, base64url (33 raw bytes).
163    pub active_vk_sec1: String,
164    /// Repo head CID re-signed by the active key. `None` until C3 (#1169).
165    pub head_at_op: Option<String>,
166    /// sha256 of the canonical payload of the previous *verified* head.
167    pub prev: Option<[u8; 32]>,
168    /// Ed25519 signature by the head-signing key over [`Self::signing_payload`].
169    pub sig: Vec<u8>,
170}
171
172impl SealedOpLogHead {
173    /// Bytes the signature covers (every field except `sig`). Field order is
174    /// stable, so `serde_json` output is deterministic.
175    pub fn signing_payload(&self) -> Result<Vec<u8>> {
176        #[derive(Serialize)]
177        struct Payload<'a> {
178            version: u16,
179            seq: u64,
180            active_kid: &'a str,
181            active_vk_sec1: &'a str,
182            head_at_op: &'a Option<String>,
183            prev: &'a Option<[u8; 32]>,
184        }
185        serde_json::to_vec(&Payload {
186            version: self.version,
187            seq: self.seq,
188            active_kid: &self.active_kid,
189            active_vk_sec1: &self.active_vk_sec1,
190            head_at_op: &self.head_at_op,
191            prev: &self.prev,
192        })
193        .context("serializing sealed-head payload")
194    }
195
196    /// Verify the head signature under `head_vk` and return the active vk.
197    pub fn verify(&self, head_vk: &Ed25519VerifyingKey) -> Result<Es256VerifyingKey> {
198        let sig = Signature::from_slice(&self.sig).context("sealed head signature is malformed")?;
199        head_vk
200            .verify(&self.signing_payload()?, &sig)
201            .context("sealed head signature does not verify under the head verifying key")?;
202        if self.version != PROJECTION_VERSION {
203            return Err(anyhow!(
204                "sealed head projection version {}: reader supports {}",
205                self.version,
206                PROJECTION_VERSION
207            ));
208        }
209        let vk_bytes = URL_SAFE_NO_PAD
210            .decode(&self.active_vk_sec1)
211            .context("active_vk_sec1 is not valid base64url")?;
212        Es256VerifyingKey::from_sec1_bytes(&vk_bytes)
213            .map_err(|e| anyhow!("invalid active_vk_sec1 in sealed head: {e}"))
214    }
215}
216
217/// The active `#atproto` generation, as resolved by a reader.
218#[derive(Clone, Debug)]
219pub struct ActiveGeneration {
220    pub seq: u64,
221    pub kid: String,
222    pub verifying_key: Es256VerifyingKey,
223    pub signing_key: Es256SigningKey,
224    pub head_at_op: Option<String>,
225}
226
227/// Source of truth for the active `#atproto` generation.
228pub trait ActiveGenerationSource: Send + Sync {
229    /// `Ok(None)` when no sealed generation is observable yet; `Err` is
230    /// fail-closed (corrupt/stale/rolled-back/unverifiable head).
231    fn active_generation(&self) -> Result<Option<ActiveGeneration>>;
232}
233
234/// A single immutable generation — preserves the old `PdsPublisher::new(store,
235/// did, signing_key)` call sites behind the source abstraction.
236pub struct FixedGenerationSource {
237    kid: String,
238    signing_key: Es256SigningKey,
239}
240
241impl FixedGenerationSource {
242    pub fn new(signing_key: Es256SigningKey) -> Self {
243        let kid = es256_kid(&signing_key);
244        Self { kid, signing_key }
245    }
246}
247
248impl ActiveGenerationSource for FixedGenerationSource {
249    fn active_generation(&self) -> Result<Option<ActiveGeneration>> {
250        Ok(Some(ActiveGeneration {
251            seq: 0,
252            kid: self.kid.clone(),
253            verifying_key: *self.signing_key.verifying_key(),
254            signing_key: self.signing_key.clone(),
255            head_at_op: None,
256        }))
257    }
258}
259
260/// Cross-process reader of the sealed op-log head. Holds NO private key
261/// material — it loads the head-signing *verifying* key from the shared state
262/// dir, so a process that can verify a head cannot forge one (F3). It tracks
263/// the highest `seq` it has accepted and rejects any regression (F2).
264pub struct SealedHeadEs256Source {
265    state_dir: PathBuf,
266    key_material_dir: PathBuf,
267    /// Highest `seq` accepted by this process. A lower head is rejected as a
268    /// rollback; the same head may be resolved repeatedly between rotations.
269    max_seq: AtomicU64,
270}
271
272impl SealedHeadEs256Source {
273    /// Construct from the shared head state dir + the ES256 key-material dir.
274    /// Holds no credentials; the verifying key is loaded from the state dir at
275    /// read time.
276    pub fn new(state_dir: &Path, key_material_dir: &Path) -> Self {
277        Self {
278            state_dir: state_dir.to_path_buf(),
279            key_material_dir: key_material_dir.to_path_buf(),
280            max_seq: AtomicU64::new(0),
281        }
282    }
283
284    fn head_path(&self) -> PathBuf {
285        self.state_dir.join(SEALED_HEAD_FILENAME)
286    }
287
288    /// Read + verify the head (no signing-material resolution).
289    fn read_head(&self) -> Result<Option<(SealedOpLogHead, Es256VerifyingKey)>> {
290        let bytes = match std::fs::read(self.head_path()) {
291            Ok(b) => b,
292            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
293            Err(e) => return Err(e).context("reading sealed op-log head"),
294        };
295        let head: SealedOpLogHead =
296            serde_json::from_slice(&bytes).context("sealed op-log head is corrupt")?;
297        let head_vk = load_head_verifying_key(&self.state_dir)?;
298        let active_vk = head.verify(&head_vk)?;
299        Ok(Some((head, active_vk)))
300    }
301}
302
303impl ActiveGenerationSource for SealedHeadEs256Source {
304    fn active_generation(&self) -> Result<Option<ActiveGeneration>> {
305        let (head, _active_vk) = match self.read_head()? {
306            Some(h) => h,
307            None => return Ok(None),
308        };
309        // F2: reject a rolled-back head against the rotator's durable anchor,
310        // including on a reader's first call after process restart.
311        let durable_max = read_durable_max_seq(&self.state_dir)?.ok_or_else(|| {
312            anyhow!(
313                "sealed head exists without durable sequence anchor {}",
314                SEALED_HEAD_MAX_SEQ_FILENAME
315            )
316        })?;
317        if head.seq < durable_max {
318            return Err(anyhow!(
319                "sealed head seq {} is older than durable high-water mark {} \
320                 (cold-start rollback or replay rejected)",
321                head.seq,
322                durable_max
323            ));
324        }
325
326        // The same sealed generation is expected to serve many publishes.
327        // Only a lower seq is a regression; equality is safe and necessary.
328        let prev_max = self.max_seq.load(Ordering::Acquire);
329        if head.seq < prev_max {
330            return Err(anyhow!(
331                "sealed head seq {} is older than the last accepted {} \
332                 (rollback rejected)",
333                head.seq,
334                prev_max
335            ));
336        }
337        // Resolve the signing key BY KID from the key-material store. The head
338        // authenticates *which* kid is active; a kid it does not name is never
339        // loaded, so a retired key cannot be presented as active.
340        let signing_key = super::key_rotation::es256_signing_key_for_kid(
341            &self.key_material_dir,
342            &head.active_kid,
343        )
344        .ok_or_else(|| {
345            anyhow!(
346                "sealed head names active kid {} but its signing key is not \
347                 materialized in the key-material dir",
348                head.active_kid
349            )
350        })?;
351        // Publish the new process-local high-water mark only after the key
352        // resolves. `fetch_max` closes the race between concurrent readers.
353        let concurrent_max = self.max_seq.fetch_max(head.seq, Ordering::AcqRel);
354        if head.seq < concurrent_max {
355            return Err(anyhow!(
356                "sealed head seq {} is older than the concurrently accepted {} \
357                 (rollback rejected)",
358                head.seq,
359                concurrent_max
360            ));
361        }
362        Ok(Some(ActiveGeneration {
363            seq: head.seq,
364            kid: head.active_kid,
365            verifying_key: *signing_key.verifying_key(),
366            signing_key,
367            head_at_op: head.head_at_op,
368        }))
369    }
370}
371
372/// Resolve the head state dir + dedicated head key, publish the verifying key,
373/// and seal the current ES256 active generation in one call. This is the
374/// rotator's full "advance the sealed head" step (boot + each rotation tick).
375pub async fn advance_sealed_head(
376    secrets_dir: &Path,
377    es256_store: &Es256SigningKeyStore,
378) -> Result<()> {
379    let state_dir = resolve_oplog_state_dir(secrets_dir)?;
380    advance_sealed_head_with_state_dir(secrets_dir, &state_dir, es256_store).await
381}
382
383pub(crate) async fn advance_sealed_head_with_state_dir(
384    secrets_dir: &Path,
385    state_dir: &Path,
386    es256_store: &Es256SigningKeyStore,
387) -> Result<()> {
388    // Publishing the head verification key is an authority marker. Do not
389    // leave it behind when first-boot ES256 persistence failed and there is
390    // no usable active signer to seal. Otherwise a repaired storage fault
391    // would be mistaken for previously published authority on the next boot.
392    // Inspect the already-authorized in-memory slot without refreshing it
393    // from disk. During a rejected/partially rolled-back promotion, disk may
394    // contain the candidate while memory deliberately retains the old active
395    // key; this function must not turn that candidate into published authority.
396    if es256_store.active_slot().is_none() {
397        anyhow::bail!("cannot advance sealed op-log head: ES256 store has no active slot");
398    }
399    let head_sk = load_or_init_head_signing_key(secrets_dir)?;
400    publish_head_verifying_key(state_dir, &head_sk)?;
401    seal_op_log_head(state_dir, &head_sk, es256_store).await
402}
403
404/// Write the sealed op-log head after a promotion (or at boot). The public
405/// operation remains async for its rotator call sites, while the ES256 store
406/// snapshot uses its synchronous `parking_lot` read guard. This is the **C2
407/// (#1168) seam**: rename-atomic at the file level; the full crash-atomic
408/// rotation ordering is C2's guarantee (see module docs).
409///
410/// Prior state is carried forward only from a *signature-verified* prior head
411/// (F5): an unverified/tampered prior does not inject `seq`/`prev`.
412pub async fn seal_op_log_head(
413    state_dir: &Path,
414    head_signing_key: &Ed25519SigningKey,
415    es256_store: &Es256SigningKeyStore,
416) -> Result<()> {
417    let active = es256_store
418        .0
419        .read()
420        .active
421        .clone()
422        .ok_or_else(|| anyhow!("cannot seal op-log head: ES256 store has no active slot"))?;
423    let signing_key = (*active.key).clone();
424    let kid = active.kid();
425    let vk_sec1 = URL_SAFE_NO_PAD.encode(signing_key.verifying_key().to_sec1_bytes());
426
427    // F5: carry the chain forward only from a verified prior head.
428    let (seq, prev) = prior_chain(state_dir, head_signing_key)?;
429
430    let mut head = SealedOpLogHead {
431        version: PROJECTION_VERSION,
432        seq,
433        active_kid: kid,
434        active_vk_sec1: vk_sec1,
435        head_at_op: None,
436        prev,
437        sig: Vec::new(),
438    };
439    let sig = head_signing_key.sign(&head.signing_payload()?);
440    head.sig = sig.to_bytes().to_vec();
441
442    atomic_write_public(
443        &state_dir.join(SEALED_HEAD_FILENAME),
444        &serde_json::to_vec(&head)?,
445    )?;
446    // Write the durable anchor after the head. If a crash lands between these
447    // renames, the previous anchor already rejects every older head; the new
448    // head remains acceptable because it is newer than that anchor.
449    write_durable_max_seq(state_dir, head.seq)?;
450    Ok(())
451}
452
453fn read_durable_max_seq(state_dir: &Path) -> Result<Option<u64>> {
454    let path = state_dir.join(SEALED_HEAD_MAX_SEQ_FILENAME);
455    let bytes = match std::fs::read(&path) {
456        Ok(bytes) => bytes,
457        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
458        Err(e) => {
459            return Err(e)
460                .with_context(|| format!("reading sealed-head high-water mark {}", path.display()))
461        }
462    };
463    let text = std::str::from_utf8(&bytes).context("sealed-head high-water mark is not UTF-8")?;
464    let seq = text
465        .parse::<u64>()
466        .context("sealed-head high-water mark is not a u64")?;
467    Ok(Some(seq))
468}
469
470fn write_durable_max_seq(state_dir: &Path, seq: u64) -> Result<()> {
471    atomic_write_public(
472        &state_dir.join(SEALED_HEAD_MAX_SEQ_FILENAME),
473        seq.to_string().as_bytes(),
474    )
475}
476
477/// Return `(seq, prev)` for the new head, carrying forward only from a
478/// signature-verified prior head (F5). An absent or unverified prior breaks the
479/// hash link, but the durable high-water mark still prevents `seq` reuse.
480fn prior_chain(
481    state_dir: &Path,
482    head_signing_key: &Ed25519SigningKey,
483) -> Result<(u64, Option<[u8; 32]>)> {
484    let durable_max = read_durable_max_seq(state_dir)?.unwrap_or(0);
485    let next_unlinked = || {
486        durable_max
487            .checked_add(1)
488            .ok_or_else(|| anyhow!("sealed-head sequence exhausted at {durable_max}"))
489            .map(|seq| (seq, None))
490    };
491    let head_vk = head_signing_key.verifying_key();
492    let bytes = match std::fs::read(state_dir.join(SEALED_HEAD_FILENAME)) {
493        Ok(b) => b,
494        Err(_) => return next_unlinked(),
495    };
496    let prior: SealedOpLogHead = match serde_json::from_slice(&bytes) {
497        Ok(h) => h,
498        Err(_) => return next_unlinked(),
499    };
500    // F5: verify the prior head before trusting its seq/prev. A tampered prior
501    // cannot inject an arbitrary sequence. The durable anchor still advances.
502    if prior.verify(&head_vk).is_err() {
503        return next_unlinked();
504    }
505    let prev_hash = match prior.signing_payload() {
506        Ok(p) => sha256_fixed(&p),
507        Err(_) => return next_unlinked(),
508    };
509    let seq = prior
510        .seq
511        .max(durable_max)
512        .checked_add(1)
513        .ok_or_else(|| anyhow!("sealed-head sequence exhausted at {}", u64::MAX))?;
514    let prev = (prior.seq >= durable_max).then_some(prev_hash);
515    Ok((seq, prev))
516}
517
518fn sha256_fixed(bytes: &[u8]) -> [u8; 32] {
519    let mut h = Sha256::new();
520    h.update(bytes);
521    h.finalize().into()
522}
523
524#[cfg(test)]
525#[allow(clippy::unwrap_used, clippy::expect_used)]
526mod tests {
527    use super::*;
528    use std::process::Command;
529    use std::time::{Duration, Instant};
530    use tempfile::TempDir;
531
532    const CROSS_PROCESS_CHILD_ENV: &str = "HYPRSTREAM_TEST_OPLOG_CHILD";
533    const CROSS_PROCESS_STATE_ENV: &str = "HYPRSTREAM_TEST_OPLOG_STATE";
534    const CROSS_PROCESS_SECRETS_ENV: &str = "HYPRSTREAM_TEST_OPLOG_SECRETS";
535    const CROSS_PROCESS_READY_ENV: &str = "HYPRSTREAM_TEST_OPLOG_READY";
536    const CROSS_PROCESS_CONTINUE_ENV: &str = "HYPRSTREAM_TEST_OPLOG_CONTINUE";
537    const CROSS_PROCESS_RESULT_ENV: &str = "HYPRSTREAM_TEST_OPLOG_RESULT";
538
539    /// A rotator fixture: dedicated head key + published verifying key + a
540    /// populated ES256 store. `state_dir` is the shared head state dir;
541    /// `secrets_dir` holds the head private key and ES256 key material.
542    struct Rotator {
543        state_dir: PathBuf,
544        secrets_dir: PathBuf,
545        head_sk: Ed25519SigningKey,
546        store: Es256SigningKeyStore,
547    }
548
549    impl Rotator {
550        fn new() -> Self {
551            let state_dir = TempDir::new().unwrap().keep();
552            let secrets_dir = TempDir::new().unwrap().keep();
553            let head_sk = Ed25519SigningKey::generate(&mut rand::rngs::OsRng);
554            publish_head_verifying_key(&state_dir, &head_sk).unwrap();
555            Self {
556                state_dir,
557                secrets_dir,
558                head_sk,
559                store: Es256SigningKeyStore::new(empty_slots()),
560            }
561        }
562
563        fn set_active(&self, sk: Es256SigningKey) {
564            *self.store.0.write() = super::super::key_rotation::Es256KeySlots {
565                active: Some(super::super::key_rotation::Es256KeySlot::new(sk, 0, 0)),
566                drain: None,
567                lead: None,
568            };
569        }
570
571        fn seal(&self) {
572            tokio::runtime::Builder::new_current_thread()
573                .build()
574                .unwrap()
575                .block_on(seal_op_log_head(
576                    &self.state_dir,
577                    &self.head_sk,
578                    &self.store,
579                ))
580                .unwrap();
581        }
582
583        fn persist_slot(&self, name: &str, sk: &Es256SigningKey) {
584            super::super::key_rotation::persist_es256_slot(
585                &self.secrets_dir,
586                name,
587                &super::super::key_rotation::Es256KeySlot::new(sk.clone(), 0, 0),
588            )
589            .unwrap();
590        }
591    }
592
593    fn empty_slots() -> super::super::key_rotation::Es256KeySlots {
594        super::super::key_rotation::Es256KeySlots {
595            active: None,
596            drain: None,
597            lead: None,
598        }
599    }
600
601    // ── F1: the seal must work inside an async runtime (panics pre-fix) ─────
602
603    #[test]
604    fn seal_runs_inside_async_runtime() {
605        // Keep exercising the async public seam even though the ES256 snapshot
606        // now uses a synchronous parking_lot guard internally.
607        let r = Rotator::new();
608        let k1 = Es256SigningKey::random(&mut rand::rngs::OsRng);
609        r.set_active(k1.clone());
610        r.persist_slot("active", &k1);
611        r.seal();
612        assert!(r.state_dir.join(SEALED_HEAD_FILENAME).exists());
613    }
614
615    // ── F2: a rolled-back head is rejected (replay-negative) ────────────────
616
617    #[test]
618    fn reader_rejects_rolled_back_head() {
619        let r = Rotator::new();
620        let k1 = Es256SigningKey::random(&mut rand::rngs::OsRng);
621        let k1_vk = *k1.verifying_key();
622        r.set_active(k1.clone());
623        r.persist_slot("active", &k1);
624        r.seal();
625        let seq1_head = std::fs::read(r.state_dir.join(SEALED_HEAD_FILENAME)).unwrap();
626
627        let reader = SealedHeadEs256Source::new(&r.state_dir, &r.secrets_dir);
628        let gen1 = reader.active_generation().unwrap().unwrap();
629        assert_eq!(gen1.verifying_key, k1_vk);
630
631        // Rotate to K2.
632        let k2 = Es256SigningKey::random(&mut rand::rngs::OsRng);
633        let k2_vk = *k2.verifying_key();
634        r.persist_slot("drain", &k1);
635        r.persist_slot("active", &k2);
636        r.set_active(k2);
637        r.seal();
638        let gen2 = reader.active_generation().unwrap().unwrap();
639        assert_eq!(gen2.verifying_key, k2_vk);
640
641        // Replay attack: overwrite the head with the still-valid signed seq=1.
642        std::fs::write(r.state_dir.join(SEALED_HEAD_FILENAME), seq1_head).unwrap();
643        let err = reader.active_generation().unwrap_err();
644        assert!(
645            err.to_string().contains("rollback"),
646            "a replayed older signed head must be rejected: {err}"
647        );
648    }
649
650    #[test]
651    fn cold_start_reader_rejects_rolled_back_head() {
652        let r = Rotator::new();
653        let k1 = Es256SigningKey::random(&mut rand::rngs::OsRng);
654        r.set_active(k1.clone());
655        r.persist_slot("active", &k1);
656        r.seal();
657        let seq1_head = std::fs::read(r.state_dir.join(SEALED_HEAD_FILENAME)).unwrap();
658
659        let k2 = Es256SigningKey::random(&mut rand::rngs::OsRng);
660        r.persist_slot("drain", &k1);
661        r.persist_slot("active", &k2);
662        r.set_active(k2);
663        r.seal();
664        assert_eq!(read_durable_max_seq(&r.state_dir).unwrap(), Some(2));
665
666        // Replay only the older, still-valid head, then construct a brand-new
667        // source to model a process cold start. The durable anchor survives.
668        std::fs::write(r.state_dir.join(SEALED_HEAD_FILENAME), seq1_head).unwrap();
669        let restarted_reader = SealedHeadEs256Source::new(&r.state_dir, &r.secrets_dir);
670        let err = restarted_reader.active_generation().unwrap_err();
671        assert!(
672            err.to_string().contains("cold-start rollback"),
673            "a cold reader must reject an older signed head: {err}"
674        );
675    }
676
677    #[test]
678    fn reader_can_resolve_the_same_head_repeatedly() {
679        let r = Rotator::new();
680        let k1 = Es256SigningKey::random(&mut rand::rngs::OsRng);
681        r.set_active(k1.clone());
682        r.persist_slot("active", &k1);
683        r.seal();
684
685        let reader = SealedHeadEs256Source::new(&r.state_dir, &r.secrets_dir);
686        assert_eq!(reader.active_generation().unwrap().unwrap().seq, 1);
687        assert_eq!(reader.active_generation().unwrap().unwrap().seq, 1);
688    }
689
690    /// #1123/#1170 acceptance test: the rotator stays in this process while a
691    /// child test process holds the reader across the K1 -> K2 rotation.
692    #[test]
693    fn cross_process_rotation_observed_via_sealed_head() {
694        if std::env::var_os(CROSS_PROCESS_CHILD_ENV).is_some() {
695            cross_process_reader_child();
696            return;
697        }
698
699        let r = Rotator::new();
700        let k1 = Es256SigningKey::random(&mut rand::rngs::OsRng);
701        let k1_kid = es256_kid(&k1);
702        r.set_active(k1.clone());
703        r.persist_slot("active", &k1);
704        r.seal();
705
706        let coordination = TempDir::new().unwrap();
707        let ready = coordination.path().join("ready");
708        let continue_path = coordination.path().join("continue");
709        let result = coordination.path().join("result");
710        let test_name = "auth::op_log::tests::cross_process_rotation_observed_via_sealed_head";
711        let mut child = Command::new(std::env::current_exe().unwrap())
712            .arg("--exact")
713            .arg(test_name)
714            .arg("--nocapture")
715            .env(CROSS_PROCESS_CHILD_ENV, "1")
716            .env(CROSS_PROCESS_STATE_ENV, &r.state_dir)
717            .env(CROSS_PROCESS_SECRETS_ENV, &r.secrets_dir)
718            .env(CROSS_PROCESS_READY_ENV, &ready)
719            .env(CROSS_PROCESS_CONTINUE_ENV, &continue_path)
720            .env(CROSS_PROCESS_RESULT_ENV, &result)
721            .spawn()
722            .unwrap();
723
724        wait_for_path(&ready);
725
726        let k2 = Es256SigningKey::random(&mut rand::rngs::OsRng);
727        let k2_kid = es256_kid(&k2);
728        r.persist_slot("drain", &k1);
729        r.persist_slot("active", &k2);
730        r.set_active(k2);
731        r.seal();
732        std::fs::write(&continue_path, b"rotate").unwrap();
733
734        let status = child.wait().unwrap();
735        assert!(status.success(), "cross-process reader child failed");
736        let observed = std::fs::read_to_string(result).unwrap();
737        assert_eq!(observed, format!("{k1_kid}\n{k2_kid}\n"));
738        assert_ne!(
739            k1_kid, k2_kid,
740            "the child must stop presenting the pre-rotation generation"
741        );
742    }
743
744    fn cross_process_reader_child() {
745        let state = PathBuf::from(std::env::var_os(CROSS_PROCESS_STATE_ENV).unwrap());
746        let secrets = PathBuf::from(std::env::var_os(CROSS_PROCESS_SECRETS_ENV).unwrap());
747        let ready = PathBuf::from(std::env::var_os(CROSS_PROCESS_READY_ENV).unwrap());
748        let continue_path = PathBuf::from(std::env::var_os(CROSS_PROCESS_CONTINUE_ENV).unwrap());
749        let result = PathBuf::from(std::env::var_os(CROSS_PROCESS_RESULT_ENV).unwrap());
750        let reader = SealedHeadEs256Source::new(&state, &secrets);
751
752        let before = reader.active_generation().unwrap().unwrap();
753        std::fs::write(&ready, b"ready").unwrap();
754        wait_for_path(&continue_path);
755        let after = reader.active_generation().unwrap().unwrap();
756        std::fs::write(result, format!("{}\n{}\n", before.kid, after.kid)).unwrap();
757    }
758
759    fn wait_for_path(path: &Path) {
760        let deadline = Instant::now() + Duration::from_secs(10);
761        while !path.exists() {
762            assert!(
763                Instant::now() < deadline,
764                "timed out waiting for {}",
765                path.display()
766            );
767            std::thread::sleep(Duration::from_millis(10));
768        }
769    }
770
771    // ── F3: a reader (verifying key only) cannot forge a head ───────────────
772
773    #[test]
774    fn reader_rejects_head_signed_by_a_different_key() {
775        let r = Rotator::new();
776        let k1 = Es256SigningKey::random(&mut rand::rngs::OsRng);
777        r.set_active(k1.clone());
778        r.persist_slot("active", &k1);
779        r.seal();
780
781        // Attacker writes a head signed by a DIFFERENT key, but the reader
782        // verifies against the published verifying key -> rejected.
783        let attacker = Ed25519SigningKey::generate(&mut rand::rngs::OsRng);
784        let mut forged = SealedOpLogHead {
785            version: PROJECTION_VERSION,
786            seq: 99,
787            active_kid: es256_kid(&k1),
788            active_vk_sec1: URL_SAFE_NO_PAD.encode(k1.verifying_key().to_sec1_bytes()),
789            head_at_op: None,
790            prev: None,
791            sig: Vec::new(),
792        };
793        forged.sig = attacker
794            .sign(&forged.signing_payload().unwrap())
795            .to_bytes()
796            .to_vec();
797        std::fs::write(
798            r.state_dir.join(SEALED_HEAD_FILENAME),
799            serde_json::to_vec(&forged).unwrap(),
800        )
801        .unwrap();
802
803        let reader = SealedHeadEs256Source::new(&r.state_dir, &r.secrets_dir);
804        let err = reader.active_generation().unwrap_err();
805        assert!(
806            err.to_string().contains("signature"),
807            "a head signed by a non-head key must fail signature verification: {err}"
808        );
809    }
810
811    // ── F5: a tampered prior head does not inject an arbitrary seq ──────────
812
813    #[test]
814    fn tampered_prior_head_does_not_inject_seq() {
815        let r = Rotator::new();
816        let k1 = Es256SigningKey::random(&mut rand::rngs::OsRng);
817        r.set_active(k1.clone());
818        r.persist_slot("active", &k1);
819        r.seal();
820
821        // Tamper with the prior head's seq (without re-signing).
822        let mut prior: SealedOpLogHead =
823            serde_json::from_slice(&std::fs::read(r.state_dir.join(SEALED_HEAD_FILENAME)).unwrap())
824                .unwrap();
825        prior.seq = 999_999;
826        std::fs::write(
827            r.state_dir.join(SEALED_HEAD_FILENAME),
828            serde_json::to_vec(&prior).unwrap(),
829        )
830        .unwrap();
831
832        // Re-seal: the tampered prior does not verify, so the hash chain
833        // restarts, but the durable sequence anchor still advances.
834        let k2 = Es256SigningKey::random(&mut rand::rngs::OsRng);
835        r.set_active(k2);
836        r.seal();
837        let head: SealedOpLogHead =
838            serde_json::from_slice(&std::fs::read(r.state_dir.join(SEALED_HEAD_FILENAME)).unwrap())
839                .unwrap();
840        assert_eq!(
841            head.seq, 2,
842            "an unverified prior must not inject or regress the new head seq"
843        );
844        assert_eq!(head.prev, None);
845    }
846
847    // ── F4/torn: a corrupt head is rejected; absent head returns None ───────
848
849    #[test]
850    fn reader_fail_closed_on_corrupt_head() {
851        let r = Rotator::new();
852        std::fs::write(r.state_dir.join(SEALED_HEAD_FILENAME), b"not json").unwrap();
853        let reader = SealedHeadEs256Source::new(&r.state_dir, &r.secrets_dir);
854        assert!(reader.active_generation().is_err());
855    }
856
857    #[test]
858    fn reader_returns_none_when_head_absent() {
859        let r = Rotator::new();
860        let reader = SealedHeadEs256Source::new(&r.state_dir, &r.secrets_dir);
861        assert!(reader.active_generation().unwrap().is_none());
862    }
863}