Skip to main content

hyprstream_core/services/
pds_record_rocksdb.rs

1//! Durable PDS record store (#910a) — the RocksDB local backend, the at9p
2//! accepted-state ingest, and the publisher/resolver built on it.
3//!
4//! Split from `services/discovery.rs` so the whole PDS persistence block
5//! gates on the `rocksdb` feature at the module boundary (the `rocksdb`
6//! feature, valkey precedent); `discovery.rs` re-exports it unchanged so
7//! callers keep addressing `services::discovery::PdsRecordStore`.
8
9use anyhow::{anyhow, bail, Context as _, Result as AnyResult};
10use async_trait::async_trait;
11use hyprstream_discovery::{RecordCarData, RecordResolver};
12use hyprstream_pds::at9p::h512;
13use hyprstream_pds::at9p_duplicity::{
14    AcceptedAt9pHead, AcceptedAt9pState, ConditionalAdvance, DuplicityGuard, WalDuplicityAlarmSink,
15    Watermark, WatermarkStore,
16};
17use hyprstream_pds::at9p_gate::{verify_did_at9p, DID_AT9P_PREFIX};
18use hyprstream_pds::car::build_record_proof_car;
19use hyprstream_pds::commit::{Commit, UnsignedCommit};
20use hyprstream_pds::ledger::{AllocationRecord, CheckpointRecord, ReceiptRecord};
21use hyprstream_pds::mst::Node;
22use hyprstream_pds::record::{ModelRecord, COLLECTION_NSID};
23use hyprstream_pds::repo_authority::is_path_form_did_web;
24use hyprstream_pds::tid::Tid;
25use sha2::Digest as _;
26use std::collections::BTreeMap;
27use std::path::{Path, PathBuf};
28use std::sync::Arc;
29
30// ============================================================================
31// PdsRecordStore — durable, keyless-read atproto record store (#910a)
32// ============================================================================
33
34/// One account's repo, loaded from the durable store: its records (keyed by
35/// `(collection, tid)`) plus the **signed commit** that was persisted at write
36/// time.
37///
38/// The MST is a deterministic, keyless function of the record set
39/// (`Node::from_records`), so it is *not* stored — it is rebuilt on demand. The
40/// only thing that must be persisted (a private key signed it) is the commit
41/// itself. Reads therefore never touch a signing key: they rebuild the MST,
42/// load this already-signed commit, and assemble a proof CAR. This mirrors how
43/// atproto works — the commit is signed **once** at write time and served
44/// verbatim thereafter.
45struct RepoState {
46    /// `<collection>/<rkey>` is the MST key; here we key by `(collection, tid)`.
47    /// One repo spans *many* collections (#910b) — all of them live in this one
48    /// map, and the MST is built over the full multi-collection key space.
49    records: BTreeMap<(String, Tid), ModelRecord>,
50    /// The signed commit persisted by the writer — served verbatim on reads,
51    /// never re-signed.
52    commit: Commit,
53    /// The exact stored bytes of `commit`. The Postgres backend compare-and-
54    /// swaps the repo head against these bytes, so a publish is gated on the
55    /// exact head it was built from — re-encoding is never involved.
56    commit_bytes: Vec<u8>,
57}
58
59// ── RocksDB key scheme ───────────────────────────────────────────────────────
60//
61// DIDs and collection NSIDs never contain NUL bytes, so `\0` is a safe,
62// unambiguous field separator for compound keys (same idea as the delimited
63// keys `RocksDbUserStore` uses, just with a compound suffix here).
64//
65//   rk\0{did}\0{collection}\0{tid}   -> canonical DAG-CBOR record bytes
66//   commit\0{did}                    -> signed-commit DAG-CBOR bytes
67//
68// The record's rkey (a TID) is derived deterministically from the repo id
69// (`PdsRecordStore::tid_for_repo`), so it is never minted or persisted. The
70// `#atproto` signing key is held ONLY by the writer (`PdsPublisher`); it is
71// never stored in this DB and never held by the resolver (#910a). The commit is
72// signed once at write time and persisted under `commit\0{did}`, so reads are a
73// keyless walk (rebuild the MST, load the signed commit, serve a proof).
74
75fn record_prefix(did: &str) -> Vec<u8> {
76    format!("rk\0{did}\0").into_bytes()
77}
78
79fn record_key(did: &str, collection: &str, tid: Tid) -> Vec<u8> {
80    format!("rk\0{did}\0{collection}\0{}", tid.encode()).into_bytes()
81}
82
83fn commit_key(did: &str) -> Vec<u8> {
84    format!("commit\0{did}").into_bytes()
85}
86
87fn at9p_state_key(subject_cid512: &str) -> Vec<u8> {
88    format!("at9p-state\0{subject_cid512}").into_bytes()
89}
90fn at9p_checkpoint_key(subject_cid512: &str) -> Vec<u8> {
91    format!("at9p-checkpoint\0{subject_cid512}").into_bytes()
92}
93
94const AT9P_STATE_MAGIC: &[u8; 8] = b"AT9PST02";
95const AT9P_STATE_HEADER_LEN: usize = 8 + 1 + 8 + 64 + 1 + 4;
96const AT9P_ACCEPTANCE_MAGIC: &[u8; 8] = b"AT9PAC01";
97const ED25519_KEY_LEN: usize = 32;
98const ED25519_SIGNATURE_LEN: usize = 64;
99const AT9P_ACCEPTANCE_PREFIX_LEN: usize =
100    AT9P_ACCEPTANCE_MAGIC.len() + ED25519_KEY_LEN + ED25519_SIGNATURE_LEN;
101const AT9P_ACCEPTANCE_KEY_AAD: &[u8] = b"hyprstream-at9p-acceptance-key/1";
102const AT9P_ACCEPTANCE_STATE_AAD: &[u8] = b"hyprstream-at9p-accepted-state/1";
103const AT9P_CHECKPOINT_MAGIC: &[u8; 8] = b"AT9PCK01";
104const AT9P_CHECKPOINT_PAYLOAD_LEN: usize = 8 + 8 + 64 + 1 + 64;
105const AT9P_CHECKPOINT_LEN: usize = AT9P_CHECKPOINT_PAYLOAD_LEN + ED25519_SIGNATURE_LEN;
106const AT9P_CHECKPOINT_AAD: &[u8] = b"hyprstream-at9p-monotonic-checkpoint/1";
107
108/// Split a full record key back into `(collection, tid)`, given the `did` it
109/// was built for.
110fn parse_record_key(key: &[u8], did: &str) -> AnyResult<(String, Tid)> {
111    let prefix = record_prefix(did);
112    let rest = key
113        .strip_prefix(prefix.as_slice())
114        .ok_or_else(|| anyhow!("record key missing expected prefix for {did:?}"))?;
115    let rest = std::str::from_utf8(rest).context("record key suffix is not UTF-8")?;
116    let (collection, tid_str) = rest
117        .split_once('\0')
118        .ok_or_else(|| anyhow!("record key missing collection/tid separator"))?;
119    let tid = Tid::parse(tid_str)?;
120    Ok((collection.to_owned(), tid))
121}
122
123/// Durable, RocksDB-backed PDS record store keyed by repo DID.
124///
125/// This is the per-account record store the DiscoveryService resolves against,
126/// and that the registry service's register/commit paths publish into (#910a).
127/// Persistence follows the same pattern as `RocksDbUserStore`
128/// (`auth/rocksdb_store.rs`): a single `rocksdb::DB` at a well-known directory,
129/// opened read-write by the *one* process that publishes, and read-only by
130/// resolvers.
131///
132/// **The store holds no signing key.** The writer (`PdsPublisher`) signs the
133/// commit once and hands this store the signed bytes; the resolver only ever
134/// reads them back. This is the security fix at the heart of #910a: a read path
135/// that needed a private key to re-sign a commit on every `getRecord` was the
136/// root of the key-exposure problem — atproto never re-signs on read.
137///
138/// RocksDB allows exactly one read-write handle per directory; on a single
139/// node the registry service (the publisher) is the sole writer, and the
140/// discovery service (the resolver) opens the same directory read-only —
141/// mirroring `RocksDbUserStore::open` / `open_readonly`. A read-only handle
142/// snapshots the manifest at open time, so [`PdsRecordStore::open_readonly`]
143/// callers that need a live view reopen per read (see [`RecordBacking::ReadOnly`]).
144pub struct PdsRecordStore {
145    backing: RecordBacking,
146    /// Trusted deployment identity that certifies the purpose-derived key on
147    /// daemon-authenticated accepted-state envelopes. This is verification
148    /// material only; the resolver still holds no signing key.
149    at9p_acceptance_identity: Option<At9pAcceptanceVerifier>,
150    at9p_advance_lock: parking_lot::Mutex<()>,
151}
152
153enum At9pAcceptanceVerifier {
154    Deployment(hyprstream_discovery::RegistryDeploymentVerifier),
155    Local(ed25519_dalek::VerifyingKey),
156}
157
158impl At9pAcceptanceVerifier {
159    fn verify_strict(&self, message: &[u8], signature: &ed25519_dalek::Signature) -> AnyResult<()> {
160        match self {
161            Self::Deployment(identity) => identity.verify_strict(message, signature),
162            Self::Local(identity) => identity
163                .verify_strict(message, signature)
164                .map_err(Into::into),
165        }
166    }
167}
168
169enum RecordBacking {
170    /// The sole writer: one long-lived, read-write RocksDB handle.
171    ReadWrite(rocksdb::DB),
172    /// A resolver: no handle is held between calls — each read opens a fresh
173    /// `open_for_read_only` handle so writes from the publisher process become
174    /// visible without a restart.
175    ReadOnly(PathBuf),
176    /// Shared RDS (Multi-AZ Postgres) backend (#1257). Both the publisher and
177    /// the resolver connect to the same RDS instance — a write via one AZ is
178    /// immediately readable by the other. **D2 invariant**: values are the
179    /// exact signed bytes the publisher commits, stored verbatim in a
180    /// projection-free BYTEA KV; nothing is normalized into SQL columns.
181    #[cfg(feature = "pds-postgres")]
182    Postgres {
183        kv: super::pds_record_pg::PgKv,
184        readonly: bool,
185    },
186}
187
188impl PdsRecordStore {
189    /// Open (or create) the durable store at `path` for read-write access.
190    /// Exactly one process should hold this — the publisher (registry
191    /// service). See the type-level docs for the single-writer rationale.
192    pub fn open(path: &Path) -> AnyResult<Self> {
193        std::fs::create_dir_all(path)
194            .with_context(|| format!("failed to create PDS store dir {path:?}"))?;
195        harden_store_dir(path);
196        let mut opts = rocksdb::Options::default();
197        opts.create_if_missing(true);
198        let db = rocksdb::DB::open(&opts, path)
199            .with_context(|| format!("failed to open PDS record store (rw) at {path:?}"))?;
200        Ok(Self {
201            backing: RecordBacking::ReadWrite(db),
202            at9p_acceptance_identity: None,
203            at9p_advance_lock: parking_lot::Mutex::new(()),
204        })
205    }
206
207    /// Open the store read-only at `path`, for resolvers that never write
208    /// (the discovery service). Verifies the directory opens successfully now
209    /// (surfacing a missing/corrupt store at startup) but does not hold the
210    /// handle — see the type-level docs. The resolver holds **no signing key**.
211    pub fn open_readonly(path: &Path) -> AnyResult<Self> {
212        // RocksDB's read-only open can create a missing directory before it
213        // discovers that no database exists. A resolver/inspection read must
214        // not turn missing retained state into a newly initialized path.
215        let metadata = std::fs::metadata(path)
216            .with_context(|| format!("PDS read-only store directory is unavailable: {path:?}"))?;
217        anyhow::ensure!(
218            metadata.is_dir(),
219            "PDS read-only store path is not a directory: {path:?}"
220        );
221        let opts = readonly_opts();
222        let _probe = rocksdb::DB::open_for_read_only(&opts, path, false)
223            .with_context(|| format!("failed to open PDS record store (ro) at {path:?}"))?;
224        Ok(Self {
225            backing: RecordBacking::ReadOnly(path.to_path_buf()),
226            at9p_acceptance_identity: None,
227            at9p_advance_lock: parking_lot::Mutex::new(()),
228        })
229    }
230
231    /// Open the store against a shared RDS (Multi-AZ Postgres) instance (#1257).
232    ///
233    /// This is the **deployed** backend: two stateless PDS services in two AZs
234    /// share one RDS instance. A write via the publisher in AZ-a is immediately
235    /// readable by the resolver in AZ-b (MVCC, no reopen dance). The store is
236    /// **FATAL-on-unavailable** — `connect` fails at startup rather than
237    /// silently degrading to a local backend.
238    ///
239    /// `rds` must already be the effective binding — callers pass
240    /// [`crate::config::RdsConfig::resolved_from_env`] output so TOML, the
241    /// role-scoped env vars, and the shared credentials directory are all
242    /// honored. Conditional writes (repo head, accepted at9p head) are SQL
243    /// compare-and-swaps, so two publishers in two AZs cannot lose an update.
244    ///
245    /// **D2 invariant**: the signed record/commit/at9p bytes are stored
246    /// verbatim in a projection-free BYTEA KV table; SQL never normalizes,
247    /// parses, or reconstructs them. This is "ship the already-signed evidence
248    /// across the seam" at zero cost when region federation arrives.
249    ///
250    /// `readonly = true` selects the resolver posture (write methods bail);
251    /// `readonly = false` selects the publisher posture.
252    #[cfg(feature = "pds-postgres")]
253    pub fn open_postgres(rds: &crate::config::RdsConfig, readonly: bool) -> AnyResult<Self> {
254        let kv = rds.connect_kv()?;
255        Ok(Self {
256            backing: RecordBacking::Postgres { kv, readonly },
257            at9p_acceptance_identity: None,
258            at9p_advance_lock: parking_lot::Mutex::new(()),
259        })
260    }
261
262    /// TEST-ONLY: open the Postgres backend against an operator-provided
263    /// scratch database (no TLS, no URL validation). Live tests use this to
264    /// hold two independent handles — the in-process model of a publisher in
265    /// AZ-a and a resolver in AZ-b sharing one RDS instance.
266    #[cfg(all(test, feature = "pds-postgres"))]
267    fn open_postgres_test(driver_url: &str, readonly: bool) -> AnyResult<Self> {
268        let kv = super::pds_record_pg::PgKv::connect_test(driver_url, "test-cell")?;
269        Ok(Self {
270            backing: RecordBacking::Postgres { kv, readonly },
271            at9p_acceptance_identity: None,
272            at9p_advance_lock: parking_lot::Mutex::new(()),
273        })
274    }
275
276    /// Pin the deployment identity that certifies accepted-state envelopes.
277    pub(crate) fn with_at9p_deployment_verifier(
278        mut self,
279        identity: hyprstream_discovery::RegistryDeploymentVerifier,
280    ) -> Self {
281        self.at9p_acceptance_identity = Some(At9pAcceptanceVerifier::Deployment(identity));
282        self
283    }
284
285    pub(crate) fn with_at9p_acceptance_identity(
286        mut self,
287        identity: ed25519_dalek::VerifyingKey,
288    ) -> Self {
289        self.at9p_acceptance_identity = Some(At9pAcceptanceVerifier::Local(identity));
290        self
291    }
292
293    /// Atomically persist a record and the repo's freshly-signed commit.
294    ///
295    /// The record and the commit that covers it advance together in one
296    /// `WriteBatch`, so a reader never observes a record whose MST root the
297    /// stored commit does not sign. Fails if this store was opened read-only.
298    ///
299    /// `expected_head` is the exact stored bytes of the commit the new head
300    /// was built from (`None` = first publish; the commit key must be absent).
301    /// The Postgres backend enforces it with a SQL compare-and-swap on the
302    /// commit key: two publishers in two AZs cannot both advance the same
303    /// head — the loser's write is rejected instead of silently committing a
304    /// head that does not cover the full record set. The RocksDB backend
305    /// needs no such guard (the OS file lock makes it single-writer) and
306    /// ignores the parameter.
307    fn put_record_and_commit(
308        &self,
309        did: &str,
310        collection: &str,
311        tid: Tid,
312        record: &ModelRecord,
313        commit: &Commit,
314        expected_head: Option<&[u8]>,
315    ) -> AnyResult<()> {
316        let RecordBacking::ReadWrite(db) = &self.backing else {
317            #[cfg(feature = "pds-postgres")]
318            if let RecordBacking::Postgres { kv, readonly } = &self.backing {
319                if *readonly {
320                    bail!("PdsRecordStore::put_record_and_commit called on a read-only Postgres store");
321                }
322                let ops = vec![(record_key(did, collection, tid), record.to_dag_cbor())];
323                let committed = kv
324                    .cas_put(
325                        &commit_key(did),
326                        expected_head,
327                        &commit.to_dag_cbor(),
328                        &ops,
329                        &[],
330                    )
331                    .context("PDS record+commit Postgres write failed")?;
332                anyhow::ensure!(
333                    committed,
334                    "PDS repo head for {did} advanced concurrently — refusing to commit \
335                     a head that does not cover the record set; reload the repo and retry \
336                     the publish"
337                );
338                return Ok(());
339            }
340            bail!("PdsRecordStore::put_record_and_commit called on a read-only store");
341        };
342        #[cfg(not(feature = "pds-postgres"))]
343        let _ = expected_head;
344        let mut batch = rocksdb::WriteBatch::default();
345        batch.put(record_key(did, collection, tid), record.to_dag_cbor());
346        batch.put(commit_key(did), commit.to_dag_cbor());
347        db.write(batch).context("PDS record+commit write failed")?;
348        Ok(())
349    }
350
351    /// Write ONLY the commit block (no record) — used by the #918 head re-sign
352    /// path to replace the persisted head's signature without changing any
353    /// record data. The MST root / `rev` / `prev` are unchanged; only `sig`
354    /// is refreshed. `expected_head` gates the Postgres compare-and-swap
355    /// exactly as in `put_record_and_commit`: a re-sign based on a head that
356    /// was concurrently advanced is rejected, never silently overwritten.
357    fn put_commit(
358        &self,
359        did: &str,
360        commit: &Commit,
361        expected_head: Option<&[u8]>,
362    ) -> AnyResult<()> {
363        let RecordBacking::ReadWrite(db) = &self.backing else {
364            #[cfg(feature = "pds-postgres")]
365            if let RecordBacking::Postgres { kv, readonly } = &self.backing {
366                if *readonly {
367                    bail!("PdsRecordStore::put_commit called on a read-only Postgres store");
368                }
369                let committed = kv
370                    .cas_put(
371                        &commit_key(did),
372                        expected_head,
373                        &commit.to_dag_cbor(),
374                        &[],
375                        &[],
376                    )
377                    .context("PDS commit-only Postgres write failed")?;
378                anyhow::ensure!(
379                    committed,
380                    "PDS repo head for {did} advanced concurrently with the re-sign — \
381                     refusing to overwrite it; reload the repo and retry"
382                );
383                return Ok(());
384            }
385            bail!("PdsRecordStore::put_commit called on a read-only store");
386        };
387        #[cfg(not(feature = "pds-postgres"))]
388        let _ = expected_head;
389        db.put(commit_key(did), commit.to_dag_cbor())
390            .context("PDS commit-only write failed")?;
391        Ok(())
392    }
393
394    fn load_at9p_state(
395        &self,
396        subject_cid512: &str,
397        acceptance_identity: &At9pAcceptanceVerifier,
398    ) -> AnyResult<Option<AcceptedAt9pState>> {
399        match &self.backing {
400            RecordBacking::ReadWrite(db) => {
401                load_at9p_state_from_db(db, subject_cid512, acceptance_identity)
402            }
403            RecordBacking::ReadOnly(path) => {
404                let db = rocksdb::DB::open_for_read_only(&readonly_opts(), path, false)?;
405                load_at9p_state_from_db(&db, subject_cid512, acceptance_identity)
406            }
407            #[cfg(feature = "pds-postgres")]
408            RecordBacking::Postgres { kv, .. } => {
409                load_at9p_state_from_pg(kv, subject_cid512, acceptance_identity)
410            }
411        }
412    }
413
414    fn load_at9p_state_with_key(
415        &self,
416        subject_cid512: &str,
417        acceptance_identity: ed25519_dalek::VerifyingKey,
418    ) -> AnyResult<Option<AcceptedAt9pState>> {
419        self.load_at9p_state(
420            subject_cid512,
421            &At9pAcceptanceVerifier::Local(acceptance_identity),
422        )
423    }
424
425    /// Typed accepted-current state read path for resolver/admission consumers.
426    /// Read-only store instances reopen RocksDB for a live view, just like PDS
427    /// record resolution. `now` enforces update expiry when supplied.
428    pub fn accepted_at9p_state(
429        &self,
430        did: &str,
431        now: Option<&str>,
432    ) -> AnyResult<Option<AcceptedAt9pState>> {
433        let subject = did
434            .strip_prefix(DID_AT9P_PREFIX)
435            .ok_or_else(|| anyhow!("identifier is not did:at9p: {did:?}"))?;
436        let acceptance_identity = self
437            .at9p_acceptance_identity
438            .as_ref()
439            .ok_or_else(|| anyhow!("accepted-state verification identity is not configured"))?;
440        let Some(state) = self.load_at9p_state(subject, acceptance_identity)? else {
441            return Ok(None);
442        };
443        anyhow::ensure!(state.did == did, "accepted at9p state DID mismatch");
444        if let Some(now) = now {
445            state.ensure_fresh(now)?;
446        }
447        Ok(Some(state))
448    }
449
450    /// Enumerate every checkpoint-verified accepted state. This is the
451    /// production startup boundary used to build native announcements; a
452    /// missing checkpoint half or any verification failure aborts the read.
453    pub fn accepted_at9p_states(&self) -> AnyResult<Vec<AcceptedAt9pState>> {
454        let acceptance_identity = self
455            .at9p_acceptance_identity
456            .as_ref()
457            .ok_or_else(|| anyhow!("accepted-state verification identity is not configured"))?;
458        let read = |db: &rocksdb::DB| -> AnyResult<Vec<AcceptedAt9pState>> {
459            let prefix = b"at9p-state\0";
460            let mut states = Vec::new();
461            for item in db.iterator(rocksdb::IteratorMode::Start) {
462                let (key, _) = item?;
463                let Some(subject) = key.strip_prefix(prefix) else {
464                    continue;
465                };
466                let subject =
467                    std::str::from_utf8(subject).context("accepted-state key is not UTF-8")?;
468                let state = load_at9p_state_from_db(db, subject, acceptance_identity)?.ok_or_else(
469                    || anyhow!("accepted-state key disappeared during snapshot read"),
470                )?;
471                states.push(state);
472            }
473            states.sort_by(|a, b| a.did.cmp(&b.did));
474            Ok(states)
475        };
476        match &self.backing {
477            RecordBacking::ReadWrite(db) => read(db),
478            RecordBacking::ReadOnly(path) => {
479                let db = rocksdb::DB::open_for_read_only(&readonly_opts(), path, false)?;
480                read(&db)
481            }
482            #[cfg(feature = "pds-postgres")]
483            RecordBacking::Postgres { kv, .. } => {
484                // One read-only transaction covering BOTH prefix scans: a
485                // concurrent accepted-state commit cannot interleave a new
486                // state envelope with an old checkpoint scan (or vice versa).
487                // Pairing and verification happen locally on the snapshot
488                // bytes, with the same half-missing/watermark/digest checks
489                // as the single-subject read path.
490                let state_prefix = b"at9p-state\0".to_vec();
491                let checkpoint_prefix = b"at9p-checkpoint\0".to_vec();
492                let state_end = super::pds_record_pg::prefix_upper_bound(&state_prefix)
493                    .ok_or_else(|| anyhow!("at9p state prefix has no successor key"))?;
494                let checkpoint_end =
495                    super::pds_record_pg::prefix_upper_bound(&checkpoint_prefix)
496                        .ok_or_else(|| anyhow!("at9p checkpoint prefix has no successor key"))?;
497                let snap = kv.read_snapshot(
498                    &[
499                        (checkpoint_prefix.clone(), checkpoint_end),
500                        (state_prefix.clone(), state_end),
501                    ],
502                    &[],
503                )?;
504                let mut ranges = snap.ranges.into_iter();
505                let (checkpoint_pairs, state_pairs) = match (ranges.next(), ranges.next()) {
506                    (Some(checkpoints), Some(states)) => (checkpoints, states),
507                    _ => bail!("RDS read_snapshot returned fewer ranges than requested"),
508                };
509                let mut checkpoints: BTreeMap<Vec<u8>, Vec<u8>> = BTreeMap::new();
510                for (key, value) in checkpoint_pairs {
511                    checkpoints.insert(
512                        key.get(checkpoint_prefix.len()..)
513                            .ok_or_else(|| {
514                                anyhow!("accepted-checkpoint key is shorter than its prefix")
515                            })?
516                            .to_vec(),
517                        value,
518                    );
519                }
520                let mut states = Vec::new();
521                for (key, value) in state_pairs {
522                    let subject_bytes = key
523                        .get(state_prefix.len()..)
524                        .ok_or_else(|| anyhow!("accepted-state key is shorter than its prefix"))?;
525                    let subject = std::str::from_utf8(subject_bytes)
526                        .context("accepted-state key is not UTF-8")?;
527                    let checkpoint = checkpoints.remove(subject_bytes).ok_or_else(|| {
528                        anyhow!("accepted at9p state exists without its monotonic checkpoint")
529                    })?;
530                    states.push(verify_at9p_pair(
531                        subject,
532                        &value,
533                        &checkpoint,
534                        acceptance_identity,
535                    )?);
536                }
537                anyhow::ensure!(
538                    checkpoints.is_empty(),
539                    "accepted at9p checkpoint exists without its state envelope"
540                );
541                states.sort_by(|a, b| a.did.cmp(&b.did));
542                Ok(states)
543            }
544        }
545    }
546
547    /// Check whether the first-boot provisioning marker (`FIRST_BOOT_KEY`) is
548    /// present in the store. The marker is written by `init-deployment-store`
549    /// and deleted atomically with the first accepted-state commit. Its
550    /// presence alongside an empty store is the ONLY lifecycle evidence that
551    /// distinguishes genuine first boot from data loss.
552    pub fn first_boot_pending(&self) -> AnyResult<bool> {
553        let check = |db: &rocksdb::DB| -> AnyResult<bool> {
554            Ok(db.get(hyprstream_discovery::FIRST_BOOT_KEY)?.is_some())
555        };
556        match &self.backing {
557            RecordBacking::ReadWrite(db) => check(db),
558            RecordBacking::ReadOnly(path) => {
559                let db = rocksdb::DB::open_for_read_only(&readonly_opts(), path, false)?;
560                check(&db)
561            }
562            #[cfg(feature = "pds-postgres")]
563            RecordBacking::Postgres { kv, .. } => {
564                Ok(kv.get(hyprstream_discovery::FIRST_BOOT_KEY)?.is_some())
565            }
566        }
567    }
568
569    /// Write the first-boot provisioning marker. Only valid on a read-write
570    /// store that has just been freshly created. Propagates the write error so
571    /// initialization cannot appear successful without the marker.
572    pub fn mark_first_boot(&self) -> AnyResult<()> {
573        let RecordBacking::ReadWrite(db) = &self.backing else {
574            #[cfg(feature = "pds-postgres")]
575            if let RecordBacking::Postgres { kv, readonly } = &self.backing {
576                if *readonly {
577                    bail!("first-boot marker write attempted on a read-only Postgres PDS store");
578                }
579                kv.put(hyprstream_discovery::FIRST_BOOT_KEY, b"")
580                    .context("failed to write first-boot provisioning marker")?;
581                return Ok(());
582            }
583            bail!("first-boot marker write attempted on a read-only PDS store");
584        };
585        db.put(hyprstream_discovery::FIRST_BOOT_KEY, b"")
586            .context("failed to write first-boot provisioning marker")?;
587        Ok(())
588    }
589
590    /// Explicitly reset the configured RDS records KV to a fresh-genesis
591    /// state. This clears only `pds_kv` and writes the first-boot marker in
592    /// the same Postgres transaction; local RocksDB and read-only stores are
593    /// never reset through this API.
594    #[cfg(feature = "pds-postgres")]
595    pub(crate) fn reset_for_fresh_genesis(&self) -> AnyResult<()> {
596        if let RecordBacking::Postgres { kv, readonly } = &self.backing {
597            anyhow::ensure!(
598                !readonly,
599                "fresh-genesis reset attempted on a read-only PDS store"
600            );
601            return kv.reset_to_first_boot(hyprstream_discovery::FIRST_BOOT_KEY);
602        }
603        bail!("fresh-genesis reset is supported only for the writable RDS records store")
604    }
605
606    fn conditional_advance_at9p_state(
607        &self,
608        expected: Option<Watermark>,
609        state: &AcceptedAt9pState,
610        acceptance_identity: &ed25519_dalek::SigningKey,
611        audit_key: &ed25519_dalek::SigningKey,
612    ) -> AnyResult<ConditionalAdvance> {
613        let RecordBacking::ReadWrite(db) = &self.backing else {
614            #[cfg(feature = "pds-postgres")]
615            if let RecordBacking::Postgres { kv, readonly } = &self.backing {
616                if *readonly {
617                    bail!(
618                        "accepted did:at9p state write attempted on a read-only Postgres PDS store"
619                    );
620                }
621                let local_verifier =
622                    At9pAcceptanceVerifier::Local(acceptance_identity.verifying_key());
623                let state_key = at9p_state_key(&state.subject_cid512);
624                let checkpoint_key = at9p_checkpoint_key(&state.subject_cid512);
625                // One snapshot read of the durable pair: enforces the
626                // watermark precondition AND yields the exact bytes the SQL
627                // compare-and-swap gates on.
628                let raw = kv.get_batch(&[state_key.clone(), checkpoint_key.clone()])?;
629                let mut raw = raw.into_iter();
630                let (raw_state, raw_checkpoint) = (raw.next().flatten(), raw.next().flatten());
631                let current = match (&raw_state, &raw_checkpoint) {
632                    (None, None) => None,
633                    (Some(_), None) => {
634                        bail!("accepted at9p state exists without its monotonic checkpoint")
635                    }
636                    (None, Some(_)) => {
637                        bail!("accepted at9p checkpoint exists without its state envelope")
638                    }
639                    (Some(envelope), Some(checkpoint)) => Some(verify_at9p_pair(
640                        &state.subject_cid512,
641                        envelope,
642                        checkpoint,
643                        &local_verifier,
644                    )?),
645                };
646                if current.as_ref().map(AcceptedAt9pState::watermark) != expected {
647                    return current.map_or_else(
648                        || bail!(
649                            "conditional accepted-state advance expected a durable head, but none exists"
650                        ),
651                        |current| Ok(ConditionalAdvance::Conflict(Box::new(current))),
652                    );
653                }
654                let encoded = encode_at9p_state(state, acceptance_identity, audit_key)?;
655                let checkpoint = encode_at9p_checkpoint(state, &encoded, acceptance_identity)?;
656                // The cross-AZ correctness mechanism: the advance is a SQL
657                // compare-and-swap on the exact current state bytes, in ONE
658                // transaction with the checkpoint write and the first-boot
659                // marker removal (mirroring the RocksDB WriteBatch below).
660                // The process-local `at9p_advance_lock` guards nothing across
661                // AZs; the row lock serializes the publishers and the loser
662                // reports Conflict instead of silently overwriting.
663                let committed = kv
664                    .cas_put(
665                        &state_key,
666                        raw_state.as_deref(),
667                        &encoded,
668                        &[(checkpoint_key, checkpoint)],
669                        &[hyprstream_discovery::FIRST_BOOT_KEY.to_vec()],
670                    )
671                    .context(
672                        "synchronous accepted did:at9p state+checkpoint Postgres commit failed",
673                    )?;
674                if !committed {
675                    // Lost the commit race between the precondition read and
676                    // the CAS: re-read and report the winner's durable state.
677                    let winner =
678                        load_at9p_state_from_pg(kv, &state.subject_cid512, &local_verifier)?;
679                    return winner.map_or_else(
680                        || bail!(
681                            "conditional accepted-state advance lost the commit race, and no durable head is readable"
682                        ),
683                        |winner| Ok(ConditionalAdvance::Conflict(Box::new(winner))),
684                    );
685                }
686                return Ok(ConditionalAdvance::Committed);
687            }
688            bail!("accepted did:at9p state write attempted on a read-only PDS store");
689        };
690        let _advance = self.at9p_advance_lock.lock();
691        let local_verifier = At9pAcceptanceVerifier::Local(acceptance_identity.verifying_key());
692        let current = load_at9p_state_from_db(db, &state.subject_cid512, &local_verifier)?;
693        if current.as_ref().map(AcceptedAt9pState::watermark) != expected {
694            return current.map_or_else(
695                || bail!("conditional accepted-state advance expected a durable head, but none exists"),
696                |current| Ok(ConditionalAdvance::Conflict(Box::new(current))),
697            );
698        }
699        let encoded = encode_at9p_state(state, acceptance_identity, audit_key)?;
700        let checkpoint = encode_at9p_checkpoint(state, &encoded, acceptance_identity)?;
701        let mut batch = rocksdb::WriteBatch::default();
702        batch.put(at9p_state_key(&state.subject_cid512), encoded);
703        batch.put(at9p_checkpoint_key(&state.subject_cid512), checkpoint);
704        // Clear the first-boot provisioning marker in the SAME synchronous
705        // batch as the accepted-state commit — so the lifecycle transition is
706        // atomic with provisioning completion. A crash between commit and
707        // marker removal is impossible: they are one RocksDB write.
708        batch.delete(hyprstream_discovery::FIRST_BOOT_KEY);
709        let mut options = rocksdb::WriteOptions::default();
710        options.set_sync(true);
711        db.write_opt(batch, &options)
712            .context("synchronous accepted did:at9p state+checkpoint commit failed")?;
713        Ok(ConditionalAdvance::Committed)
714    }
715
716    /// The stable record-key TID for `repo_id`, derived **deterministically**
717    /// from the repo id (#910a M1).
718    ///
719    /// The rkey must be a pure function of `repo_id`: two distinct repos must
720    /// never alias onto one `(collection, tid)` key (which would silently and
721    /// permanently clobber one repo's record with the other's), and the same
722    /// repo must always resolve to the same key so a re-publish is a pointer
723    /// advance, not a new record. A wall-clock `Tid::now()` fails both (same
724    /// microsecond → identical TID; and it needs a persisted mint with a
725    /// get-then-put race). Instead derive the TID's 53-bit micros field and
726    /// 10-bit clock id from `SHA-256(repo_id)` — a hash, not a timestamp:
727    /// these records are not time-ordered, so 63 bits of collision resistance
728    /// is exactly what we want, with no mint, no persistence, and no race.
729    pub fn tid_for_repo(repo_id: &str) -> Tid {
730        // SHA-256 output is always 32 bytes, so these indexes never panic and
731        // need no fallible `try_into`.
732        let d = sha2::Sha256::digest(repo_id.as_bytes());
733        let micros = u64::from_be_bytes([d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]]);
734        let clock_id = u16::from_be_bytes([d[8], d[9]]);
735        // Preserve the spelling already stored in RocksDB and signed into
736        // native MSTs before the public TID codec correction. The old codec
737        // encoded this 63-bit hash as base32(raw << 1). Lift the derivation
738        // into the corrected integer representation, rather than changing
739        // existing keys or teaching the public parser an ambiguous format.
740        // from_micros always leaves bit 63 clear, so this shift cannot lose
741        // entropy or exceed the public TID's 64-bit range.
742        Tid::from_raw(Tid::from_micros(micros, clock_id).to_raw() << 1)
743    }
744
745    /// Load the full record set + signed commit for `did`, or `None` if the
746    /// account has never published (no records, or no persisted commit).
747    fn load_repo(&self, did: &str) -> AnyResult<Option<RepoState>> {
748        match &self.backing {
749            RecordBacking::ReadWrite(db) => Self::load_repo_from(db, did),
750            RecordBacking::ReadOnly(path) => {
751                let opts = readonly_opts();
752                let db = rocksdb::DB::open_for_read_only(&opts, path, false)
753                    .with_context(|| format!("failed to reopen PDS record store at {path:?}"))?;
754                Self::load_repo_from(&db, did)
755            }
756            #[cfg(feature = "pds-postgres")]
757            RecordBacking::Postgres { kv, .. } => Self::load_repo_from_pg(kv, did),
758        }
759    }
760
761    fn load_repo_from(db: &rocksdb::DB, did: &str) -> AnyResult<Option<RepoState>> {
762        let prefix = record_prefix(did);
763        let mut records = BTreeMap::new();
764        // `prefix_iterator` without a configured `prefix_extractor` degrades to
765        // a full forward scan from `prefix` — bound it manually.
766        for item in db.prefix_iterator(&prefix) {
767            let (key, value) = item.context("PDS store record scan failed")?;
768            if !key.starts_with(prefix.as_slice()) {
769                break;
770            }
771            let (collection, tid) = parse_record_key(&key, did)?;
772            let record = ModelRecord::from_dag_cbor(&value)
773                .with_context(|| format!("corrupt PDS record for {did}/{collection}/{tid}"))?;
774            records.insert((collection, tid), record);
775        }
776        // "Never published" == no records for this DID.
777        if records.is_empty() {
778            return Ok(None);
779        }
780        // A published repo always has a persisted signed commit (records and
781        // the commit that signs their MST root advance together atomically —
782        // see `put_record_and_commit`). If the commit is absent the store is
783        // inconsistent (e.g. a legacy pre-#910a-rework write); we cannot serve
784        // a keyless proof without it, so report "not published" rather than
785        // re-sign (the resolver holds no key by design).
786        let Some(commit_bytes) = db
787            .get(commit_key(did))
788            .context("PDS store commit read failed")?
789        else {
790            tracing::warn!(
791                did,
792                "PDS repo has records but no signed commit — refusing to serve (resolver holds no key)"
793            );
794            return Ok(None);
795        };
796        let commit = Commit::from_dag_cbor(&commit_bytes)
797            .with_context(|| format!("corrupt signed commit for {did}"))?;
798        Ok(Some(RepoState {
799            records,
800            commit,
801            commit_bytes,
802        }))
803    }
804
805    /// Postgres counterpart of [`load_repo_from`] (#1257). The record range
806    /// scan and the commit get run in ONE read-only transaction (see
807    /// [`super::pds_record_pg::PgKv::read_snapshot`]), so a concurrent writer
808    /// cannot interleave old records with a new signed commit — the
809    /// reconstructed MST root always verifies against the commit it is served
810    /// with. Identical semantics otherwise, D2-preserving (signed bytes
811    /// verbatim).
812    #[cfg(feature = "pds-postgres")]
813    fn load_repo_from_pg(
814        kv: &super::pds_record_pg::PgKv,
815        did: &str,
816    ) -> AnyResult<Option<RepoState>> {
817        let prefix = record_prefix(did);
818        let mut records = BTreeMap::new();
819        let upper = super::pds_record_pg::prefix_upper_bound(&prefix);
820        let (pairs, commit_bytes) = if let Some(end) = &upper {
821            let snap = kv.read_snapshot(&[(prefix.clone(), end.clone())], &[commit_key(did)])?;
822            let mut values = snap.values.into_iter();
823            (
824                snap.ranges.into_iter().next().unwrap_or_default(),
825                values.next().flatten(),
826            )
827        } else {
828            // Unreachable for well-formed prefixes (they end in NUL, so an
829            // upper bound always exists); kept as a defensive fallback.
830            let pairs = kv
831                .all_pairs()?
832                .into_iter()
833                .filter(|(k, _)| k.starts_with(prefix.as_slice()))
834                .collect();
835            let commit_bytes = kv
836                .get(&commit_key(did))
837                .context("PDS Postgres commit read failed")?;
838            (pairs, commit_bytes)
839        };
840        for (key, value) in pairs {
841            let (collection, tid) = parse_record_key(&key, did)?;
842            let record = ModelRecord::from_dag_cbor(&value)
843                .with_context(|| format!("corrupt PDS record for {did}/{collection}/{tid}"))?;
844            records.insert((collection, tid), record);
845        }
846        if records.is_empty() {
847            return Ok(None);
848        }
849        let Some(commit_bytes) = commit_bytes else {
850            tracing::warn!(
851                did,
852                "PDS repo has records but no signed commit — refusing to serve (resolver holds no key)"
853            );
854            return Ok(None);
855        };
856        let commit = Commit::from_dag_cbor(&commit_bytes)
857            .with_context(|| format!("corrupt signed commit for {did}"))?;
858        Ok(Some(RepoState {
859            records,
860            commit,
861            commit_bytes,
862        }))
863    }
864
865    /// Build the MST over a repo's record set. The MST is a deterministic,
866    /// keyless function of the record keys/CIDs, so this reproduces exactly the
867    /// tree whose root the persisted commit signs.
868    ///
869    /// All of a repo's records — across **all** collections — share ONE MST,
870    /// keyed by `<collection>/<rkey>` exactly as atproto specifies (#910b).
871    /// One repo, one tree, one signed root: adding a record in any collection
872    /// changes the single root the commit signs.
873    fn build_tree(records: &BTreeMap<(String, Tid), ModelRecord>) -> Node {
874        let keyed: BTreeMap<String, hyprstream_pds::cid::Cid> = records
875            .iter()
876            .map(|((collection, tid), rec)| (format!("{collection}/{}", tid.encode()), rec.cid()))
877            .collect();
878        Node::from_keyed_records(&keyed)
879    }
880}
881
882fn acceptance_message(domain: &[u8], payload: &[u8]) -> Vec<u8> {
883    let mut message = Vec::with_capacity(domain.len() + payload.len());
884    message.extend_from_slice(domain);
885    message.extend_from_slice(payload);
886    message
887}
888
889#[derive(Clone, Copy, Debug, Eq, PartialEq)]
890struct At9pCheckpoint {
891    watermark: Watermark,
892    envelope_digest: [u8; 64],
893}
894
895fn checkpoint_message(subject: &str, payload: &[u8]) -> AnyResult<Vec<u8>> {
896    let len = u32::try_from(subject.len()).context("at9p subject exceeds u32")?;
897    let mut out = Vec::with_capacity(AT9P_CHECKPOINT_AAD.len() + 4 + subject.len() + payload.len());
898    out.extend_from_slice(AT9P_CHECKPOINT_AAD);
899    out.extend_from_slice(&len.to_be_bytes());
900    out.extend_from_slice(subject.as_bytes());
901    out.extend_from_slice(payload);
902    Ok(out)
903}
904
905fn encode_at9p_checkpoint(
906    state: &AcceptedAt9pState,
907    envelope: &[u8],
908    identity: &ed25519_dalek::SigningKey,
909) -> AnyResult<Vec<u8>> {
910    use ed25519_dalek::Signer as _;
911    let mut payload = Vec::with_capacity(AT9P_CHECKPOINT_LEN);
912    payload.extend_from_slice(AT9P_CHECKPOINT_MAGIC);
913    payload.extend_from_slice(&state.epoch.to_be_bytes());
914    payload.extend_from_slice(&state.head_digest);
915    payload.push(u8::from(state.terminal));
916    payload.extend_from_slice(&h512(envelope));
917    let sig = identity.sign(&checkpoint_message(&state.subject_cid512, &payload)?);
918    payload.extend_from_slice(&sig.to_bytes());
919    Ok(payload)
920}
921
922fn decode_at9p_checkpoint(
923    subject: &str,
924    bytes: &[u8],
925    identity: &At9pAcceptanceVerifier,
926) -> AnyResult<At9pCheckpoint> {
927    anyhow::ensure!(
928        bytes.len() == AT9P_CHECKPOINT_LEN,
929        "accepted at9p checkpoint length mismatch"
930    );
931    let payload = bytes
932        .get(..AT9P_CHECKPOINT_PAYLOAD_LEN)
933        .ok_or_else(|| anyhow!("accepted at9p checkpoint payload missing"))?;
934    anyhow::ensure!(
935        payload.starts_with(AT9P_CHECKPOINT_MAGIC),
936        "accepted at9p checkpoint has bad version"
937    );
938    let sig = ed25519_dalek::Signature::from_bytes(
939        bytes
940            .get(AT9P_CHECKPOINT_PAYLOAD_LEN..)
941            .ok_or_else(|| anyhow!("accepted at9p checkpoint signature missing"))?
942            .try_into()?,
943    );
944    identity
945        .verify_strict(&checkpoint_message(subject, payload)?, &sig)
946        .context("accepted at9p monotonic checkpoint signature rejected")?;
947    let epoch = u64::from_be_bytes(
948        payload
949            .get(8..16)
950            .ok_or_else(|| anyhow!("checkpoint epoch missing"))?
951            .try_into()?,
952    );
953    let record_digest = payload
954        .get(16..80)
955        .ok_or_else(|| anyhow!("checkpoint digest missing"))?
956        .try_into()?;
957    let terminal = match payload.get(80) {
958        Some(0) => false,
959        Some(1) => true,
960        _ => bail!("invalid checkpoint terminal flag"),
961    };
962    Ok(At9pCheckpoint {
963        watermark: Watermark {
964            epoch,
965            record_digest,
966            terminal,
967        },
968        envelope_digest: payload
969            .get(81..145)
970            .ok_or_else(|| anyhow!("checkpoint envelope digest missing"))?
971            .try_into()?,
972    })
973}
974
975/// Decode and cross-verify one durable accepted-state pair (state envelope +
976/// monotonic checkpoint) against the acceptance identity. Both halves must be
977/// present, the checkpoint watermark must equal the body watermark, and the
978/// checkpoint digest must bind the exact envelope bytes — any mismatch is a
979/// hard consistency failure, never "absent".
980fn verify_at9p_pair(
981    subject: &str,
982    envelope: &[u8],
983    checkpoint: &[u8],
984    identity: &At9pAcceptanceVerifier,
985) -> AnyResult<AcceptedAt9pState> {
986    let state = decode_at9p_state(subject, envelope, identity)?;
987    let checkpoint = decode_at9p_checkpoint(subject, checkpoint, identity)?;
988    anyhow::ensure!(
989        checkpoint.watermark == state.watermark(),
990        "accepted at9p checkpoint/body watermark mismatch"
991    );
992    anyhow::ensure!(
993        checkpoint.envelope_digest == h512(envelope),
994        "accepted at9p checkpoint/state envelope mismatch"
995    );
996    Ok(state)
997}
998
999fn load_at9p_state_from_db(
1000    db: &rocksdb::DB,
1001    subject: &str,
1002    identity: &At9pAcceptanceVerifier,
1003) -> AnyResult<Option<AcceptedAt9pState>> {
1004    let snapshot = db.snapshot();
1005    let envelope = snapshot.get(at9p_state_key(subject))?;
1006    let checkpoint = snapshot.get(at9p_checkpoint_key(subject))?;
1007    match (envelope, checkpoint) {
1008        (None, None) => Ok(None),
1009        (Some(_), None) => bail!("accepted at9p state exists without its monotonic checkpoint"),
1010        (None, Some(_)) => bail!("accepted at9p checkpoint exists without its state envelope"),
1011        (Some(envelope), Some(checkpoint)) => Ok(Some(verify_at9p_pair(
1012            subject,
1013            &envelope,
1014            &checkpoint,
1015            identity,
1016        )?)),
1017    }
1018}
1019
1020/// Postgres counterpart of [`load_at9p_state_from_db`]. Uses `get_batch` for
1021/// a snapshot-consistent read of the state envelope + checkpoint (#1257).
1022#[cfg(feature = "pds-postgres")]
1023fn load_at9p_state_from_pg(
1024    kv: &super::pds_record_pg::PgKv,
1025    subject: &str,
1026    identity: &At9pAcceptanceVerifier,
1027) -> AnyResult<Option<AcceptedAt9pState>> {
1028    let vals = kv.get_batch(&[at9p_state_key(subject), at9p_checkpoint_key(subject)])?;
1029    match (&vals[0], &vals[1]) {
1030        (None, None) => Ok(None),
1031        (Some(_), None) => bail!("accepted at9p state exists without its monotonic checkpoint"),
1032        (None, Some(_)) => bail!("accepted at9p checkpoint exists without its state envelope"),
1033        (Some(envelope), Some(checkpoint)) => Ok(Some(verify_at9p_pair(
1034            subject, envelope, checkpoint, identity,
1035        )?)),
1036    }
1037}
1038
1039fn encode_at9p_state_body(state: &AcceptedAt9pState) -> AnyResult<Vec<u8>> {
1040    let (kind, head) = match state.head() {
1041        AcceptedAt9pHead::Genesis(capsule) => (0u8, capsule.to_dag_cbor()?),
1042        AcceptedAt9pHead::Update(record) => (1u8, record.to_dag_cbor()?),
1043    };
1044    let head_len = u32::try_from(head.len()).context("accepted at9p head exceeds u32")?;
1045    let mut out = Vec::with_capacity(AT9P_STATE_HEADER_LEN + head.len());
1046    out.extend_from_slice(AT9P_STATE_MAGIC);
1047    out.push(kind);
1048    out.extend_from_slice(&state.epoch.to_be_bytes());
1049    out.extend_from_slice(&state.head_digest);
1050    out.push(u8::from(state.terminal));
1051    out.extend_from_slice(&head_len.to_be_bytes());
1052    out.extend_from_slice(&head);
1053    Ok(out)
1054}
1055
1056/// Wrap one complete state value in a daemon-authenticated envelope. The
1057/// stable deployment identity certifies the purpose-derived audit key, and the
1058/// audit key signs the exact state bytes. A different internally valid at9p
1059/// fork therefore cannot be substituted by someone who can edit RocksDB.
1060fn encode_at9p_state(
1061    state: &AcceptedAt9pState,
1062    acceptance_identity: &ed25519_dalek::SigningKey,
1063    audit_key: &ed25519_dalek::SigningKey,
1064) -> AnyResult<Vec<u8>> {
1065    use ed25519_dalek::Signer as _;
1066
1067    let body = encode_at9p_state_body(state)?;
1068    let audit_vk = audit_key.verifying_key().to_bytes();
1069    let key_signature =
1070        acceptance_identity.sign(&acceptance_message(AT9P_ACCEPTANCE_KEY_AAD, &audit_vk));
1071    let state_signature = audit_key.sign(&acceptance_message(AT9P_ACCEPTANCE_STATE_AAD, &body));
1072    let mut out =
1073        Vec::with_capacity(AT9P_ACCEPTANCE_PREFIX_LEN + body.len() + ED25519_SIGNATURE_LEN);
1074    out.extend_from_slice(AT9P_ACCEPTANCE_MAGIC);
1075    out.extend_from_slice(&audit_vk);
1076    out.extend_from_slice(&key_signature.to_bytes());
1077    out.extend_from_slice(&body);
1078    out.extend_from_slice(&state_signature.to_bytes());
1079    Ok(out)
1080}
1081
1082fn decode_at9p_state(
1083    subject_cid512: &str,
1084    bytes: &[u8],
1085    acceptance_identity: &At9pAcceptanceVerifier,
1086) -> AnyResult<AcceptedAt9pState> {
1087    anyhow::ensure!(
1088        bytes.len() >= AT9P_ACCEPTANCE_PREFIX_LEN + AT9P_STATE_HEADER_LEN + ED25519_SIGNATURE_LEN,
1089        "accepted at9p envelope is truncated"
1090    );
1091    anyhow::ensure!(
1092        bytes.starts_with(AT9P_ACCEPTANCE_MAGIC),
1093        "accepted at9p envelope has bad version"
1094    );
1095    let audit_vk_bytes: [u8; ED25519_KEY_LEN] = bytes
1096        .get(AT9P_ACCEPTANCE_MAGIC.len()..AT9P_ACCEPTANCE_MAGIC.len() + ED25519_KEY_LEN)
1097        .ok_or_else(|| anyhow!("accepted at9p envelope missing audit key"))?
1098        .try_into()?;
1099    let key_sig_start = AT9P_ACCEPTANCE_MAGIC.len() + ED25519_KEY_LEN;
1100    let key_sig_end = key_sig_start + ED25519_SIGNATURE_LEN;
1101    let key_signature = ed25519_dalek::Signature::from_bytes(
1102        bytes
1103            .get(key_sig_start..key_sig_end)
1104            .ok_or_else(|| anyhow!("accepted at9p envelope missing key certificate"))?
1105            .try_into()?,
1106    );
1107    acceptance_identity
1108        .verify_strict(
1109            &acceptance_message(AT9P_ACCEPTANCE_KEY_AAD, &audit_vk_bytes),
1110            &key_signature,
1111        )
1112        .context("accepted at9p audit-key certificate rejected")?;
1113    let audit_vk = ed25519_dalek::VerifyingKey::from_bytes(&audit_vk_bytes)
1114        .context("accepted at9p envelope has malformed audit key")?;
1115    let state_sig_start = bytes.len() - ED25519_SIGNATURE_LEN;
1116    let body = bytes
1117        .get(AT9P_ACCEPTANCE_PREFIX_LEN..state_sig_start)
1118        .ok_or_else(|| anyhow!("accepted at9p envelope missing state body"))?;
1119    let state_signature = ed25519_dalek::Signature::from_bytes(
1120        bytes
1121            .get(state_sig_start..)
1122            .ok_or_else(|| anyhow!("accepted at9p envelope missing state signature"))?
1123            .try_into()?,
1124    );
1125    audit_vk
1126        .verify_strict(
1127            &acceptance_message(AT9P_ACCEPTANCE_STATE_AAD, body),
1128            &state_signature,
1129        )
1130        .context("accepted at9p daemon acceptance signature rejected")?;
1131    decode_at9p_state_body(subject_cid512, body)
1132}
1133
1134fn decode_at9p_state_body(subject_cid512: &str, bytes: &[u8]) -> AnyResult<AcceptedAt9pState> {
1135    anyhow::ensure!(
1136        bytes.len() >= AT9P_STATE_HEADER_LEN,
1137        "accepted at9p state is truncated"
1138    );
1139    anyhow::ensure!(
1140        bytes.starts_with(AT9P_STATE_MAGIC),
1141        "accepted at9p state has bad version"
1142    );
1143    let kind = *bytes
1144        .get(8)
1145        .ok_or_else(|| anyhow!("accepted at9p state missing kind"))?;
1146    let epoch = u64::from_be_bytes(
1147        bytes
1148            .get(9..17)
1149            .ok_or_else(|| anyhow!("accepted at9p state missing epoch"))?
1150            .try_into()?,
1151    );
1152    let digest: [u8; 64] = bytes
1153        .get(17..81)
1154        .ok_or_else(|| anyhow!("accepted at9p state missing digest"))?
1155        .try_into()?;
1156    let terminal = match bytes
1157        .get(81)
1158        .ok_or_else(|| anyhow!("accepted at9p state missing terminal flag"))?
1159    {
1160        0 => false,
1161        1 => true,
1162        _ => bail!("accepted at9p state has invalid terminal flag"),
1163    };
1164    let head_len = u32::from_be_bytes(
1165        bytes
1166            .get(82..86)
1167            .ok_or_else(|| anyhow!("accepted at9p state missing length"))?
1168            .try_into()?,
1169    ) as usize;
1170    anyhow::ensure!(
1171        bytes.len() == AT9P_STATE_HEADER_LEN + head_len,
1172        "accepted at9p state length mismatch"
1173    );
1174    let head = bytes
1175        .get(AT9P_STATE_HEADER_LEN..)
1176        .ok_or_else(|| anyhow!("accepted at9p state missing head"))?;
1177    let state = match kind {
1178        0 => AcceptedAt9pState::from_persisted_genesis(head)?,
1179        1 => AcceptedAt9pState::from_persisted_update(head)?,
1180        _ => bail!("accepted at9p state has unknown head kind {kind}"),
1181    };
1182    anyhow::ensure!(
1183        state.subject_cid512 == subject_cid512,
1184        "accepted at9p state subject mismatch"
1185    );
1186    anyhow::ensure!(
1187        state.epoch == epoch,
1188        "accepted at9p state epoch/body mismatch"
1189    );
1190    anyhow::ensure!(
1191        state.head_digest == digest,
1192        "accepted at9p state digest/body mismatch"
1193    );
1194    anyhow::ensure!(
1195        state.terminal == terminal,
1196        "accepted at9p state terminal/body mismatch"
1197    );
1198    Ok(state)
1199}
1200
1201#[derive(Clone)]
1202struct RocksAt9pStateStore {
1203    store: Arc<PdsRecordStore>,
1204    acceptance_identity: ed25519_dalek::SigningKey,
1205    audit_key: ed25519_dalek::SigningKey,
1206}
1207
1208impl WatermarkStore for RocksAt9pStateStore {
1209    fn get(&self, subject_cid512: &str) -> AnyResult<Option<AcceptedAt9pState>> {
1210        self.store
1211            .load_at9p_state_with_key(subject_cid512, self.acceptance_identity.verifying_key())
1212    }
1213
1214    fn conditional_advance(
1215        &self,
1216        expected: Option<Watermark>,
1217        state: &AcceptedAt9pState,
1218    ) -> AnyResult<ConditionalAdvance> {
1219        self.store.conditional_advance_at9p_state(
1220            expected,
1221            state,
1222            &self.acceptance_identity,
1223            &self.audit_key,
1224        )
1225    }
1226}
1227
1228/// Sole production ownership boundary for accepted current `did:at9p` state.
1229/// It GATE-seeds genesis, reloads predecessors exclusively from durable state,
1230/// calls the duplicity guard for every successor, and atomically commits the
1231/// complete accepted head. It does not dial, select reach, or admit carriers.
1232pub struct At9pStateIngest {
1233    guard: DuplicityGuard<RocksAt9pStateStore, WalDuplicityAlarmSink>,
1234}
1235
1236impl At9pStateIngest {
1237    pub fn open(
1238        store: Arc<PdsRecordStore>,
1239        alarm_path: &Path,
1240        acceptance_identity: ed25519_dalek::SigningKey,
1241        audit_ed_sk: ed25519_dalek::SigningKey,
1242        pq_sk: hyprstream_rpc::crypto::pq::MlDsaSigningKey,
1243    ) -> AnyResult<Self> {
1244        let guard = DuplicityGuard::with_durable_alarm(
1245            RocksAt9pStateStore {
1246                store,
1247                acceptance_identity,
1248                audit_key: audit_ed_sk.clone(),
1249            },
1250            alarm_path,
1251            audit_ed_sk,
1252            pq_sk,
1253        )?;
1254        Ok(Self { guard })
1255    }
1256
1257    pub fn ingest_genesis(&self, did: &str, capsule_bytes: &[u8]) -> AnyResult<AcceptedAt9pState> {
1258        let verified = verify_did_at9p(did, capsule_bytes)
1259            .map_err(|e| anyhow!("did:at9p genesis ingest rejected: {e}"))?;
1260        self.guard.seed_genesis(&verified)?;
1261        self.accepted_state(did, None)
1262    }
1263
1264    pub fn ingest_successor(
1265        &self,
1266        did: &str,
1267        update_bytes: &[u8],
1268        now: &str,
1269    ) -> AnyResult<AcceptedAt9pState> {
1270        let subject = did
1271            .strip_prefix(DID_AT9P_PREFIX)
1272            .ok_or_else(|| anyhow!("identifier is not did:at9p: {did:?}"))?;
1273        let candidate = hyprstream_pds::at9p::UpdateRecord::from_dag_cbor(update_bytes)?;
1274        anyhow::ensure!(
1275            candidate.subject_cid512 == subject,
1276            "update subject does not match ingest DID"
1277        );
1278        self.guard.admit_successor(&candidate, now)?;
1279        self.accepted_state(did, Some(now))
1280    }
1281
1282    /// Read the complete accepted snapshot. Supplying `now` enforces successor
1283    /// expiry before the state is exposed to a consumer.
1284    pub fn accepted_state(&self, did: &str, now: Option<&str>) -> AnyResult<AcceptedAt9pState> {
1285        let subject = did
1286            .strip_prefix(DID_AT9P_PREFIX)
1287            .ok_or_else(|| anyhow!("identifier is not did:at9p: {did:?}"))?;
1288        let state = self
1289            .guard
1290            .accepted_state(subject)?
1291            .ok_or_else(|| anyhow!("no accepted did:at9p state for {did}"))?;
1292        anyhow::ensure!(state.did == did, "accepted at9p state DID mismatch");
1293        if let Some(now) = now {
1294            state.ensure_fresh(now)?;
1295        }
1296        Ok(state)
1297    }
1298}
1299
1300fn readonly_opts() -> rocksdb::Options {
1301    let mut opts = rocksdb::Options::default();
1302    opts.create_if_missing(false);
1303    opts
1304}
1305
1306/// Restrict the record-store directory to owner-only (0700) on unix.
1307///
1308/// Defence in depth: the record *values* (and the signed commit) are what a
1309/// resolver serves as authentic PDS state — a store another local user could
1310/// write would let them inject records/commits the node then serves. On
1311/// non-unix this is a no-op (the store path is expected to be an OS-managed
1312/// private dir there).
1313fn harden_store_dir(path: &Path) {
1314    #[cfg(unix)]
1315    {
1316        use std::os::unix::fs::PermissionsExt;
1317        // Best-effort: this is defence-in-depth (the private key never lands in
1318        // this DB, and the path never resolves to /tmp — H2). If the chmod
1319        // fails (e.g. the dir is owned by a different uid in a split-uid setup),
1320        // warn rather than hard-fail startup.
1321        if let Err(e) = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) {
1322            tracing::warn!("could not set 0700 on PDS store dir {path:?}: {e}");
1323        }
1324    }
1325    #[cfg(not(unix))]
1326    let _ = path;
1327}
1328
1329// ============================================================================
1330// PdsPublisher — the register/commit write side of #910a
1331// ============================================================================
1332
1333/// Publishes/advances a repo's `ai.hyprstream.model` PDS record on this node,
1334/// over a read-write [`PdsRecordStore`]. Constructed once by the registry
1335/// service (the sole writer) and called from `handle_register` /
1336/// `handle_commit_with_author`.
1337///
1338/// This is the **only** holder of the `#atproto` P-256 private key: it signs
1339/// the repo's commit once, here, at write time, and persists the signed bytes.
1340/// The resolver never sees the key.
1341struct StoreGenerationSource(Arc<crate::auth::key_rotation::Es256SigningKeyStore>);
1342
1343impl crate::auth::ActiveGenerationSource for StoreGenerationSource {
1344    fn active_generation(&self) -> AnyResult<Option<crate::auth::ActiveGeneration>> {
1345        Ok(self
1346            .0
1347            .active_slot()
1348            .map(|slot| crate::auth::ActiveGeneration {
1349                seq: 0,
1350                kid: slot.kid(),
1351                verifying_key: *slot.key.verifying_key(),
1352                signing_key: slot.key.as_ref().clone(),
1353                head_at_op: None,
1354            }))
1355    }
1356}
1357
1358pub trait IntoPdsGenerationSource {
1359    fn into_pds_generation_source(
1360        self,
1361    ) -> (
1362        Arc<dyn crate::auth::ActiveGenerationSource>,
1363        Option<Arc<crate::auth::key_rotation::Es256SigningKeyStore>>,
1364    );
1365}
1366
1367impl IntoPdsGenerationSource for p256::ecdsa::SigningKey {
1368    fn into_pds_generation_source(
1369        self,
1370    ) -> (
1371        Arc<dyn crate::auth::ActiveGenerationSource>,
1372        Option<Arc<crate::auth::key_rotation::Es256SigningKeyStore>>,
1373    ) {
1374        (
1375            Arc::new(crate::auth::FixedGenerationSource::new(self)),
1376            None,
1377        )
1378    }
1379}
1380
1381impl IntoPdsGenerationSource for Arc<crate::auth::key_rotation::Es256SigningKeyStore> {
1382    fn into_pds_generation_source(
1383        self,
1384    ) -> (
1385        Arc<dyn crate::auth::ActiveGenerationSource>,
1386        Option<Arc<crate::auth::key_rotation::Es256SigningKeyStore>>,
1387    ) {
1388        (
1389            Arc::new(StoreGenerationSource(Arc::clone(&self))),
1390            Some(self),
1391        )
1392    }
1393}
1394
1395pub struct PdsPublisher {
1396    store: Arc<PdsRecordStore>,
1397    at9p_state: Option<At9pStateIngest>,
1398    /// This node's own `did:key` identity — the `repo` at-uri authority for
1399    /// every record this node publishes (single-node, self-hosted PDS; a
1400    /// per-account DID scheme is out of scope for #910a).
1401    did: String,
1402    /// Source of truth for the active `#atproto` ES256 generation. Resolved
1403    /// **at sign time**, not at construction (#1123/C4): under `--ipc` this
1404    /// reads the sealed op-log head so a rotation performed by the OAuth
1405    /// process is observed here without any event-delivery mechanism. A frozen
1406    /// construction-time copy would keep signing with a retired key after a
1407    /// rotation. See [`crate::auth::ActiveGenerationSource`].
1408    generation_source: Arc<dyn crate::auth::ActiveGenerationSource>,
1409    /// Optional process-local store used only to install the promotion hook.
1410    /// Production signing still resolves the sealed generation source.
1411    es256_store: Option<Arc<crate::auth::key_rotation::Es256SigningKeyStore>>,
1412    /// Serializes the load→rebuild→sign→persist critical section (#910b).
1413    publish_lock: parking_lot::Mutex<()>,
1414}
1415
1416impl PdsPublisher {
1417    /// Construct with a fixed signing key. Preserved for tests and simple
1418    /// embedders; production wiring uses [`Self::with_generation_source`] so
1419    /// the publisher observes rotations.
1420    pub fn new<S: IntoPdsGenerationSource>(
1421        store: Arc<PdsRecordStore>,
1422        did: String,
1423        source: S,
1424    ) -> Self {
1425        let (generation_source, es256_store) = source.into_pds_generation_source();
1426        Self {
1427            store,
1428            at9p_state: None,
1429            did,
1430            generation_source,
1431            es256_store,
1432            publish_lock: parking_lot::Mutex::new(()),
1433        }
1434    }
1435
1436    /// Construct with a live generation source — the production shape (#1123).
1437    /// The publisher resolves the active `#atproto` key from `source` at every
1438    /// sign, so it never freezes a retired key.
1439    pub fn with_generation_source(
1440        store: Arc<PdsRecordStore>,
1441        did: String,
1442        source: Arc<dyn crate::auth::ActiveGenerationSource>,
1443    ) -> Self {
1444        Self {
1445            store,
1446            at9p_state: None,
1447            did,
1448            generation_source: source,
1449            es256_store: None,
1450            publish_lock: parking_lot::Mutex::new(()),
1451        }
1452    }
1453
1454    pub fn with_es256_store(
1455        mut self,
1456        store: Arc<crate::auth::key_rotation::Es256SigningKeyStore>,
1457    ) -> Self {
1458        self.es256_store = Some(store);
1459        self
1460    }
1461
1462    /// Install the daemon-owned accepted-state ingest boundary. Production
1463    /// construction always calls this; the optional form keeps isolated PDS
1464    /// pointer-store tests free of unrelated audit-key setup.
1465    pub fn with_at9p_state_ingest(mut self, ingest: At9pStateIngest) -> Self {
1466        self.at9p_state = Some(ingest);
1467        self
1468    }
1469
1470    /// Connect this publisher to the shared ES256 store's in-process
1471    /// lead→active transition. A weak reference avoids a publisher→store→
1472    /// publisher reference cycle; if its target has been dropped, the hook
1473    /// errors so rotation remains pending. Cross-`--ipc` delivery is deferred
1474    /// to #1123.
1475    pub fn install_es256_promotion_hook(self: &Arc<Self>) -> AnyResult<()> {
1476        let publisher = Arc::downgrade(self);
1477        let store = self
1478            .es256_store
1479            .as_ref()
1480            .ok_or_else(|| anyhow!("no process-local ES256 store for promotion hook"))?;
1481        store.set_promotion_hook(Arc::new(move |key| {
1482            let publisher = publisher
1483                .upgrade()
1484                .ok_or_else(|| anyhow!("PDS publisher dropped before ES256 promotion re-sign"))?;
1485            publisher.resign_head_with_key(&key).map(|_| ())
1486        }));
1487        // Reconcile once at installation too: if promotion happened before the
1488        // registry factory installed the callback, an idle persisted head must
1489        // still be brought under the current active key.
1490        self.resign_head().map(|_| ())
1491    }
1492
1493    pub fn ingest_at9p_genesis(
1494        &self,
1495        did: &str,
1496        capsule_bytes: &[u8],
1497    ) -> AnyResult<AcceptedAt9pState> {
1498        self.at9p_state
1499            .as_ref()
1500            .ok_or_else(|| anyhow!("accepted did:at9p ingest is not configured"))?
1501            .ingest_genesis(did, capsule_bytes)
1502    }
1503
1504    /// The sole read-write record store this publisher owns. Exposed for the
1505    /// in-daemon at9p renewal timer, which must reuse the LIVE handles —
1506    /// never re-open the store (the RocksDB directory LOCK excludes the
1507    /// second read-write opener this process already is).
1508    pub fn at9p_record_store(&self) -> &Arc<PdsRecordStore> {
1509        &self.store
1510    }
1511
1512    /// The daemon-owned accepted-state ingest boundary, when configured.
1513    /// Renewal admission goes through this exact boundary (same guard, audit
1514    /// WAL, and checkpoint CAS as the RPC path).
1515    pub fn at9p_state_ingest(&self) -> Option<&At9pStateIngest> {
1516        self.at9p_state.as_ref()
1517    }
1518
1519    pub fn ingest_at9p_successor(
1520        &self,
1521        did: &str,
1522        update_bytes: &[u8],
1523        now: &str,
1524    ) -> AnyResult<AcceptedAt9pState> {
1525        self.at9p_state
1526            .as_ref()
1527            .ok_or_else(|| anyhow!("accepted did:at9p ingest is not configured"))?
1528            .ingest_successor(did, update_bytes, now)
1529    }
1530
1531    pub fn accepted_at9p_state(
1532        &self,
1533        did: &str,
1534        now: Option<&str>,
1535    ) -> AnyResult<AcceptedAt9pState> {
1536        self.at9p_state
1537            .as_ref()
1538            .ok_or_else(|| anyhow!("accepted did:at9p ingest is not configured"))?
1539            .accepted_state(did, now)
1540    }
1541
1542    /// Publish (or advance) the `ai.hyprstream.model` record for `repo_id`,
1543    /// pointing it at `current_oid_hex` (a git OID, hex-encoded).
1544    ///
1545    /// Collection-parameterized publication lives in
1546    /// [`Self::publish_record`]; this is the `ai.hyprstream.model` wrapper the
1547    /// registry service calls — behavior unchanged from #910a.
1548    ///
1549    /// Best-effort by design: the caller (registry service) should log and
1550    /// continue on error rather than fail the git operation that triggered
1551    /// this — the local git commit is the durable source of truth; a missed
1552    /// PDS publish is caught up by the next successful one (same
1553    /// eventual-consistency contract `git::promote` documents).
1554    pub fn publish(&self, repo_id: &str, current_oid_hex: &str) -> AnyResult<()> {
1555        self.publish_record(COLLECTION_NSID, repo_id, current_oid_hex)
1556    }
1557
1558    /// Publish (or advance) a record in `collection` for `record_id`, pointing
1559    /// it at `current_oid_hex` (a git OID, hex-encoded). This is the
1560    /// collection-parameterized publication path (#910b): the repo's ONE MST
1561    /// spans every collection, and the ONE commit signed here covers them all.
1562    ///
1563    /// The record's rkey is derived deterministically from `record_id`
1564    /// (`PdsRecordStore::tid_for_repo`) and reused on every call, so this
1565    /// always *replaces* the existing record's value rather than creating a
1566    /// new one. (rkeys are scoped per collection in atproto, so the same
1567    /// `record_id` in two collections is two distinct records at
1568    /// `<collection-a>/<rkey>` and `<collection-b>/<rkey>`.) The repo's MST is
1569    /// rebuilt over its full multi-collection record set, a new commit is
1570    /// signed **once** here (rev strictly advances; `prev` points at the
1571    /// previous commit's CID — the atproto pointer advance), and the record
1572    /// and signed commit are persisted together.
1573    pub fn publish_record(
1574        &self,
1575        collection: &str,
1576        record_id: &str,
1577        current_oid_hex: &str,
1578    ) -> AnyResult<()> {
1579        let current_oid_cid = git_oid_to_cid_string(current_oid_hex)?;
1580        self.publish_pointer(collection, record_id, current_oid_cid)
1581    }
1582
1583    /// Publish (or advance) a holder-controlled `ai.hyprstream.ledger.allocation`
1584    /// record — the held entitlement (#924).
1585    ///
1586    /// The allocation's canonical DAG-CBOR bytes are content-addressed and the
1587    /// resulting CID becomes the pointer this repo's MST carries under the
1588    /// `ai.hyprstream.ledger.allocation` collection; the holder's signed PDS
1589    /// commit is the only signature the record needs (it lives in the holder's
1590    /// own repo). The rkey is derived deterministically from the record's grant
1591    /// CID, so re-publishing the same grant advances the same record.
1592    pub fn publish_ledger_allocation(&self, rec: &AllocationRecord) -> AnyResult<()> {
1593        self.publish_pointer(
1594            hyprstream_pds::ledger::allocation::COLLECTION_NSID,
1595            &rec.grant,
1596            hyprstream_pds::cid::Cid::from_dag_cbor(&rec.to_dag_cbor()).encode(),
1597        )
1598    }
1599
1600    /// Publish (or advance) a holder-controlled `ai.hyprstream.ledger.receipt`
1601    /// record — a dual-signed usage record (#924).
1602    ///
1603    /// The rkey is derived from the receipt's `transferId`, giving at-least-once
1604    /// (idempotent) receipt delivery: re-publishing the same transfer advances
1605    /// the same record rather than creating a duplicate.
1606    pub fn publish_ledger_receipt(&self, rec: &ReceiptRecord) -> AnyResult<()> {
1607        self.publish_pointer(
1608            hyprstream_pds::ledger::receipt::COLLECTION_NSID,
1609            &rec.transfer_id,
1610            hyprstream_pds::cid::Cid::from_dag_cbor(&rec.to_dag_cbor()).encode(),
1611        )
1612    }
1613
1614    /// Publish (or advance) a holder-controlled `ai.hyprstream.ledger.checkpoint`
1615    /// record — a signed ledger head (#924).
1616    ///
1617    /// `ledger_id` is the stable key of the ledger whose head this checkpoint
1618    /// commits, so successive checkpoints advance the one head pointer.
1619    pub fn publish_ledger_checkpoint(
1620        &self,
1621        ledger_id: &str,
1622        rec: &CheckpointRecord,
1623    ) -> AnyResult<()> {
1624        self.publish_pointer(
1625            hyprstream_pds::ledger::checkpoint::COLLECTION_NSID,
1626            ledger_id,
1627            hyprstream_pds::cid::Cid::from_dag_cbor(&rec.to_dag_cbor()).encode(),
1628        )
1629    }
1630
1631    /// The shared publish critical section: write (or advance) the pointer record
1632    /// in `collection` for `record_id`, pointing it at the already-formed
1633    /// `current_oid_cid` (`format: "cid"` string). Every collection — the git
1634    /// model pointer and the ledger records alike — shares the repo's ONE MST and
1635    /// the ONE commit signed here (#910b).
1636    fn publish_pointer(
1637        &self,
1638        collection: &str,
1639        record_id: &str,
1640        current_oid_cid: String,
1641    ) -> AnyResult<()> {
1642        // ── #1159 freeze: durable store-boundary guard ─────────────────────
1643        // This is the single chokepoint every `PdsPublisher` durable write
1644        // funnels through, and it persists `self.did` as the repo authority
1645        // (`commit\0{did}` / `record\0{did}...` RocksDB keys). Production
1646        // construction supplies a node `did:key` (factories.rs), but
1647        // `PdsPublisher::new` accepts an arbitrary `String` DID, so a caller
1648        // handing it a path-form `did:web:{authority}:users:{name}` would
1649        // otherwise anchor a permanent, spec-invalid repo identifier in durable
1650        // storage — exactly the shape #1159 freezes. Reject it here, at the
1651        // durable-write boundary, WITHOUT applying the full repo-authority
1652        // allowlist (which would reject the legitimate node `did:key` design).
1653        if is_path_form_did_web(&self.did) {
1654            bail!(
1655                "PdsPublisher authority {} is a path-form did:web identifier; \
1656                 the atproto profile requires host-form did:web, and path-form \
1657                 account minting is frozen (#1159)",
1658                self.did,
1659            );
1660        }
1661
1662        // The collection NSID becomes both a `\0`-delimited RocksDB key field
1663        // and the `<collection>/<rkey>` MST key prefix — reject anything that
1664        // would be ambiguous in either encoding.
1665        if collection.is_empty() || collection.contains(['\0', '/']) {
1666            bail!("invalid collection NSID {collection:?}");
1667        }
1668        let tid = PdsRecordStore::tid_for_repo(record_id);
1669        let record = ModelRecord::new(
1670            format!("at://{}", self.did),
1671            current_oid_cid,
1672            atproto_datetime_now(),
1673        )?;
1674
1675        // ── critical section: load → rebuild → sign → persist ──────────────
1676        // Serialized so every signed commit covers ALL prior writes (see
1677        // `publish_lock`). A panicked publish is benign — the store write is
1678        // a single atomic WriteBatch, so the store stays consistent and the
1679        // next publisher proceeds (parking_lot has no poisoning).
1680        let _guard = self.publish_lock.lock();
1681
1682        // Load the existing repo state (records + previous signed commit) so we
1683        // can rebuild the full MST and advance the commit pointer.
1684        let existing = self.store.load_repo(&self.did)?;
1685        let mut records = existing
1686            .as_ref()
1687            .map(|r| r.records.clone())
1688            .unwrap_or_default();
1689        records.insert((collection.to_owned(), tid), record.clone());
1690
1691        // Rebuild the MST (deterministic, keyless) over the full
1692        // multi-collection record set and sign the commit ONCE over its root.
1693        let tree = PdsRecordStore::build_tree(&records);
1694        let root = tree.root_cid();
1695        let (rev, prev) = match existing.as_ref().map(|r| &r.commit) {
1696            Some(prev_commit) => (next_rev(prev_commit.rev), Some(prev_commit.cid())),
1697            None => (Tid::now(), None),
1698        };
1699        let unsigned = UnsignedCommit::new(self.did.clone(), root, rev, prev);
1700        // Resolve the active `#atproto` generation at sign time (#1123/C4).
1701        // Under `--ipc` this reads the sealed op-log head written by the OAuth
1702        // rotation task, so a rotation on the other side is observed here with
1703        // no propagation mechanism. Fail-closed: if no verified generation is
1704        // observable, decline to sign rather than fall back to a stale key.
1705        let generation = self
1706            .generation_source
1707            .active_generation()
1708            .context("resolving active #atproto generation")?
1709            .ok_or_else(|| {
1710                anyhow!(
1711                    "no active #atproto generation observable; declining to sign \
1712                     (sealed op-log head not present or unverifiable)"
1713                )
1714            })?;
1715        let commit = Commit::sign(&unsigned, &generation.signing_key);
1716
1717        // Gate the durable write on the exact head this publish was built
1718        // from: on the Postgres backend the commit-key compare-and-swap
1719        // rejects the write if another publisher (in either AZ) advanced the
1720        // head since `load_repo`, so a commit never silently lands without
1721        // covering the full record set (#1257 cross-AZ invariant).
1722        let expected_head = existing.as_ref().map(|r| r.commit_bytes.as_slice());
1723        self.store.put_record_and_commit(
1724            &self.did,
1725            collection,
1726            tid,
1727            &record,
1728            &commit,
1729            expected_head,
1730        )
1731    }
1732
1733    /// #918 re-sign-on-rotation: re-sign the EXISTING persisted repo head with
1734    /// the authenticated active `#atproto` generation, leaving the MST root,
1735    /// `rev`, and `prev` unchanged. Cross-process maintenance uses the same
1736    /// sealed generation source as ordinary publication, never the optional
1737    /// process-local key cache. The guarded in-process promotion hook uses
1738    /// `resign_head_with_key` with its not-yet-published candidate instead.
1739    ///
1740    /// Holds the `publish_lock` to serialize with concurrent publishes. Returns
1741    /// `Ok(true)` if the head was re-signed; no repo or an already-current
1742    /// signature is an exact no-op. Missing/unverifiable authority fails closed
1743    /// when a repo exists. The writer's maintenance tick attempts reconciliation;
1744    /// downtime or failed ticks can outlast overlap, so recovery does not imply
1745    /// uninterrupted public-read availability during that interval.
1746    pub fn resign_head(&self) -> AnyResult<bool> {
1747        let _guard = self.publish_lock.lock();
1748        // Initial boot can precede OAuth's first sealed head. With no published
1749        // repo there is nothing to sign; a concurrent first publish resolves its
1750        // own authenticated generation under the normal publish lock.
1751        let Some(repo) = self.store.load_repo(&self.did)? else {
1752            return Ok(false);
1753        };
1754        let generation = self
1755            .generation_source
1756            .active_generation()
1757            .context("resolving active #atproto generation for head re-sign")?
1758            .ok_or_else(|| anyhow!("no active #atproto generation for head re-sign"))?;
1759        self.resign_loaded_head(&repo, &generation.signing_key)
1760    }
1761
1762    /// Re-sign the existing head with an explicit candidate key while the
1763    /// rotation store's write guard keeps that key hidden from DID-document
1764    /// readers. Only after this succeeds does rotation install the candidate
1765    /// as live active, making publication and persisted-head re-sign one
1766    /// externally atomic transition.
1767    fn resign_head_with_key(&self, key: &p256::ecdsa::SigningKey) -> AnyResult<bool> {
1768        let _guard = self.publish_lock.lock();
1769        let Some(repo) = self.store.load_repo(&self.did)? else {
1770            return Ok(false); // no repo yet — nothing to re-sign
1771        };
1772        self.resign_loaded_head(&repo, key)
1773    }
1774
1775    /// Both callers hold publish_lock from load through authenticated key
1776    /// resolution (or guarded candidate selection) and the exact-head CAS.
1777    fn resign_loaded_head(
1778        &self,
1779        repo: &RepoState,
1780        key: &p256::ecdsa::SigningKey,
1781    ) -> AnyResult<bool> {
1782        if repo.commit.verify(key.verifying_key()).is_ok() {
1783            return Ok(false);
1784        }
1785        // Re-sign over the same canonical unsigned bytes — only the `sig`
1786        // changes. The commit CID changes because it includes `sig`; the next
1787        // normal publish will link `prev` to this re-signed head's CID. The
1788        // re-sign is gated on the exact head bytes it was computed from, so a
1789        // concurrent publish in either AZ cannot be silently overwritten by
1790        // the refreshed signature.
1791        let unsigned = repo.commit.unsigned();
1792        let re_signed = Commit::sign(&unsigned, key);
1793        self.store
1794            .put_commit(&self.did, &re_signed, Some(&repo.commit_bytes))?;
1795        Ok(true)
1796    }
1797}
1798
1799/// The next commit `rev`: the current wall-clock TID, or one past the previous
1800/// rev if the clock has not advanced past it.
1801///
1802/// atproto requires `rev` to strictly increase across a repo's commits; two
1803/// publishes within the same microsecond (or a clock that went backwards) must
1804/// still advance.
1805fn next_rev(prev: Tid) -> Tid {
1806    let now = Tid::now();
1807    if now > prev {
1808        now
1809    } else {
1810        Tid::from_raw(prev.to_raw().saturating_add(1))
1811    }
1812}
1813
1814/// Encode a git OID (hex string, SHA-1 or SHA-256) as a `format: "cid"`
1815/// string for `ModelRecord::currentOid`.
1816///
1817/// TODO(#395): this is a placeholder encoding (a CIDv1 `raw` codec over the
1818/// OID's raw bytes, i.e. a *hash of the OID*, not an identity-preserving
1819/// encoding of it) — good enough to be deterministic and format-valid today,
1820/// but the real git-OID↔CID grammar belongs to the CID encoder work tracked
1821/// there. Once that lands, swap this call for the real encoder; nothing else
1822/// in the publish path needs to change.
1823fn git_oid_to_cid_string(oid_hex: &str) -> AnyResult<String> {
1824    let oid_bytes =
1825        hex::decode(oid_hex).with_context(|| format!("git OID {oid_hex:?} is not valid hex"))?;
1826    Ok(hyprstream_pds::cid::Cid::from_raw(&oid_bytes).encode())
1827}
1828
1829/// The current UTC time in the atproto `datetime` lexicon format
1830/// (ISO-8601, millisecond precision, trailing `Z`).
1831fn atproto_datetime_now() -> String {
1832    chrono::Utc::now()
1833        .format("%Y-%m-%dT%H:%M:%S%.3fZ")
1834        .to_string()
1835}
1836
1837/// `RecordResolver` over a [`PdsRecordStore`]. Builds CAR proofs via the
1838/// reused `hyprstream_pds::car::build_record_proof_car` — no CAR/MST/commit
1839/// code is reinvented here, and **no signing happens on the read path**: the
1840/// commit served in the proof is the one the writer signed at publish time.
1841pub struct PdsRecordResolver {
1842    store: Arc<PdsRecordStore>,
1843    /// The rotation store feeding the root did:web document and that document's
1844    /// repo DID. Local reads resolve exactly its bounded publication authority.
1845    rotation: Option<(Arc<crate::auth::key_rotation::Es256SigningKeyStore>, String)>,
1846}
1847
1848impl PdsRecordResolver {
1849    pub fn new(store: Arc<PdsRecordStore>) -> Self {
1850        Self {
1851            store,
1852            rotation: None,
1853        }
1854    }
1855
1856    /// Install the ES256 rotation store + the root did:web repo DID.
1857    pub fn with_es256_rotation(
1858        mut self,
1859        store: Arc<crate::auth::key_rotation::Es256SigningKeyStore>,
1860        node_did: String,
1861    ) -> Self {
1862        self.rotation = Some((store, node_did));
1863        self
1864    }
1865
1866    fn local_generation_guard(
1867        &self,
1868        did: &str,
1869    ) -> Option<parking_lot::RwLockReadGuard<'_, crate::auth::key_rotation::Es256KeySlots>> {
1870        let (store, node_did) = self.rotation.as_ref()?;
1871        (did == node_did).then(|| store.0.read())
1872    }
1873
1874    pub fn accepted_at9p_state(
1875        &self,
1876        did: &str,
1877        now: &str,
1878    ) -> AnyResult<Option<AcceptedAt9pState>> {
1879        self.store.accepted_at9p_state(did, Some(now))
1880    }
1881}
1882
1883/// Rebuild the repo's MST (keyless) and assemble a proof CAR for one record,
1884/// served against the STORED signed commit — no re-signing.
1885///
1886/// **Fail-closed root-consistency check (#910b):** the rebuilt tree's root must
1887/// equal the root the persisted commit signs (`commit.data`). Records and their
1888/// covering commit are written atomically, so a mismatch means the store is
1889/// corrupt or was tampered with — serving would hand out a record the signed
1890/// commit does not actually cover (the proof could not verify anyway, but we
1891/// refuse loudly rather than emit a broken CAR). This is an error, not
1892/// NOT_FOUND: an inconsistent store must never look like a merely-absent record.
1893fn proof_car(
1894    repo: &RepoState,
1895    did: &str,
1896    collection: &str,
1897    tid: Tid,
1898    record: &ModelRecord,
1899) -> AnyResult<Vec<u8>> {
1900    let tree = PdsRecordStore::build_tree(&repo.records);
1901    let rebuilt_root = tree.root_cid();
1902    if rebuilt_root != repo.commit.data {
1903        bail!(
1904            "PDS store inconsistent for {did}: rebuilt MST root {rebuilt_root} \
1905             != signed commit root {} — refusing to serve",
1906            repo.commit.data
1907        );
1908    }
1909    let proof = tree
1910        .proof(collection, &tid)
1911        .ok_or_else(|| anyhow!("record present but no MST proof — store inconsistent"))?;
1912    let (_root_data, node_blocks) = tree.to_node_data_with_blocks();
1913    Ok(build_record_proof_car(
1914        &repo.commit,
1915        &proof,
1916        &node_blocks,
1917        record,
1918    ))
1919}
1920
1921#[async_trait(?Send)]
1922impl RecordResolver for PdsRecordResolver {
1923    async fn resolve_record(
1924        &self,
1925        did: &str,
1926        collection: &str,
1927        rkey: &str,
1928    ) -> anyhow::Result<Option<RecordCarData>> {
1929        let tid = match Tid::parse(rkey) {
1930            Ok(t) => t,
1931            // A malformed rkey simply names no record — report NOT_FOUND, not an
1932            // error, so a bad address looks the same as an absent one.
1933            Err(_) => return Ok(None),
1934        };
1935
1936        // Pair with promotion's write guard through the complete head snapshot
1937        // and proof assembly. A reader therefore sees either generation K or
1938        // generation K', never the K'-signed head during the hidden transition.
1939        if let Some((store, _)) = self
1940            .rotation
1941            .as_ref()
1942            .filter(|(_, local_did)| did == local_did)
1943        {
1944            store.refresh_from_disk();
1945        }
1946        let _generation_guard = self.local_generation_guard(did);
1947        let Some(repo) = self.store.load_repo(did)? else {
1948            return Ok(None);
1949        };
1950        let key = (collection.to_owned(), tid);
1951        let Some(record) = repo.records.get(&key).cloned() else {
1952            return Ok(None);
1953        };
1954
1955        // Rebuild the MST (keyless) and serve the STORED signed commit — no
1956        // re-signing. `proof_car` fail-closes if the rebuilt root does not
1957        // match the root the persisted commit signs.
1958        let car = proof_car(&repo, did, collection, tid, &record)?;
1959
1960        let uri = format!("at://{did}/{collection}/{rkey}");
1961        Ok(Some(RecordCarData { uri, car }))
1962    }
1963
1964    async fn resolve_repo(&self, did: &str) -> anyhow::Result<Option<RecordCarData>> {
1965        if let Some((store, _)) = self
1966            .rotation
1967            .as_ref()
1968            .filter(|(_, local_did)| did == local_did)
1969        {
1970            store.refresh_from_disk();
1971        }
1972        let _generation_guard = self.local_generation_guard(did);
1973        let Some(repo) = self.store.load_repo(did)? else {
1974            return Ok(None);
1975        };
1976
1977        // Full-repo CAR: a self-contained helper for the WHOLE repo (commit +
1978        // all MST nodes + all record blocks) does not exist in the PDS `car`
1979        // module yet (only `build_record_proof_car` for a single record). Rather
1980        // than reinvent the CAR/MST assembly here, return the proof CAR for ONE
1981        // record (the first, deterministically) as the repo-root proof. This is
1982        // documented as a deferral: getRepo currently returns a single-record
1983        // proof rooted at the same signed commit; a true all-records CAR is a
1984        // follow-up (needs a `build_repo_car` helper in hyprstream-pds).
1985        let Some(((collection, tid), record)) = repo.records.iter().next() else {
1986            return Ok(None);
1987        };
1988        let collection = collection.clone();
1989        let tid = *tid;
1990        let record = record.clone();
1991
1992        let car = proof_car(&repo, did, &collection, tid, &record)?;
1993
1994        let uri = format!("at://{did}/{collection}/{}", tid.encode());
1995        Ok(Some(RecordCarData { uri, car }))
1996    }
1997
1998    async fn resolve_verifying_keys(
1999        &self,
2000        did: &str,
2001    ) -> anyhow::Result<Option<hyprstream_pds::commit::PublishedAtprotoKeys>> {
2002        let Some((store, local_did)) = self.rotation.as_ref() else {
2003            return Ok(None);
2004        };
2005        if did != local_did {
2006            return Ok(None);
2007        }
2008        // `slots_snapshot` is a single rotation generation and is the exact
2009        // source used by root_did_document. No absent value loosens this check:
2010        // an authority without an active slot cannot be resolved.
2011        let slots = store.slots_snapshot();
2012        let Some(active) = slots.active else {
2013            return Ok(None);
2014        };
2015        let drain = slots
2016            .drain
2017            .map(|slot| (*slot.key.verifying_key(), slot.nbf, slot.exp));
2018        let lead = slots
2019            .lead
2020            .map(|slot| (*slot.key.verifying_key(), slot.nbf, slot.exp));
2021        Ok(Some(
2022            hyprstream_pds::commit::PublishedAtprotoKeys::from_published_slots(
2023                *active.key.verifying_key(),
2024                drain,
2025                lead,
2026            )?,
2027        ))
2028    }
2029}
2030
2031#[cfg(test)]
2032#[allow(
2033    clippy::unwrap_used,
2034    clippy::expect_used,
2035    clippy::indexing_slicing,
2036    clippy::panic
2037)]
2038mod pds_store_tests {
2039    use super::*;
2040    use hyprstream_pds::car::{parse_car_v1, verify_record_proof};
2041    use p256::ecdsa::{SigningKey, VerifyingKey};
2042
2043    /// Build a test `Es256SigningKeyStore` holding `sk` as the single active
2044    /// slot, so `PdsPublisher::new` can resolve it at sign time (#918
2045    /// re-sign-on-rotation: the publisher holds the store, not a frozen key).
2046    fn test_es256_store(sk: SigningKey) -> Arc<crate::auth::key_rotation::Es256SigningKeyStore> {
2047        use crate::auth::key_rotation::{Es256KeySlot, Es256KeySlots, Es256SigningKeyStore};
2048        Arc::new(Es256SigningKeyStore::new(Es256KeySlots {
2049            drain: None,
2050            active: Some(Es256KeySlot::new(sk, 0, i64::MAX)),
2051            lead: None,
2052        }))
2053    }
2054
2055    /// A 40-hex-char (SHA-1-shaped) git OID for the publish path.
2056    const SAMPLE_OID: &str = "1111111111111111111111111111111111111111";
2057    const SAMPLE_OID_2: &str = "2222222222222222222222222222222222222222";
2058    const DID: &str = "did:key:zTestNode";
2059
2060    /// Extract and decode the signed commit from a proof CAR (its single root).
2061    fn commit_from_car(car: &[u8]) -> Commit {
2062        let (roots, blocks) = parse_car_v1(car).expect("parse CAR");
2063        let root = roots.first().copied().expect("CAR has a root commit");
2064        let (_cid, bytes) = blocks
2065            .iter()
2066            .find(|(cid, _)| *cid == root)
2067            .expect("commit block present");
2068        Commit::from_dag_cbor(bytes).expect("decode commit")
2069    }
2070
2071    /// Minimal single-threaded block_on so these tests don't need a full
2072    /// `#[tokio::test]` runtime just to drive one `async_trait(?Send)` call.
2073    fn block_on<F: std::future::Future>(fut: F) -> F::Output {
2074        tokio::runtime::Builder::new_current_thread()
2075            .build()
2076            .expect("build current-thread runtime")
2077            .block_on(fut)
2078    }
2079
2080    /// #1123/#1170 acceptance test at the publisher level: the publisher
2081    /// resolves the active `#atproto` generation from the sealed op-log head at
2082    /// **sign time**, so a rotation performed by the (separate) OAuth process
2083    /// is observed here with no event delivery. A frozen construction-time key
2084    /// — the pre-C4 shape — would sign the second commit with the retired K1
2085    /// and this test would fail on the final assertion.
2086    #[test]
2087    fn publisher_resolves_live_generation_across_rotation() {
2088        use crate::auth::key_rotation::{
2089            persist_es256_slot, Es256KeySlot, Es256KeySlots, Es256SigningKeyStore,
2090        };
2091        use crate::auth::op_log::{
2092            publish_head_verifying_key, seal_op_log_head, SealedHeadEs256Source,
2093        };
2094        use ed25519_dalek::SigningKey as Ed25519SigningKey;
2095
2096        let pds_dir = tempfile::tempdir().expect("pds tempdir");
2097        let secrets_dir = tempfile::tempdir().expect("secrets tempdir");
2098        let state_dir = tempfile::tempdir().expect("oplog state tempdir");
2099        let secrets = secrets_dir.path();
2100        let state = state_dir.path();
2101        // Dedicated head key; the public verifying key is published to the
2102        // shared state dir (the registry loads only this public key).
2103        let head_sk = Ed25519SigningKey::generate(&mut rand::rngs::OsRng);
2104        publish_head_verifying_key(state, &head_sk).unwrap();
2105
2106        // Process A (the rotator): K1 active, persisted, head sealed (async).
2107        let k1 = SigningKey::random(&mut rand::rngs::OsRng);
2108        let k1_vk: VerifyingKey = *k1.verifying_key();
2109        let k1_slot = Es256KeySlot::new(k1, 0, 0);
2110        let es256_store = Es256SigningKeyStore::new(Es256KeySlots {
2111            active: Some(k1_slot.clone()),
2112            drain: None,
2113            lead: None,
2114        });
2115        persist_es256_slot(secrets, "active", &k1_slot).unwrap();
2116        block_on(seal_op_log_head(state, &head_sk, &es256_store)).expect("seal");
2117
2118        // Process B (the registry/publisher): resolves the generation from the
2119        // sealed head — the `--ipc` cross-process read path. Holds no private
2120        // head material.
2121        let pds_store = Arc::new(PdsRecordStore::open(pds_dir.path()).expect("open rw"));
2122        let source: Arc<dyn crate::auth::ActiveGenerationSource> =
2123            Arc::new(SealedHeadEs256Source::new(state, secrets));
2124        let publisher =
2125            PdsPublisher::with_generation_source(Arc::clone(&pds_store), DID.to_owned(), source);
2126
2127        publisher.publish("repo-a", SAMPLE_OID).expect("publish K1");
2128        let commit1 = pds_store
2129            .load_repo(DID)
2130            .expect("load")
2131            .expect("repo exists")
2132            .commit;
2133        assert!(commit1.verify(&k1_vk).is_ok(), "first commit signed by K1");
2134
2135        // Process A rotates: K1 → drain, K2 → active. Persist + re-seal.
2136        let k2 = SigningKey::random(&mut rand::rngs::OsRng);
2137        let k2_vk: VerifyingKey = *k2.verifying_key();
2138        let k2_slot = Es256KeySlot::new(k2, 0, 0);
2139        persist_es256_slot(secrets, "drain", &k1_slot).unwrap();
2140        persist_es256_slot(secrets, "active", &k2_slot).unwrap();
2141        es256_store.0.write().active = Some(k2_slot);
2142        block_on(seal_op_log_head(state, &head_sk, &es256_store)).expect("seal");
2143
2144        // Same publisher instance, new commit. It MUST be signed by K2 — the
2145        // publisher observed the rotation through the re-sealed head.
2146        publisher
2147            .publish("repo-a", SAMPLE_OID_2)
2148            .expect("publish K2");
2149        let commit2 = pds_store
2150            .load_repo(DID)
2151            .expect("load")
2152            .expect("repo exists")
2153            .commit;
2154        assert!(
2155            commit2.verify(&k2_vk).is_ok(),
2156            "post-rotation commit must be signed by K2"
2157        );
2158        // Negative case: the post-rotation commit MUST NOT verify under the
2159        // retired pre-rotation key K1.
2160        assert!(
2161            commit2.verify(&k1_vk).is_err(),
2162            "publisher must not sign with the retired pre-rotation key K1"
2163        );
2164    }
2165
2166    #[test]
2167    fn keyless_read_serves_writer_signed_commit() {
2168        // The writer signs once; a resolver constructed with NO key returns a
2169        // proof whose commit verifies against the published P-256 key.
2170        let dir = tempfile::tempdir().expect("tempdir");
2171        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2172        let vk: VerifyingKey = *sk.verifying_key();
2173
2174        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2175        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
2176        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
2177
2178        // Resolver holds no key at all.
2179        let resolver = PdsRecordResolver::new(Arc::clone(&store));
2180        let tid = PdsRecordStore::tid_for_repo("repo-a");
2181        let got = block_on(resolver.resolve_record(DID, COLLECTION_NSID, &tid.encode()))
2182            .expect("resolve ok")
2183            .expect("record present");
2184
2185        let commit = commit_from_car(&got.car);
2186        commit
2187            .verify(&vk)
2188            .expect("writer-signed commit must verify against the published #atproto key");
2189    }
2190
2191    #[test]
2192    fn publisher_signs_new_commits_with_active_not_lead_key() {
2193        let dir = tempfile::tempdir().expect("tempdir");
2194        let active = SigningKey::random(&mut rand::rngs::OsRng);
2195        let active_vk = VerifyingKey::from(&active);
2196        let lead = SigningKey::random(&mut rand::rngs::OsRng);
2197        let lead_vk = VerifyingKey::from(&lead);
2198        let now = chrono::Utc::now().timestamp();
2199        let keys = Arc::new(crate::auth::key_rotation::Es256SigningKeyStore::new(
2200            crate::auth::key_rotation::Es256KeySlots {
2201                drain: None,
2202                active: Some(crate::auth::key_rotation::Es256KeySlot::new(
2203                    active,
2204                    now - 60,
2205                    now + 60,
2206                )),
2207                lead: Some(crate::auth::key_rotation::Es256KeySlot::new(
2208                    lead,
2209                    now - 30,
2210                    now + 120,
2211                )),
2212            },
2213        ));
2214        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open store"));
2215        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), keys);
2216        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
2217        let repo = store.load_repo(DID).expect("load").expect("repo");
2218        repo.commit
2219            .verify(&active_vk)
2220            .expect("new commit must use the active key");
2221        assert!(
2222            repo.commit.verify(&lead_vk).is_err(),
2223            "a published lead key must never sign a new commit"
2224        );
2225    }
2226
2227    /// #918 production-path rotation re-sign: publish with K, run the real
2228    /// lead→active promotion (which invokes the installed publisher hook),
2229    /// then assert the STORED head verifies under K' while the un-re-signed
2230    /// control does not.
2231    #[tokio::test]
2232    async fn rotation_resigns_stored_head_with_new_key() {
2233        use crate::auth::key_rotation::{
2234            rotate_es256_keys, Es256KeySlot, Es256KeySlots, Es256SigningKeyStore,
2235        };
2236
2237        let dir = tempfile::tempdir().expect("tempdir");
2238        let pds_dir = dir.path().join("pds");
2239        let secrets_dir = dir.path().join("secrets");
2240        std::fs::create_dir_all(&secrets_dir).expect("create secrets dir");
2241        let now = chrono::Utc::now().timestamp();
2242        let old_sk = SigningKey::random(&mut rand::rngs::OsRng);
2243        let old_vk: VerifyingKey = *old_sk.verifying_key();
2244        let new_sk = SigningKey::random(&mut rand::rngs::OsRng);
2245        let new_vk: VerifyingKey = *new_sk.verifying_key();
2246
2247        let store = Arc::new(PdsRecordStore::open(&pds_dir).expect("open rw"));
2248        // Capture the pre-rotation commit for the control assertion.
2249        let es256_store = Arc::new(Es256SigningKeyStore::new(Es256KeySlots {
2250            drain: None,
2251            active: Some(Es256KeySlot::new(old_sk, now - 100, now + 1)),
2252            lead: Some(Es256KeySlot::new(new_sk, now - 1, now + 30 * 86_400)),
2253        }));
2254        let publisher = Arc::new(PdsPublisher::new(
2255            Arc::clone(&store),
2256            DID.to_owned(),
2257            Arc::clone(&es256_store),
2258        ));
2259        publisher
2260            .install_es256_promotion_hook()
2261            .expect("install promotion hook");
2262        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
2263        let pre_rotation_commit = store
2264            .load_repo(DID)
2265            .expect("load")
2266            .expect("repo present")
2267            .commit;
2268
2269        // Run the production promotion function. Its in-store notification
2270        // invokes PdsPublisher::resign_head; the test does not call it directly.
2271        let promoted = rotate_es256_keys(
2272            &crate::config::OAuthConfig::default(),
2273            &secrets_dir,
2274            &es256_store,
2275            now,
2276        )
2277        .await;
2278        assert!(promoted, "the prepared lead key must be promoted");
2279
2280        // The STORED head now verifies under K' (the current active key).
2281        let post_rotation_commit = store
2282            .load_repo(DID)
2283            .expect("load")
2284            .expect("repo present")
2285            .commit;
2286        assert_eq!(
2287            post_rotation_commit.rev, pre_rotation_commit.rev,
2288            "rev unchanged"
2289        );
2290        assert_eq!(
2291            post_rotation_commit.data, pre_rotation_commit.data,
2292            "MST root unchanged"
2293        );
2294        assert_eq!(
2295            post_rotation_commit.unsigned().to_dag_cbor(),
2296            pre_rotation_commit.unsigned().to_dag_cbor(),
2297            "rotation must re-sign the exact same canonical unsigned DAG-CBOR bytes"
2298        );
2299        assert_ne!(
2300            post_rotation_commit.sig, pre_rotation_commit.sig,
2301            "signature must differ (re-signed with K')"
2302        );
2303        post_rotation_commit
2304            .verify(&new_vk)
2305            .expect("re-signed head must verify under K' (the new active key)");
2306
2307        // Control: the pre-rotation head (signed by K) does NOT verify under K'.
2308        assert!(
2309            pre_rotation_commit.verify(&new_vk).is_err(),
2310            "the un-re-signed head must fail under K'"
2311        );
2312        // And the re-signed head does NOT verify under K (the old key).
2313        assert!(
2314            post_rotation_commit.verify(&old_vk).is_err(),
2315            "the re-signed head must fail under K (the old key)"
2316        );
2317    }
2318
2319    #[tokio::test]
2320    async fn failed_rotation_keeps_published_key_and_head_consistent_then_retries() {
2321        use crate::auth::key_rotation::{
2322            rotate_es256_keys, Es256KeySlot, Es256KeySlots, Es256SigningKeyStore,
2323        };
2324
2325        let dir = tempfile::tempdir().expect("tempdir");
2326        let pds_dir = dir.path().join("pds");
2327        let secrets_dir = dir.path().join("secrets");
2328        std::fs::create_dir_all(&secrets_dir).expect("create secrets dir");
2329        let now = chrono::Utc::now().timestamp();
2330        let old_sk = SigningKey::random(&mut rand::rngs::OsRng);
2331        let old_vk: VerifyingKey = *old_sk.verifying_key();
2332        let new_sk = SigningKey::random(&mut rand::rngs::OsRng);
2333        let new_vk: VerifyingKey = *new_sk.verifying_key();
2334
2335        let store = Arc::new(PdsRecordStore::open(&pds_dir).expect("open rw"));
2336        let es256_store = Arc::new(Es256SigningKeyStore::new(Es256KeySlots {
2337            drain: None,
2338            active: Some(Es256KeySlot::new(old_sk, now - 100, now + 1)),
2339            lead: Some(Es256KeySlot::new(new_sk, now - 1, now + 30 * 86_400)),
2340        }));
2341        let publisher = Arc::new(PdsPublisher::new(
2342            Arc::clone(&store),
2343            DID.to_owned(),
2344            Arc::clone(&es256_store),
2345        ));
2346        publisher
2347            .install_es256_promotion_hook()
2348            .expect("install promotion hook");
2349        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
2350
2351        let fail_once = Arc::new(std::sync::atomic::AtomicBool::new(true));
2352        let injected_failure = Arc::clone(&fail_once);
2353        let weak_publisher = Arc::downgrade(&publisher);
2354        es256_store.set_promotion_hook(Arc::new(move |key| {
2355            if injected_failure.swap(false, std::sync::atomic::Ordering::AcqRel) {
2356                bail!("injected persisted-head re-sign failure");
2357            }
2358            let publisher = weak_publisher
2359                .upgrade()
2360                .ok_or_else(|| anyhow!("publisher dropped during retry test"))?;
2361            publisher.resign_head_with_key(&key).map(|_| ())
2362        }));
2363
2364        assert!(
2365            !rotate_es256_keys(
2366                &crate::config::OAuthConfig::default(),
2367                &secrets_dir,
2368                &es256_store,
2369                now,
2370            )
2371            .await,
2372            "a failed head re-sign must leave promotion pending"
2373        );
2374        let failed_head = store
2375            .load_repo(DID)
2376            .expect("load")
2377            .expect("repo present")
2378            .commit;
2379        failed_head
2380            .verify(&old_vk)
2381            .expect("head must remain signed by the still-published old key");
2382        assert!(failed_head.verify(&new_vk).is_err());
2383        assert_eq!(
2384            es256_store.active_key().unwrap().verifying_key(),
2385            &old_vk,
2386            "candidate key must not become visible after re-sign failure"
2387        );
2388
2389        assert!(
2390            rotate_es256_keys(
2391                &crate::config::OAuthConfig::default(),
2392                &secrets_dir,
2393                &es256_store,
2394                now,
2395            )
2396            .await,
2397            "the next tick must reconcile and finish promotion"
2398        );
2399        let reconciled_head = store
2400            .load_repo(DID)
2401            .expect("load")
2402            .expect("repo present")
2403            .commit;
2404        reconciled_head
2405            .verify(&new_vk)
2406            .expect("retried head must match the newly published key");
2407        assert!(reconciled_head.verify(&old_vk).is_err());
2408        assert_eq!(es256_store.active_key().unwrap().verifying_key(), &new_vk);
2409    }
2410
2411    #[tokio::test]
2412    async fn dropped_publisher_hook_keeps_rotation_pending_and_head_consistent() {
2413        use crate::auth::key_rotation::{
2414            rotate_es256_keys, Es256KeySlot, Es256KeySlots, Es256SigningKeyStore,
2415        };
2416
2417        let dir = tempfile::tempdir().expect("tempdir");
2418        let pds_dir = dir.path().join("pds");
2419        let secrets_dir = dir.path().join("secrets");
2420        std::fs::create_dir_all(&secrets_dir).expect("create secrets dir");
2421        let now = chrono::Utc::now().timestamp();
2422        let old_sk = SigningKey::random(&mut rand::rngs::OsRng);
2423        let old_vk: VerifyingKey = *old_sk.verifying_key();
2424        let new_sk = SigningKey::random(&mut rand::rngs::OsRng);
2425        let new_vk: VerifyingKey = *new_sk.verifying_key();
2426
2427        let store = Arc::new(PdsRecordStore::open(&pds_dir).expect("open rw"));
2428        let es256_store = Arc::new(Es256SigningKeyStore::new(Es256KeySlots {
2429            drain: None,
2430            active: Some(Es256KeySlot::new(old_sk, now - 100, now + 1)),
2431            lead: Some(Es256KeySlot::new(new_sk, now - 1, now + 30 * 86_400)),
2432        }));
2433        let publisher = Arc::new(PdsPublisher::new(
2434            Arc::clone(&store),
2435            DID.to_owned(),
2436            Arc::clone(&es256_store),
2437        ));
2438        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
2439        publisher
2440            .install_es256_promotion_hook()
2441            .expect("install promotion hook");
2442        drop(publisher);
2443
2444        assert!(
2445            !rotate_es256_keys(
2446                &crate::config::OAuthConfig::default(),
2447                &secrets_dir,
2448                &es256_store,
2449                now,
2450            )
2451            .await,
2452            "a stale weak hook must leave promotion pending"
2453        );
2454
2455        let slots = es256_store.0.read();
2456        assert_eq!(slots.active.as_ref().unwrap().key.verifying_key(), &old_vk);
2457        assert_eq!(slots.lead.as_ref().unwrap().key.verifying_key(), &new_vk);
2458        assert!(slots.drain.is_none());
2459        drop(slots);
2460
2461        let head = store
2462            .load_repo(DID)
2463            .expect("load")
2464            .expect("repo present")
2465            .commit;
2466        head.verify(&old_vk)
2467            .expect("head must remain signed by the still-live old key");
2468        assert!(
2469            head.verify(&new_vk).is_err(),
2470            "candidate key must not sign or become live after publisher drop"
2471        );
2472    }
2473
2474    #[test]
2475    fn repo_reader_waits_for_complete_rotation_generation() {
2476        use crate::auth::key_rotation::{
2477            rotate_es256_keys, Es256KeySlot, Es256KeySlots, Es256SigningKeyStore,
2478        };
2479        use std::sync::{mpsc, Barrier};
2480        use std::time::Duration;
2481
2482        let dir = tempfile::tempdir().expect("tempdir");
2483        let pds_dir = dir.path().join("pds");
2484        let secrets_dir = dir.path().join("secrets");
2485        std::fs::create_dir_all(&secrets_dir).expect("create secrets dir");
2486        let now = chrono::Utc::now().timestamp();
2487        let old_sk = SigningKey::random(&mut rand::rngs::OsRng);
2488        let new_sk = SigningKey::random(&mut rand::rngs::OsRng);
2489        let new_vk: VerifyingKey = *new_sk.verifying_key();
2490
2491        let store = Arc::new(PdsRecordStore::open(&pds_dir).expect("open rw"));
2492        let es256_store = Arc::new(Es256SigningKeyStore::new(Es256KeySlots {
2493            drain: None,
2494            active: Some(Es256KeySlot::new(old_sk, now - 100, now + 1)),
2495            lead: Some(Es256KeySlot::new(new_sk, now - 1, now + 30 * 86_400)),
2496        }));
2497        let publisher = Arc::new(PdsPublisher::new(
2498            Arc::clone(&store),
2499            DID.to_owned(),
2500            Arc::clone(&es256_store),
2501        ));
2502        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
2503        let resolver = Arc::new(
2504            PdsRecordResolver::new(Arc::clone(&store))
2505                .with_es256_rotation(Arc::clone(&es256_store), DID.to_owned()),
2506        );
2507
2508        // Pause after K' has been persisted as the repo head but before the
2509        // rotation write guard installs K' as the live/published key.
2510        let transition_barrier = Arc::new(Barrier::new(2));
2511        let hook_barrier = Arc::clone(&transition_barrier);
2512        let (head_written_tx, head_written_rx) = mpsc::channel();
2513        let weak_publisher = Arc::downgrade(&publisher);
2514        es256_store.set_promotion_hook(Arc::new(move |key| {
2515            let publisher = weak_publisher
2516                .upgrade()
2517                .ok_or_else(|| anyhow!("publisher dropped during rotation test"))?;
2518            publisher.resign_head_with_key(&key)?;
2519            head_written_tx
2520                .send(())
2521                .map_err(|_| anyhow!("test observer dropped"))?;
2522            hook_barrier.wait();
2523            Ok(())
2524        }));
2525
2526        let rotation_store = Arc::clone(&es256_store);
2527        let rotation_secrets = secrets_dir.clone();
2528        let rotation = std::thread::spawn(move || {
2529            block_on(rotate_es256_keys(
2530                &crate::config::OAuthConfig::default(),
2531                &rotation_secrets,
2532                &rotation_store,
2533                now,
2534            ))
2535        });
2536        head_written_rx
2537            .recv_timeout(Duration::from_secs(2))
2538            .expect("hook wrote candidate head");
2539
2540        let observer_store = Arc::clone(&es256_store);
2541        let (observed_tx, observed_rx) = mpsc::channel();
2542        let reader = std::thread::spawn(move || {
2543            let repo = block_on(resolver.resolve_repo(DID))
2544                .expect("resolve repo")
2545                .expect("repo present");
2546            let key = observer_store.active_key().expect("active key");
2547            observed_tx
2548                .send((commit_from_car(&repo.car), key))
2549                .expect("send observation");
2550        });
2551        let early_observation = observed_rx.recv_timeout(Duration::from_millis(100));
2552        let reader_was_blocked = matches!(&early_observation, Err(mpsc::RecvTimeoutError::Timeout));
2553        transition_barrier.wait();
2554        assert!(rotation.join().expect("rotation thread"));
2555        reader.join().expect("reader thread");
2556        assert!(
2557            reader_was_blocked,
2558            "repo reader must block while candidate head and live key generations differ"
2559        );
2560        let (commit, observed_key) = match early_observation {
2561            Ok(observation) => observation,
2562            Err(mpsc::RecvTimeoutError::Timeout) => observed_rx
2563                .recv_timeout(Duration::from_secs(2))
2564                .expect("post-transition observation"),
2565            Err(mpsc::RecvTimeoutError::Disconnected) => {
2566                panic!("reader disconnected before observation")
2567            }
2568        };
2569        assert_eq!(observed_key.verifying_key(), &new_vk);
2570        commit
2571            .verify(&new_vk)
2572            .expect("reader must observe the K'-signed generation");
2573    }
2574
2575    #[test]
2576    fn local_did_web_authority_admits_pre_rotation_commit_through_drain() {
2577        let dir = tempfile::tempdir().expect("tempdir");
2578        let secrets_dir = dir.path().join("secrets");
2579        std::fs::create_dir_all(&secrets_dir).expect("create secrets");
2580        let now = chrono::Utc::now().timestamp();
2581        let old = SigningKey::random(&mut rand::rngs::OsRng);
2582        let new = SigningKey::random(&mut rand::rngs::OsRng);
2583        let es256_store = Arc::new(crate::auth::key_rotation::Es256SigningKeyStore::new(
2584            crate::auth::key_rotation::Es256KeySlots {
2585                drain: None,
2586                active: Some(crate::auth::key_rotation::Es256KeySlot::new(
2587                    old,
2588                    now - 60,
2589                    now + 1,
2590                )),
2591                lead: Some(crate::auth::key_rotation::Es256KeySlot::new(
2592                    new,
2593                    now - 1,
2594                    now + 86_400,
2595                )),
2596            },
2597        ));
2598        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2599        let publisher =
2600            PdsPublisher::new(Arc::clone(&store), DID.to_owned(), Arc::clone(&es256_store));
2601        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
2602        let resolver =
2603            PdsRecordResolver::new(store).with_es256_rotation(es256_store, DID.to_owned());
2604
2605        assert!(block_on(crate::auth::key_rotation::rotate_es256_keys(
2606            &crate::config::OAuthConfig::default(),
2607            &secrets_dir,
2608            resolver.rotation.as_ref().expect("rotation").0.as_ref(),
2609            now,
2610        )));
2611        let keys = block_on(resolver.resolve_verifying_keys(DID))
2612            .expect("resolve root authority")
2613            .expect("matching did:web authority");
2614        let repo = block_on(resolver.resolve_repo(DID))
2615            .expect("resolve repo")
2616            .expect("repo");
2617        let commit = commit_from_car(&repo.car);
2618        commit
2619            .verify_against_published_keys(&keys, now)
2620            .expect("K-signed commit must verify under published drain slot");
2621        assert!(
2622            block_on(resolver.resolve_verifying_keys("did:web:foreign.example"))
2623                .expect("foreign resolution is unsupported")
2624                .is_none(),
2625            "foreign DID resolution stays unsupported so ingest can fail closed"
2626        );
2627        let index = hyprstream_discovery::placement_index::PlacementIndex::new();
2628        block_on(index.ingest_did(&resolver, DID))
2629            .expect("production placement ingest must accept the drain-authorized commit");
2630    }
2631
2632    #[test]
2633    fn publish_read_full_proof_verifies() {
2634        // End-to-end: publish → getRecord → the FULL record proof (commit sig +
2635        // MST path + record CID) verifies keyless against the published key.
2636        let dir = tempfile::tempdir().expect("tempdir");
2637        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2638        let vk: VerifyingKey = *sk.verifying_key();
2639
2640        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2641        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
2642        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
2643
2644        // Reconstruct the proof the same way the resolver does (store holds no
2645        // key), and run the offline verifier against the published key.
2646        let repo = store
2647            .load_repo(DID)
2648            .expect("load ok")
2649            .expect("repo present");
2650        let tid = PdsRecordStore::tid_for_repo("repo-a");
2651        let record = repo
2652            .records
2653            .get(&(COLLECTION_NSID.to_owned(), tid))
2654            .cloned()
2655            .expect("record present");
2656        let tree = PdsRecordStore::build_tree(&repo.records);
2657        let proof = tree.proof(COLLECTION_NSID, &tid).expect("proof");
2658        verify_record_proof(&repo.commit, &vk, &proof, &record)
2659            .expect("full keyless proof must verify");
2660    }
2661
2662    #[test]
2663    fn pointer_advance_bumps_rev_and_links_prev() {
2664        // A re-publish must advance the commit: a new rev and prev = the old
2665        // commit's CID.
2666        let dir = tempfile::tempdir().expect("tempdir");
2667        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2668        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2669        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
2670
2671        publisher.publish("repo-a", SAMPLE_OID).expect("publish 1");
2672        let first = store.load_repo(DID).expect("load").expect("repo").commit;
2673        assert!(first.prev.is_none(), "genesis commit has no prev");
2674
2675        publisher
2676            .publish("repo-a", SAMPLE_OID_2)
2677            .expect("publish 2");
2678        let second = store.load_repo(DID).expect("load").expect("repo").commit;
2679        assert_eq!(
2680            second.prev,
2681            Some(first.cid()),
2682            "the advanced commit must link the previous commit"
2683        );
2684        assert!(second.rev > first.rev, "rev must strictly advance");
2685    }
2686
2687    #[test]
2688    fn readonly_roster_store_open_never_creates_missing_directories() -> AnyResult<()> {
2689        let root = tempfile::tempdir()?;
2690        for path in [
2691            root.path().join("missing"),
2692            root.path().join("absent-parent/nested/store"),
2693        ] {
2694            assert!(PdsRecordStore::open_readonly(&path).is_err());
2695            assert!(!path.exists());
2696            assert_eq!(std::fs::read_dir(root.path())?.count(), 0);
2697        }
2698        let file = root.path().join("not-a-directory");
2699        std::fs::write(&file, b"preserve existing bytes")?;
2700        assert!(PdsRecordStore::open_readonly(&file).is_err());
2701        assert_eq!(std::fs::read(&file)?, b"preserve existing bytes");
2702        assert_eq!(std::fs::read_dir(root.path())?.count(), 1);
2703        Ok(())
2704    }
2705
2706    #[test]
2707    fn readonly_store_survives_restart_and_sees_writer_updates() {
2708        let dir = tempfile::tempdir().expect("tempdir");
2709        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2710        let vk: VerifyingKey = *sk.verifying_key();
2711
2712        {
2713            let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2714            let publisher =
2715                PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
2716            publisher.publish("repo-b", SAMPLE_OID).expect("publish");
2717            // Writer handle dropped here — simulates a process restart.
2718        }
2719
2720        let ro = Arc::new(PdsRecordStore::open_readonly(dir.path()).expect("open ro"));
2721        let resolver = PdsRecordResolver::new(Arc::clone(&ro));
2722        let got = block_on(resolver.resolve_repo(DID))
2723            .expect("resolve_repo ok")
2724            .expect("repo present after reopen");
2725        assert!(got.uri.starts_with(&format!("at://{DID}/")));
2726        // The signed commit survived the reopen and still verifies.
2727        commit_from_car(&got.car)
2728            .verify(&vk)
2729            .expect("commit persisted across restart must verify");
2730    }
2731
2732    #[test]
2733    fn readonly_store_rejects_writes() {
2734        let dir = tempfile::tempdir().expect("tempdir");
2735        // Create the directory with a real DB first so open_readonly succeeds.
2736        drop(PdsRecordStore::open(dir.path()).expect("open rw"));
2737
2738        let ro = PdsRecordStore::open_readonly(dir.path()).expect("open ro");
2739        let record = ModelRecord::new(
2740            "at://did:key:zX",
2741            format!("bafyreiexample{SAMPLE_OID}"),
2742            "2026-06-23T12:34:56.789Z",
2743        )
2744        .expect("record");
2745        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2746        let tree = PdsRecordStore::build_tree(&BTreeMap::new());
2747        let unsigned = UnsignedCommit::new("did:key:zX", tree.root_cid(), Tid::now(), None);
2748        let commit = Commit::sign(&unsigned, &sk);
2749        let err = ro
2750            .put_record_and_commit(
2751                "did:key:zX",
2752                COLLECTION_NSID,
2753                Tid::now(),
2754                &record,
2755                &commit,
2756                None,
2757            )
2758            .unwrap_err();
2759        assert!(err.to_string().contains("read-only"));
2760    }
2761
2762    /// A second collection for multi-collection tests (#910b) — the G3 (#908)
2763    /// name collection this slice exists to unblock.
2764    const NAME_COLLECTION: &str = "ai.hyprstream.name";
2765
2766    /// Resolve `(collection, record_id)` via a keyless resolver and verify the
2767    /// full record proof offline against `vk`: commit signature + MST path from
2768    /// the signed root + record CID.
2769    fn resolve_and_verify(
2770        store: &Arc<PdsRecordStore>,
2771        vk: &VerifyingKey,
2772        collection: &str,
2773        record_id: &str,
2774    ) {
2775        let tid = PdsRecordStore::tid_for_repo(record_id);
2776        let resolver = PdsRecordResolver::new(Arc::clone(store));
2777        let got = block_on(resolver.resolve_record(DID, collection, &tid.encode()))
2778            .expect("resolve ok")
2779            .expect("record present");
2780        assert_eq!(got.uri, format!("at://{DID}/{collection}/{}", tid.encode()));
2781        commit_from_car(&got.car)
2782            .verify(vk)
2783            .expect("commit verifies");
2784
2785        // Full offline proof, reconstructed the way the resolver does it.
2786        let repo = store
2787            .load_repo(DID)
2788            .expect("load ok")
2789            .expect("repo present");
2790        let record = repo
2791            .records
2792            .get(&(collection.to_owned(), tid))
2793            .cloned()
2794            .expect("record present in store");
2795        let tree = PdsRecordStore::build_tree(&repo.records);
2796        assert_eq!(
2797            tree.root_cid(),
2798            repo.commit.data,
2799            "rebuilt multi-collection root must match the signed commit root"
2800        );
2801        let proof = tree.proof(collection, &tid).expect("proof");
2802        verify_record_proof(&repo.commit, vk, &proof, &record)
2803            .expect("full keyless proof must verify");
2804    }
2805
2806    #[test]
2807    fn multi_collection_round_trip() {
2808        // Records in two collections under ONE DID, covered by ONE signed
2809        // commit — each independently resolvable with a valid proof (#910b).
2810        let dir = tempfile::tempdir().expect("tempdir");
2811        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2812        let vk: VerifyingKey = *sk.verifying_key();
2813
2814        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2815        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
2816        publisher
2817            .publish("repo-a", SAMPLE_OID)
2818            .expect("publish model");
2819        publisher
2820            .publish_record(NAME_COLLECTION, "name-a", SAMPLE_OID_2)
2821            .expect("publish name");
2822
2823        // One repo, one commit: both records live under the same signed root.
2824        let repo = store.load_repo(DID).expect("load").expect("repo");
2825        assert_eq!(repo.records.len(), 2, "both collections in one repo");
2826
2827        resolve_and_verify(&store, &vk, COLLECTION_NSID, "repo-a");
2828        resolve_and_verify(&store, &vk, NAME_COLLECTION, "name-a");
2829    }
2830
2831    #[test]
2832    fn ledger_multi_collection_publish_resolves() {
2833        // #924: an allocation + a receipt written under ONE holder DID as two new
2834        // ledger collections, covered by ONE signed commit and each independently
2835        // resolvable via the #910b path. The receipt binds a *pairwise* spender
2836        // id (not a root DID), exercising the pseudonymity constraint end-to-end.
2837        use hyprstream_pds::ledger::{
2838            allocation as alloc_ns, receipt as receipt_ns, AllocationRecord, GrantClass,
2839            ReceiptRecord, Unit,
2840        };
2841
2842        let dir = tempfile::tempdir().expect("tempdir");
2843        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2844        let vk: VerifyingKey = *sk.verifying_key();
2845
2846        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2847        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
2848
2849        let alloc = AllocationRecord::new(
2850            "bafyreiexamplegrantcid1234567890abcdef",
2851            Unit {
2852                code: "compute-second".into(),
2853                issuer: "did:web:issuer.example.com".into(),
2854            },
2855            1_000,
2856            7,
2857            "did:web:issuer.example.com",
2858            "did:key:zHolderSubject",
2859            GrantClass::Prepaid,
2860        )
2861        .expect("valid allocation");
2862        let receipt = ReceiptRecord::new(
2863            alloc.cid().encode(),
2864            "pairwise:cell42:8f3ad9c0e1",
2865            "did:key:zHostNode",
2866            "transfer-01HXYZ",
2867            25,
2868        )
2869        .expect("valid receipt");
2870
2871        publisher
2872            .publish_ledger_allocation(&alloc)
2873            .expect("publish allocation");
2874        publisher
2875            .publish_ledger_receipt(&receipt)
2876            .expect("publish receipt");
2877
2878        // One repo, one signed commit, two ledger collections.
2879        let repo = store.load_repo(DID).expect("load").expect("repo");
2880        assert_eq!(repo.records.len(), 2, "allocation + receipt in one repo");
2881
2882        // Each ledger collection resolves with a valid keyless proof.
2883        resolve_and_verify(&store, &vk, alloc_ns::COLLECTION_NSID, &alloc.grant);
2884        resolve_and_verify(
2885            &store,
2886            &vk,
2887            receipt_ns::COLLECTION_NSID,
2888            &receipt.transfer_id,
2889        );
2890
2891        // The published pointer's currentOid is the ledger record's content CID —
2892        // the record is content-addressed, so its bytes are recoverable/verifiable.
2893        let alloc_tid = PdsRecordStore::tid_for_repo(&alloc.grant);
2894        let ptr = repo
2895            .records
2896            .get(&(alloc_ns::COLLECTION_NSID.to_owned(), alloc_tid))
2897            .cloned()
2898            .expect("allocation pointer present");
2899        assert_eq!(
2900            ptr.current_oid,
2901            alloc.cid().encode(),
2902            "pointer must address the allocation record's content CID"
2903        );
2904    }
2905
2906    #[test]
2907    fn cross_collection_isolation() {
2908        // Writing collection B must not corrupt proofs for collection A: after
2909        // the B write, A still resolves with a valid proof under the (new)
2910        // signed root, and A's record bytes are untouched.
2911        let dir = tempfile::tempdir().expect("tempdir");
2912        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2913        let vk: VerifyingKey = *sk.verifying_key();
2914
2915        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2916        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
2917        publisher
2918            .publish("repo-a", SAMPLE_OID)
2919            .expect("publish model");
2920        let tid_a = PdsRecordStore::tid_for_repo("repo-a");
2921        let record_a_before = store
2922            .load_repo(DID)
2923            .expect("load")
2924            .expect("repo")
2925            .records
2926            .get(&(COLLECTION_NSID.to_owned(), tid_a))
2927            .cloned()
2928            .expect("record a");
2929
2930        publisher
2931            .publish_record(NAME_COLLECTION, "name-a", SAMPLE_OID_2)
2932            .expect("publish name");
2933
2934        // A's record bytes are unchanged and its proof verifies under the
2935        // advanced commit.
2936        let repo = store.load_repo(DID).expect("load").expect("repo");
2937        let record_a_after = repo
2938            .records
2939            .get(&(COLLECTION_NSID.to_owned(), tid_a))
2940            .cloned()
2941            .expect("record a still present");
2942        assert_eq!(
2943            record_a_before, record_a_after,
2944            "B write must not touch A's record"
2945        );
2946        resolve_and_verify(&store, &vk, COLLECTION_NSID, "repo-a");
2947        resolve_and_verify(&store, &vk, NAME_COLLECTION, "name-a");
2948
2949        // The name record is NOT addressable under the model collection: same
2950        // rkey, different collection → a different MST key.
2951        let tid_name = PdsRecordStore::tid_for_repo("name-a");
2952        let resolver = PdsRecordResolver::new(Arc::clone(&store));
2953        let cross = block_on(resolver.resolve_record(DID, COLLECTION_NSID, &tid_name.encode()))
2954            .expect("resolve ok");
2955        assert!(
2956            cross.is_none(),
2957            "collections must not alias onto each other"
2958        );
2959    }
2960
2961    #[test]
2962    fn concurrent_publishes_serialize_and_stay_consistent() {
2963        // Concurrent writers — including writers to DIFFERENT collections of
2964        // the same DID — must serialize on the publish lock so the final
2965        // commit covers every record, and every record's proof verifies
2966        // against that single signed root (#910b).
2967        let dir = tempfile::tempdir().expect("tempdir");
2968        let sk = SigningKey::random(&mut rand::rngs::OsRng);
2969        let vk: VerifyingKey = *sk.verifying_key();
2970
2971        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
2972        let publisher = Arc::new(PdsPublisher::new(
2973            Arc::clone(&store),
2974            DID.to_owned(),
2975            test_es256_store(sk),
2976        ));
2977
2978        const WRITERS_PER_COLLECTION: usize = 4;
2979        let mut handles = Vec::new();
2980        for i in 0..WRITERS_PER_COLLECTION {
2981            for collection in [COLLECTION_NSID, NAME_COLLECTION] {
2982                let publisher = Arc::clone(&publisher);
2983                handles.push(std::thread::spawn(move || {
2984                    publisher
2985                        .publish_record(collection, &format!("rec-{i}"), SAMPLE_OID)
2986                        .expect("concurrent publish");
2987                }));
2988            }
2989        }
2990        for h in handles {
2991            h.join().expect("writer thread");
2992        }
2993
2994        let repo = store.load_repo(DID).expect("load").expect("repo");
2995        assert_eq!(
2996            repo.records.len(),
2997            WRITERS_PER_COLLECTION * 2,
2998            "every concurrent write across both collections must survive"
2999        );
3000        // The stored commit signs the FULL record set (no lost update).
3001        let tree = PdsRecordStore::build_tree(&repo.records);
3002        assert_eq!(
3003            tree.root_cid(),
3004            repo.commit.data,
3005            "final signed commit must cover all concurrent writes"
3006        );
3007        // And every record still proves against that one commit.
3008        for i in 0..WRITERS_PER_COLLECTION {
3009            resolve_and_verify(&store, &vk, COLLECTION_NSID, &format!("rec-{i}"));
3010            resolve_and_verify(&store, &vk, NAME_COLLECTION, &format!("rec-{i}"));
3011        }
3012    }
3013
3014    #[test]
3015    fn root_consistency_check_fails_closed() {
3016        // A record the stored commit does NOT cover must make reads fail
3017        // loudly (Err), never serve a broken proof and never report a clean
3018        // NOT_FOUND.
3019        let dir = tempfile::tempdir().expect("tempdir");
3020        let sk = SigningKey::random(&mut rand::rngs::OsRng);
3021
3022        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
3023        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
3024        publisher.publish("repo-a", SAMPLE_OID).expect("publish");
3025
3026        // Corrupt the store: write a second record while re-persisting the
3027        // STALE commit (whose root covers only the first record).
3028        let stale_commit = store.load_repo(DID).expect("load").expect("repo").commit;
3029        let rogue = ModelRecord::new(
3030            format!("at://{DID}"),
3031            "bafyreiexampleoid00000000000000000000",
3032            "2026-06-23T12:34:56.789Z",
3033        )
3034        .expect("record");
3035        store
3036            .put_record_and_commit(
3037                DID,
3038                NAME_COLLECTION,
3039                PdsRecordStore::tid_for_repo("rogue"),
3040                &rogue,
3041                &stale_commit,
3042                None,
3043            )
3044            .expect("raw write");
3045
3046        let resolver = PdsRecordResolver::new(Arc::clone(&store));
3047        let tid_a = PdsRecordStore::tid_for_repo("repo-a");
3048        let err = block_on(resolver.resolve_record(DID, COLLECTION_NSID, &tid_a.encode()))
3049            .expect_err("inconsistent store must fail closed");
3050        assert!(
3051            err.to_string().contains("inconsistent"),
3052            "error must name the inconsistency: {err}"
3053        );
3054        // resolve_repo fails closed the same way.
3055        assert!(block_on(resolver.resolve_repo(DID)).is_err());
3056    }
3057
3058    #[test]
3059    fn publish_record_rejects_malformed_collection() {
3060        let dir = tempfile::tempdir().expect("tempdir");
3061        let sk = SigningKey::random(&mut rand::rngs::OsRng);
3062        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
3063        let publisher = PdsPublisher::new(Arc::clone(&store), DID.to_owned(), test_es256_store(sk));
3064        for bad in ["", "with/slash", "with\0nul"] {
3065            assert!(
3066                publisher.publish_record(bad, "repo-a", SAMPLE_OID).is_err(),
3067                "collection {bad:?} must be rejected"
3068            );
3069        }
3070    }
3071
3072    // Frozen spellings from the pre-correction base32(raw << 1) codec.
3073    // In particular repo-a exercises bit 63 of the corrected representation.
3074    const LEGACY_RKEYS: [(&str, &str); 3] = [
3075        ("repo-a", "i4pvpmy7trq2g"),
3076        ("repo-b", "42al4ks2j6lfu"),
3077        ("name-a", "65wkrauzdcfsw"),
3078    ];
3079
3080    #[test]
3081    fn tid_for_repo_preserves_legacy_storage_spelling() {
3082        for (id, spelling) in LEGACY_RKEYS {
3083            let tid = PdsRecordStore::tid_for_repo(id);
3084            assert_eq!(tid.encode(), spelling);
3085            assert_eq!(Tid::parse(spelling).unwrap(), tid);
3086            assert_eq!(tid.to_raw() & 1, 0);
3087        }
3088        assert!(PdsRecordStore::tid_for_repo("repo-a").to_raw() > i64::MAX as u64);
3089    }
3090
3091    #[test]
3092    fn legacy_store_upgrade_preserves_keys_and_republishes_without_duplicates() {
3093        let dir = tempfile::tempdir().unwrap();
3094        let sk = SigningKey::random(&mut rand::rngs::OsRng);
3095        let vk = *sk.verifying_key();
3096        let collections = [COLLECTION_NSID, COLLECTION_NSID, NAME_COLLECTION];
3097        let record = ModelRecord::new(
3098            format!("at://{DID}"),
3099            git_oid_to_cid_string(SAMPLE_OID).unwrap(),
3100            "2026-07-19T00:00:00.000Z",
3101        )
3102        .unwrap();
3103        // Seed the pre-upgrade wire/disk representation directly: no current
3104        // tid_for_repo, record_key, or build_tree helper can mask a regression.
3105        let mut keyed = BTreeMap::new();
3106        let mut expected_keys = std::collections::BTreeSet::new();
3107        let legacy_commit = {
3108            let db = rocksdb::DB::open_default(dir.path()).unwrap();
3109            let mut batch = rocksdb::WriteBatch::default();
3110            for ((_, spelling), collection) in LEGACY_RKEYS.iter().zip(collections) {
3111                let key = format!("rk\0{DID}\0{collection}\0{spelling}").into_bytes();
3112                batch.put(&key, record.to_dag_cbor());
3113                expected_keys.insert(key);
3114                keyed.insert(format!("{collection}/{spelling}"), record.cid());
3115            }
3116            let root = Node::from_keyed_records(&keyed).root_cid();
3117            // Parsing the old on-wire revision preserves its exact signed
3118            // bytes even though the codec's internal raw representation changed.
3119            let commit = Commit::sign(
3120                &UnsignedCommit::new(
3121                    DID.to_owned(),
3122                    root,
3123                    Tid::parse("3jzfcijpj2z2a").unwrap(),
3124                    None,
3125                ),
3126                &sk,
3127            );
3128            batch.put(format!("commit\0{DID}"), commit.to_dag_cbor());
3129            db.write(batch).unwrap();
3130            commit
3131        };
3132        let mut previous = legacy_commit.cid();
3133        for oid in [SAMPLE_OID_2, SAMPLE_OID] {
3134            let store = Arc::new(PdsRecordStore::open(dir.path()).unwrap());
3135            let loaded = store.load_repo(DID).unwrap().unwrap();
3136            assert_eq!(loaded.commit.cid(), previous);
3137            assert_eq!(loaded.records.len(), LEGACY_RKEYS.len());
3138            assert_eq!(
3139                PdsRecordStore::build_tree(&loaded.records).root_cid(),
3140                loaded.commit.data
3141            );
3142            loaded.commit.verify(&vk).unwrap();
3143            for ((id, _), collection) in LEGACY_RKEYS.iter().zip(collections) {
3144                resolve_and_verify(&store, &vk, collection, id);
3145            }
3146            let publisher =
3147                PdsPublisher::new(store.clone(), DID.to_owned(), test_es256_store(sk.clone()));
3148            for ((id, spelling), collection) in LEGACY_RKEYS.iter().zip(collections) {
3149                publisher.publish_record(collection, id, oid).unwrap();
3150                let repo = store.load_repo(DID).unwrap().unwrap();
3151                assert_eq!(repo.records.len(), LEGACY_RKEYS.len());
3152                assert_eq!(repo.commit.prev, Some(previous));
3153                assert_eq!(
3154                    repo.records[&(collection.to_owned(), Tid::parse(spelling).unwrap())]
3155                        .current_oid,
3156                    git_oid_to_cid_string(oid).unwrap()
3157                );
3158                resolve_and_verify(&store, &vk, collection, id);
3159                previous = repo.commit.cid();
3160            }
3161            let RecordBacking::ReadWrite(db) = &store.backing else {
3162                unreachable!()
3163            };
3164            let actual_keys: std::collections::BTreeSet<_> = db
3165                .prefix_iterator(record_prefix(DID))
3166                .map(|entry| entry.unwrap().0.to_vec())
3167                .take_while(|key| key.starts_with(&record_prefix(DID)))
3168                .collect();
3169            assert_eq!(
3170                actual_keys, expected_keys,
3171                "upgrade must overwrite the original keys"
3172            );
3173            drop(publisher);
3174            drop(store);
3175            let readonly = Arc::new(PdsRecordStore::open_readonly(dir.path()).unwrap());
3176            for ((id, _), collection) in LEGACY_RKEYS.iter().zip(collections) {
3177                resolve_and_verify(&readonly, &vk, collection, id);
3178            }
3179        }
3180    }
3181
3182    #[test]
3183    fn tid_for_repo_is_deterministic_and_distinct() {
3184        let first = PdsRecordStore::tid_for_repo("repo-a");
3185        let second = PdsRecordStore::tid_for_repo("repo-a");
3186        assert_eq!(first, second, "the same repo must always get the same rkey");
3187        let other = PdsRecordStore::tid_for_repo("repo-b");
3188        assert_ne!(first, other, "distinct repos must not alias onto one rkey");
3189    }
3190
3191    struct At9pSigner {
3192        ed: ed25519_dalek::SigningKey,
3193        pq: hyprstream_rpc::crypto::pq::MlDsaSigningKey,
3194        pair: hyprstream_pds::at9p::HybridKeyPair,
3195    }
3196
3197    fn at9p_signer(tag: u8) -> At9pSigner {
3198        use hyprstream_rpc::crypto::pq::{ml_dsa_generate_keypair, ml_dsa_vk_bytes};
3199        let mut seed = [0u8; 32];
3200        seed[0] = tag;
3201        seed[31] = tag.wrapping_add(17);
3202        let ed = ed25519_dalek::SigningKey::from_bytes(&seed);
3203        let (pq, pq_vk) = ml_dsa_generate_keypair();
3204        let pair = hyprstream_pds::at9p::HybridKeyPair::new(
3205            ed.verifying_key().to_bytes().to_vec(),
3206            ml_dsa_vk_bytes(&pq_vk),
3207        )
3208        .expect("hybrid pair");
3209        At9pSigner { ed, pq, pair }
3210    }
3211
3212    fn at9p_body(
3213        current: &At9pSigner,
3214        next: &[&At9pSigner],
3215        service_tag: &str,
3216    ) -> hyprstream_pds::at9p::CapsuleBody {
3217        use hyprstream_pds::at9p::{
3218            CapsuleBody, ServiceEndpoint, ServiceEntry, ServiceType, Transport,
3219        };
3220        let endpoint = ServiceEndpoint::new(Transport::Iroh, format!("iroh://{service_tag}"))
3221            .expect("endpoint");
3222        let service =
3223            ServiceEntry::new("#ns", ServiceType::NinePExport, endpoint).expect("service");
3224        let mut body =
3225            CapsuleBody::new(vec![current.pair.clone()], vec![service]).expect("capsule body");
3226        body.next_key_commitments = next
3227            .iter()
3228            .map(|signer| signer.pair.commitment_digest())
3229            .collect();
3230        body
3231    }
3232
3233    fn audit_keys() -> (
3234        ed25519_dalek::SigningKey,
3235        hyprstream_rpc::crypto::pq::MlDsaSigningKey,
3236    ) {
3237        (
3238            ed25519_dalek::SigningKey::from_bytes(&[91u8; 32]),
3239            hyprstream_rpc::crypto::pq::ml_dsa_sk_from_seed(&[92u8; 32]),
3240        )
3241    }
3242
3243    fn production_at9p_publisher(store: Arc<PdsRecordStore>, alarm: &Path) -> PdsPublisher {
3244        let (audit_ed, audit_pq) = audit_keys();
3245        let acceptance_identity = ed25519_dalek::SigningKey::from_bytes(&[90u8; 32]);
3246        let ingest = At9pStateIngest::open(
3247            Arc::clone(&store),
3248            alarm,
3249            acceptance_identity,
3250            audit_ed,
3251            audit_pq,
3252        )
3253        .expect("open accepted-state owner");
3254        PdsPublisher::new(
3255            store,
3256            DID.to_owned(),
3257            test_es256_store(SigningKey::random(&mut rand::rngs::OsRng)),
3258        )
3259        .with_at9p_state_ingest(ingest)
3260    }
3261
3262    #[test]
3263    fn independent_production_guards_share_rocks_cas() {
3264        use hyprstream_pds::at9p_duplicity::{Admission, DuplicityGuard};
3265        use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
3266        use std::sync::Barrier;
3267
3268        const NOW: &str = "2026-07-16T12:00:00Z";
3269        let dir = tempfile::tempdir().expect("tempdir");
3270        let db_path = dir.path().join("pds");
3271        let store = Arc::new(PdsRecordStore::open(&db_path).expect("open"));
3272        let (g, n1, n2) = (at9p_signer(41), at9p_signer(42), at9p_signer(43));
3273        let genesis =
3274            sign_capsule(at9p_body(&g, &[&n1], "genesis"), &g.ed, &g.pq).expect("genesis");
3275        let genesis_bytes = genesis.to_dag_cbor().expect("genesis bytes");
3276        let cid = genesis.cid512().expect("cid");
3277        let did = format!("{DID_AT9P_PREFIX}{cid}");
3278        let verified = verify_did_at9p(&did, &genesis_bytes).expect("verified genesis");
3279        let acceptance_identity = ed25519_dalek::SigningKey::from_bytes(&[90u8; 32]);
3280        let make_guard = |alarm: &str| {
3281            let (audit_ed, audit_pq) = audit_keys();
3282            DuplicityGuard::with_durable_alarm(
3283                RocksAt9pStateStore {
3284                    store: Arc::clone(&store),
3285                    acceptance_identity: acceptance_identity.clone(),
3286                    audit_key: audit_ed.clone(),
3287                },
3288                dir.path().join(alarm),
3289                audit_ed,
3290                audit_pq,
3291            )
3292            .expect("guard")
3293        };
3294        let guard_a = Arc::new(make_guard("alarm-a"));
3295        let guard_b = Arc::new(make_guard("alarm-b"));
3296        guard_a.seed_genesis(&verified).expect("seed");
3297        let candidate_a = sign_update_record(
3298            cid.clone(),
3299            1,
3300            h512(&genesis_bytes),
3301            at9p_body(&n1, &[&n2], "winner-a"),
3302            "2099-01-01T00:00:00Z".to_owned(),
3303            &n1.ed,
3304            &n1.pq,
3305        )
3306        .expect("candidate a");
3307        let candidate_b = sign_update_record(
3308            cid,
3309            1,
3310            h512(&genesis_bytes),
3311            at9p_body(&n1, &[&n2], "winner-b"),
3312            "2099-01-01T00:00:00Z".to_owned(),
3313            &n1.ed,
3314            &n1.pq,
3315        )
3316        .expect("candidate b");
3317        let barrier = Arc::new(Barrier::new(3));
3318        let run = |guard: Arc<DuplicityGuard<_, _>>,
3319                   candidate: hyprstream_pds::at9p::UpdateRecord| {
3320            let barrier = Arc::clone(&barrier);
3321            std::thread::spawn(move || {
3322                barrier.wait();
3323                guard.admit_successor(&candidate, NOW)
3324            })
3325        };
3326        let thread_a = run(guard_a, candidate_a);
3327        let thread_b = run(guard_b, candidate_b);
3328        barrier.wait();
3329        let results = [
3330            thread_a.join().expect("thread a"),
3331            thread_b.join().expect("thread b"),
3332        ];
3333        assert_eq!(
3334            results
3335                .iter()
3336                .filter(|result| matches!(result, Ok(Admission::Advanced(_))))
3337                .count(),
3338            1,
3339            "the RocksDB CAS must allow exactly one independently locked guard to advance"
3340        );
3341        assert_eq!(
3342            store
3343                .load_at9p_state_with_key(
3344                    &did[DID_AT9P_PREFIX.len()..],
3345                    acceptance_identity.verifying_key()
3346                )
3347                .expect("load")
3348                .expect("state")
3349                .epoch,
3350            1
3351        );
3352    }
3353
3354    #[test]
3355    fn production_at9p_ingest_rotation_forks_expiry_terminal_and_restart() {
3356        use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
3357
3358        const NOW: &str = "2026-07-16T12:00:00Z";
3359        const FUTURE: &str = "2099-01-01T00:00:00Z";
3360        let dir = tempfile::tempdir().expect("tempdir");
3361        let db_path = dir.path().join("pds");
3362        let alarm_path = dir.path().join("at9p-alarm.wal");
3363        let (genesis_signer, n1, n2, n3) = (
3364            at9p_signer(1),
3365            at9p_signer(2),
3366            at9p_signer(3),
3367            at9p_signer(4),
3368        );
3369        let genesis = sign_capsule(
3370            at9p_body(&genesis_signer, &[&n1], "genesis"),
3371            &genesis_signer.ed,
3372            &genesis_signer.pq,
3373        )
3374        .expect("signed genesis");
3375        let genesis_bytes = genesis.to_dag_cbor().expect("genesis bytes");
3376        let did = format!("{DID_AT9P_PREFIX}{}", genesis.cid512().expect("cid"));
3377
3378        let u1 = sign_update_record(
3379            genesis.cid512().expect("cid"),
3380            1,
3381            hyprstream_pds::at9p::h512(&genesis_bytes),
3382            at9p_body(&n1, &[&n2], "rotated"),
3383            FUTURE.to_owned(),
3384            &n1.ed,
3385            &n1.pq,
3386        )
3387        .expect("u1");
3388
3389        {
3390            let store = Arc::new(PdsRecordStore::open(&db_path).expect("open rw"));
3391            let publisher = production_at9p_publisher(store, &alarm_path);
3392            let seeded = publisher
3393                .ingest_at9p_genesis(&did, &genesis_bytes)
3394                .expect("GATE seed");
3395            assert_eq!(seeded.epoch, 0);
3396            assert_eq!(seeded.current.subject_keys[0], genesis_signer.pair);
3397
3398            let accepted = publisher
3399                .ingest_at9p_successor(&did, &u1.to_dag_cbor().expect("u1 bytes"), NOW)
3400                .expect("accept rotation");
3401            assert_eq!(accepted.epoch, 1);
3402            assert_eq!(accepted.current.subject_keys[0], n1.pair);
3403            assert_ne!(accepted.current.subject_keys[0], genesis_signer.pair);
3404            assert_eq!(
3405                accepted.current.services[0].endpoint.address,
3406                "iroh://rotated"
3407            );
3408
3409            // Same-epoch, separately valid fork: fail closed and durably alarm.
3410            let fork = sign_update_record(
3411                genesis.cid512().expect("cid"),
3412                1,
3413                hyprstream_pds::at9p::h512(&genesis_bytes),
3414                at9p_body(&n1, &[&n2], "same-epoch-fork"),
3415                FUTURE.to_owned(),
3416                &n1.ed,
3417                &n1.pq,
3418            )
3419            .expect("fork");
3420            assert!(publisher
3421                .ingest_at9p_successor(&did, &fork.to_dag_cbor().expect("fork bytes"), NOW)
3422                .is_err());
3423            let alarm_after_same = std::fs::metadata(&alarm_path).expect("alarm WAL").len();
3424
3425            // A higher record whose link is not the durable head is never chosen.
3426            let off_head = sign_update_record(
3427                genesis.cid512().expect("cid"),
3428                2,
3429                [44u8; 64],
3430                at9p_body(&n2, &[&n3], "off-head"),
3431                FUTURE.to_owned(),
3432                &n2.ed,
3433                &n2.pq,
3434            )
3435            .expect("off-head");
3436            assert!(publisher
3437                .ingest_at9p_successor(&did, &off_head.to_dag_cbor().expect("bytes"), NOW)
3438                .is_err());
3439            assert!(std::fs::metadata(&alarm_path).expect("alarm WAL").len() > alarm_after_same);
3440
3441            // Expiry rejects without changing the accepted head.
3442            let expired = sign_update_record(
3443                genesis.cid512().expect("cid"),
3444                2,
3445                accepted.head_digest,
3446                at9p_body(&n2, &[&n3], "expired"),
3447                "2026-01-01T00:00:00Z".to_owned(),
3448                &n2.ed,
3449                &n2.pq,
3450            )
3451            .expect("expired");
3452            assert!(publisher
3453                .ingest_at9p_successor(&did, &expired.to_dag_cbor().expect("bytes"), NOW)
3454                .is_err());
3455            assert_eq!(
3456                publisher
3457                    .accepted_at9p_state(&did, Some(NOW))
3458                    .expect("state")
3459                    .epoch,
3460                1
3461            );
3462
3463            // A valid final rotation persists terminal state; later successors
3464            // fail closed before any genesis/key fallback.
3465            let terminal = sign_update_record(
3466                genesis.cid512().expect("cid"),
3467                2,
3468                accepted.head_digest,
3469                at9p_body(&n2, &[], "terminal"),
3470                FUTURE.to_owned(),
3471                &n2.ed,
3472                &n2.pq,
3473            )
3474            .expect("terminal");
3475            let terminal_state = publisher
3476                .ingest_at9p_successor(&did, &terminal.to_dag_cbor().expect("bytes"), NOW)
3477                .expect("accept terminal");
3478            assert!(terminal_state.terminal);
3479            let after_terminal = sign_update_record(
3480                genesis.cid512().expect("cid"),
3481                3,
3482                terminal_state.head_digest,
3483                at9p_body(&n3, &[&n3], "forbidden"),
3484                FUTURE.to_owned(),
3485                &n3.ed,
3486                &n3.pq,
3487            )
3488            .expect("syntactically genuine successor");
3489            assert!(publisher
3490                .ingest_at9p_successor(&did, &after_terminal.to_dag_cbor().expect("bytes"), NOW)
3491                .is_err());
3492        }
3493
3494        // Reopen verifies the signed alarm WAL and recovers the complete atomic
3495        // state. Re-seed and rollback remain refused after restart.
3496        let store = Arc::new(PdsRecordStore::open(&db_path).expect("reopen rw"));
3497        let publisher = production_at9p_publisher(store, &alarm_path);
3498        let recovered = publisher
3499            .accepted_at9p_state(&did, Some(NOW))
3500            .expect("recover state");
3501        assert_eq!(recovered.epoch, 2);
3502        assert!(recovered.terminal);
3503        assert_eq!(recovered.current.subject_keys[0], n2.pair);
3504        assert!(publisher.ingest_at9p_genesis(&did, &genesis_bytes).is_err());
3505        assert!(publisher
3506            .ingest_at9p_successor(&did, &u1.to_dag_cbor().expect("u1 bytes"), NOW)
3507            .is_err());
3508    }
3509
3510    #[test]
3511    fn accepted_at9p_read_rejects_watermark_body_mutation() {
3512        use hyprstream_pds::at9p_sign::sign_capsule;
3513        let dir = tempfile::tempdir().expect("tempdir");
3514        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
3515        let signer = at9p_signer(11);
3516        let capsule = sign_capsule(at9p_body(&signer, &[], "immutable"), &signer.ed, &signer.pq)
3517            .expect("capsule");
3518        let bytes = capsule.to_dag_cbor().expect("bytes");
3519        let cid = capsule.cid512().expect("cid");
3520        let did = format!("{DID_AT9P_PREFIX}{cid}");
3521        let publisher = production_at9p_publisher(Arc::clone(&store), &dir.path().join("alarm"));
3522        publisher.ingest_at9p_genesis(&did, &bytes).expect("seed");
3523
3524        let RecordBacking::ReadWrite(db) = &store.backing else {
3525            panic!("writer")
3526        };
3527        let key = at9p_state_key(&cid);
3528        let mut encoded = db.get(&key).expect("get").expect("stored state");
3529        encoded[AT9P_ACCEPTANCE_PREFIX_LEN + 16] ^= 1;
3530        // Mutate persisted epoch without changing the retained head body or
3531        // recomputing the daemon acceptance signature.
3532        db.put(&key, encoded).expect("inject mutation");
3533        assert!(
3534            publisher.accepted_at9p_state(&did, None).is_err(),
3535            "a partial watermark/body representation must fail closed"
3536        );
3537    }
3538
3539    #[test]
3540    fn accepted_checkpoint_rejects_old_envelope_and_reverse_mismatch() {
3541        use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
3542        let dir = tempfile::tempdir().expect("tempdir");
3543        let db_path = dir.path().join("pds");
3544        let alarm = dir.path().join("alarm");
3545        let store = Arc::new(PdsRecordStore::open(&db_path).expect("open"));
3546        let (g, n1, n2) = (at9p_signer(31), at9p_signer(32), at9p_signer(33));
3547        let genesis =
3548            sign_capsule(at9p_body(&g, &[&n1], "genesis"), &g.ed, &g.pq).expect("genesis");
3549        let genesis_bytes = genesis.to_dag_cbor().expect("bytes");
3550        let cid = genesis.cid512().expect("cid");
3551        let did = format!("{DID_AT9P_PREFIX}{cid}");
3552        let publisher = production_at9p_publisher(Arc::clone(&store), &alarm);
3553        publisher
3554            .ingest_at9p_genesis(&did, &genesis_bytes)
3555            .expect("seed");
3556        let RecordBacking::ReadWrite(db) = &store.backing else {
3557            panic!("writer")
3558        };
3559        let state_key = at9p_state_key(&cid);
3560        let checkpoint_key = at9p_checkpoint_key(&cid);
3561        let old_state = db.get(&state_key).unwrap().unwrap();
3562        let old_checkpoint = db.get(&checkpoint_key).unwrap().unwrap();
3563        let update = sign_update_record(
3564            cid.clone(),
3565            1,
3566            h512(&genesis_bytes),
3567            at9p_body(&n1, &[&n2], "rotated"),
3568            "2099-01-01T00:00:00Z".to_owned(),
3569            &n1.ed,
3570            &n1.pq,
3571        )
3572        .expect("update");
3573        publisher
3574            .ingest_at9p_successor(&did, &update.to_dag_cbor().unwrap(), "2026-07-16T12:00:00Z")
3575            .expect("advance");
3576        let new_state = db.get(&state_key).unwrap().unwrap();
3577        let new_checkpoint = db.get(&checkpoint_key).unwrap().unwrap();
3578        db.delete(&state_key).expect("simulate state-half missing");
3579        assert!(publisher.accepted_at9p_state(&did, None).is_err());
3580        db.put(&state_key, &new_state).expect("restore state");
3581        db.delete(&checkpoint_key)
3582            .expect("simulate checkpoint-half missing");
3583        assert!(publisher.accepted_at9p_state(&did, None).is_err());
3584        db.put(&checkpoint_key, &new_checkpoint)
3585            .expect("restore checkpoint");
3586        db.put(&state_key, old_state).expect("old envelope replay");
3587        assert!(publisher.accepted_at9p_state(&did, None).is_err());
3588        db.put(&state_key, new_state).expect("restore body");
3589        db.put(&checkpoint_key, old_checkpoint)
3590            .expect("old checkpoint replay");
3591        assert!(publisher.accepted_at9p_state(&did, None).is_err());
3592        db.put(&checkpoint_key, new_checkpoint)
3593            .expect("restore checkpoint");
3594        assert_eq!(publisher.accepted_at9p_state(&did, None).unwrap().epoch, 1);
3595        drop(publisher);
3596        drop(store);
3597        let reopened = Arc::new(PdsRecordStore::open(&db_path).expect("reopen"));
3598        assert_eq!(
3599            production_at9p_publisher(reopened, &alarm)
3600                .accepted_at9p_state(&did, None)
3601                .unwrap()
3602                .epoch,
3603            1
3604        );
3605        let acceptance_identity = ed25519_dalek::SigningKey::from_bytes(&[90u8; 32]);
3606        let readonly = PdsRecordStore::open_readonly(&db_path)
3607            .expect("open real production read boundary")
3608            .with_at9p_acceptance_identity(acceptance_identity.verifying_key());
3609        let states = readonly
3610            .accepted_at9p_states()
3611            .expect("enumerate checkpoint-verified accepted states");
3612        assert_eq!(states.len(), 1);
3613        assert_eq!(states[0].did, did);
3614        assert_eq!(states[0].epoch, 1);
3615    }
3616
3617    #[test]
3618    fn accepted_at9p_read_rejects_attacker_selected_signed_fork() {
3619        use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
3620
3621        let dir = tempfile::tempdir().expect("tempdir");
3622        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
3623        let (genesis_signer, n1, n2) = (at9p_signer(21), at9p_signer(22), at9p_signer(23));
3624        let genesis = sign_capsule(
3625            at9p_body(&genesis_signer, &[&n1], "genesis"),
3626            &genesis_signer.ed,
3627            &genesis_signer.pq,
3628        )
3629        .expect("genesis");
3630        let genesis_bytes = genesis.to_dag_cbor().expect("genesis bytes");
3631        let cid = genesis.cid512().expect("cid");
3632        let did = format!("{DID_AT9P_PREFIX}{cid}");
3633        let publisher = production_at9p_publisher(Arc::clone(&store), &dir.path().join("alarm"));
3634        publisher
3635            .ingest_at9p_genesis(&did, &genesis_bytes)
3636            .expect("seed");
3637
3638        // This update is internally canonical, correctly linked, and signed by
3639        // the identity's committed key. It was never selected by this daemon.
3640        let fork = sign_update_record(
3641            cid.clone(),
3642            1,
3643            hyprstream_pds::at9p::h512(&genesis_bytes),
3644            at9p_body(&n1, &[&n2], "attacker-selected-fork"),
3645            "2099-01-01T00:00:00Z".to_owned(),
3646            &n1.ed,
3647            &n1.pq,
3648        )
3649        .expect("signed fork");
3650        let fork_state =
3651            AcceptedAt9pState::from_persisted_update(&fork.to_dag_cbor().expect("fork bytes"))
3652                .expect("internally valid fork state");
3653
3654        // A process that can rewrite RocksDB but lacks the stable deployment
3655        // identity can only certify the substituted value with its own key.
3656        let attacker_identity = ed25519_dalek::SigningKey::from_bytes(&[70u8; 32]);
3657        let attacker_audit = ed25519_dalek::SigningKey::from_bytes(&[71u8; 32]);
3658        let forged = encode_at9p_state(&fork_state, &attacker_identity, &attacker_audit)
3659            .expect("forge self-consistent envelope");
3660        let RecordBacking::ReadWrite(db) = &store.backing else {
3661            panic!("writer")
3662        };
3663        db.put(at9p_state_key(&cid), forged)
3664            .expect("inject attacker-selected fork");
3665        assert!(
3666            publisher.accepted_at9p_state(&did, None).is_err(),
3667            "an identity-valid fork without daemon acceptance must fail closed"
3668        );
3669    }
3670
3671    // ── #1159 freeze: durable publisher rejects path-form authorities ───────
3672
3673    /// A `PdsPublisher` whose authority is a path-form `did:web` must refuse to
3674    /// publish — the durable write would anchor a permanent, spec-invalid repo
3675    /// identifier (`commit\0{did}` / `record\0{did}` keys). Production supplies
3676    /// a node `did:key`; this guards the public `PdsPublisher::new(did)` surface.
3677    #[test]
3678    fn publish_rejects_path_form_did_web_authority() {
3679        let dir = tempfile::tempdir().expect("tempdir");
3680        let sk = SigningKey::random(&mut rand::rngs::OsRng);
3681        let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
3682        let path_form_did = "did:web:accounts.example:users:alice".to_owned();
3683        let publisher = PdsPublisher::new(Arc::clone(&store), path_form_did.clone(), sk);
3684
3685        let err = publisher.publish("repo-a", SAMPLE_OID).unwrap_err();
3686        assert!(
3687            err.to_string().contains("path-form did:web"),
3688            "expected path-form rejection, got: {err}",
3689        );
3690
3691        // No durable anchor was written: the store holds no commit for this DID.
3692        let RecordBacking::ReadWrite(db) = &store.backing else {
3693            panic!("read-write store")
3694        };
3695        assert!(
3696            db.get(commit_key(&path_form_did)).unwrap().is_none(),
3697            "no commit key must be persisted for a path-form authority",
3698        );
3699        let tid = PdsRecordStore::tid_for_repo("repo-a");
3700        assert!(
3701            db.get(record_key(&path_form_did, COLLECTION_NSID, tid))
3702                .unwrap()
3703                .is_none(),
3704            "no record key must be persisted for a path-form authority",
3705        );
3706    }
3707
3708    /// Host-form `did:web` and node `did:key` authorities still publish
3709    /// normally — the guard rejects ONLY path-form did:web.
3710    #[test]
3711    fn publish_accepts_host_form_did_web_and_did_key() {
3712        let dir = tempfile::tempdir().expect("tempdir");
3713        let sk = SigningKey::random(&mut rand::rngs::OsRng);
3714
3715        for accepted in ["did:web:alice.example.com", DID] {
3716            let store = Arc::new(PdsRecordStore::open(dir.path()).expect("open rw"));
3717            let publisher = PdsPublisher::new(store, accepted.to_owned(), sk.clone());
3718            publisher
3719                .publish("repo-a", SAMPLE_OID)
3720                .unwrap_or_else(|e| panic!("host-form/key authority {accepted} must publish: {e}"));
3721        }
3722    }
3723}
3724
3725/// Live Postgres-backed store tests (#1257). Every test skips itself green
3726/// unless the operator provides a scratch database via
3727/// `HYPRSTREAM_POSTGRES_TEST_URL_FILE` — a file path, never a direct env var,
3728/// mirroring the production file-backed credential model (metal v1.1). These
3729/// cover exactly the semantics where RocksDB and Postgres could diverge:
3730/// cross-handle visibility (the two-AZ invariant), repo-head compare-and-swap,
3731/// concurrent accepted-state advance, and the first-boot lifecycle marker.
3732#[cfg(all(test, feature = "pds-postgres"))]
3733#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
3734mod pg_tests {
3735    use super::*;
3736    use p256::ecdsa::SigningKey;
3737
3738    const COLLECTION: &str = "ai.hyprstream.model";
3739    const NOW: &str = "2026-07-16T12:00:00Z";
3740    const FUTURE: &str = "2099-01-01T00:00:00Z";
3741
3742    fn test_url() -> Option<String> {
3743        let path = std::env::var_os("HYPRSTREAM_POSTGRES_TEST_URL_FILE")?;
3744        let url = std::fs::read_to_string(&path)
3745            .unwrap_or_else(|e| panic!("read test URL file {}: {e}", path.to_string_lossy()));
3746        let url = url.trim();
3747        assert!(!url.is_empty(), "test URL file is empty");
3748        Some(url.to_owned())
3749    }
3750
3751    macro_rules! require_db {
3752        () => {{
3753            let Some(url) = test_url() else {
3754                eprintln!(
3755                    "skipping PdsRecordStore Postgres test: \
3756                     HYPRSTREAM_POSTGRES_TEST_URL_FILE unset"
3757                );
3758                return;
3759            };
3760            url
3761        }};
3762    }
3763
3764    /// A DID unique to this process run so repeated runs against a shared
3765    /// scratch database never collide with their own prior writes.
3766    fn unique_did(label: &str) -> String {
3767        let nanos = std::time::SystemTime::now()
3768            .duration_since(std::time::UNIX_EPOCH)
3769            .map(|d| d.as_nanos())
3770            .unwrap_or(0);
3771        format!("did:key:zPg{label}{}{nanos}", std::process::id())
3772    }
3773
3774    fn open_pair(url: &str) -> (PdsRecordStore, PdsRecordStore) {
3775        let writer = PdsRecordStore::open_postgres_test(url, false).expect("open writer handle");
3776        let reader = PdsRecordStore::open_postgres_test(url, true).expect("open reader handle");
3777        (writer, reader)
3778    }
3779
3780    fn record_for(did: &str, tag: &str) -> ModelRecord {
3781        ModelRecord::new(
3782            format!("at://{did}"),
3783            format!("bafyreipgtest{tag}{}", "0".repeat(40 - tag.len())),
3784            "2026-06-23T12:34:56.789Z",
3785        )
3786        .expect("record")
3787    }
3788
3789    /// Sign the commit covering `records`, chained on `prev`.
3790    fn sign_covering_commit(
3791        did: &str,
3792        records: &BTreeMap<(String, Tid), ModelRecord>,
3793        prev: Option<&Commit>,
3794        sk: &SigningKey,
3795    ) -> Commit {
3796        let tree = PdsRecordStore::build_tree(records);
3797        let (rev, prev_cid) = match prev {
3798            Some(prev) => (next_rev(prev.rev), Some(prev.cid())),
3799            None => (Tid::now(), None),
3800        };
3801        let unsigned = UnsignedCommit::new(did.to_owned(), tree.root_cid(), rev, prev_cid);
3802        Commit::sign(&unsigned, sk)
3803    }
3804
3805    // ── at9p fixtures (mirrors of the pds_store_tests helpers, which are
3806    //    private to that sibling module) ────────────────────────────────────
3807
3808    struct At9pSigner {
3809        ed: ed25519_dalek::SigningKey,
3810        pq: hyprstream_rpc::crypto::pq::MlDsaSigningKey,
3811        pair: hyprstream_pds::at9p::HybridKeyPair,
3812    }
3813
3814    fn at9p_signer(tag: u8) -> At9pSigner {
3815        use hyprstream_rpc::crypto::pq::{ml_dsa_generate_keypair, ml_dsa_vk_bytes};
3816        let mut seed = [0u8; 32];
3817        seed[0] = tag;
3818        seed[31] = tag.wrapping_add(17);
3819        let ed = ed25519_dalek::SigningKey::from_bytes(&seed);
3820        let (pq, pq_vk) = ml_dsa_generate_keypair();
3821        let pair = hyprstream_pds::at9p::HybridKeyPair::new(
3822            ed.verifying_key().to_bytes().to_vec(),
3823            ml_dsa_vk_bytes(&pq_vk),
3824        )
3825        .expect("hybrid pair");
3826        At9pSigner { ed, pq, pair }
3827    }
3828
3829    fn at9p_body(
3830        current: &At9pSigner,
3831        next: &[&At9pSigner],
3832        service_tag: &str,
3833    ) -> hyprstream_pds::at9p::CapsuleBody {
3834        use hyprstream_pds::at9p::{
3835            CapsuleBody, ServiceEndpoint, ServiceEntry, ServiceType, Transport,
3836        };
3837        let endpoint = ServiceEndpoint::new(Transport::Iroh, format!("iroh://{service_tag}"))
3838            .expect("endpoint");
3839        let service =
3840            ServiceEntry::new("#ns", ServiceType::NinePExport, endpoint).expect("service");
3841        let mut body =
3842            CapsuleBody::new(vec![current.pair.clone()], vec![service]).expect("capsule body");
3843        body.next_key_commitments = next
3844            .iter()
3845            .map(|signer| signer.pair.commitment_digest())
3846            .collect();
3847        body
3848    }
3849
3850    fn acceptance_identity() -> ed25519_dalek::SigningKey {
3851        ed25519_dalek::SigningKey::from_bytes(&[90u8; 32])
3852    }
3853
3854    fn audit_ed() -> ed25519_dalek::SigningKey {
3855        ed25519_dalek::SigningKey::from_bytes(&[91u8; 32])
3856    }
3857
3858    fn audit_pq() -> hyprstream_rpc::crypto::pq::MlDsaSigningKey {
3859        hyprstream_rpc::crypto::pq::ml_dsa_sk_from_seed(&[92u8; 32])
3860    }
3861
3862    fn pg_at9p_publisher(store: Arc<PdsRecordStore>, alarm: &std::path::Path) -> PdsPublisher {
3863        use crate::auth::key_rotation::{Es256KeySlot, Es256KeySlots, Es256SigningKeyStore};
3864        let es256 = Arc::new(Es256SigningKeyStore::new(Es256KeySlots {
3865            drain: None,
3866            active: Some(Es256KeySlot::new(
3867                SigningKey::random(&mut rand::rngs::OsRng),
3868                0,
3869                i64::MAX,
3870            )),
3871            lead: None,
3872        }));
3873        let ingest = At9pStateIngest::open(
3874            Arc::clone(&store),
3875            alarm,
3876            acceptance_identity(),
3877            audit_ed(),
3878            audit_pq(),
3879        )
3880        .expect("open accepted-state owner");
3881        PdsPublisher::new(store, "did:key:zPgTestNode".to_owned(), es256)
3882            .with_at9p_state_ingest(ingest)
3883    }
3884
3885    // ── tests ───────────────────────────────────────────────────────────────
3886
3887    #[test]
3888    fn live_two_handle_persistence_and_visibility() {
3889        let url = require_db!();
3890        let did = unique_did("Vis");
3891        let sk = SigningKey::random(&mut rand::rngs::OsRng);
3892        let tid = PdsRecordStore::tid_for_repo("repo-vis");
3893        let record = record_for(&did, "vis");
3894        let mut records = BTreeMap::new();
3895        records.insert((COLLECTION.to_owned(), tid), record.clone());
3896        let commit = sign_covering_commit(&did, &records, None, &sk);
3897
3898        {
3899            let (writer, reader) = open_pair(&url);
3900            writer
3901                .put_record_and_commit(&did, COLLECTION, tid, &record, &commit, None)
3902                .expect("publish via AZ-a handle");
3903
3904            // Cross-AZ invariant: the write is visible to the resolver handle.
3905            let repo = reader
3906                .load_repo(&did)
3907                .expect("read via AZ-b handle")
3908                .expect("repo visible through the second handle");
3909            assert_eq!(
3910                repo.records.get(&(COLLECTION.to_owned(), tid)),
3911                Some(&record)
3912            );
3913            assert_eq!(repo.commit_bytes, commit.to_dag_cbor());
3914
3915            // The resolver posture is read-only.
3916            let err = reader
3917                .put_record_and_commit(&did, COLLECTION, tid, &record, &commit, None)
3918                .expect_err("resolver handle must reject writes");
3919            assert!(err.to_string().contains("read-only"), "got: {err}");
3920        } // both handles (and their bridges) drop here
3921
3922        // Persistence: a fresh handle on a fresh bridge sees the same bytes.
3923        let reopened = PdsRecordStore::open_postgres_test(&url, true).expect("reopen reader");
3924        let repo = reopened
3925            .load_repo(&did)
3926            .expect("reload")
3927            .expect("repo persists across handle close");
3928        assert_eq!(repo.commit_bytes, commit.to_dag_cbor());
3929
3930        // An unknown DID is absent, never an error and never a torn read.
3931        assert!(reopened
3932            .load_repo(&unique_did("Absent"))
3933            .expect("absent repo read")
3934            .is_none());
3935    }
3936
3937    #[test]
3938    fn live_repo_head_cas_rejects_stale_publish() {
3939        let url = require_db!();
3940        let did = unique_did("Cas");
3941        let sk = SigningKey::random(&mut rand::rngs::OsRng);
3942        let writer_a = PdsRecordStore::open_postgres_test(&url, false).expect("open publisher A");
3943        let writer_b = PdsRecordStore::open_postgres_test(&url, false).expect("open publisher B");
3944
3945        // A's genesis publish (no prior head).
3946        let tid1 = PdsRecordStore::tid_for_repo("repo-1");
3947        let rec1 = record_for(&did, "r1");
3948        let mut records = BTreeMap::new();
3949        records.insert((COLLECTION.to_owned(), tid1), rec1.clone());
3950        let commit1 = sign_covering_commit(&did, &records, None, &sk);
3951        writer_a
3952            .put_record_and_commit(&did, COLLECTION, tid1, &rec1, &commit1, None)
3953            .expect("genesis publish");
3954        let head1 = writer_a
3955            .load_repo(&did)
3956            .expect("load")
3957            .map(|r| r.commit_bytes)
3958            .expect("head 1 durable");
3959
3960        // B reads the same head through its own handle.
3961        let repo_b = writer_b
3962            .load_repo(&did)
3963            .expect("load via B")
3964            .expect("head 1 visible via B");
3965        assert_eq!(repo_b.commit_bytes, head1);
3966
3967        // A advances the head (covers r1 + r2).
3968        let tid2 = PdsRecordStore::tid_for_repo("repo-2");
3969        let rec2 = record_for(&did, "r2");
3970        records.insert((COLLECTION.to_owned(), tid2), rec2.clone());
3971        let commit2 = sign_covering_commit(&did, &records, Some(&commit1), &sk);
3972        writer_a
3973            .put_record_and_commit(&did, COLLECTION, tid2, &rec2, &commit2, Some(&head1))
3974            .expect("advance with the current head commits");
3975
3976        // B publishes from the STALE head: rejected, never silently committed.
3977        let tid3 = PdsRecordStore::tid_for_repo("repo-3");
3978        let rec3 = record_for(&did, "r3");
3979        let mut stale_records = BTreeMap::new();
3980        stale_records.insert((COLLECTION.to_owned(), tid1), rec1.clone());
3981        stale_records.insert((COLLECTION.to_owned(), tid3), rec3.clone());
3982        let commit3 = sign_covering_commit(&did, &stale_records, Some(&commit1), &sk);
3983        let err = writer_b
3984            .put_record_and_commit(&did, COLLECTION, tid3, &rec3, &commit3, Some(&head1))
3985            .expect_err("publish from a stale head must be rejected");
3986        assert!(
3987            err.to_string().contains("advanced concurrently"),
3988            "error must name the lost race: {err}"
3989        );
3990
3991        // The durable head still covers exactly A's record set — no clobber.
3992        let durable = writer_b
3993            .load_repo(&did)
3994            .expect("reload")
3995            .expect("repo present");
3996        assert!(durable.records.contains_key(&(COLLECTION.to_owned(), tid2)));
3997        assert!(!durable.records.contains_key(&(COLLECTION.to_owned(), tid3)));
3998
3999        // B reloads, rebases on the current head, and its retry commits.
4000        let tid4 = PdsRecordStore::tid_for_repo("repo-4");
4001        let rec4 = record_for(&did, "r4");
4002        let mut rebased = durable.records.clone();
4003        rebased.insert((COLLECTION.to_owned(), tid4), rec4.clone());
4004        let commit4 = sign_covering_commit(&did, &rebased, Some(&durable.commit), &sk);
4005        writer_b
4006            .put_record_and_commit(
4007                &did,
4008                COLLECTION,
4009                tid4,
4010                &rec4,
4011                &commit4,
4012                Some(&durable.commit_bytes),
4013            )
4014            .expect("retry on the current head commits");
4015    }
4016
4017    #[test]
4018    fn live_at9p_concurrent_advance_has_exactly_one_winner() {
4019        use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
4020        use std::sync::Barrier;
4021
4022        let url = require_db!();
4023        let (g, n1) = (at9p_signer(61), at9p_signer(62));
4024        let genesis =
4025            sign_capsule(at9p_body(&g, &[&n1], "genesis"), &g.ed, &g.pq).expect("genesis");
4026        let genesis_bytes = genesis.to_dag_cbor().expect("genesis bytes");
4027        let cid = genesis.cid512().expect("cid");
4028        let did = format!("{DID_AT9P_PREFIX}{cid}");
4029        let dir = tempfile::tempdir().expect("tempdir");
4030
4031        // Produce the epoch-1 accepted state on a scratch local store (same
4032        // fixtures → same deterministic state) without touching Postgres.
4033        let state0;
4034        let state1;
4035        {
4036            let scratch_dir = tempfile::tempdir().expect("scratch tempdir");
4037            let scratch = Arc::new(
4038                PdsRecordStore::open(&scratch_dir.path().join("pds")).expect("scratch store"),
4039            );
4040            let publisher = pg_at9p_publisher(scratch, &scratch_dir.path().join("alarm"));
4041            state0 = publisher
4042                .ingest_at9p_genesis(&did, &genesis_bytes)
4043                .expect("scratch genesis");
4044            let update = sign_update_record(
4045                cid.clone(),
4046                1,
4047                state0.head_digest,
4048                at9p_body(&n1, &[], "rotated"),
4049                FUTURE.to_owned(),
4050                &n1.ed,
4051                &n1.pq,
4052            )
4053            .expect("update");
4054            state1 = publisher
4055                .ingest_at9p_successor(&did, &update.to_dag_cbor().expect("update bytes"), NOW)
4056                .expect("scratch successor");
4057        }
4058
4059        // Seed the genesis state into Postgres through handle A.
4060        let store_a = Arc::new(
4061            PdsRecordStore::open_postgres_test(&url, false)
4062                .expect("open pg A")
4063                .with_at9p_acceptance_identity(acceptance_identity().verifying_key()),
4064        );
4065        let store_b = Arc::new(
4066            PdsRecordStore::open_postgres_test(&url, false)
4067                .expect("open pg B")
4068                .with_at9p_acceptance_identity(acceptance_identity().verifying_key()),
4069        );
4070        let publisher_a = pg_at9p_publisher(Arc::clone(&store_a), &dir.path().join("alarm-a"));
4071        publisher_a
4072            .ingest_at9p_genesis(&did, &genesis_bytes)
4073            .expect("seed genesis into Postgres");
4074        // Cross-handle visibility of the accepted state before the race.
4075        let seen = store_b
4076            .accepted_at9p_state(&did, None)
4077            .expect("read via B")
4078            .expect("genesis visible via B");
4079        assert_eq!(seen.watermark(), state0.watermark());
4080
4081        // Two publishers race the same conditional advance from the genesis
4082        // watermark: exactly one commits; the loser reports Conflict instead
4083        // of overwriting (#1405 review: no lost updates across AZs).
4084        let barrier = Barrier::new(2);
4085        let outcome = std::thread::scope(|scope| {
4086            let a = scope.spawn(|| {
4087                barrier.wait();
4088                store_a.conditional_advance_at9p_state(
4089                    Some(state0.watermark()),
4090                    &state1,
4091                    &acceptance_identity(),
4092                    &audit_ed(),
4093                )
4094            });
4095            let b = scope.spawn(|| {
4096                barrier.wait();
4097                store_b.conditional_advance_at9p_state(
4098                    Some(state0.watermark()),
4099                    &state1,
4100                    &acceptance_identity(),
4101                    &audit_ed(),
4102                )
4103            });
4104            [
4105                a.join().expect("racer A panicked"),
4106                b.join().expect("racer B panicked"),
4107            ]
4108        });
4109        let outcomes = outcome;
4110        let committed = outcomes
4111            .iter()
4112            .filter(|o| matches!(o, Ok(ConditionalAdvance::Committed)))
4113            .count();
4114        let mut conflicts = outcomes.iter().filter_map(|o| match o {
4115            Ok(ConditionalAdvance::Conflict(current)) => Some(current.as_ref()),
4116            _ => None,
4117        });
4118        assert_eq!(
4119            committed, 1,
4120            "exactly one publisher commits the advance: {outcomes:?}"
4121        );
4122        let loser = conflicts
4123            .next()
4124            .expect("exactly one publisher reports the conflict")
4125            .watermark();
4126        assert!(
4127            conflicts.next().is_none(),
4128            "at most one conflict: {outcomes:?}"
4129        );
4130        assert_eq!(
4131            loser,
4132            state1.watermark(),
4133            "the loser's conflict reports the winner's durable state"
4134        );
4135
4136        // The durable head is the epoch-1 state, readable through both handles.
4137        let durable = store_b
4138            .accepted_at9p_state(&did, None)
4139            .expect("final read")
4140            .expect("state present");
4141        assert_eq!(durable.watermark(), state1.watermark());
4142    }
4143
4144    #[test]
4145    fn live_first_boot_marker_visible_across_handles() {
4146        let url = require_db!();
4147        let (writer, reader) = open_pair(&url);
4148        writer
4149            .mark_first_boot()
4150            .expect("write marker via writer handle");
4151        assert!(
4152            writer.first_boot_pending().expect("read via writer"),
4153            "writer observes its own marker"
4154        );
4155        assert!(
4156            reader.first_boot_pending().expect("read via reader"),
4157            "the marker is visible through the resolver handle"
4158        );
4159    }
4160}