Skip to main content

hyprstream_discovery/
checkpointed_pds.rs

1//! Concrete reader for daemon-authenticated, monotonic-checkpointed accepted
2//! `did:at9p` state. This module is private so production authority cannot be
3//! implemented, constructed, or installed by a downstream crate.
4
5use anyhow::{bail, Context as _, Result};
6use hyprstream_pds::at9p::h512;
7use hyprstream_pds::at9p_duplicity::{AcceptedAt9pState, Watermark};
8use hyprstream_pds::at9p_gate::DID_AT9P_PREFIX;
9use std::path::Path;
10#[cfg(feature = "rocksdb")]
11use std::path::PathBuf;
12use std::sync::Arc;
13
14const AT9P_STATE_MAGIC: &[u8; 8] = b"AT9PST02";
15const AT9P_STATE_HEADER_LEN: usize = 8 + 1 + 8 + 64 + 1 + 4;
16const AT9P_ACCEPTANCE_MAGIC: &[u8; 8] = b"AT9PAC01";
17const ED25519_KEY_LEN: usize = 32;
18const ED25519_SIGNATURE_LEN: usize = 64;
19const AT9P_ACCEPTANCE_PREFIX_LEN: usize =
20    AT9P_ACCEPTANCE_MAGIC.len() + ED25519_KEY_LEN + ED25519_SIGNATURE_LEN;
21const AT9P_ACCEPTANCE_KEY_AAD: &[u8] = b"hyprstream-at9p-acceptance-key/1";
22const AT9P_ACCEPTANCE_STATE_AAD: &[u8] = b"hyprstream-at9p-accepted-state/1";
23const AT9P_CHECKPOINT_MAGIC: &[u8; 8] = b"AT9PCK01";
24const AT9P_CHECKPOINT_PAYLOAD_LEN: usize = 8 + 8 + 64 + 1 + 64;
25const AT9P_CHECKPOINT_LEN: usize = AT9P_CHECKPOINT_PAYLOAD_LEN + ED25519_SIGNATURE_LEN;
26const AT9P_CHECKPOINT_AAD: &[u8] = b"hyprstream-at9p-monotonic-checkpoint/1";
27
28#[cfg(feature = "rocksdb")]
29pub(super) struct CheckpointedPdsAcceptedStateSource {
30    path: PathBuf,
31    acceptance_identity: Arc<dyn AcceptanceVerifier>,
32    network_bootstrap: bool,
33}
34
35trait AcceptanceVerifier: Send + Sync {
36    fn verify_strict(&self, message: &[u8], signature: &ed25519_dalek::Signature) -> Result<()>;
37}
38
39impl AcceptanceVerifier for crate::service::RegistryDeploymentVerifier {
40    fn verify_strict(&self, message: &[u8], signature: &ed25519_dalek::Signature) -> Result<()> {
41        self.verify_strict(message, signature)
42    }
43}
44
45#[cfg(test)]
46impl AcceptanceVerifier for ed25519_dalek::VerifyingKey {
47    fn verify_strict(&self, message: &[u8], signature: &ed25519_dalek::Signature) -> Result<()> {
48        self.verify_strict(message, signature).map_err(Into::into)
49    }
50}
51
52#[cfg(feature = "rocksdb")]
53impl CheckpointedPdsAcceptedStateSource {
54    pub(super) fn open(
55        path: &Path,
56        acceptance_identity: crate::service::RegistryDeploymentVerifier,
57    ) -> Result<Self> {
58        // The monotonic checkpoint history is security state. A missing
59        // database is indistinguishable from deletion of that history, so it
60        // must never be recreated at resolver bootstrap. First boot is an
61        // explicit deployment-provisioning step before services may start.
62        let _probe = rocksdb::DB::open_for_read_only(&readonly_opts(), path, false)
63            .with_context(|| format!("failed to open checkpointed PDS store at {path:?}"))?;
64        Ok(Self {
65            path: path.to_path_buf(),
66            acceptance_identity: Arc::new(acceptance_identity),
67            network_bootstrap: false,
68        })
69    }
70
71    #[cfg(test)]
72    pub(super) fn open_test(
73        path: &Path,
74        acceptance_identity: ed25519_dalek::VerifyingKey,
75    ) -> Result<Self> {
76        let _probe = rocksdb::DB::open_for_read_only(&readonly_opts(), path, false)
77            .with_context(|| format!("failed to open checkpointed PDS store at {path:?}"))?;
78        Ok(Self {
79            path: path.to_path_buf(),
80            acceptance_identity: Arc::new(acceptance_identity),
81            network_bootstrap: false,
82        })
83    }
84
85    pub(super) fn with_network_bootstrap(mut self, required: bool) -> Self {
86        self.network_bootstrap = required;
87        self
88    }
89
90    fn bootstrap_states(&self) -> Result<Vec<AcceptedAt9pState>> {
91        let db = rocksdb::DB::open_for_read_only(&readonly_opts(), &self.path, false)?;
92        let mut states = Vec::new();
93        for entry in db.prefix_iterator(b"at9p-state\0") {
94            let (key, _) = entry?;
95            let Some(subject) = key.strip_prefix(b"at9p-state\0") else {
96                break;
97            };
98            let subject = std::str::from_utf8(subject)?;
99            if let Some(state) =
100                load_at9p_state_from_db(&db, subject, self.acceptance_identity.as_ref())?
101            {
102                states.push(state);
103            }
104        }
105        Ok(states)
106    }
107
108    pub(super) fn accepted_state(&self, did: &str) -> Result<Option<AcceptedAt9pState>> {
109        let subject = did
110            .strip_prefix(DID_AT9P_PREFIX)
111            .ok_or_else(|| anyhow::anyhow!("identifier is not did:at9p: {did:?}"))?;
112        let db = rocksdb::DB::open_for_read_only(&readonly_opts(), &self.path, false)?;
113        let state = load_at9p_state_from_db(&db, subject, self.acceptance_identity.as_ref())?;
114        if let Some(state) = &state {
115            anyhow::ensure!(state.did == did, "accepted at9p state DID mismatch");
116        }
117        Ok(state)
118    }
119}
120
121// ---------------------------------------------------------------------------
122// PostgreSQL (RDS) backend — the same verified envelopes in the shared `pds_kv`
123// BYTEA shell. Compiled only with the `postgres` feature.
124// ---------------------------------------------------------------------------
125
126/// Accepted-state authority over the networked RDS Postgres store, selected
127/// when the deployment's `[rds]` binding is configured. Uses the same
128/// `hyprstream_pds::pgsql_kv::PgKv` connection/TLS/pool layer as the app
129/// record store and the same [`verify_state_pair`] verifier as the RocksDB
130/// reader — no divergent verification.
131#[cfg(feature = "postgres")]
132pub(super) struct CheckpointedPgAcceptedStateSource {
133    kv: hyprstream_pds::pgsql_kv::PgKv,
134    acceptance_identity: Arc<dyn AcceptanceVerifier>,
135    network_bootstrap: bool,
136}
137
138#[cfg(feature = "postgres")]
139impl CheckpointedPgAcceptedStateSource {
140    /// READ-ONLY connect to the provisioned records store. FATAL on any error
141    /// (including an unprovisioned store): a configured PostgreSQL authority
142    /// never falls back to the local RocksDB store, and resolver startup never
143    /// recreates security history.
144    pub(super) fn connect(
145        records: &hyprstream_pds::rds::RdsConfig,
146        acceptance_identity: crate::service::RegistryDeploymentVerifier,
147    ) -> Result<Self> {
148        let kv = records
149            .connect_kv_readonly()
150            .context("failed to open checkpointed PDS Postgres store")?;
151        Ok(Self {
152            kv,
153            acceptance_identity: Arc::new(acceptance_identity),
154            network_bootstrap: false,
155        })
156    }
157
158    #[cfg(test)]
159    pub(super) fn connect_test(
160        kv: hyprstream_pds::pgsql_kv::PgKv,
161        acceptance_identity: ed25519_dalek::VerifyingKey,
162    ) -> Self {
163        Self {
164            kv,
165            acceptance_identity: Arc::new(acceptance_identity),
166            network_bootstrap: false,
167        }
168    }
169
170    pub(super) fn with_network_bootstrap(mut self, required: bool) -> Self {
171        self.network_bootstrap = required;
172        self
173    }
174
175    /// The snapshot pair read for one subject — the Postgres counterpart of
176    /// `load_at9p_state_from_db`'s `rocksdb::DB::snapshot()` (one read-only
177    /// REPEATABLE READ transaction inside `get_batch`).
178    fn load_state(&self, subject: &str) -> Result<Option<AcceptedAt9pState>> {
179        let mut values = self
180            .kv
181            .get_batch(&[state_key(subject), checkpoint_key(subject)])
182            .context("checkpointed PDS Postgres read failed")?;
183        let checkpoint = values.pop().flatten();
184        let envelope = values.pop().flatten();
185        verify_state_pair(
186            subject,
187            envelope,
188            checkpoint,
189            self.acceptance_identity.as_ref(),
190        )
191    }
192
193    fn bootstrap_states(&self) -> Result<Vec<AcceptedAt9pState>> {
194        let state_prefix = b"at9p-state\0".to_vec();
195        let state_upper = hyprstream_pds::pgsql_kv::prefix_upper_bound(&state_prefix)
196            .ok_or_else(|| anyhow::anyhow!("at9p state prefix has no upper bound"))?;
197        let checkpoint_prefix = b"at9p-checkpoint\0".to_vec();
198        let checkpoint_upper = hyprstream_pds::pgsql_kv::prefix_upper_bound(&checkpoint_prefix)
199            .ok_or_else(|| anyhow::anyhow!("at9p checkpoint prefix has no upper bound"))?;
200        let snap = self
201            .kv
202            .read_snapshot(
203                &[
204                    (state_prefix, state_upper),
205                    (checkpoint_prefix, checkpoint_upper),
206                ],
207                &[],
208            )
209            .context("checkpointed PDS Postgres snapshot read failed")?;
210        let mut ranges = snap.ranges.into_iter();
211        let (states, checkpoints) = match (ranges.next(), ranges.next()) {
212            (Some(states), Some(checkpoints)) => (states, checkpoints),
213            _ => bail!("RDS read_snapshot returned fewer ranges than requested"),
214        };
215        let checkpoints: std::collections::BTreeMap<Vec<u8>, Vec<u8>> =
216            checkpoints.into_iter().collect();
217        let mut out = Vec::new();
218        for (key, envelope) in states {
219            let Some(subject) = key.strip_prefix(b"at9p-state\0".as_slice()) else {
220                continue;
221            };
222            let subject = std::str::from_utf8(subject)?;
223            let checkpoint = checkpoints.get(checkpoint_key(subject).as_slice()).cloned();
224            if let Some(state) = verify_state_pair(
225                subject,
226                Some(envelope),
227                checkpoint,
228                self.acceptance_identity.as_ref(),
229            )? {
230                out.push(state);
231            }
232        }
233        Ok(out)
234    }
235}
236
237#[cfg(feature = "postgres")]
238impl super::service::AcceptedStateSource for CheckpointedPgAcceptedStateSource {
239    fn accepted_state(&self, did: &str) -> Result<Option<AcceptedAt9pState>> {
240        let subject = did
241            .strip_prefix(DID_AT9P_PREFIX)
242            .ok_or_else(|| anyhow::anyhow!("identifier is not did:at9p: {did:?}"))?;
243        let state = self.load_state(subject)?;
244        if let Some(state) = &state {
245            anyhow::ensure!(state.did == did, "accepted at9p state DID mismatch");
246        }
247        Ok(state)
248    }
249
250    fn accepted_states(&self) -> Result<Vec<AcceptedAt9pState>> {
251        self.bootstrap_states()
252    }
253
254    fn bootstrap_endpoints(
255        &self,
256        service_name: &str,
257    ) -> Result<Option<Vec<crate::state_store::AnnouncedEndpoint>>> {
258        if !self.network_bootstrap || !matches!(service_name, "discovery" | "policy") {
259            return Ok(None);
260        }
261        let key = hyprstream_service::global_trust_store()
262            .resolve_one(service_name)
263            .ok_or_else(|| {
264                anyhow::anyhow!("trusted bootstrap {service_name} response key is missing")
265            })?;
266        super::service::project_bootstrap_endpoint(&self.bootstrap_states()?, service_name, &key)
267            .map(|endpoint| Some(vec![endpoint]))
268    }
269}
270
271/// Select the deployment's accepted-state authority backend from the resolved
272/// records-role binding. A configured PostgreSQL binding selects the networked
273/// store and FAILS CLOSED on any error — there is never a silent local
274/// fallback (a local store that missed the accepted-state history would
275/// silently deny every deployment identity). Unconfigured keeps the local
276/// RocksDB checkpoint store.
277#[cfg_attr(not(feature = "postgres"), allow(unused_variables))]
278pub(super) fn open_deployment_accepted_state_source(
279    store_path: &Path,
280    identity: crate::service::RegistryDeploymentVerifier,
281    records: &hyprstream_pds::rds::RdsConfig,
282    network_bootstrap: bool,
283) -> Result<Arc<dyn super::service::AcceptedStateSource>> {
284    if records.is_configured() {
285        #[cfg(feature = "postgres")]
286        {
287            return Ok(Arc::new(
288                CheckpointedPgAcceptedStateSource::connect(records, identity)?
289                    .with_network_bootstrap(network_bootstrap),
290            ));
291        }
292        #[cfg(not(feature = "postgres"))]
293        bail!(
294            "records RDS Postgres is configured but this binary lacks the discovery `postgres` feature;              refusing to silently fall back to the local store"
295        );
296    }
297    #[cfg(feature = "rocksdb")]
298    {
299        Ok(Arc::new(
300            CheckpointedPdsAcceptedStateSource::open(store_path, identity)?
301                .with_network_bootstrap(network_bootstrap),
302        ))
303    }
304    #[cfg(not(feature = "rocksdb"))]
305    {
306        // Fail closed: without `rocksdb` there is no local concrete authority.
307        let _ = store_path;
308        bail!("checkpointed PDS accepted-state authority requires the `rocksdb` feature")
309    }
310}
311
312/// RocksDB key for the first-boot provisioning marker.
313///
314/// Written by [`initialize_deployment_store`] (and any other path that creates
315/// a fresh empty store) and deleted by the registry service **in the same
316/// synchronous `WriteBatch` as the first accepted-state commit** — so the
317/// lifecycle transition is atomic with provisioning completion, not a
318/// best-effort filesystem side-effect. Its presence alongside an empty store
319/// is the ONLY lifecycle evidence that distinguishes a genuine first boot
320/// (store freshly provisioned, no accepted state written yet) from a
321/// steady-state store that lost its data. The QUIC startup gate reads it via
322/// [`PdsRecordStore::first_boot_pending`] in the app crate.
323#[cfg(feature = "rocksdb")]
324pub const FIRST_BOOT_KEY: &[u8] = b"first-boot-pending";
325
326/// Outcome of an explicit deployment checkpoint-store initialization.
327#[cfg(feature = "rocksdb")]
328#[derive(Clone, Copy, Debug, Eq, PartialEq)]
329pub enum DeploymentStoreInit {
330    /// A safely absent store was created and the first-boot marker written.
331    Created,
332    /// A healthy retained store was validated and reused; nothing was
333    /// re-initialized, rewritten, or discarded.
334    Reused,
335}
336
337/// Explicitly create the empty checkpoint store for a newly provisioned
338/// deployment. Resolver startup never calls this: a missing store there is
339/// treated as lost security history and fails closed.
340///
341/// Also writes the [`FIRST_BOOT_KEY`] so the QUIC startup gate can recognize
342/// this as a genuine first boot (defer-eligible) rather than data loss. The
343/// registry deletes the key (in the same RocksDB batch as the first
344/// accepted-state commit) once provisioning completes.
345///
346/// Reboot-idempotency contract:
347/// - absent path: create the store, write the marker, report
348///   [`DeploymentStoreInit::Created`];
349/// - retained path: reuse it only when it opens as a valid store (the
350///   RocksDB open runs recovery and rejects non-database/corrupt state) and
351///   serves a real read, then report [`DeploymentStoreInit::Reused`]
352///   without writing anything;
353/// - any other retained state (empty directory, garbage, unreadable store)
354///   fails closed with an error — invalid retained state never becomes a
355///   silent re-initialization.
356#[cfg(feature = "rocksdb")]
357pub(crate) fn initialize_deployment_store() -> Result<DeploymentStoreInit> {
358    let path = hyprstream_service::deployment_data_dir()?.join("pds-store");
359    initialize_deployment_store_at(&path)
360}
361
362/// Path-explicit core of [`initialize_deployment_store`], split out so the
363/// reboot-idempotency contract is testable against scratch directories.
364#[cfg(feature = "rocksdb")]
365fn initialize_deployment_store_at(path: &Path) -> Result<DeploymentStoreInit> {
366    if path.exists() {
367        // An existing but empty directory is retained state that carries no
368        // history, and RocksDB's own probe would leave partial artifacts in
369        // it — refuse it up front instead of initializing into it.
370        anyhow::ensure!(
371            !(path.is_dir() && std::fs::read_dir(path)?.next().is_none()),
372            "refusing to initialize over empty retained checkpointed PDS store directory at {}",
373            path.display()
374        );
375        let mut opts = rocksdb::Options::default();
376        opts.create_if_missing(false);
377        let db = rocksdb::DB::open(&opts, path).with_context(|| {
378            format!(
379                "refusing to initialize over invalid retained checkpointed PDS store at {}",
380                path.display()
381            )
382        })?;
383        // Force a real read: a store that opens but cannot serve its
384        // lifecycle key must fail closed here, not later at the QUIC gate.
385        db.get(FIRST_BOOT_KEY).with_context(|| {
386            format!(
387                "retained checkpointed PDS store at {} failed readback validation",
388                path.display()
389            )
390        })?;
391        return Ok(DeploymentStoreInit::Reused);
392    }
393    let mut opts = rocksdb::Options::default();
394    opts.create_if_missing(true);
395    let db = rocksdb::DB::open(&opts, path)
396        .with_context(|| format!("failed to initialize checkpointed PDS store at {path:?}"))?;
397    // Write the first-boot marker as a durable RocksDB key (not a filesystem
398    // side-effect) so its deletion can be atomic with the first accepted-state
399    // commit. A failed put propagates — provisioning must not appear successful
400    // if the marker wasn't written.
401    db.put(FIRST_BOOT_KEY, b"")
402        .with_context(|| format!("failed to write first-boot marker in store at {path:?}"))?;
403    Ok(DeploymentStoreInit::Created)
404}
405
406#[cfg(feature = "rocksdb")]
407impl super::service::AcceptedStateSource for CheckpointedPdsAcceptedStateSource {
408    fn accepted_state(&self, did: &str) -> Result<Option<AcceptedAt9pState>> {
409        self.accepted_state(did)
410    }
411
412    /// Every verified accepted state in the store — the roster universe the
413    /// derived-admission uniqueness rule (#1652) enumerates. This is the
414    /// deployment's provisioned identities AND any foreign record admitted
415    /// through the public ingest RPC; provenance is not distinguishable in
416    /// the store, so membership alone is never deployment authorization.
417    fn accepted_states(&self) -> Result<Vec<AcceptedAt9pState>> {
418        self.bootstrap_states()
419    }
420
421    fn bootstrap_endpoints(
422        &self,
423        service_name: &str,
424    ) -> Result<Option<Vec<crate::state_store::AnnouncedEndpoint>>> {
425        if !self.network_bootstrap || !matches!(service_name, "discovery" | "policy") {
426            return Ok(None);
427        }
428        let key = hyprstream_service::global_trust_store()
429            .resolve_one(service_name)
430            .ok_or_else(|| {
431                anyhow::anyhow!("trusted bootstrap {service_name} response key is missing")
432            })?;
433        super::service::project_bootstrap_endpoint(&self.bootstrap_states()?, service_name, &key)
434            .map(|endpoint| Some(vec![endpoint]))
435    }
436}
437
438#[derive(Clone, Copy, Debug, Eq, PartialEq)]
439struct At9pCheckpoint {
440    watermark: Watermark,
441    envelope_digest: [u8; 64],
442}
443
444#[cfg(feature = "rocksdb")]
445fn readonly_opts() -> rocksdb::Options {
446    let mut opts = rocksdb::Options::default();
447    opts.create_if_missing(false);
448    opts
449}
450
451fn state_key(subject: &str) -> Vec<u8> {
452    format!("at9p-state\0{subject}").into_bytes()
453}
454
455fn checkpoint_key(subject: &str) -> Vec<u8> {
456    format!("at9p-checkpoint\0{subject}").into_bytes()
457}
458
459fn acceptance_message(domain: &[u8], payload: &[u8]) -> Vec<u8> {
460    let mut message = Vec::with_capacity(domain.len() + payload.len());
461    message.extend_from_slice(domain);
462    message.extend_from_slice(payload);
463    message
464}
465
466fn checkpoint_message(subject: &str, payload: &[u8]) -> Result<Vec<u8>> {
467    let len = u32::try_from(subject.len()).context("at9p subject exceeds u32")?;
468    let mut out = Vec::with_capacity(AT9P_CHECKPOINT_AAD.len() + 4 + subject.len() + payload.len());
469    out.extend_from_slice(AT9P_CHECKPOINT_AAD);
470    out.extend_from_slice(&len.to_be_bytes());
471    out.extend_from_slice(subject.as_bytes());
472    out.extend_from_slice(payload);
473    Ok(out)
474}
475
476#[cfg(feature = "rocksdb")]
477fn load_at9p_state_from_db(
478    db: &rocksdb::DB,
479    subject: &str,
480    identity: &dyn AcceptanceVerifier,
481) -> Result<Option<AcceptedAt9pState>> {
482    let snapshot = db.snapshot();
483    let envelope = snapshot.get(state_key(subject))?;
484    let checkpoint = snapshot.get(checkpoint_key(subject))?;
485    verify_state_pair(subject, envelope, checkpoint, identity)
486}
487
488/// The backend-independent half of accepted-state reads: both stores hand the
489/// state envelope and monotonic checkpoint read from ONE snapshot to this
490/// verifier. There is exactly one verification path; a new backend never
491/// re-implements it.
492fn verify_state_pair(
493    subject: &str,
494    envelope: Option<Vec<u8>>,
495    checkpoint: Option<Vec<u8>>,
496    identity: &dyn AcceptanceVerifier,
497) -> Result<Option<AcceptedAt9pState>> {
498    match (envelope, checkpoint) {
499        (None, None) => Ok(None),
500        (Some(_), None) => bail!("accepted at9p state exists without its monotonic checkpoint"),
501        (None, Some(_)) => bail!("accepted at9p checkpoint exists without its state envelope"),
502        (Some(envelope), Some(checkpoint)) => {
503            let state = decode_state(subject, &envelope, identity)?;
504            let checkpoint = decode_checkpoint(subject, &checkpoint, identity)?;
505            anyhow::ensure!(
506                checkpoint.watermark == state.watermark(),
507                "accepted at9p checkpoint/body watermark mismatch"
508            );
509            anyhow::ensure!(
510                checkpoint.envelope_digest == h512(&envelope),
511                "accepted at9p checkpoint/state envelope mismatch"
512            );
513            Ok(Some(state))
514        }
515    }
516}
517
518fn decode_checkpoint(
519    subject: &str,
520    bytes: &[u8],
521    identity: &dyn AcceptanceVerifier,
522) -> Result<At9pCheckpoint> {
523    anyhow::ensure!(
524        bytes.len() == AT9P_CHECKPOINT_LEN,
525        "accepted at9p checkpoint length mismatch"
526    );
527    let payload = bytes
528        .get(..AT9P_CHECKPOINT_PAYLOAD_LEN)
529        .ok_or_else(|| anyhow::anyhow!("accepted at9p checkpoint payload missing"))?;
530    anyhow::ensure!(
531        payload.starts_with(AT9P_CHECKPOINT_MAGIC),
532        "accepted at9p checkpoint has bad version"
533    );
534    let signature = ed25519_dalek::Signature::from_bytes(
535        bytes
536            .get(AT9P_CHECKPOINT_PAYLOAD_LEN..)
537            .ok_or_else(|| anyhow::anyhow!("accepted at9p checkpoint signature missing"))?
538            .try_into()?,
539    );
540    identity
541        .verify_strict(&checkpoint_message(subject, payload)?, &signature)
542        .context("accepted at9p monotonic checkpoint signature rejected")?;
543    let epoch = u64::from_be_bytes(
544        payload
545            .get(8..16)
546            .ok_or_else(|| anyhow::anyhow!("checkpoint epoch missing"))?
547            .try_into()?,
548    );
549    let record_digest = payload
550        .get(16..80)
551        .ok_or_else(|| anyhow::anyhow!("checkpoint digest missing"))?
552        .try_into()?;
553    let terminal = match payload.get(80) {
554        Some(0) => false,
555        Some(1) => true,
556        _ => bail!("invalid checkpoint terminal flag"),
557    };
558    Ok(At9pCheckpoint {
559        watermark: Watermark {
560            epoch,
561            record_digest,
562            terminal,
563        },
564        envelope_digest: payload
565            .get(81..145)
566            .ok_or_else(|| anyhow::anyhow!("checkpoint envelope digest missing"))?
567            .try_into()?,
568    })
569}
570
571fn decode_state(
572    subject: &str,
573    bytes: &[u8],
574    acceptance_identity: &dyn AcceptanceVerifier,
575) -> Result<AcceptedAt9pState> {
576    anyhow::ensure!(
577        bytes.len() >= AT9P_ACCEPTANCE_PREFIX_LEN + AT9P_STATE_HEADER_LEN + ED25519_SIGNATURE_LEN,
578        "accepted at9p envelope is truncated"
579    );
580    anyhow::ensure!(
581        bytes.starts_with(AT9P_ACCEPTANCE_MAGIC),
582        "accepted at9p envelope has bad version"
583    );
584    let audit_key_bytes: [u8; ED25519_KEY_LEN] = bytes
585        .get(AT9P_ACCEPTANCE_MAGIC.len()..AT9P_ACCEPTANCE_MAGIC.len() + ED25519_KEY_LEN)
586        .ok_or_else(|| anyhow::anyhow!("accepted at9p envelope missing audit key"))?
587        .try_into()?;
588    let key_signature_start = AT9P_ACCEPTANCE_MAGIC.len() + ED25519_KEY_LEN;
589    let key_signature_end = key_signature_start + ED25519_SIGNATURE_LEN;
590    let key_signature = ed25519_dalek::Signature::from_bytes(
591        bytes
592            .get(key_signature_start..key_signature_end)
593            .ok_or_else(|| anyhow::anyhow!("accepted at9p envelope missing key certificate"))?
594            .try_into()?,
595    );
596    acceptance_identity
597        .verify_strict(
598            &acceptance_message(AT9P_ACCEPTANCE_KEY_AAD, &audit_key_bytes),
599            &key_signature,
600        )
601        .context("accepted at9p audit-key certificate rejected")?;
602    let audit_key = ed25519_dalek::VerifyingKey::from_bytes(&audit_key_bytes)
603        .context("accepted at9p envelope has malformed audit key")?;
604    let state_signature_start = bytes.len() - ED25519_SIGNATURE_LEN;
605    let body = bytes
606        .get(AT9P_ACCEPTANCE_PREFIX_LEN..state_signature_start)
607        .ok_or_else(|| anyhow::anyhow!("accepted at9p envelope missing state body"))?;
608    let state_signature = ed25519_dalek::Signature::from_bytes(
609        bytes
610            .get(state_signature_start..)
611            .ok_or_else(|| anyhow::anyhow!("accepted at9p envelope missing state signature"))?
612            .try_into()?,
613    );
614    audit_key
615        .verify_strict(
616            &acceptance_message(AT9P_ACCEPTANCE_STATE_AAD, body),
617            &state_signature,
618        )
619        .context("accepted at9p daemon acceptance signature rejected")?;
620    decode_state_body(subject, body)
621}
622
623fn decode_state_body(subject: &str, bytes: &[u8]) -> Result<AcceptedAt9pState> {
624    anyhow::ensure!(
625        bytes.len() >= AT9P_STATE_HEADER_LEN,
626        "accepted at9p state is truncated"
627    );
628    anyhow::ensure!(
629        bytes.starts_with(AT9P_STATE_MAGIC),
630        "accepted at9p state has bad version"
631    );
632    let kind = *bytes
633        .get(8)
634        .ok_or_else(|| anyhow::anyhow!("accepted at9p state missing kind"))?;
635    let epoch = u64::from_be_bytes(
636        bytes
637            .get(9..17)
638            .ok_or_else(|| anyhow::anyhow!("accepted at9p state missing epoch"))?
639            .try_into()?,
640    );
641    let digest: [u8; 64] = bytes
642        .get(17..81)
643        .ok_or_else(|| anyhow::anyhow!("accepted at9p state missing digest"))?
644        .try_into()?;
645    let terminal = match bytes.get(81) {
646        Some(0) => false,
647        Some(1) => true,
648        _ => bail!("accepted at9p state has invalid terminal flag"),
649    };
650    let head_len = u32::from_be_bytes(
651        bytes
652            .get(82..86)
653            .ok_or_else(|| anyhow::anyhow!("accepted at9p state missing length"))?
654            .try_into()?,
655    ) as usize;
656    anyhow::ensure!(
657        bytes.len() == AT9P_STATE_HEADER_LEN + head_len,
658        "accepted at9p state length mismatch"
659    );
660    let head = bytes
661        .get(AT9P_STATE_HEADER_LEN..)
662        .ok_or_else(|| anyhow::anyhow!("accepted at9p state missing head"))?;
663    let state = match kind {
664        0 => AcceptedAt9pState::from_persisted_genesis(head)?,
665        1 => AcceptedAt9pState::from_persisted_update(head)?,
666        _ => bail!("accepted at9p state has unknown head kind {kind}"),
667    };
668    anyhow::ensure!(
669        state.subject_cid512 == subject,
670        "accepted at9p state subject mismatch"
671    );
672    anyhow::ensure!(
673        state.epoch == epoch,
674        "accepted at9p state epoch/body mismatch"
675    );
676    anyhow::ensure!(
677        state.head_digest == digest,
678        "accepted at9p state digest/body mismatch"
679    );
680    anyhow::ensure!(
681        state.terminal == terminal,
682        "accepted at9p state terminal/body mismatch"
683    );
684    Ok(state)
685}
686
687#[cfg(all(test, feature = "rocksdb"))]
688pub(super) fn write_test_state(
689    path: &Path,
690    state: &AcceptedAt9pState,
691    identity: &ed25519_dalek::SigningKey,
692) -> Result<()> {
693    let audit = ed25519_dalek::SigningKey::from_bytes(&[0x43; 32]);
694    let (envelope, checkpoint) = tests::encode_fixture(state, identity, &audit);
695    let mut options = rocksdb::Options::default();
696    options.create_if_missing(true);
697    let db = rocksdb::DB::open(&options, path)?;
698    let mut batch = rocksdb::WriteBatch::default();
699    batch.put(state_key(&state.subject_cid512), envelope);
700    batch.put(checkpoint_key(&state.subject_cid512), checkpoint);
701    db.write(batch)?;
702    Ok(())
703}
704
705/// Remove one accepted state from the test store — the store-wipe half of a
706/// simulated service-identity re-initialization (#1652 churn tests: the
707/// successor state is then written by [`write_test_state`]).
708#[cfg(all(test, feature = "rocksdb"))]
709pub(super) fn remove_test_state(path: &Path, state: &AcceptedAt9pState) -> Result<()> {
710    let mut options = rocksdb::Options::default();
711    options.create_if_missing(true);
712    let db = rocksdb::DB::open(&options, path)?;
713    let mut batch = rocksdb::WriteBatch::default();
714    batch.delete(state_key(&state.subject_cid512));
715    batch.delete(checkpoint_key(&state.subject_cid512));
716    db.write(batch)?;
717    Ok(())
718}
719
720#[cfg(all(test, feature = "rocksdb"))]
721#[allow(clippy::expect_used, clippy::unwrap_used)]
722mod tests {
723    use super::*;
724    use ed25519_dalek::Signer as _;
725    use hyprstream_crypto::pq::{ml_dsa_generate_keypair, ml_dsa_vk_bytes};
726    use hyprstream_pds::at9p::{
727        CapsuleBody, HybridKeyPair, ServiceEndpoint, ServiceEntry, ServiceType,
728        Transport as At9pTransport,
729    };
730    use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
731
732    pub(super) fn accepted_state() -> AcceptedAt9pState {
733        let signing = ed25519_dalek::SigningKey::from_bytes(&[0x41; 32]);
734        let (pq_signing, pq_verifying) = ml_dsa_generate_keypair();
735        let keys = HybridKeyPair::new(
736            signing.verifying_key().to_bytes().to_vec(),
737            ml_dsa_vk_bytes(&pq_verifying),
738        )
739        .expect("hybrid keys");
740        let endpoint = ServiceEndpoint::new(At9pTransport::Iroh, "iroh://checkpointed-reach")
741            .expect("service endpoint");
742        let service = ServiceEntry::new("#checkpointed", ServiceType::NinePExport, endpoint)
743            .expect("service entry");
744        let capsule = sign_capsule(
745            CapsuleBody::new(vec![keys], vec![service]).expect("capsule body"),
746            &signing,
747            &pq_signing,
748        )
749        .expect("signed capsule");
750        AcceptedAt9pState::from_persisted_genesis(&capsule.to_dag_cbor().expect("capsule bytes"))
751            .expect("accepted genesis")
752    }
753
754    /// Reboot-idempotency: a safely absent store is created with the
755    /// first-boot marker, and a still-pending first boot re-runs as reuse.
756    #[test]
757    fn init_creates_absent_store_then_reuses_pending_first_boot() {
758        let dir = tempfile::tempdir().expect("tempdir");
759        let path = dir.path().join("pds-store");
760        assert_eq!(
761            initialize_deployment_store_at(&path).expect("fresh init"),
762            DeploymentStoreInit::Created
763        );
764        let mut opts = rocksdb::Options::default();
765        opts.create_if_missing(false);
766        {
767            let db = rocksdb::DB::open(&opts, &path).expect("reopen");
768            assert!(db.get(FIRST_BOOT_KEY).expect("marker read").is_some());
769        }
770        assert_eq!(
771            initialize_deployment_store_at(&path).expect("reboot init"),
772            DeploymentStoreInit::Reused
773        );
774        let db = rocksdb::DB::open(&opts, &path).expect("reopen after reuse");
775        assert!(
776            db.get(FIRST_BOOT_KEY).expect("marker read").is_some(),
777            "reuse of a pending first boot must not rewrite the marker"
778        );
779    }
780
781    /// Reboot-idempotency: a healthy retained store (provisioning completed,
782    /// marker deleted, real state committed) is reused with zero writes.
783    #[test]
784    fn init_reuses_healthy_retained_store_without_writes() {
785        let dir = tempfile::tempdir().expect("tempdir");
786        let path = dir.path().join("pds-store");
787        assert_eq!(
788            initialize_deployment_store_at(&path).expect("fresh init"),
789            DeploymentStoreInit::Created
790        );
791        let mut opts = rocksdb::Options::default();
792        opts.create_if_missing(false);
793        {
794            let db = rocksdb::DB::open(&opts, &path).expect("reopen");
795            db.delete(FIRST_BOOT_KEY)
796                .expect("registry deletes the marker");
797            db.put(b"retained-key", b"retained-value")
798                .expect("commit retained state");
799        }
800        assert_eq!(
801            initialize_deployment_store_at(&path).expect("reboot init"),
802            DeploymentStoreInit::Reused
803        );
804        let db = rocksdb::DB::open(&opts, &path).expect("reopen after reuse");
805        assert_eq!(
806            db.get(b"retained-key").expect("readback"),
807            Some(b"retained-value".to_vec()),
808            "reuse must preserve retained state"
809        );
810        assert!(
811            db.get(FIRST_BOOT_KEY).expect("marker read").is_none(),
812            "reuse must not resurrect the first-boot marker"
813        );
814    }
815
816    /// Fail closed on invalid retained state: neither garbage at the store
817    /// path nor an empty retained directory may be clobbered or silently
818    /// initialized over, and the retained bytes stay untouched.
819    #[test]
820    fn init_fails_closed_on_invalid_retained_state() {
821        let dir = tempfile::tempdir().expect("tempdir");
822        let garbage = dir.path().join("garbage-store");
823        std::fs::create_dir(&garbage).expect("mkdir");
824        std::fs::write(garbage.join("CURRENT"), b"not-a-rocksdb-store").expect("garbage");
825        let err = initialize_deployment_store_at(&garbage)
826            .expect_err("invalid retained state must fail closed");
827        assert!(
828            format!("{err:#}").contains("refusing to initialize"),
829            "fail-closed error names the refusal: {err:#}"
830        );
831        assert_eq!(
832            std::fs::read(garbage.join("CURRENT")).expect("garbage survives"),
833            b"not-a-rocksdb-store",
834            "failure must not clobber retained bytes"
835        );
836        let empty = dir.path().join("empty-store");
837        std::fs::create_dir(&empty).expect("mkdir");
838        assert!(
839            initialize_deployment_store_at(&empty).is_err(),
840            "an empty retained directory is not a safely absent store"
841        );
842        assert!(
843            std::fs::read_dir(&empty).expect("readdir").next().is_none(),
844            "failure must not initialize into the retained directory"
845        );
846    }
847
848    fn encode_body(state: &AcceptedAt9pState) -> Vec<u8> {
849        let (kind, head) = match state.head() {
850            hyprstream_pds::at9p_duplicity::AcceptedAt9pHead::Genesis(capsule) => {
851                (0, capsule.to_dag_cbor().expect("genesis bytes"))
852            }
853            hyprstream_pds::at9p_duplicity::AcceptedAt9pHead::Update(update) => {
854                (1, update.to_dag_cbor().expect("update bytes"))
855            }
856        };
857        let mut out = Vec::new();
858        out.extend_from_slice(AT9P_STATE_MAGIC);
859        out.push(kind);
860        out.extend_from_slice(&state.epoch.to_be_bytes());
861        out.extend_from_slice(&state.head_digest);
862        out.push(u8::from(state.terminal));
863        out.extend_from_slice(
864            &u32::try_from(head.len())
865                .expect("head length")
866                .to_be_bytes(),
867        );
868        out.extend_from_slice(&head);
869        out
870    }
871
872    #[test]
873    fn update_fixture_encodes_and_decodes_update_kind() {
874        let genesis = accepted_state();
875        let signing = ed25519_dalek::SigningKey::from_bytes(&[0x41; 32]);
876        let (pq_signing, pq_verifying) = ml_dsa_generate_keypair();
877        let keys = HybridKeyPair::new(
878            signing.verifying_key().to_bytes().to_vec(),
879            ml_dsa_vk_bytes(&pq_verifying),
880        )
881        .expect("hybrid keys");
882        let endpoint = ServiceEndpoint::new(At9pTransport::Iroh, "iroh://updated-reach")
883            .expect("service endpoint");
884        let service = ServiceEntry::new("#updated", ServiceType::NinePExport, endpoint)
885            .expect("service entry");
886        let update = sign_update_record(
887            genesis.subject_cid512.clone(),
888            1,
889            genesis.head_digest,
890            CapsuleBody::new(vec![keys], vec![service]).expect("capsule body"),
891            "2099-01-01T00:00:00Z".to_owned(),
892            &signing,
893            &pq_signing,
894        )
895        .expect("signed update");
896        let state =
897            AcceptedAt9pState::from_persisted_update(&update.to_dag_cbor().expect("update bytes"))
898                .expect("accepted update");
899        assert_eq!(encode_body(&state)[AT9P_STATE_MAGIC.len()], 1);
900
901        let identity = ed25519_dalek::SigningKey::from_bytes(&[0x45; 32]);
902        let audit = ed25519_dalek::SigningKey::from_bytes(&[0x46; 32]);
903        let (envelope, _) = encode_fixture(&state, &identity, &audit);
904        let decoded = decode_state(&state.subject_cid512, &envelope, &identity.verifying_key())
905            .expect("decode update fixture");
906        assert!(matches!(
907            decoded.head(),
908            hyprstream_pds::at9p_duplicity::AcceptedAt9pHead::Update(_)
909        ));
910    }
911
912    pub(super) fn encode_fixture(
913        state: &AcceptedAt9pState,
914        identity: &ed25519_dalek::SigningKey,
915        audit: &ed25519_dalek::SigningKey,
916    ) -> (Vec<u8>, Vec<u8>) {
917        let body = encode_body(state);
918        let audit_key = audit.verifying_key().to_bytes();
919        let key_signature = identity.sign(&acceptance_message(AT9P_ACCEPTANCE_KEY_AAD, &audit_key));
920        let state_signature = audit.sign(&acceptance_message(AT9P_ACCEPTANCE_STATE_AAD, &body));
921        let mut envelope = Vec::new();
922        envelope.extend_from_slice(AT9P_ACCEPTANCE_MAGIC);
923        envelope.extend_from_slice(&audit_key);
924        envelope.extend_from_slice(&key_signature.to_bytes());
925        envelope.extend_from_slice(&body);
926        envelope.extend_from_slice(&state_signature.to_bytes());
927
928        let mut payload = Vec::new();
929        payload.extend_from_slice(AT9P_CHECKPOINT_MAGIC);
930        payload.extend_from_slice(&state.epoch.to_be_bytes());
931        payload.extend_from_slice(&state.head_digest);
932        payload.push(u8::from(state.terminal));
933        payload.extend_from_slice(&h512(&envelope));
934        let checkpoint_signature = identity.sign(
935            &checkpoint_message(&state.subject_cid512, &payload).expect("checkpoint message"),
936        );
937        let mut checkpoint = payload;
938        checkpoint.extend_from_slice(&checkpoint_signature.to_bytes());
939        (envelope, checkpoint)
940    }
941
942    #[test]
943    fn concrete_checkpointed_pds_reader_accepts_only_matching_daemon_checkpoint() {
944        let dir = tempfile::tempdir().expect("tempdir");
945        let identity = ed25519_dalek::SigningKey::from_bytes(&[0x42; 32]);
946        let audit = ed25519_dalek::SigningKey::from_bytes(&[0x43; 32]);
947        let state = accepted_state();
948        let (envelope, checkpoint) = encode_fixture(&state, &identity, &audit);
949        let mut options = rocksdb::Options::default();
950        options.create_if_missing(true);
951        let db = rocksdb::DB::open(&options, dir.path()).expect("open fixture store");
952        db.put(state_key(&state.subject_cid512), envelope)
953            .expect("write accepted state");
954        db.put(checkpoint_key(&state.subject_cid512), checkpoint)
955            .expect("write checkpoint");
956        drop(db);
957
958        let source =
959            CheckpointedPdsAcceptedStateSource::open_test(dir.path(), identity.verifying_key())
960                .expect("open checkpointed source");
961        let recovered = source
962            .accepted_state(&state.did)
963            .expect("read checkpointed state")
964            .expect("state present");
965        assert_eq!(recovered.watermark(), state.watermark());
966
967        let wrong_identity = ed25519_dalek::SigningKey::from_bytes(&[0x44; 32]);
968        let rejected = CheckpointedPdsAcceptedStateSource::open_test(
969            dir.path(),
970            wrong_identity.verifying_key(),
971        )
972        .expect("open same store")
973        .accepted_state(&state.did);
974        assert!(
975            rejected.is_err(),
976            "caller-selected checkpoint identity accepted"
977        );
978    }
979}
980
981// ---------------------------------------------------------------------------
982// Postgres authority tests: the resolver-side reader against a real scratch
983// database (HYPRSTREAM_POSTGRES_TEST_URL_FILE), plus the fail-closed selection
984// contract that needs no database at all.
985// ---------------------------------------------------------------------------
986
987#[cfg(all(test, feature = "postgres", feature = "rocksdb"))]
988#[allow(clippy::expect_used, clippy::unwrap_used)]
989mod pg_tests {
990    use super::tests::{accepted_state, encode_fixture};
991    use super::*;
992    use crate::service::AcceptedStateSource as _;
993    use hyprstream_pds::pgsql_kv::PgKv;
994
995    /// Read the test database URL from a FILE, never from a direct env var
996    /// (metal v1.1 acceptance check 1 — the password never transits the
997    /// process environment).
998    fn test_url() -> Option<String> {
999        let path = std::env::var_os("HYPRSTREAM_POSTGRES_TEST_URL_FILE")?;
1000        let url = std::fs::read_to_string(&path)
1001            .unwrap_or_else(|e| panic!("read test URL file {}: {e}", path.to_string_lossy()));
1002        let url = url.trim();
1003        assert!(!url.is_empty(), "test URL file is empty");
1004        Some(url.to_owned())
1005    }
1006
1007    macro_rules! require_db {
1008        () => {{
1009            let Some(url) = test_url() else {
1010                eprintln!(
1011                    "skipping checkpointed PDS Postgres test: \
1012                     HYPRSTREAM_POSTGRES_TEST_URL_FILE unset"
1013                );
1014                return;
1015            };
1016            url
1017        }};
1018    }
1019
1020    fn deployment_verifier(
1021        seed: u8,
1022    ) -> (
1023        ed25519_dalek::SigningKey,
1024        crate::service::RegistryDeploymentVerifier,
1025    ) {
1026        let ed = ed25519_dalek::SigningKey::from_bytes(&[seed; 32]);
1027        let pq = hyprstream_crypto::pq::ml_dsa_sk_from_seed(&[seed; 32]);
1028        let verifier =
1029            crate::service::RegistryDeploymentVerifier::for_test_deployment_root(&ed, &pq)
1030                .expect("test deployment verifier");
1031        (ed, verifier)
1032    }
1033
1034    /// Run test-side DDL against the scratch server over a direct NoTls
1035    /// connection (database lifecycle for the unprovisioned gate).
1036    fn with_direct_connection(
1037        url: &str,
1038        ddl: impl Fn(
1039            tokio_postgres::Client,
1040        ) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>>,
1041    ) {
1042        let rt = tokio::runtime::Builder::new_current_thread()
1043            .enable_all()
1044            .build()
1045            .expect("ddl runtime");
1046        rt.block_on(async move {
1047            let config: tokio_postgres::Config = url.parse().expect("ddl URL parses");
1048            let (client, connection) = config
1049                .connect(tokio_postgres::NoTls)
1050                .await
1051                .expect("ddl connect");
1052            let handle = tokio::spawn(async move {
1053                if let Err(e) = connection.await {
1054                    eprintln!("ddl connection error: {e}");
1055                }
1056            });
1057            ddl(client).await;
1058            handle.await.expect("ddl connection join");
1059        });
1060    }
1061
1062    /// The repro's causal chain: accepted state written to PostgreSQL by the
1063    /// writer path is served read-only by the resolver authority — same
1064    /// envelopes, same checkpoint pairing, same verifier.
1065    #[test]
1066    fn live_pg_authority_serves_writer_committed_state() {
1067        let url = require_db!();
1068        let (identity, _) = deployment_verifier(0x61);
1069        let audit = ed25519_dalek::SigningKey::from_bytes(&[0x62; 32]);
1070        let state = accepted_state();
1071        let (envelope, checkpoint) = encode_fixture(&state, &identity, &audit);
1072
1073        // Writer posture: migrates and commits the pair.
1074        let writer = PgKv::connect_test(&url, "test-cell").expect("writer connect");
1075        writer
1076            .put(&state_key(&state.subject_cid512), &envelope)
1077            .expect("write state");
1078        writer
1079            .put(&checkpoint_key(&state.subject_cid512), &checkpoint)
1080            .expect("write checkpoint");
1081        drop(writer);
1082
1083        // Resolver posture: read-only, no migration.
1084        let reader = CheckpointedPgAcceptedStateSource::connect_test(
1085            PgKv::connect_test_readonly(&url).expect("readonly connect"),
1086            identity.verifying_key(),
1087        );
1088        let recovered = reader
1089            .accepted_state(&state.did)
1090            .expect("read accepted state")
1091            .expect("state present");
1092        assert_eq!(recovered.watermark(), state.watermark());
1093        let roster = reader.accepted_states().expect("roster read");
1094        assert!(
1095            roster
1096                .iter()
1097                .any(|s| s.subject_cid512 == state.subject_cid512),
1098            "roster must include the writer-committed state"
1099        );
1100
1101        // The verifier is binding: a caller-selected wrong identity rejects.
1102        let wrong = ed25519_dalek::SigningKey::from_bytes(&[0x63; 32]);
1103        let rejected = CheckpointedPgAcceptedStateSource::connect_test(
1104            PgKv::connect_test_readonly(&url).expect("readonly connect"),
1105            wrong.verifying_key(),
1106        )
1107        .accepted_state(&state.did);
1108        assert!(
1109            rejected.is_err(),
1110            "caller-selected checkpoint identity accepted"
1111        );
1112    }
1113
1114    /// A state envelope without its monotonic checkpoint is torn security
1115    /// state — the reader must refuse it, exactly like the RocksDB reader.
1116    #[test]
1117    fn live_pg_authority_rejects_torn_pair() {
1118        let url = require_db!();
1119        let (identity, _) = deployment_verifier(0x64);
1120        let audit = ed25519_dalek::SigningKey::from_bytes(&[0x65; 32]);
1121        let state = accepted_state();
1122        let (envelope, _) = encode_fixture(&state, &identity, &audit);
1123
1124        let nanos = std::time::SystemTime::now()
1125            .duration_since(std::time::UNIX_EPOCH)
1126            .map(|d| d.as_nanos())
1127            .unwrap_or(0);
1128        let subject = format!("torn-{}-{nanos}", std::process::id());
1129        let writer = PgKv::connect_test(&url, "test-cell").expect("writer connect");
1130        writer
1131            .put(&state_key(&subject), &envelope)
1132            .expect("write lone state");
1133
1134        let reader = CheckpointedPgAcceptedStateSource::connect_test(
1135            PgKv::connect_test_readonly(&url).expect("readonly connect"),
1136            identity.verifying_key(),
1137        );
1138        let error = match reader.accepted_state(&format!("did:at9p:{subject}")) {
1139            Ok(_) => panic!("torn pair must be rejected"),
1140            Err(error) => error,
1141        };
1142        assert!(
1143            error
1144                .to_string()
1145                .contains("without its monotonic checkpoint"),
1146            "unexpected error: {error}"
1147        );
1148
1149        // Isolation: the lone fixture row lives under the roster-enumerated
1150        // `at9p-state\0` prefix on a shared scratch database, and serial
1151        // execution alone does not remove persisted fixtures — a later roster
1152        // read would trip on the torn subject. Remove it explicitly.
1153        with_direct_connection(&url, |client| {
1154            let key = state_key(&subject);
1155            Box::pin(async move {
1156                client
1157                    .execute("DELETE FROM pds_kv WHERE key = $1", &[&key])
1158                    .await
1159                    .expect("cleanup delete");
1160            })
1161        });
1162    }
1163
1164    /// Resolver startup never recreates history: an unprovisioned database
1165    /// (no pds_kv at all) fails the read-only connect closed.
1166    #[test]
1167    fn live_pg_authority_fails_closed_when_unprovisioned() {
1168        let url = require_db!();
1169        let nanos = std::time::SystemTime::now()
1170            .duration_since(std::time::UNIX_EPOCH)
1171            .map(|d| d.as_nanos())
1172            .unwrap_or(0);
1173        let scratch = format!("pg_unprovisioned_{}_{nanos}", std::process::id());
1174        // A genuinely empty scratch database on the same server: the shared
1175        // test database keeps its provisioned pds_kv.
1176        with_direct_connection(&url, |client| {
1177            let scratch = scratch.clone();
1178            Box::pin(async move {
1179                client
1180                    .execute(&format!("CREATE DATABASE {scratch}"), &[])
1181                    .await
1182                    .expect("create unprovisioned scratch database");
1183            })
1184        });
1185        let (base, query) = match url.split_once('?') {
1186            Some((base, query)) => (base, format!("?{query}")),
1187            None => (url.as_str(), String::new()),
1188        };
1189        let (head, _) = base.rsplit_once('/').expect("test URL has a database path");
1190        let scoped = format!("{head}/{scratch}{query}");
1191        let error = match PgKv::connect_test_readonly(&scoped) {
1192            Ok(_) => panic!("unprovisioned read-only connect unexpectedly succeeded"),
1193            Err(error) => error,
1194        };
1195        assert!(
1196            format!("{error:#}").contains("not provisioned"),
1197            "unexpected error: {error:#}"
1198        );
1199        with_direct_connection(&url, |client| {
1200            let scratch = scratch.clone();
1201            Box::pin(async move {
1202                client
1203                    .execute(&format!("DROP DATABASE {scratch}"), &[])
1204                    .await
1205                    .expect("drop unprovisioned scratch database");
1206            })
1207        });
1208    }
1209
1210    /// Fail-closed selection: a configured RDS binding that cannot be read
1211    /// must error WITHOUT creating or consulting the local RocksDB store.
1212    #[test]
1213    fn configured_postgres_selection_never_falls_back_to_local() {
1214        let (_ed, verifier) = deployment_verifier(0x66);
1215        let dir = tempfile::tempdir().expect("tempdir");
1216        // url_file unreadable → the binding is configured but broken.
1217        let records = hyprstream_pds::rds::RdsConfig {
1218            url_file: Some(dir.path().join("missing-records-url")),
1219            root_cert_file: Some(dir.path().join("missing-rds-ca.pem")),
1220            cell_id: "test-cell".to_owned(),
1221        };
1222        assert!(records.is_configured());
1223        let local_store = dir.path().join("pds-store");
1224        let error =
1225            match open_deployment_accepted_state_source(&local_store, verifier, &records, false) {
1226                Ok(_) => panic!("a broken configured PostgreSQL binding unexpectedly succeeded"),
1227                Err(error) => error,
1228            };
1229        assert!(
1230            format!("{error:#}").contains("RDS url_file"),
1231            "unexpected error: {error:#}"
1232        );
1233        assert!(
1234            !local_store.exists(),
1235            "the failed configured-Postgres path created a local store"
1236        );
1237    }
1238}