Skip to main content

hyprstream_discovery/
service.rs

1//! Discovery service — exposes EndpointRegistry over RPC.
2//!
3//! Allows remote clients to discover registered services, their endpoints,
4//! socket kinds, and schemas via the standard REQ/REP transport.
5
6#[cfg(all(test, feature = "rocksdb", not(target_arch = "wasm32")))]
7#[path = "network_bootstrap_tests.rs"]
8mod network_bootstrap_tests;
9
10#[cfg(all(test, feature = "rocksdb", not(target_arch = "wasm32")))]
11#[path = "event_network_bootstrap_tests.rs"]
12mod event_network_bootstrap_tests;
13
14#[cfg(all(test, feature = "rocksdb", not(target_arch = "wasm32")))]
15#[path = "moql_checkpoint_admission_tests.rs"]
16mod moql_checkpoint_admission_tests;
17
18use async_trait::async_trait;
19use hyprstream_rpc::browser_provisioning::{
20    BrowserCarrierProfile, BrowserCurrentnessVerifier, BrowserProvisioningDocument,
21    BrowserProvisioningMaterial, BrowserProvisioningRequest, BrowserRequestBinding,
22    BrowserRouteRole, BrowserTransportSecurity,
23};
24use hyprstream_rpc::registry::{self, EndpointRegistry, SocketKind};
25use hyprstream_rpc::resolver::{Resolver, ResolverProfile, ServiceQuery};
26use hyprstream_rpc::rpc_client::{CallOptions, RpcClient};
27use hyprstream_rpc::service::{EnvelopeContext, RequestService};
28use hyprstream_rpc::stream_consumer::StreamHandle;
29use hyprstream_rpc::transport::{EndpointType, TransportConfig};
30use hyprstream_rpc::{SigningKey, VerifyingKey};
31
32use crate::generated::discovery_client::{
33    dispatch_discovery, serialize_response, DiscoveryHandler,
34};
35use crate::placement_index::PlacementIndex;
36use crate::scheduling;
37use crate::state_store::{
38    unix_millis_now, AnnouncedEndpoint, CachedEntityStatement, CachedEnvelopeKeyset,
39    DiscoveryState, DiscoveryStateStore, LiveAllocatable, MemoryStateStore, ANNOUNCED_ENDPOINT_TTL,
40};
41use hyprstream_rpc_std::discovery_client::{
42    AuthMetadata, AuthMetadataList, DiscoveryResponseVariant, EndpointInfo, EntityStatement,
43    EnvelopeKeyset, ErrorInfo, GetRecordRequest, IssuerList, NodeLiveness, PingInfo,
44    PlacementCandidate, PlacementCandidateSet, QueryCandidatesRequest, RecordCar,
45    RegisterEntityStatementRequest, RegisterEnvelopeKeysetRequest, Resource, ServiceAnnouncement,
46    ServiceEndpoints, ServiceList, ServiceSummary,
47};
48
49use anyhow::{Context, Result};
50use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
51use hyprstream_rpc::identity::Did;
52use hyprstream_util::ttl_cache::TtlCache;
53use std::collections::BTreeSet;
54#[cfg(any(test, feature = "test-fixtures"))]
55use std::collections::HashMap;
56use std::net::SocketAddr;
57use std::str::FromStr;
58use std::sync::Arc;
59use std::time::{Duration, Instant};
60use tracing::{info, trace};
61
62/// #524 P1 — liveness heartbeat TTL: a node with no live/fresh
63/// `reportNodeLiveness` entry is hard-excluded from `queryCandidates`
64/// (ratified decision — see the epic's issue comments), never just flagged
65/// stale. 45s sits mid-range of the 30-60s window floated in earlier design
66/// notes; named here so it's easy to retune.
67const LIVENESS_TTL: Duration = Duration::from_secs(45);
68
69/// Liveness cache capacity / inline-reap budget. Generous fleet-scale
70/// headroom; eviction is O(log n) and happens inline on every insert/get, so a
71/// large bound doesn't cost anything until it's actually full.
72const LIVENESS_CACHE_MAX_ENTRIES: usize = 16_384;
73const LIVENESS_CACHE_REAP_BUDGET: usize = 32;
74/// Bound retry work for an admitted DID whose repo is absent, invalid, or does
75/// not yet contain a node record. This prevents heartbeat-rate resolver polls
76/// while allowing eventual recovery when a placement record is later published.
77const PLACEMENT_INGEST_RETRY_TTL: Duration = Duration::from_secs(300);
78const CANDIDATE_QUERY_CONCURRENCY: usize = 16;
79const CANDIDATE_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
80
81#[derive(Clone, Copy, PartialEq, Eq)]
82enum PlacementIngestStatus {
83    Pending,
84    Complete,
85    Cancelled,
86}
87
88struct PlacementIngestGuard(Arc<parking_lot::Mutex<PlacementIngestStatus>>);
89
90impl Drop for PlacementIngestGuard {
91    fn drop(&mut self) {
92        let mut status = self.0.lock();
93        if *status == PlacementIngestStatus::Pending {
94            *status = PlacementIngestStatus::Cancelled;
95        }
96    }
97}
98
99/// Default bound applied to `queryCandidates` when the caller passes
100/// `maxCandidates == 0` (unspecified) — keeps an unscoped query from returning
101/// the entire fleet in one response.
102const DEFAULT_MAX_CANDIDATES: usize = 100;
103
104/// Private checkpoint-bound projection used only while Discovery validates an
105/// announcement against the daemon-owned accepted-state source.
106#[derive(Debug, Clone, PartialEq, Eq)]
107struct AcceptedStateEvidence {
108    service_did: Did,
109    digest: [u8; 64],
110    epoch: u64,
111    expires_at_unix_ms: i64,
112    response_ed25519: [u8; 32],
113    response_ml_dsa65: Vec<u8>,
114}
115
116#[derive(Debug, Clone, PartialEq, Eq)]
117struct AnchoredKemRecipient {
118    key_id: String,
119    recipient: hyprstream_rpc::crypto::hybrid_kem::RecipientPublic,
120    not_after_unix_ms: i64,
121}
122
123#[derive(Debug, Clone, PartialEq, Eq)]
124struct ServiceCandidate {
125    service_name: String,
126    service_did: Did,
127    response_verifying_key: [u8; 32],
128    response_ml_dsa65: Vec<u8>,
129    response_key_id: String,
130    request_kem_recipient: Option<AnchoredKemRecipient>,
131    transport: TransportConfig,
132    capabilities: BTreeSet<String>,
133    accepted_state: AcceptedStateEvidence,
134    source_signer: [u8; 32],
135    expires_at_unix_ms: i64,
136}
137
138#[derive(Debug, Clone, PartialEq, Eq)]
139struct CandidateDecision {
140    candidate_fingerprint: String,
141    accepted: bool,
142    reason: String,
143}
144
145#[derive(Debug, Clone, PartialEq, Eq)]
146struct ResolutionEvidence {
147    accepted_state_digest: [u8; 64],
148    accepted_state_epoch: u64,
149    selected_fingerprint: String,
150    ordered_decisions: Vec<CandidateDecision>,
151}
152
153/// Private raw selection result. It can become production authority only via
154/// `ResolvedService::mint`, which requires the typed accepted-state witness.
155#[derive(Debug, Clone)]
156struct SelectedService {
157    service_name: String,
158    service_did: Did,
159    response_verifying_key: VerifyingKey,
160    response_ml_dsa65: Vec<u8>,
161    response_key_id: String,
162    request_kem_recipient: AnchoredKemRecipient,
163    transport: TransportConfig,
164    expires_at_unix_ms: i64,
165    evidence: ResolutionEvidence,
166}
167
168impl SelectedService {
169    fn same_authority(&self, other: &Self) -> bool {
170        self.service_name == other.service_name
171            && self.service_did == other.service_did
172            && self.response_verifying_key == other.response_verifying_key
173            && self.response_ml_dsa65 == other.response_ml_dsa65
174            && self.response_key_id == other.response_key_id
175            && self.request_kem_recipient == other.request_kem_recipient
176            && self.evidence.accepted_state_digest == other.evidence.accepted_state_digest
177            && self.evidence.accepted_state_epoch == other.evidence.accepted_state_epoch
178    }
179    fn service_name(&self) -> &str {
180        &self.service_name
181    }
182
183    fn service_did(&self) -> &Did {
184        &self.service_did
185    }
186
187    fn response_verifying_key(&self) -> VerifyingKey {
188        self.response_verifying_key
189    }
190
191    fn response_ml_dsa65(&self) -> &[u8] {
192        &self.response_ml_dsa65
193    }
194
195    fn response_key_id(&self) -> &str {
196        &self.response_key_id
197    }
198    fn request_kem_recipient(&self) -> &AnchoredKemRecipient {
199        &self.request_kem_recipient
200    }
201
202    fn transport(&self) -> &TransportConfig {
203        &self.transport
204    }
205
206    fn evidence(&self) -> &ResolutionEvidence {
207        &self.evidence
208    }
209
210    fn expires_at_unix_ms(&self) -> i64 {
211        self.expires_at_unix_ms
212    }
213
214    fn ensure_fresh(&self, now_unix_ms: i64) -> Result<()> {
215        anyhow::ensure!(
216            now_unix_ms < self.expires_at_unix_ms,
217            "resolved service snapshot expired; re-resolution required"
218        );
219        anyhow::ensure!(
220            now_unix_ms < self.request_kem_recipient.not_after_unix_ms,
221            "resolved request KEM recipient expired; re-resolution required"
222        );
223        Ok(())
224    }
225
226    fn crypto_stores(
227        &self,
228    ) -> Result<(
229        Arc<dyn hyprstream_rpc::crypto::hybrid_kem::KemTrustStore>,
230        Arc<dyn hyprstream_rpc::envelope::PqTrustStore>,
231    )> {
232        let ed = self.response_verifying_key.to_bytes();
233        let mut kem = hyprstream_rpc::crypto::hybrid_kem::KeyedKemTrustStore::new();
234        kem.bind(ed, self.request_kem_recipient.recipient.clone());
235        let pq_key = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&self.response_ml_dsa65)
236            .map_err(|e| anyhow::anyhow!("invalid resolved ML-DSA-65 response key: {e}"))?;
237        let mut pq = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
238        pq.bind(ed, &pq_key);
239        Ok((Arc::new(kem), Arc::new(pq)))
240    }
241}
242
243fn canonical_service_name(name: &str) -> Result<String> {
244    let canonical = name.trim().to_ascii_lowercase();
245    anyhow::ensure!(!canonical.is_empty(), "service name is empty");
246    anyhow::ensure!(
247        canonical
248            .bytes()
249            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'),
250        "service name contains non-canonical characters"
251    );
252    anyhow::ensure!(canonical == name, "service name is not canonical");
253    Ok(canonical)
254}
255
256fn network_reach(transport: &TransportConfig) -> bool {
257    matches!(
258        transport.endpoint,
259        EndpointType::Quic { .. } | EndpointType::Iroh { .. }
260    )
261}
262
263fn native_iroh_reach(transport: &TransportConfig) -> bool {
264    matches!(transport.endpoint, EndpointType::Iroh { .. })
265}
266
267fn browser_webtransport_reach(transport: &TransportConfig) -> bool {
268    matches!(transport.endpoint, EndpointType::Quic { .. })
269}
270
271fn transport_fingerprint(transport: &TransportConfig) -> String {
272    blake3::hash(format!("{transport:?}").as_bytes())
273        .to_hex()
274        .to_string()
275}
276
277fn hash_framed(hasher: &mut blake3::Hasher, bytes: &[u8]) {
278    hasher.update(&(bytes.len() as u64).to_be_bytes());
279    hasher.update(bytes);
280}
281
282fn candidate_fingerprint(candidate: &ServiceCandidate) -> String {
283    let mut h = blake3::Hasher::new();
284    hash_framed(&mut h, candidate.service_name.as_bytes());
285    hash_framed(&mut h, candidate.service_did.as_str().as_bytes());
286    h.update(&candidate.response_verifying_key);
287    hash_framed(&mut h, &candidate.response_ml_dsa65);
288    hash_framed(&mut h, candidate.response_key_id.as_bytes());
289    if let Some(kem) = &candidate.request_kem_recipient {
290        hash_framed(&mut h, kem.key_id.as_bytes());
291        hash_framed(&mut h, &kem.recipient.encode());
292    }
293    hash_framed(
294        &mut h,
295        transport_fingerprint(&candidate.transport).as_bytes(),
296    );
297    for capability in &candidate.capabilities {
298        hash_framed(&mut h, capability.as_bytes());
299    }
300    h.update(&candidate.accepted_state.digest);
301    h.update(&candidate.accepted_state.epoch.to_be_bytes());
302    hash_framed(
303        &mut h,
304        candidate.accepted_state.service_did.as_str().as_bytes(),
305    );
306    h.update(&candidate.accepted_state.expires_at_unix_ms.to_be_bytes());
307    h.update(&candidate.accepted_state.response_ed25519);
308    hash_framed(&mut h, &candidate.accepted_state.response_ml_dsa65);
309    h.update(&candidate.source_signer);
310    h.update(&candidate.expires_at_unix_ms.to_be_bytes());
311    if let Some(kem) = &candidate.request_kem_recipient {
312        h.update(&kem.not_after_unix_ms.to_be_bytes());
313    }
314    h.finalize().to_hex().to_string()
315}
316
317fn rejection_reason(
318    query: &ServiceQuery,
319    candidate: &ServiceCandidate,
320    now_unix_ms: i64,
321) -> Option<&'static str> {
322    if canonical_service_name(&candidate.service_name)
323        .ok()
324        .as_deref()
325        != Some(query.service_name.as_str())
326    {
327        return Some("service-name-mismatch");
328    }
329    if !candidate.service_did.is_did_at9p()
330        || candidate.service_did != candidate.accepted_state.service_did
331    {
332        return Some("missing-or-mismatched-accepted-identity");
333    }
334    if candidate.source_signer != candidate.accepted_state.response_ed25519 {
335        return Some("announcement-signer-not-current");
336    }
337    if candidate.response_verifying_key != candidate.accepted_state.response_ed25519
338        || candidate.response_ml_dsa65 != candidate.accepted_state.response_ml_dsa65
339    {
340        return Some("response-key-not-current");
341    }
342    if !candidate
343        .response_key_id
344        .starts_with(&format!("{}#", candidate.service_did))
345        || hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&candidate.response_ml_dsa65).is_err()
346    {
347        return Some("missing-or-invalid-response-key-material");
348    }
349    if !query
350        .required_capabilities
351        .is_subset(&candidate.capabilities)
352    {
353        return Some("missing-capability");
354    }
355    if candidate.expires_at_unix_ms <= now_unix_ms
356        || candidate.accepted_state.expires_at_unix_ms <= now_unix_ms
357    {
358        return Some("expired-candidate-or-state");
359    }
360    match query.profile {
361        ResolverProfile::NetworkDiscovery if !network_reach(&candidate.transport) => {
362            return Some("local-reach-for-network-profile");
363        }
364        ResolverProfile::NativeIrohRequired if !native_iroh_reach(&candidate.transport) => {
365            return Some("non-iroh-reach-for-native-profile");
366        }
367        ResolverProfile::BrowserWebTransport
368            if !browser_webtransport_reach(&candidate.transport) =>
369        {
370            return Some("non-quic-reach-for-browser-profile");
371        }
372        ResolverProfile::LocalInproc
373            if !matches!(candidate.transport.endpoint, EndpointType::Inproc { .. }) =>
374        {
375            return Some("non-inproc-reach-for-local-profile");
376        }
377        ResolverProfile::Ipc
378            if !matches!(
379                candidate.transport.endpoint,
380                EndpointType::Ipc { .. } | EndpointType::SystemdFd { .. }
381            ) =>
382        {
383            return Some("non-ipc-reach-for-ipc-profile");
384        }
385        _ => {}
386    }
387    let Some(kem) = &candidate.request_kem_recipient else {
388        return Some("missing-request-kem-recipient");
389    };
390    if !kem
391        .key_id
392        .starts_with(&format!("{}#", candidate.service_did))
393        || kem.not_after_unix_ms <= now_unix_ms
394        || kem.recipient.suite_id
395            != hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768
396        || kem.recipient.eks.len() != kem.recipient.suite_id.components().len()
397        || hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::decode(&kem.recipient.encode())
398            .is_err()
399    {
400        return Some("invalid-expired-or-nonhybrid-kem-recipient");
401    }
402    None
403}
404
405fn select_service_candidate(
406    query: &ServiceQuery,
407    candidates: Vec<ServiceCandidate>,
408    now_unix_ms: i64,
409) -> Result<SelectedService> {
410    let mut decisions = Vec::with_capacity(candidates.len());
411    let mut valid = Vec::new();
412    for candidate in candidates {
413        let fingerprint = candidate_fingerprint(&candidate);
414        if let Some(reason) = rejection_reason(query, &candidate, now_unix_ms) {
415            decisions.push(CandidateDecision {
416                candidate_fingerprint: fingerprint,
417                accepted: false,
418                reason: reason.to_owned(),
419            });
420        } else {
421            decisions.push(CandidateDecision {
422                candidate_fingerprint: fingerprint.clone(),
423                accepted: true,
424                reason: "validated".to_owned(),
425            });
426            valid.push((fingerprint, candidate));
427        }
428    }
429    anyhow::ensure!(!valid.is_empty(), "no validated service candidates");
430    let authority = &valid[0].1;
431    anyhow::ensure!(
432        valid.iter().all(|(_, candidate)| {
433            candidate.service_did == authority.service_did
434                && candidate.response_verifying_key == authority.response_verifying_key
435                && candidate.response_ml_dsa65 == authority.response_ml_dsa65
436                && candidate.response_key_id == authority.response_key_id
437                && candidate.request_kem_recipient == authority.request_kem_recipient
438                && candidate.accepted_state == authority.accepted_state
439        }),
440        "ambiguous validated service authority"
441    );
442    valid.sort_by(|(af, a), (bf, b)| {
443        b.accepted_state
444            .epoch
445            .cmp(&a.accepted_state.epoch)
446            .then_with(|| b.expires_at_unix_ms.cmp(&a.expires_at_unix_ms))
447            .then_with(|| a.service_did.cmp(&b.service_did))
448            .then_with(|| {
449                transport_fingerprint(&a.transport).cmp(&transport_fingerprint(&b.transport))
450            })
451            .then_with(|| af.cmp(bf))
452    });
453    valid.dedup_by(|(af, _), (bf, _)| af == bf);
454    let (selected_fingerprint, selected) = valid.remove(0);
455    decisions.sort_by(|a, b| a.candidate_fingerprint.cmp(&b.candidate_fingerprint));
456    let response_verifying_key = VerifyingKey::from_bytes(&selected.response_verifying_key)
457        .map_err(|e| anyhow::anyhow!("invalid selected response verifying key: {e}"))?;
458    let request_kem_recipient = selected
459        .request_kem_recipient
460        .ok_or_else(|| anyhow::anyhow!("validated candidate lost request KEM recipient"))?;
461    let expires_at_unix_ms = selected
462        .expires_at_unix_ms
463        .min(selected.accepted_state.expires_at_unix_ms)
464        .min(request_kem_recipient.not_after_unix_ms);
465    Ok(SelectedService {
466        service_name: selected.service_name,
467        service_did: selected.service_did,
468        response_verifying_key,
469        response_ml_dsa65: selected.response_ml_dsa65,
470        response_key_id: selected.response_key_id,
471        request_kem_recipient,
472        transport: selected.transport,
473        expires_at_unix_ms,
474        evidence: ResolutionEvidence {
475            accepted_state_digest: selected.accepted_state.digest,
476            accepted_state_epoch: selected.accepted_state.epoch,
477            selected_fingerprint,
478            ordered_decisions: decisions,
479        },
480    })
481}
482
483fn select_service_candidates(
484    query: &ServiceQuery,
485    mut candidates: Vec<ServiceCandidate>,
486    now_unix_ms: i64,
487) -> Result<Vec<SelectedService>> {
488    let mut ordered = Vec::new();
489    loop {
490        let selected = match select_service_candidate(query, candidates.clone(), now_unix_ms) {
491            Ok(selected) => selected,
492            Err(error) if !ordered.is_empty() && error.to_string().contains("no validated") => {
493                break;
494            }
495            Err(error) => return Err(error),
496        };
497        let fingerprint = selected.evidence.selected_fingerprint.clone();
498        candidates.retain(|candidate| candidate_fingerprint(candidate) != fingerprint);
499        ordered.push(selected);
500    }
501    Ok(ordered)
502}
503
504/// Convert the wire `SelectorOp` (generated from `discovery.capnp`) to the
505/// shared scheduling-substrate `SelectorOp` (`crate::scheduling`, #628).
506/// Deliberately exhaustive (no catch-all) so a future wire variant fails to
507/// compile here instead of silently matching nothing.
508fn to_scheduling_op(
509    op: hyprstream_rpc_std::discovery_client::SelectorOp,
510) -> scheduling::SelectorOp {
511    use hyprstream_rpc_std::discovery_client::SelectorOp as Wire;
512    match op {
513        Wire::In => scheduling::SelectorOp::In,
514        Wire::NotIn => scheduling::SelectorOp::NotIn,
515        Wire::Exists => scheduling::SelectorOp::Exists,
516        Wire::DoesNotExist => scheduling::SelectorOp::DoesNotExist,
517        Wire::Gt => scheduling::SelectorOp::Gt,
518        Wire::Lt => scheduling::SelectorOp::Lt,
519    }
520}
521
522// ============================================================================
523// AuthorizationProvider trait (D5)
524// ============================================================================
525
526/// Trait for pluggable authorization checks.
527///
528/// Decouples DiscoveryService from PolicyClient. The main `hyprstream` crate
529/// provides a `PolicyAuthProvider` that wraps `PolicyClient`.
530#[async_trait(?Send)]
531pub trait AuthorizationProvider: Send + Sync {
532    /// Verified ingress context, including original holder evidence for
533    /// service-mediated checks. Policy-backed providers override this seam.
534    async fn check_context(
535        &self,
536        ctx: &EnvelopeContext,
537        resource: &str,
538        operation: &str,
539    ) -> Result<bool> {
540        self.check(
541            &ctx.subject().to_string(),
542            "*",
543            resource,
544            operation,
545            ctx.jwt_token(),
546        )
547        .await
548    }
549
550    async fn check_batch_context(
551        &self,
552        ctx: &EnvelopeContext,
553        resources: &[String],
554        operation: &str,
555    ) -> Result<Vec<bool>> {
556        self.check_batch(
557            &ctx.subject().to_string(),
558            "*",
559            resources,
560            operation,
561            ctx.jwt_token(),
562        )
563        .await
564    }
565
566    /// One bounded authorization vector; implementations may amortize RPC
567    /// overhead but must retain one decision per resource in input order.
568    async fn check_batch(
569        &self,
570        subject: &str,
571        domain: &str,
572        resources: &[String],
573        operation: &str,
574        bearer: Option<&str>,
575    ) -> Result<Vec<bool>> {
576        anyhow::ensure!(resources.len() <= 256, "authorization batch exceeds 256");
577        let mut allowed = Vec::with_capacity(resources.len());
578        for resource in resources {
579            allowed.push(
580                self.check(subject, domain, resource, operation, bearer)
581                    .await
582                    .unwrap_or(false),
583            );
584        }
585        Ok(allowed)
586    }
587
588    /// Check if a subject is authorized for the given operation on a resource.
589    async fn check(
590        &self,
591        subject: &str,
592        domain: &str,
593        resource: &str,
594        operation: &str,
595        bearer: Option<&str>,
596    ) -> Result<bool>;
597}
598
599// ============================================================================
600// RecordResolver trait (#431)
601// ============================================================================
602
603/// A record returned as a CARv1 proof (the in-memory form of `RecordCar`).
604///
605/// `uri` is the at:// URI this CAR answers; `car` is the CARv1 bytes
606/// (roots = [commit CID]; blocks = commit + MST path + record). The caller
607/// validates it offline via `hyprstream_pds::car::verify_record_proof` — the
608/// DiscoveryService is an untrusted relay, so integrity comes from the signed
609/// proof, never from trusting the responder.
610#[derive(Clone, Debug)]
611pub struct RecordCarData {
612    pub uri: String,
613    pub car: Vec<u8>,
614}
615
616/// Trait for resolving atproto records/repos from the node's local PDS.
617///
618/// Decouples DiscoveryService from `hyprstream-pds` (a pure crypto/metadata
619/// crate with no networking) and from the per-account record stores held by the
620/// main `hyprstream` crate. The main crate provides a `PdsRecordResolver` that
621/// builds CAR proofs via `hyprstream_pds::car::build_record_proof_car`.
622///
623/// IMPORTANT: a resolver MUST NOT perform access control — confidentiality is
624/// enforced by the DiscoveryService handler's Casbin check on the target
625/// DID/collection *before* the resolver is consulted. The resolver only answers
626/// "does this record exist, and what is its signed CAR proof".
627#[async_trait(?Send)]
628pub trait RecordResolver: Send + Sync {
629    /// Resolve a single record to a CAR proof. `collection` is the NSID
630    /// (e.g. `ai.hyprstream.model`), `rkey` the TID record key. Returns
631    /// `Ok(None)` when the DID/collection/rkey does not name a stored record.
632    async fn resolve_record(
633        &self,
634        did: &str,
635        collection: &str,
636        rkey: &str,
637    ) -> Result<Option<RecordCarData>>;
638
639    /// Resolve a full repo CAR for a DID (commit + MST + all records).
640    /// Returns `Ok(None)` when no repo is stored for the DID.
641    async fn resolve_repo(&self, did: &str) -> Result<Option<RecordCarData>>;
642
643    /// Resolve the DID's `#atproto` P-256 verifying key — the key a repo CAR's
644    /// commit is signed by — or `Ok(None)` when this resolver cannot establish
645    /// one.
646    ///
647    /// This is the **signature-verification seam** for verified-by-construction
648    /// ingest: [`PlacementIndex::ingest_did`] requires a key and verifies the
649    /// ingested repo CAR's commit signature against it *before* any record
650    /// enters the index. A failed resolution, including `Ok(None)`, is rejected
651    /// at that trust boundary; it never authorizes an unverified fallback.
652    /// The default keeps non-participating resolvers source-compatible while
653    /// preserving fail-closed ingest.
654    async fn resolve_verifying_key(&self, _did: &str) -> Result<Option<p256::ecdsa::VerifyingKey>> {
655        Ok(None)
656    }
657
658    /// Resolve every bounded `#atproto` verification slot currently published
659    /// for `did`. Implementations that only know one trusted active key retain
660    /// the fail-closed default through [`RecordResolver::resolve_verifying_key`].
661    async fn resolve_verifying_keys(
662        &self,
663        did: &str,
664    ) -> Result<Option<hyprstream_pds::commit::PublishedAtprotoKeys>> {
665        Ok(self
666            .resolve_verifying_key(did)
667            .await?
668            .map(hyprstream_pds::commit::PublishedAtprotoKeys::single))
669    }
670}
671
672// ============================================================================
673// DiscoveryService
674// ============================================================================
675
676/// Checkpoint-verifying accepted-current-state read used by production
677/// resolution. Implemented by the daemon-owned PDS reader from #1004.
678pub(super) trait AcceptedStateSource: Send + Sync {
679    fn accepted_state(
680        &self,
681        did: &str,
682    ) -> Result<Option<hyprstream_pds::at9p_duplicity::AcceptedAt9pState>>;
683
684    /// Every verified accepted state in the store (the full roster universe:
685    /// provisioned deployment identities and foreign ingested records alike).
686    /// Required — there is deliberately no silent empty default — because the
687    /// #1652 derived-admission uniqueness rule enumerates it per lookup.
688    fn accepted_states(&self) -> Result<Vec<hyprstream_pds::at9p_duplicity::AcceptedAt9pState>>;
689
690    fn bootstrap_endpoints(&self, _service_name: &str) -> Result<Option<Vec<AnnouncedEndpoint>>> {
691        Ok(None)
692    }
693}
694
695/// Opaque production authority minted only while holding the checkpoint/PDS
696/// accepted-state object. Raw announcement candidates never have this type.
697#[derive(Clone)]
698struct ResolvedService {
699    selected: SelectedService,
700}
701
702impl ResolvedService {
703    fn mint(
704        selected: SelectedService,
705        state: &hyprstream_pds::at9p_duplicity::AcceptedAt9pState,
706    ) -> Result<Self> {
707        anyhow::ensure!(
708            selected.service_did().as_str() == state.did,
709            "selected service DID is not the checkpoint-verified subject"
710        );
711        anyhow::ensure!(
712            selected.evidence().accepted_state_epoch == state.epoch,
713            "selected service epoch is not checkpoint current"
714        );
715        anyhow::ensure!(
716            selected.evidence().accepted_state_digest == state.head_digest,
717            "selected service digest is not checkpoint current"
718        );
719        let current_key = state
720            .current
721            .subject_keys
722            .iter()
723            .find(|key| {
724                key.ed25519_pub.as_slice()
725                    == selected.response_verifying_key().to_bytes().as_slice()
726            })
727            .ok_or_else(|| {
728                anyhow::anyhow!("selected response key is not one of the accepted current keys")
729            })?;
730        anyhow::ensure!(
731            selected.response_ml_dsa65() == current_key.mldsa65_pub.as_slice(),
732            "selected PQ response key is not the atomically-bound accepted current key"
733        );
734        anyhow::ensure!(
735            state
736                .current
737                .services
738                .iter()
739                .any(|service| service.id == format!("#{}", selected.service_name())),
740            "selected service is not present in accepted current state"
741        );
742        Ok(Self { selected })
743    }
744    fn same_authority(&self, other: &Self) -> bool {
745        self.selected.same_authority(&other.selected)
746    }
747}
748
749impl std::ops::Deref for ResolvedService {
750    type Target = SelectedService;
751    fn deref(&self) -> &Self::Target {
752        &self.selected
753    }
754}
755
756struct CurrentStreamHandle {
757    inner: Box<dyn StreamHandle>,
758    resolver: Arc<DiscoveryServiceResolver>,
759    snapshot: ResolvedService,
760    invalidated: bool,
761}
762
763#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)]
764#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))]
765impl StreamHandle for CurrentStreamHandle {
766    async fn next_payload(&mut self) -> Result<Option<hyprstream_rpc::streaming::StreamPayload>> {
767        anyhow::ensure!(!self.invalidated, "stream snapshot is invalidated");
768        if let Err(error) = self.resolver.ensure_current(&self.snapshot).await {
769            self.invalidated = true;
770            return Err(error);
771        }
772        self.inner.next_payload().await
773    }
774    async fn cancel(&mut self) -> Result<()> {
775        anyhow::ensure!(!self.invalidated, "stream snapshot is invalidated");
776        if let Err(error) = self.resolver.ensure_current(&self.snapshot).await {
777            self.invalidated = true;
778            return Err(error);
779        }
780        self.inner.cancel().await
781    }
782    fn stream_id(&self) -> &str {
783        self.inner.stream_id()
784    }
785    fn is_completed(&self) -> bool {
786        self.invalidated || self.inner.is_completed()
787    }
788}
789
790/// Cloneable production resolver installed after Discovery bootstrap.
791struct DiscoveryServiceResolver {
792    state_store: Arc<dyn DiscoveryStateStore>,
793    accepted_state_source: Arc<dyn AcceptedStateSource>,
794    discovery_client: Option<hyprstream_rpc_std::discovery_client::DiscoveryClient>,
795}
796
797/// Parse an `at://<did>/<collection>/<rkey>` URI into its three components.
798///
799/// The DID itself may contain `:` (e.g. `did:web:alice.example.com`,
800/// `did:plc:abc123`) but no `/`, so splitting the post-`at://` remainder on `/`
801/// yields exactly `[did, collection, rkey]`. Returns an error for any other shape.
802fn parse_at_uri(uri: &str) -> Result<(String, String, String)> {
803    let rest = uri
804        .strip_prefix("at://")
805        .ok_or_else(|| anyhow::anyhow!("at-uri must start with \"at://\": {uri:?}"))?;
806    let mut parts = rest.splitn(3, '/');
807    let did = parts.next().filter(|s| !s.is_empty());
808    let collection = parts.next().filter(|s| !s.is_empty());
809    let rkey = parts.next().filter(|s| !s.is_empty());
810    match (did, collection, rkey) {
811        (Some(d), Some(c), Some(r)) => Ok((d.to_owned(), c.to_owned(), r.to_owned())),
812        _ => anyhow::bail!("at-uri must be at://<did>/<collection>/<rkey>: {uri:?}"),
813    }
814}
815
816fn unix_seconds_now() -> i64 {
817    use std::time::{SystemTime, UNIX_EPOCH};
818    SystemTime::now()
819        .duration_since(UNIX_EPOCH)
820        .map(|d| d.as_secs() as i64)
821        .unwrap_or(0)
822}
823
824/// Discovery service that exposes EndpointRegistry over RPC.
825pub struct DiscoveryService {
826    /// Timestamp when the service was created
827    started_at: Instant,
828    /// Ed25519 signing key for envelope signing
829    signing_key: Arc<SigningKey>,
830    /// JWT verifying key for service JWT verification (derived from root via HKDF)
831    jwt_verifying_key: hyprstream_rpc::prelude::VerifyingKey,
832    /// JWT key source for client JWT verification (RequestService trait)
833    jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
834    /// OAuth issuer URL for RFC 9728 metadata (None = not configured)
835    oauth_issuer_url: Option<String>,
836    /// Expected audience for JWT validation (resource URL)
837    expected_audience: Option<String>,
838    /// Authorization provider (None = no authorization)
839    auth_provider: Option<Box<dyn AuthorizationProvider>>,
840    /// Record resolver backing getRecord/getRepo (#431). None = no local PDS,
841    /// so getRecord/getRepo report NOT_FOUND for everything.
842    record_resolver: Option<Arc<dyn RecordResolver>>,
843    accepted_state_source: Option<Arc<dyn AcceptedStateSource>>,
844    /// Backend-neutral volatile state. Memory is the safe single-process/WASM
845    /// default; configured native deployments may install shared/tiered stores.
846    state_store: Arc<dyn DiscoveryStateStore>,
847    /// Pre-computed TLS endorsement: Sign(tls_key, ed25519_pubkey || domain).
848    /// Empty when TLS endorsement is not available (e.g. self-signed certs).
849    tls_endorsement: Vec<u8>,
850    /// Domain the TLS endorsement covers (empty when no endorsement).
851    tls_domain: String,
852    /// #524 P1 — durable placement directory (labels/declared-resources/group
853    /// consents), refreshed by polling `record_resolver` lazily for an admitted
854    /// heartbeat DID with no ingested `NodeRecord` yet.
855    placement_index: PlacementIndex,
856    /// Bounded retry gate for first-seen placement repository polls. A DID is
857    /// marked before resolver access, so absent/invalid/non-node repos cannot
858    /// turn heartbeat frequency into unbounded work.
859    placement_ingest_attempts: TtlCache<Did, Arc<parking_lot::Mutex<PlacementIngestStatus>>>,
860    // Infrastructure (for Spawnable)
861    transport: TransportConfig,
862}
863
864/// Owned publisher sharing Discovery's actual state and verification sources.
865pub struct DiscoverySelfAnnouncer {
866    validator: DiscoveryService,
867}
868
869impl DiscoverySelfAnnouncer {
870    pub async fn publish(&self, announcement: &ServiceAnnouncement) -> Result<()> {
871        anyhow::ensure!(
872            announcement.service_name == "discovery",
873            "self publication is only for Discovery"
874        );
875        anyhow::ensure!(
876            announcement.service_did.is_did_at9p(),
877            "self publication requires an accepted identity"
878        );
879        let mut ctx = EnvelopeContext::from_callback_service(0, "discovery");
880        ctx.cnf = self.validator.signing_key.verifying_key().to_bytes();
881        self.validator
882            .store_announcement(&ctx, announcement)
883            .await?;
884        Ok(())
885    }
886}
887
888impl DiscoveryService {
889    pub fn self_announcer(&self) -> Result<DiscoverySelfAnnouncer> {
890        let source = self
891            .accepted_state_source
892            .clone()
893            .ok_or_else(|| anyhow::anyhow!("self publication requires accepted-state authority"))?;
894        let mut validator = Self::new(
895            self.signing_key.clone(),
896            self.jwt_verifying_key,
897            self.transport.clone(),
898        );
899        validator.state_store = self.state_store.clone();
900        validator.accepted_state_source = Some(source);
901        validator.jwt_key_source = self.jwt_key_source.clone();
902        validator.expected_audience = self.expected_audience.clone();
903        Ok(DiscoverySelfAnnouncer { validator })
904    }
905
906    async fn store_announcement(
907        &self,
908        ctx: &EnvelopeContext,
909        data: &ServiceAnnouncement,
910    ) -> Result<DiscoveryResponseVariant> {
911        info!(
912            "Discovery: service '{}' announced {} endpoint: {} (from {})",
913            data.service_name,
914            data.socket_kind,
915            data.endpoint,
916            ctx.subject()
917        );
918
919        let svc_name = data.service_name.clone();
920        let sock_kind = data.socket_kind.clone();
921        let endpoint = data.endpoint.clone();
922        let service_jwt = data.service_jwt.clone().unwrap_or_default();
923        let identity_bound = !data.service_did.as_str().is_empty()
924            || !data.accepted_state_digest.is_empty()
925            || !data.request_kem_recipient.is_empty();
926        if identity_bound {
927            anyhow::ensure!(
928                !service_jwt.is_empty(),
929                "identity-bound announcement requires a verified service JWT"
930            );
931            anyhow::ensure!(
932                data.service_did.is_did_at9p()
933                    && data.accepted_state_digest.len() == 64
934                    && !data.capabilities.is_empty()
935                    && data
936                        .response_key_id
937                        .starts_with(&format!("{}#", data.service_did))
938                    && data
939                        .request_kem_key_id
940                        .starts_with(&format!("{}#", data.service_did))
941                    && data.expires_at_unix_ms > unix_millis_now(),
942                "identity-bound announcement metadata is incomplete or expired"
943            );
944            let recipient = hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::decode(
945                &data.request_kem_recipient,
946            )?;
947            anyhow::ensure!(
948                recipient.suite_id
949                    == hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768
950                    && recipient.eks.len() == recipient.suite_id.components().len(),
951                "identity-bound announcement requires suite-complete hybrid KEM material"
952            );
953            match data.socket_kind.as_str() {
954                "quic" => {
955                    parse_announced_quic(&data.endpoint)?;
956                }
957                "iroh" => {
958                    parse_announced_iroh(&data.endpoint)?;
959                }
960                _ => {
961                    anyhow::bail!("identity-bound network announcement requires QUIC or Iroh reach")
962                }
963            }
964        }
965
966        // R3: Verify service JWT signature + subject matches serviceName.
967        // Full JWT verification (not decode_unverified) to prevent forged identities.
968        if !service_jwt.is_empty() {
969            // Service JWTs are minted hybrid (ML-DSA-65-Ed25519); the composite
970            // kid resolves through the key source (ledger pairs + the CA
971            // composite pair). Classical EdDSA remains accepted here for
972            // tokens minted before the hybrid cutover.
973            let is_composite = hyprstream_rpc::auth::jwt::header_alg(&service_jwt)
974                .ok()
975                .flatten()
976                .is_some_and(|alg| alg == "ML-DSA-65-Ed25519");
977            let verified = if is_composite {
978                let dispatch =
979                    hyprstream_rpc::auth::jwt::parse_composite_dispatch(&service_jwt, &["wit+jwt"])
980                        .map_err(|e| {
981                            tracing::warn!("Service JWT dispatch failed in announce: {}", e);
982                            anyhow::anyhow!("Invalid service JWT in announce: {}", e)
983                        })?;
984                let pair = self
985                    .jwt_key_source
986                    .as_ref()
987                    .and_then(|ks| ks.composite_pair(dispatch.kid()))
988                    .ok_or_else(|| {
989                        anyhow::anyhow!("Invalid service JWT in announce: unknown composite kid")
990                    })?;
991                // Signing-domain separation: announcements assert a service
992                // identity, which is certified by the Policy/CA domain only —
993                // the ledger's Policy slot and the derived CA pair (registered
994                // under the Policy role). The OAuth-role pair signs browser
995                // and workload WITs and must not be able to certify a service
996                // announcement.
997                anyhow::ensure!(
998                    pair.role() == hyprstream_rpc::auth::CompositePairRole::Policy,
999                    "Invalid service JWT in announce: composite pair role is not authorized \
1000                     for service certification"
1001                );
1002                hyprstream_rpc::auth::jwt::decode_composite(
1003                    &service_jwt,
1004                    pair.ml_dsa(),
1005                    pair.ed25519(),
1006                    self.expected_audience.as_deref(),
1007                    &dispatch,
1008                )
1009                .map_err(|e| {
1010                    tracing::warn!("Service JWT verification failed in announce: {}", e);
1011                    anyhow::anyhow!("Invalid service JWT in announce: {}", e)
1012                })?
1013            } else {
1014                hyprstream_rpc::auth::jwt::decode_with_key(
1015                    &service_jwt,
1016                    &self.jwt_verifying_key,
1017                    self.expected_audience.as_deref(),
1018                )
1019                .map_err(|e| {
1020                    tracing::warn!("Service JWT verification failed in announce: {}", e);
1021                    anyhow::anyhow!("Invalid service JWT in announce: {}", e)
1022                })?
1023            };
1024            // Check that sub matches "service:{serviceName}"
1025            let expected_sub = format!("service:{}", svc_name);
1026            if verified.sub != expected_sub {
1027                anyhow::bail!(
1028                    "Service JWT subject mismatch: expected '{}', got '{}'",
1029                    expected_sub,
1030                    verified.sub
1031                );
1032            }
1033            anyhow::ensure!(
1034                verified.cnf_key_bytes() == Some(ctx.cnf),
1035                "service JWT confirmation key does not match verified announcement signer"
1036            );
1037        }
1038
1039        let now_unix_ms = unix_millis_now();
1040        let heartbeat_expiry =
1041            now_unix_ms.saturating_add(ANNOUNCED_ENDPOINT_TTL.as_millis() as i64);
1042        // Legacy clients omit identity expiry (Cap'n Proto defaults it to 0).
1043        // Only identity-bound announcements carry a signed expiry constraint.
1044        let expires_at_unix_ms = if identity_bound {
1045            data.expires_at_unix_ms
1046        } else {
1047            heartbeat_expiry
1048        };
1049        let mut live_until_unix_ms = expires_at_unix_ms.min(heartbeat_expiry);
1050        if identity_bound {
1051            if let Some(source) = &self.accepted_state_source {
1052                let state = source
1053                    .accepted_state(data.service_did.as_str())?
1054                    .ok_or_else(|| {
1055                        anyhow::anyhow!("announcement DID has no accepted-current state")
1056                    })?;
1057                anyhow::ensure!(
1058                    state
1059                        .current
1060                        .subject_keys
1061                        .iter()
1062                        .any(|key| key.ed25519_pub == ctx.cnf),
1063                    "announcement signer is not an accepted current subject key"
1064                );
1065                let service = state
1066                    .current
1067                    .services
1068                    .iter()
1069                    .find(|entry| entry.id == format!("#{svc_name}"))
1070                    .ok_or_else(|| anyhow::anyhow!("announcement service is not accepted"))?;
1071                // Checkpoint discipline (the same rule as the fixed bootstrap
1072                // roles): only capsule-signed material binds request
1073                // encryption and Iroh reach. A legacy identity whose accepted
1074                // entry carries no signed request KEM cannot authorize one, so
1075                // an identity-bound announcement over it must not present
1076                // unbound KEM or reach material.
1077                let kem = service.endpoint.request_kem.as_ref().ok_or_else(|| {
1078                    anyhow::anyhow!(
1079                        "accepted service entry lacks a signed request KEM; \
1080                         reprovision service identity"
1081                    )
1082                })?;
1083                anyhow::ensure!(
1084                    kem == &data.request_kem_recipient,
1085                    "announcement KEM differs from accepted service"
1086                );
1087                if data.socket_kind == "iroh" {
1088                    anyhow::ensure!(
1089                        service.endpoint.address == data.endpoint,
1090                        "announcement reach differs from accepted service"
1091                    );
1092                }
1093                anyhow::ensure!(
1094                    state.epoch == data.accepted_state_epoch
1095                        && state.head_digest.as_slice() == data.accepted_state_digest.as_slice(),
1096                    "announcement does not match accepted-current state"
1097                );
1098                live_until_unix_ms = live_until_unix_ms.min(accepted_expiry_unix_ms(&state)?);
1099            }
1100        }
1101        anyhow::ensure!(
1102            live_until_unix_ms > now_unix_ms,
1103            "announcement effective lifetime is already expired"
1104        );
1105
1106        let replacement = AnnouncedEndpoint {
1107            socket_kind: sock_kind.clone(),
1108            endpoint: endpoint.clone(),
1109            service_did: data.service_did.clone(),
1110            capabilities: data.capabilities.iter().cloned().collect(),
1111            accepted_state_digest: data.accepted_state_digest.clone(),
1112            accepted_state_epoch: data.accepted_state_epoch,
1113            response_key_id: data.response_key_id.clone(),
1114            request_kem_key_id: data.request_kem_key_id.clone(),
1115            request_kem_recipient: data.request_kem_recipient.clone(),
1116            expires_at_unix_ms,
1117            source_signer: ctx.cnf,
1118            live_until_unix_ms,
1119        };
1120        self.state_store
1121            .put_announcement(&svc_name, replacement)
1122            .await?;
1123
1124        Ok(DiscoveryResponseVariant::AnnounceResult)
1125    }
1126
1127    /// Populate this replica's verified placement projection for an admitted
1128    /// shared live node. Failed/absent repos use the existing bounded retry gate.
1129    async fn ensure_placement_ingested(&self, node: &Did) -> bool {
1130        if self.placement_index.record_uri(node.as_str()).is_some() {
1131            return true;
1132        }
1133        let attempt = Arc::new(parking_lot::Mutex::new(PlacementIngestStatus::Pending));
1134        let attempt = if self.placement_ingest_attempts.insert_if_absent(
1135            node.clone(),
1136            Arc::clone(&attempt),
1137            PLACEMENT_INGEST_RETRY_TTL,
1138        ) {
1139            attempt
1140        } else {
1141            let Some(existing) = self.placement_ingest_attempts.get(node) else {
1142                return false;
1143            };
1144            {
1145                let mut status = existing.lock();
1146                match *status {
1147                    PlacementIngestStatus::Complete => return true,
1148                    PlacementIngestStatus::Pending => return false,
1149                    PlacementIngestStatus::Cancelled => *status = PlacementIngestStatus::Pending,
1150                }
1151            }
1152            existing
1153        };
1154        // A cancelled query must not turn an unfinished ingest into a cached
1155        // absence. Its next query can retry; concurrent queries fail closed.
1156        let _guard = PlacementIngestGuard(Arc::clone(&attempt));
1157        if let Some(resolver) = &self.record_resolver {
1158            if let Err(e) = self
1159                .placement_index
1160                .ingest_did(resolver.as_ref(), node.as_str())
1161                .await
1162            {
1163                tracing::warn!(
1164                    node = %node,
1165                    error = %e,
1166                    "placement directory ingestion failed for live node (liveness still recorded)"
1167                );
1168            }
1169        }
1170        *attempt.lock() = PlacementIngestStatus::Complete;
1171        true
1172    }
1173
1174    /// Create a new discovery service with infrastructure.
1175    ///
1176    /// `signing_key` is used for envelope signing (should be the per-service key
1177    /// derived via HKDF). `jwt_verifying_key` is the root-derived JWT verifying
1178    /// key for verifying service JWTs (derived from root via HKDF "hyprstream-jwt-v1").
1179    pub fn new(
1180        signing_key: Arc<SigningKey>,
1181        jwt_verifying_key: hyprstream_rpc::prelude::VerifyingKey,
1182        transport: TransportConfig,
1183    ) -> Self {
1184        Self {
1185            started_at: Instant::now(),
1186            signing_key,
1187            jwt_verifying_key,
1188            jwt_key_source: None,
1189            oauth_issuer_url: None,
1190            expected_audience: None,
1191            auth_provider: None,
1192            record_resolver: None,
1193            accepted_state_source: None,
1194            state_store: MemoryStateStore::production_default(),
1195            tls_endorsement: Vec::new(),
1196            tls_domain: String::new(),
1197            placement_index: PlacementIndex::new(),
1198            placement_ingest_attempts: TtlCache::new(
1199                LIVENESS_CACHE_MAX_ENTRIES,
1200                LIVENESS_CACHE_REAP_BUDGET,
1201            ),
1202            transport,
1203        }
1204    }
1205
1206    /// Install the configured volatile-state backend before the service is shared.
1207    pub fn with_state(mut self, state: DiscoveryState) -> Self {
1208        self.state_store = state.into_inner();
1209        self
1210    }
1211
1212    /// Set the pre-computed TLS endorsement and domain.
1213    ///
1214    /// The endorsement is `Sign(tls_private_key, "TLS_ENDORSEMENT_V1" || ed25519_pubkey || domain)`,
1215    /// computed once at startup by the service factory where TLS key materials are available.
1216    /// Discovery never touches the TLS private key directly.
1217    pub fn with_tls_endorsement(mut self, endorsement: Vec<u8>, domain: String) -> Self {
1218        self.tls_endorsement = endorsement;
1219        self.tls_domain = domain;
1220        self
1221    }
1222
1223    /// Set the OAuth issuer URL for RFC 9728 auth metadata responses.
1224    pub fn with_oauth_issuer(mut self, url: String) -> Self {
1225        self.oauth_issuer_url = Some(url);
1226        self
1227    }
1228
1229    /// Set the expected audience for JWT validation.
1230    pub fn with_expected_audience(mut self, audience: String) -> Self {
1231        self.expected_audience = Some(audience);
1232        self
1233    }
1234
1235    /// Set the JWT key source for client JWT verification.
1236    pub fn with_jwt_key_source(
1237        mut self,
1238        src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
1239    ) -> Self {
1240        self.jwt_key_source = Some(src);
1241        self
1242    }
1243
1244    /// Set the authorization provider for access control.
1245    pub fn with_auth_provider(mut self, provider: Box<dyn AuthorizationProvider>) -> Self {
1246        self.auth_provider = Some(provider);
1247        self
1248    }
1249
1250    /// Set the record resolver backing getRecord/getRepo (#431).
1251    pub fn with_record_resolver(mut self, resolver: Arc<dyn RecordResolver>) -> Self {
1252        self.record_resolver = Some(resolver);
1253        self
1254    }
1255
1256    #[cfg(test)]
1257    fn with_accepted_state_source(mut self, source: Arc<dyn AcceptedStateSource>) -> Self {
1258        self.accepted_state_source = Some(source);
1259        self
1260    }
1261
1262    /// Prove that downstream crates cannot access any production authority
1263    /// construction or installation seam.
1264    ///
1265    /// ```compile_fail
1266    /// use hyprstream_discovery::AcceptedStateSource;
1267    /// ```
1268    ///
1269    /// ```compile_fail
1270    /// use hyprstream_discovery::DiscoveryService;
1271    /// fn inject(service: DiscoveryService) {
1272    ///     service.with_accepted_state_source(todo!());
1273    /// }
1274    /// ```
1275    ///
1276    /// ```compile_fail
1277    /// use hyprstream_discovery::DiscoveryService;
1278    /// fn install_caller_authority(service: &DiscoveryService) {
1279    ///     service.install_production_resolver().unwrap();
1280    /// }
1281    /// ```
1282    ///
1283    /// ```compile_fail
1284    /// use hyprstream_discovery::DiscoveryService;
1285    /// fn choose_store_and_key(service: &mut DiscoveryService) {
1286    ///     service.install_checkpointed_pds_resolver(todo!(), todo!()).unwrap();
1287    /// }
1288    /// ```
1289    ///
1290    /// ```compile_fail
1291    /// use hyprstream_discovery::CheckpointedPdsAuthority;
1292    /// ```
1293    ///
1294    /// ```compile_fail
1295    /// use hyprstream_discovery::DiscoveryService;
1296    /// fn ambient_production_constructor() {
1297    ///     let _ = DiscoveryService::new_production(todo!(), todo!(), todo!());
1298    /// }
1299    /// ```
1300    ///
1301    /// ```compile_fail
1302    /// use hyprstream_service::DiscoveryBootstrapAuthority;
1303    /// fn forge(path: std::path::PathBuf, key: ed25519_dalek::VerifyingKey) {
1304    ///     let _ = DiscoveryBootstrapAuthority { store_path: path, acceptance_identity: key };
1305    /// }
1306    /// ```
1307    ///
1308    /// ```compile_fail
1309    /// fn duplicate(authority: hyprstream_service::DiscoveryBootstrapAuthority) {
1310    ///     let _ = authority.clone();
1311    /// }
1312    /// ```
1313    ///
1314    /// ```compile_fail
1315    /// use hyprstream_discovery::AuthenticatedDiscoveryBootstrap;
1316    /// fn forge(
1317    ///     store_path: std::path::PathBuf,
1318    ///     acceptance_identity: ed25519_dalek::VerifyingKey,
1319    /// ) -> AuthenticatedDiscoveryBootstrap {
1320    ///     AuthenticatedDiscoveryBootstrap { seal: (), store_path, acceptance_identity }
1321    /// }
1322    /// ```
1323    ///
1324    /// ```compile_fail
1325    /// use hyprstream_service::AuthenticatedRegistryDeploymentIdentity;
1326    /// fn forge(key: ed25519_dalek::VerifyingKey) -> AuthenticatedRegistryDeploymentIdentity {
1327    ///     AuthenticatedRegistryDeploymentIdentity { verifying_key: key }
1328    /// }
1329    /// ```
1330    ///
1331    /// ```compile_fail
1332    /// use hyprstream_service::AuthenticatedRegistryDeploymentIdentity;
1333    /// fn mint(key: ed25519_dalek::VerifyingKey) {
1334    ///     let _ = AuthenticatedRegistryDeploymentIdentity::mint(key);
1335    /// }
1336    /// ```
1337    ///
1338    /// ```compile_fail
1339    /// fn duplicate(identity: hyprstream_service::AuthenticatedRegistryDeploymentIdentity) {
1340    ///     let _ = identity.clone();
1341    /// }
1342    /// ```
1343    ///
1344    /// ```compile_fail
1345    /// # use ed25519_dalek::SigningKey;
1346    /// let signing = SigningKey::from_bytes(&[7; 32]);
1347    /// let mut context = hyprstream_service::ServiceContext::new(
1348    ///     signing.clone(), signing.verifying_key(), false, "caller".into()
1349    /// );
1350    /// context.authenticate_registry_deployment_credential("caller-issued-jwt")?;
1351    /// # Ok::<(), anyhow::Error>(())
1352    /// ```
1353    ///
1354    /// ```compile_fail
1355    /// # use ed25519_dalek::SigningKey;
1356    /// let signing = SigningKey::from_bytes(&[7; 32]);
1357    /// let mut context = hyprstream_service::ServiceContext::new(
1358    ///     signing.clone(), signing.verifying_key(), false, "caller".into()
1359    /// );
1360    /// context.authenticate_registry_deployment_credential()?;
1361    /// context.take_authenticated_registry_identity()?;
1362    /// let _raw = context.authenticated_registry_identity()?;
1363    /// # Ok::<(), anyhow::Error>(())
1364    /// ```
1365    ///
1366    /// ```compile_fail
1367    /// fn old_discovery_chain(
1368    ///     identity: hyprstream_service::AuthenticatedRegistryDeploymentIdentity,
1369    ///     client: hyprstream_rpc_std::discovery_client::DiscoveryClient,
1370    /// ) {
1371    ///     let authority = hyprstream_discovery::authenticate_discovery_bootstrap(identity).unwrap();
1372    ///     hyprstream_discovery::DiscoveryService::bootstrap_authenticated_process(authority, client).unwrap();
1373    /// }
1374    /// ```
1375    ///
1376    /// ```compile_fail
1377    /// fn extract(verifier: hyprstream_discovery::RegistryDeploymentVerifier) {
1378    ///     let _: ed25519_dalek::VerifyingKey = verifier.into_verifying_key();
1379    /// }
1380    /// ```
1381    ///
1382    /// ```compile_fail
1383    /// use hyprstream_discovery::RegistryDeploymentVerifier;
1384    /// fn forge(key: ed25519_dalek::VerifyingKey) -> RegistryDeploymentVerifier {
1385    ///     RegistryDeploymentVerifier { verifying_key: key }
1386    /// }
1387    /// ```
1388    ///
1389    /// The complete former public composition chain is unavailable to an
1390    /// external crate.
1391    /// ```compile_fail
1392    /// use hyprstream_discovery::DiscoveryService;
1393    /// use hyprstream_service::ServiceContext;
1394    /// fn inject(context: ServiceContext, client: hyprstream_rpc_std::discovery_client::DiscoveryClient) {
1395    ///     context.seal_discovery_bootstrap_authority().unwrap();
1396    ///     let authority = context
1397    ///         .consume_discovery_bootstrap_authority(|path, key| Ok((path.to_owned(), key)))
1398    ///         .unwrap();
1399    ///     DiscoveryService::install_process_bootstrap(&context, client).unwrap();
1400    ///     drop(authority);
1401    /// }
1402    /// ```
1403    #[cfg(not(target_arch = "wasm32"))]
1404    // Without `rocksdb` every authority arm fails closed before any use, so
1405    // the remainder of this body is intentionally unreachable there.
1406    #[cfg_attr(
1407        not(any(feature = "rocksdb", feature = "postgres")),
1408        allow(unreachable_code, unused_variables)
1409    )]
1410    fn bootstrap_authenticated_process(
1411        authority: ProcessBootstrapAuthority,
1412        discovery_client: hyprstream_rpc_std::discovery_client::DiscoveryClient,
1413        records: &hyprstream_pds::rds::RdsConfig,
1414    ) -> Result<()> {
1415        let authority = {
1416            let mut state = PROCESS_BOOTSTRAP_AUTHORITY.lock();
1417            match std::mem::replace(&mut *state, ProcessBootstrapAuthorityState::Consumed) {
1418                ProcessBootstrapAuthorityState::Sealed => {}
1419                previous => {
1420                    *state = previous;
1421                    anyhow::bail!("authenticated Discovery bootstrap is unavailable or consumed");
1422                }
1423            }
1424            authority
1425        };
1426        let deployment_identity = match &authority.acceptance_identity {
1427            ProcessAcceptanceIdentity::Deployment(identity) => Some(identity.clone()),
1428            #[cfg(test)]
1429            ProcessAcceptanceIdentity::Test(_) => None,
1430        };
1431        let source: Arc<dyn AcceptedStateSource> = match authority.acceptance_identity {
1432            #[cfg(any(feature = "rocksdb", feature = "postgres"))]
1433            ProcessAcceptanceIdentity::Deployment(identity) => crate::checkpointed_pds::open_deployment_accepted_state_source(
1434                &authority.store_path,
1435                identity,
1436                records,
1437                authority.network_required,
1438            )?,
1439            // Fail closed: without a concrete store backend there is no
1440            // accepted-state authority — never a silent empty resolver.
1441            #[cfg(not(any(feature = "rocksdb", feature = "postgres")))]
1442            ProcessAcceptanceIdentity::Deployment(_) => anyhow::bail!(
1443                "checkpointed PDS accepted-state authority requires the `rocksdb` or `postgres` feature"
1444            ),
1445            #[cfg(all(test, feature = "rocksdb"))]
1446            ProcessAcceptanceIdentity::Test(identity) => Arc::new(
1447                crate::checkpointed_pds::CheckpointedPdsAcceptedStateSource::open_test(
1448                    &authority.store_path,
1449                    identity,
1450                )?,
1451            ),
1452            #[cfg(all(test, not(feature = "rocksdb")))]
1453            ProcessAcceptanceIdentity::Test(_) => anyhow::bail!(
1454                "checkpointed PDS accepted-state authority requires the `rocksdb` feature"
1455            ),
1456        };
1457        PROCESS_ACCEPTED_STATE_SOURCE
1458            .set(Arc::clone(&source))
1459            .map_err(|_| anyhow::anyhow!("process Discovery authority is already consumed"))?;
1460        PROCESS_NATIVE_NETWORK_REQUIRED
1461            .set(authority.network_required)
1462            .map_err(|_| anyhow::anyhow!("process native profile is already installed"))?;
1463        if let Some(identity) = deployment_identity {
1464            PROCESS_REGISTRY_VERIFIER.set(identity).map_err(|_| {
1465                anyhow::anyhow!("deployment registry verifier is already installed")
1466            })?;
1467        }
1468        PRODUCTION_RESOLVER
1469            .set(Arc::new(DiscoveryServiceResolver {
1470                state_store: MemoryStateStore::production_default(),
1471                accepted_state_source: source,
1472                discovery_client: Some(discovery_client),
1473            }))
1474            .map_err(|_| anyhow::anyhow!("production service resolver is already installed"))?;
1475
1476        Ok(())
1477    }
1478
1479    /// Attach the process-pinned source to the Discovery daemon. The source
1480    /// was fixed and consumed before inventory factories were invoked.
1481    pub fn attach_process_accepted_state_source(&mut self) -> Result<()> {
1482        self.accepted_state_source = Some(
1483            PROCESS_ACCEPTED_STATE_SOURCE
1484                .get()
1485                .cloned()
1486                .ok_or_else(|| anyhow::anyhow!("process Discovery authority is not installed"))?,
1487        );
1488        Ok(())
1489    }
1490
1491    #[cfg(test)]
1492    fn production_resolver(&self) -> Result<DiscoveryServiceResolver> {
1493        Ok(DiscoveryServiceResolver {
1494            state_store: Arc::clone(&self.state_store),
1495            accepted_state_source: self.accepted_state_source.clone().ok_or_else(|| {
1496                anyhow::anyhow!("Discovery accepted-state source is not installed")
1497            })?,
1498            discovery_client: None,
1499        })
1500    }
1501
1502    /// Get the global EndpointRegistry (D9: graceful error, not panic).
1503    fn reg(&self) -> Result<impl std::ops::Deref<Target = EndpointRegistry> + '_> {
1504        registry::try_global().ok_or_else(|| anyhow::anyhow!("EndpointRegistry not initialized"))
1505    }
1506}
1507
1508// ============================================================================
1509// Resolver implementation — DiscoveryService IS the authoritative resolver
1510// ============================================================================
1511
1512#[async_trait]
1513impl Resolver for DiscoveryService {
1514    async fn resolve(&self, name: &str, kind: SocketKind) -> anyhow::Result<TransportConfig> {
1515        if let Some(transport) = self.resolve_announced_endpoint(name, kind).await? {
1516            return Ok(transport);
1517        }
1518
1519        // Cross-pod QUIC peers must come from fresh service announcements. The
1520        // registry fallback for QUIC is a non-dialable bootstrap placeholder.
1521        if kind == SocketKind::Quic {
1522            anyhow::bail!("no fresh announced QUIC endpoint for service '{name}'");
1523        }
1524
1525        self.reg()?.try_endpoint(name, kind)
1526    }
1527}
1528
1529fn accepted_expiry_unix_ms(
1530    state: &hyprstream_pds::at9p_duplicity::AcceptedAt9pState,
1531) -> Result<i64> {
1532    let expiry = state.expires_at.as_deref().ok_or_else(|| {
1533        anyhow::anyhow!(
1534            "genesis-only accepted state has no bounded expiry; refusing production resolution"
1535        )
1536    })?;
1537    Ok(chrono::DateTime::parse_from_rfc3339(expiry)
1538        .map_err(|e| anyhow::anyhow!("invalid accepted-state expiry: {e}"))?
1539        .timestamp_millis())
1540}
1541
1542/// Fixed bootstrap roles use only checkpoint-authenticated capsule contents.
1543/// Carrier addresses and caller-provided public keys never mint authority.
1544#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
1545pub(super) fn project_bootstrap_endpoint(
1546    states: &[hyprstream_pds::at9p_duplicity::AcceptedAt9pState],
1547    service_name: &str,
1548    key: &VerifyingKey,
1549) -> Result<AnnouncedEndpoint> {
1550    anyhow::ensure!(
1551        matches!(service_name, "discovery" | "policy"),
1552        "not a fixed bootstrap role"
1553    );
1554    let service_id = format!("#{service_name}");
1555    let mut matching = states.iter().filter(|state| {
1556        state
1557            .current
1558            .services
1559            .iter()
1560            .any(|entry| entry.id == service_id)
1561            && state
1562                .current
1563                .subject_keys
1564                .iter()
1565                .any(|subject| subject.ed25519_pub == key.as_bytes())
1566    });
1567    let state = matching
1568        .next()
1569        .ok_or_else(|| anyhow::anyhow!("no accepted bootstrap identity for {service_name}"))?;
1570    anyhow::ensure!(
1571        matching.next().is_none(),
1572        "ambiguous accepted bootstrap identity for {service_name}"
1573    );
1574    let expiry = accepted_expiry_unix_ms(state)?;
1575    anyhow::ensure!(
1576        state.epoch > 0 && expiry > unix_millis_now(),
1577        "bootstrap identity is unbounded or expired"
1578    );
1579    let entry = state
1580        .current
1581        .services
1582        .iter()
1583        .find(|entry| entry.id == service_id)
1584        .ok_or_else(|| anyhow::anyhow!("bootstrap service disappeared"))?;
1585    anyhow::ensure!(
1586        entry.endpoint.transport == hyprstream_pds::at9p::Transport::Iroh,
1587        "bootstrap service requires signed Iroh reach"
1588    );
1589    parse_announced_iroh(&entry.endpoint.address)?;
1590    let kem = entry.endpoint.request_kem.as_ref().ok_or_else(|| {
1591        anyhow::anyhow!("bootstrap service lacks signed request KEM; reprovision service identity")
1592    })?;
1593    let recipient = hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::decode(kem)?;
1594    recipient.validate()?;
1595    anyhow::ensure!(
1596        recipient.suite_id == hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
1597        "bootstrap requires hybrid KEM"
1598    );
1599    let subject_key = state
1600        .current
1601        .subject_keys
1602        .iter()
1603        .find(|subject| subject.ed25519_pub == key.as_bytes())
1604        .ok_or_else(|| anyhow::anyhow!("bootstrap response key disappeared"))?;
1605    hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&subject_key.mldsa65_pub)?;
1606    Ok(AnnouncedEndpoint {
1607        socket_kind: "iroh".to_owned(),
1608        endpoint: entry.endpoint.address.clone(),
1609        service_did: Did::from(state.did.clone()),
1610        capabilities: ["hyprstream-rpc/1".to_owned()].into_iter().collect(),
1611        accepted_state_digest: state.head_digest.to_vec(),
1612        accepted_state_epoch: state.epoch,
1613        response_key_id: format!("{}#response", state.did),
1614        request_kem_key_id: format!("{}#mesh-kem", state.did),
1615        request_kem_recipient: kem.clone(),
1616        expires_at_unix_ms: expiry,
1617        source_signer: key.to_bytes(),
1618        live_until_unix_ms: expiry,
1619    })
1620}
1621
1622impl DiscoveryServiceResolver {
1623    #[cfg(test)]
1624    async fn resolve_service(&self, query: ServiceQuery) -> Result<ResolvedService> {
1625        let selected = select_service_candidate(
1626            &query,
1627            self.acquire_candidates(&query).await?,
1628            unix_millis_now(),
1629        )?;
1630        let state = self
1631            .accepted_state_source
1632            .accepted_state(selected.service_did().as_str())?
1633            .ok_or_else(|| anyhow::anyhow!("accepted state disappeared during resolution"))?;
1634        ResolvedService::mint(selected, &state)
1635    }
1636
1637    async fn acquire_candidates(&self, query: &ServiceQuery) -> Result<Vec<ServiceCandidate>> {
1638        let bootstrap = if query.profile == ResolverProfile::NativeIrohRequired {
1639            self.accepted_state_source
1640                .bootstrap_endpoints(&query.service_name)?
1641        } else {
1642            None
1643        };
1644        let entries = if let Some(entries) = bootstrap {
1645            entries
1646        } else if let Some(client) = &self.discovery_client {
1647            client
1648                .get_endpoints(&query.service_name)
1649                .await?
1650                .endpoints
1651                .into_iter()
1652                .filter_map(|endpoint| {
1653                    let source_signer: [u8; 32] = endpoint.source_signer.try_into().ok()?;
1654                    Some(AnnouncedEndpoint {
1655                        socket_kind: endpoint.socket_kind,
1656                        endpoint: endpoint.endpoint,
1657                        service_did: endpoint.service_did,
1658                        capabilities: endpoint.capabilities.into_iter().collect(),
1659                        accepted_state_digest: endpoint.accepted_state_digest,
1660                        accepted_state_epoch: endpoint.accepted_state_epoch,
1661                        response_key_id: endpoint.response_key_id,
1662                        request_kem_key_id: endpoint.request_kem_key_id,
1663                        request_kem_recipient: endpoint.request_kem_recipient,
1664                        expires_at_unix_ms: endpoint.expires_at_unix_ms,
1665                        source_signer,
1666                        live_until_unix_ms: endpoint
1667                            .expires_at_unix_ms
1668                            .min(unix_millis_now() + ANNOUNCED_ENDPOINT_TTL.as_millis() as i64),
1669                    })
1670                })
1671                .collect()
1672        } else {
1673            self.state_store
1674                .announcements_for(&query.service_name, unix_millis_now())
1675                .await?
1676        };
1677
1678        let mut candidates = Vec::new();
1679        for entry in entries {
1680            // The backend/remote Discovery already checked the volatile lease
1681            // on its receipt clock. Signed/current authority is checked below.
1682            if entry.service_did.as_str().is_empty() || entry.accepted_state_digest.len() != 64 {
1683                continue;
1684            }
1685            let Some(state) = self
1686                .accepted_state_source
1687                .accepted_state(entry.service_did.as_str())?
1688            else {
1689                continue;
1690            };
1691            if state.did != entry.service_did.as_str()
1692                || state.epoch != entry.accepted_state_epoch
1693                || state.head_digest.as_slice() != entry.accepted_state_digest.as_slice()
1694            {
1695                continue;
1696            }
1697            let Ok(state_expiry) = accepted_expiry_unix_ms(&state) else {
1698                continue;
1699            };
1700            let Some(current_key) = state
1701                .current
1702                .subject_keys
1703                .iter()
1704                .find(|key| key.ed25519_pub.as_slice() == entry.source_signer)
1705            else {
1706                continue;
1707            };
1708            let Ok(response_ed25519) = current_key.ed25519_pub.as_slice().try_into() else {
1709                continue;
1710            };
1711            let Ok(recipient) = hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::decode(
1712                &entry.request_kem_recipient,
1713            ) else {
1714                continue;
1715            };
1716            let Ok(transport) = announced_endpoint_to_transport(&entry) else {
1717                continue;
1718            };
1719            let mut digest = [0u8; 64];
1720            digest.copy_from_slice(&entry.accepted_state_digest);
1721            candidates.push(ServiceCandidate {
1722                service_name: query.service_name.clone(),
1723                service_did: entry.service_did,
1724                response_verifying_key: response_ed25519,
1725                response_ml_dsa65: current_key.mldsa65_pub.clone(),
1726                response_key_id: entry.response_key_id,
1727                request_kem_recipient: Some(AnchoredKemRecipient {
1728                    key_id: entry.request_kem_key_id,
1729                    recipient,
1730                    not_after_unix_ms: entry.expires_at_unix_ms,
1731                }),
1732                transport,
1733                capabilities: entry.capabilities,
1734                accepted_state: AcceptedStateEvidence {
1735                    service_did: Did::from(state.did),
1736                    digest,
1737                    epoch: state.epoch,
1738                    expires_at_unix_ms: state_expiry,
1739                    response_ed25519,
1740                    response_ml_dsa65: current_key.mldsa65_pub.clone(),
1741                },
1742                source_signer: entry.source_signer,
1743                expires_at_unix_ms: entry.expires_at_unix_ms,
1744            });
1745        }
1746        Ok(candidates)
1747    }
1748}
1749
1750impl DiscoveryServiceResolver {
1751    async fn resolve_service_candidates(
1752        &self,
1753        query: ServiceQuery,
1754    ) -> Result<Vec<ResolvedService>> {
1755        let candidates = self.acquire_candidates(&query).await?;
1756        let selected = select_service_candidates(&query, candidates, unix_millis_now())?;
1757        let mut resolved = Vec::with_capacity(selected.len());
1758        for item in selected {
1759            let state = self
1760                .accepted_state_source
1761                .accepted_state(item.service_did().as_str())?
1762                .ok_or_else(|| anyhow::anyhow!("accepted state disappeared during resolution"))?;
1763            resolved.push(ResolvedService::mint(item, &state)?);
1764        }
1765        Ok(resolved)
1766    }
1767
1768    async fn ensure_current(&self, resolved: &ResolvedService) -> Result<()> {
1769        resolved.ensure_fresh(unix_millis_now())?;
1770        let state = self
1771            .accepted_state_source
1772            .accepted_state(resolved.service_did().as_str())?
1773            .ok_or_else(|| anyhow::anyhow!("accepted state disappeared; re-resolution required"))?;
1774        anyhow::ensure!(
1775            state.epoch == resolved.evidence().accepted_state_epoch
1776                && state.head_digest == resolved.evidence().accepted_state_digest,
1777            "accepted state advanced or forked; re-resolution required"
1778        );
1779        let state_expiry = accepted_expiry_unix_ms(&state)?;
1780        anyhow::ensure!(
1781            unix_millis_now() < state_expiry,
1782            "accepted state expired; re-resolution required"
1783        );
1784        Ok(())
1785    }
1786
1787    async fn browser_provisioning(
1788        &self,
1789        request: BrowserProvisioningRequest,
1790    ) -> Result<BrowserProvisioningDocument> {
1791        request.validate()?;
1792        match request.carrier_profile {
1793            BrowserCarrierProfile::OwnedHybridWebTransport => {
1794                anyhow::ensure!(
1795                    request.capability == "hyprstream-rpc/1"
1796                        && request.scope == request.service_name,
1797                    "owned browser RPC provisioning requires hyprstream-rpc/1 and exact service scope"
1798                );
1799            }
1800            BrowserCarrierProfile::StandardPublicRelay => {
1801                anyhow::ensure!(
1802                    request.capability == "hyprstream-moq/1",
1803                    "standard-public-relay provisioning requires hyprstream-moq/1"
1804                );
1805            }
1806        }
1807
1808        let query = ServiceQuery::new(
1809            request.service_name.clone(),
1810            [request.capability.clone()],
1811            ResolverProfile::BrowserWebTransport,
1812            1,
1813        )?;
1814        let resolved = self
1815            .resolve_service_candidates(query)
1816            .await?
1817            .into_iter()
1818            .next()
1819            .ok_or_else(|| anyhow::anyhow!("resolver returned no validated browser reach"))?;
1820        self.ensure_current(&resolved).await?;
1821        let state = self
1822            .accepted_state_source
1823            .accepted_state(resolved.service_did().as_str())?
1824            .ok_or_else(|| {
1825                anyhow::anyhow!("accepted state disappeared during browser provisioning")
1826            })?;
1827        anyhow::ensure!(
1828            state.epoch == resolved.evidence().accepted_state_epoch
1829                && state.head_digest == resolved.evidence().accepted_state_digest,
1830            "accepted state advanced during browser provisioning; re-resolution required"
1831        );
1832        let accepted_service = state
1833            .current
1834            .services
1835            .iter()
1836            .find(|service| service.id == format!("#{}", request.service_name))
1837            .ok_or_else(|| {
1838                anyhow::anyhow!("accepted current state does not contain requested service")
1839            })?;
1840
1841        let (origin, route, certificate_hashes, route_role, transport_security, encrypted_objects) =
1842            match request.carrier_profile {
1843                BrowserCarrierProfile::OwnedHybridWebTransport => {
1844                    let EndpointType::Quic {
1845                        addr,
1846                        server_name,
1847                        auth,
1848                    } = resolved.transport().endpoint.clone()
1849                    else {
1850                        anyhow::bail!(
1851                            "browser provisioning requires selected QUIC/WebTransport network reach"
1852                        );
1853                    };
1854                    // URL parsers elide the default HTTPS port. Emit that same
1855                    // canonical form before signing the browser projection.
1856                    let authority = if addr.port() == 443 {
1857                        server_name.clone()
1858                    } else {
1859                        format!("{server_name}:{}", addr.port())
1860                    };
1861                    let origin = format!("https://{authority}/");
1862                    let route = format!(
1863                        "https://{authority}{}",
1864                        hyprstream_rpc::browser_provisioning::BROWSER_RPC_PATH
1865                    );
1866                    (
1867                        origin,
1868                        route,
1869                        auth.accept_cert_hashes().to_vec(),
1870                        BrowserRouteRole::Origin,
1871                        BrowserTransportSecurity::OwnedHybridRequired,
1872                        false,
1873                    )
1874                }
1875                BrowserCarrierProfile::StandardPublicRelay => {
1876                    anyhow::ensure!(
1877                        accepted_service.endpoint.transport == hyprstream_pds::at9p::Transport::Moq,
1878                        "accepted service does not select the MoQT carrier"
1879                    );
1880                    let relay = accepted_service.endpoint.relay.clone().ok_or_else(|| {
1881                        anyhow::anyhow!(
1882                            "accepted service has no signed standard public relay route"
1883                        )
1884                    })?;
1885                    (
1886                        accepted_service.endpoint.address.clone(),
1887                        relay,
1888                        Vec::new(),
1889                        BrowserRouteRole::Relay,
1890                        BrowserTransportSecurity::ClassicalUntrusted,
1891                        true,
1892                    )
1893                }
1894            };
1895
1896        let now = unix_millis_now();
1897        let expires_at_unix_ms = resolved
1898            .expires_at_unix_ms()
1899            .min(now.saturating_add(30_000));
1900        anyhow::ensure!(
1901            expires_at_unix_ms > now,
1902            "resolved browser provisioning expired before projection"
1903        );
1904        Ok(BrowserProvisioningDocument::from_material(
1905            BrowserProvisioningMaterial {
1906                service_name: request.service_name,
1907                service_did: resolved.service_did().as_str().to_owned(),
1908                service_origin: origin,
1909                webtransport_url: route,
1910                capability: request.capability,
1911                scope: request.scope,
1912                carrier_profile: request.carrier_profile,
1913                route_role,
1914                transport_security,
1915                response_key_id: resolved.response_key_id().to_owned(),
1916                response_ed25519: resolved.response_verifying_key().to_bytes(),
1917                response_ml_dsa65: resolved.response_ml_dsa65().to_vec(),
1918                request_kem_key_id: resolved.request_kem_recipient().key_id.clone(),
1919                request_kem_recipient: resolved.request_kem_recipient().recipient.clone(),
1920                accepted_state_digest: resolved.evidence().accepted_state_digest,
1921                accepted_state_epoch: resolved.evidence().accepted_state_epoch,
1922                expires_at_unix_ms,
1923                certificate_hashes,
1924                encrypted_objects_required: encrypted_objects,
1925            },
1926        ))
1927    }
1928
1929    async fn verify_browser_binding(&self, binding: &BrowserRequestBinding) -> Result<()> {
1930        binding.validate_shape()?;
1931        let now = unix_millis_now();
1932        anyhow::ensure!(
1933            binding.expires_at_unix_ms > now,
1934            "sealed browser provisioning binding expired before dispatch"
1935        );
1936        let current = self
1937            .browser_provisioning(BrowserProvisioningRequest::new(
1938                binding.service_name.clone(),
1939                binding.capability.clone(),
1940                binding.scope.clone(),
1941                binding.carrier_profile,
1942            )?)
1943            .await?;
1944        anyhow::ensure!(
1945            binding.service_did == current.service_did
1946                && binding.service_origin == current.service_origin
1947                && binding.webtransport_url == current.webtransport_url
1948                && binding.certificate_hashes == current.certificate_hashes
1949                && binding.response_key_id == current.response_key_id
1950                && binding.request_kem_key_id == current.request_kem_key_id
1951                && binding.accepted_state_epoch == current.accepted_state_epoch
1952                && binding.accepted_state_digest == current.accepted_state_digest
1953                && binding.expires_at_unix_ms <= current.expires_at_unix_ms,
1954            "accepted-current browser authority changed before dispatch"
1955        );
1956        let response_key = URL_SAFE_NO_PAD
1957            .decode(&current.response_ed25519)
1958            .context("invalid current response key projection")?;
1959        let kem = URL_SAFE_NO_PAD
1960            .decode(&current.request_kem_recipient)
1961            .context("invalid current request KEM projection")?;
1962        anyhow::ensure!(
1963            binding.matches_response_key(&response_key)? && binding.matches_request_kem(&kem)?,
1964            "accepted-current browser key material changed before dispatch"
1965        );
1966        Ok(())
1967    }
1968}
1969
1970struct ProductionBrowserCurrentnessVerifier {
1971    resolver: Arc<DiscoveryServiceResolver>,
1972}
1973
1974#[async_trait]
1975impl BrowserCurrentnessVerifier for ProductionBrowserCurrentnessVerifier {
1976    async fn ensure_current(&self, binding: &BrowserRequestBinding) -> Result<()> {
1977        self.resolver.verify_browser_binding(binding).await
1978    }
1979}
1980
1981static PROCESS_ACCEPTED_STATE_SOURCE: std::sync::OnceLock<Arc<dyn AcceptedStateSource>> =
1982    std::sync::OnceLock::new();
1983static PRODUCTION_RESOLVER: std::sync::OnceLock<Arc<DiscoveryServiceResolver>> =
1984    std::sync::OnceLock::new();
1985static PROCESS_REGISTRY_VERIFIER: std::sync::OnceLock<RegistryDeploymentVerifier> =
1986    std::sync::OnceLock::new();
1987static PROCESS_NATIVE_NETWORK_REQUIRED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1988#[cfg(not(target_arch = "wasm32"))]
1989static PROCESS_BOOTSTRAP_CARRIER: std::sync::OnceLock<
1990    hyprstream_rpc::transport::iroh_substrate::IrohSubstrate,
1991> = std::sync::OnceLock::new();
1992
1993/// Transport profile selected by authenticated process bootstrap.
1994pub fn native_network_required() -> bool {
1995    PROCESS_NATIVE_NETWORK_REQUIRED
1996        .get()
1997        .copied()
1998        .unwrap_or(false)
1999}
2000
2001/// Resolve native Event reach and its accepted-current server witness together.
2002pub async fn production_moq_event_target() -> Result<(
2003    TransportConfig,
2004    hyprstream_rpc::stream_info::MoqlServerIdentity,
2005)> {
2006    let resolver = PRODUCTION_RESOLVER
2007        .get()
2008        .ok_or_else(|| anyhow::anyhow!("production resolver is not installed"))?;
2009    let resolved = resolver
2010        .resolve_service_candidates(ServiceQuery::network_moq("event")?)
2011        .await?
2012        .into_iter()
2013        .next()
2014        .ok_or_else(|| anyhow::anyhow!("no native Event reach"))?;
2015    resolver.ensure_current(&resolved).await?;
2016    Ok((
2017        resolved.transport().clone(),
2018        hyprstream_rpc::stream_info::MoqlServerIdentity {
2019            did: resolved.service_did().as_str().to_owned(),
2020            epoch: resolved.evidence().accepted_state_epoch,
2021            head_digest: resolved.evidence().accepted_state_digest.to_vec(),
2022            expires_at_unix_ms: resolved.expires_at_unix_ms(),
2023            ed25519: resolved.response_verifying_key().to_bytes(),
2024            ml_dsa65: resolved.response_ml_dsa65().to_vec(),
2025        },
2026    ))
2027}
2028
2029/// Project one checkpoint-verified accepted state into the admission-facing
2030/// [`AcceptedIdentityState`], or `None` when it does not parse into a bounded
2031/// valid projection. Shared by the per-DID admission authority and the
2032/// roster-wide derivation source (#1652) so both see identical projections.
2033pub(crate) fn project_accepted_identity_state(
2034    state: &hyprstream_pds::at9p_duplicity::AcceptedAt9pState,
2035) -> Option<hyprstream_rpc::transport::moql_admission::AcceptedIdentityState> {
2036    use hyprstream_rpc::transport::moql_admission::{AcceptedIdentityState, AcceptedSubjectKey};
2037    let expires = chrono::DateTime::parse_from_rfc3339(state.expires_at.as_deref()?)
2038        .ok()?
2039        .timestamp_millis();
2040    let subject_keys = state
2041        .current
2042        .subject_keys
2043        .iter()
2044        .map(|key| {
2045            Some(AcceptedSubjectKey {
2046                ed25519: key.ed25519_pub.as_slice().try_into().ok()?,
2047                ml_dsa_65: (!key.mldsa65_pub.is_empty()).then(|| key.mldsa65_pub.clone())?,
2048            })
2049        })
2050        .collect::<Option<Vec<_>>>()?;
2051    // Project the capsule's `#<name>` service ids verbatim (#1652): the
2052    // state is already in hand, so DID→service-name derivation at the
2053    // admission sites costs zero extra store reads.
2054    let service_ids = state
2055        .current
2056        .services
2057        .iter()
2058        .map(|service| service.id.clone())
2059        .collect::<Vec<_>>();
2060    (!subject_keys.is_empty()).then_some(AcceptedIdentityState {
2061        epoch: state.epoch,
2062        head_digest: state.head_digest,
2063        subject_keys,
2064        service_ids,
2065        expires_at_unix_ms: Some(expires),
2066    })
2067}
2068
2069/// Project only bounded, valid checkpoint state into the MoQL authority.
2070/// Every admission and live-session recheck rereads the pinned source.
2071pub fn production_moql_accepted_state_authority(
2072) -> Result<Arc<dyn hyprstream_rpc::transport::moql_admission::AcceptedStateAuthority>> {
2073    let source = PROCESS_ACCEPTED_STATE_SOURCE
2074        .get()
2075        .cloned()
2076        .ok_or_else(|| anyhow::anyhow!("production accepted-state authority is not installed"))?;
2077    Ok(Arc::new(move |did: &str| {
2078        let state = source.accepted_state(did).ok().flatten()?;
2079        project_accepted_identity_state(&state)
2080    }))
2081}
2082
2083/// The process's deployment-roster source for #1652 derived admission:
2084/// enumerates every verified accepted state (with DIDs) so grant derivation
2085/// can enforce cross-store uniqueness — the one property a per-DID read
2086/// cannot see. Sourced from the same pinned checkpoint store as the
2087/// per-DID authority; a read failure yields an empty roster, which denies.
2088/// Served through a TTL cache: the live-session recheck runs every 100 ms
2089/// per session and must not re-verify the whole store each time.
2090pub fn production_deployment_roster(
2091) -> Result<Arc<dyn crate::admission_roster::DeploymentRosterSource>> {
2092    let source = PROCESS_ACCEPTED_STATE_SOURCE
2093        .get()
2094        .cloned()
2095        .ok_or_else(|| anyhow::anyhow!("production accepted-state authority is not installed"))?;
2096    let load: std::sync::Arc<dyn Fn() -> Vec<crate::admission_roster::RosterEntry> + Send + Sync> =
2097        Arc::new(move || {
2098            source
2099                .accepted_states()
2100                .ok()
2101                .map(|states| {
2102                    states
2103                        .iter()
2104                        .filter_map(|state| {
2105                            project_accepted_identity_state(state)
2106                                .map(|projected| (state.did.clone(), projected))
2107                        })
2108                        .collect::<Vec<_>>()
2109                })
2110                .unwrap_or_default()
2111        });
2112    Ok(Arc::new(crate::admission_roster::CachedRoster::new(
2113        load,
2114        crate::admission_roster::ROSTER_CACHE_TTL,
2115    )))
2116}
2117
2118const DEPLOYMENT_CA_ROOT_PATH: &str = "/etc/hyprstream/trust/deployment-ca.hybrid";
2119const DEPLOYMENT_AUTHORITY_LOG_PATH: &str = "/etc/hyprstream/trust/deployment-authority.log.json";
2120const DEPLOYMENT_AUTHORITY_CHECKPOINT_PATH: &str =
2121    "/etc/hyprstream/trust/deployment-authority.head.json";
2122const REGISTRY_DEPLOYMENT_CREDENTIAL_PATH: &str =
2123    "/run/hyprstream/credentials/registry-service.jwt";
2124const DEPLOYMENT_TRUST_DIR_ENV: &str = "HYPRSTREAM_DEPLOYMENT_TRUST_DIR";
2125const SYSTEMD_CREDENTIALS_DIRECTORY_ENV: &str = "CREDENTIALS_DIRECTORY";
2126const DEPLOYMENT_CA_ROOT_FILE: &str = "deployment-ca.hybrid";
2127const DEPLOYMENT_AUTHORITY_LOG_FILE: &str = "deployment-authority.log.json";
2128const DEPLOYMENT_AUTHORITY_CHECKPOINT_FILE: &str = "deployment-authority.head.json";
2129const REGISTRY_DEPLOYMENT_CREDENTIAL_FILE: &str = "registry-service.jwt";
2130const REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE: &str = "hyprstream.registry-deployment.v1";
2131const REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE: &str = "urn:hyprstream:service:registry";
2132const REGISTRY_DEPLOYMENT_CREDENTIAL_MAX_TTL_SECONDS: i64 = 3_600;
2133const REGISTRY_DEPLOYMENT_CREDENTIAL_CLOCK_SKEW_SECONDS: i64 = 60;
2134const ED25519_PUBLIC_KEY_BYTES: usize = 32;
2135const ML_DSA_65_PUBLIC_KEY_BYTES: usize = 1_952;
2136const DEPLOYMENT_CA_ROOT_BYTES: usize = ED25519_PUBLIC_KEY_BYTES + ML_DSA_65_PUBLIC_KEY_BYTES;
2137const HYBRID_JWT_SIGNATURE_BYTES: usize = 3_309 + 64;
2138const AUTHORITY_LOG_SCHEMA: &str = "hyprstream.deployment-authority-log.v1";
2139const AUTHORITY_CHECKPOINT_SCHEMA: &str = "hyprstream.deployment-authority-checkpoint.v1";
2140const REGISTRY_DELEGATION_SCHEMA: &str = "hyprstream.registry-delegation.v1";
2141const MAX_AUTHORITY_LOG_OPERATIONS: usize = 128;
2142const MAX_REGISTRY_DELEGATION_BYTES: usize = 256 * 1024;
2143const MAX_DEPLOYMENT_CLOUD_SECRET_BYTES: usize = 64 * 1024;
2144const REGISTRY_DELEGATION_ABILITY: &str = "mint-registry-jwt";
2145const SERVICE_KEY_ENROLLMENT_SCHEMA: &str = "hyprstream.service-key-enrollment.v1";
2146const SERVICE_KEY_ENROLLMENT_ABILITY: &str = "enroll-service-key";
2147const SERVICE_KEY_ENROLLMENT_KEY_TYPE: &str = "hybrid-ed25519-mldsa65";
2148const SERVICE_KEY_ENROLLMENT_MAX_TTL_SECONDS: i64 = 3_600;
2149/// Signature context (AAD) binding an enrollment attestation to its schema.
2150const SERVICE_KEY_ENROLLMENT_SIGNATURE_CONTEXT: &[u8] = b"hyprstream.service-key-enrollment.v1";
2151/// Fixed allowlist per hyprstream#1562: registry stays JWT-enrolled and
2152/// policy-CA/authority keys are out of scope, so exactly these services may be
2153/// enrolled by a delegated signer.
2154pub const SERVICE_KEY_ENROLLMENT_ALLOWED_SERVICES: [&str; 2] = ["discovery", "policy"];
2155const MAX_SERVICE_KEY_ENROLLMENT_BYTES: usize = 256 * 1024;
2156
2157/// Signed DidOp authority history embedded in a registry delegation.
2158#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2159#[serde(deny_unknown_fields)]
2160pub struct DeploymentAuthorityLog {
2161    pub schema: String,
2162    pub deployment_domain: String,
2163    pub did: String,
2164    pub operations_b64: Vec<String>,
2165}
2166
2167/// Independently provisioned expected head of the deployment authority log.
2168///
2169/// This checkpoint is deliberately separate from the remotely supplied log:
2170/// a signature-valid historical prefix cannot redefine the expected sequence
2171/// or CID and thereby reactivate retired authority keys.
2172#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2173#[serde(deny_unknown_fields)]
2174pub struct DeploymentAuthorityCheckpoint {
2175    pub schema: String,
2176    pub deployment_domain: String,
2177    pub did: String,
2178    pub sequence: u64,
2179    pub head_cid: String,
2180}
2181
2182/// Root/current-authority UCAN delegation embedded in delegated registry JWTs.
2183#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2184#[serde(deny_unknown_fields)]
2185pub struct RegistryDelegationArtifact {
2186    pub schema: String,
2187    pub deployment_domain: String,
2188    pub authority_log_did: String,
2189    pub delegated_public_key_b64: String,
2190    pub ucan_b64: String,
2191}
2192
2193/// Chain-signed attestation binding a discovery/policy service's live hybrid
2194/// public key (the exact 1984-byte `bootstrap-pubkeys` entry) to the
2195/// deployment authority, minted at activation by the delegated signer
2196/// (hyprstream#1562). Public trust material: installed mode 0644, per-service.
2197#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2198#[serde(deny_unknown_fields)]
2199pub struct ServiceKeyEnrollmentArtifact {
2200    pub schema: String,
2201    pub deployment_domain: String,
2202    pub service: String,
2203    /// Base64 of the exact 1984-byte hybrid public key (32-byte Ed25519
2204    /// followed by 1952-byte ML-DSA-65).
2205    pub hybrid_public_key_b64: String,
2206    pub not_before: i64,
2207    pub expires_at: i64,
2208    /// The two-capability delegation authorizing the signing key.
2209    pub delegation: RegistryDelegationArtifact,
2210    /// Base64 of the hybrid Ed25519+ML-DSA-65 signature by the delegated
2211    /// signer over [`ServiceKeyEnrollmentArtifact::signing_bytes`].
2212    pub signature_b64: String,
2213}
2214
2215/// The signed body of a [`ServiceKeyEnrollmentArtifact`]: every field except
2216/// the signature, serialized with fixed struct-field order so the mint and the
2217/// production verifier derive identical bytes.
2218#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2219#[serde(deny_unknown_fields)]
2220pub struct ServiceKeyEnrollmentSigningBody {
2221    pub schema: String,
2222    pub deployment_domain: String,
2223    pub service: String,
2224    pub hybrid_public_key_b64: String,
2225    pub not_before: i64,
2226    pub expires_at: i64,
2227    pub delegation: RegistryDelegationArtifact,
2228}
2229
2230impl ServiceKeyEnrollmentArtifact {
2231    /// The artifact with an empty signature, ready to be signed.
2232    pub fn unsigned(body: ServiceKeyEnrollmentSigningBody) -> Self {
2233        Self {
2234            schema: body.schema,
2235            deployment_domain: body.deployment_domain,
2236            service: body.service,
2237            hybrid_public_key_b64: body.hybrid_public_key_b64,
2238            not_before: body.not_before,
2239            expires_at: body.expires_at,
2240            delegation: body.delegation,
2241            signature_b64: String::new(),
2242        }
2243    }
2244
2245    /// Canonical bytes the delegated signer signs and the verifier checks.
2246    pub fn signing_bytes(&self) -> Result<Vec<u8>> {
2247        let body = ServiceKeyEnrollmentSigningBody {
2248            schema: self.schema.clone(),
2249            deployment_domain: self.deployment_domain.clone(),
2250            service: self.service.clone(),
2251            hybrid_public_key_b64: self.hybrid_public_key_b64.clone(),
2252            not_before: self.not_before,
2253            expires_at: self.expires_at,
2254            delegation: self.delegation.clone(),
2255        };
2256        serde_json::to_vec(&body)
2257            .map_err(|error| anyhow::anyhow!("encoding enrollment signing body: {error}"))
2258    }
2259}
2260
2261/// Non-optional Ed25519 + ML-DSA-65 deployment trust root.
2262///
2263/// This type has no classical-only constructor. Both the OS-owned and
2264/// DID-anchored providers must parse both halves before a registry credential
2265/// can be authenticated.
2266#[derive(Clone)]
2267pub(crate) struct HybridDeploymentCa {
2268    ed25519: VerifyingKey,
2269    ml_dsa_65: hyprstream_rpc::crypto::pq::MlDsaVerifyingKey,
2270}
2271
2272impl HybridDeploymentCa {
2273    pub(crate) fn from_public_key_bytes(ed25519: &[u8], ml_dsa_65: &[u8]) -> Result<Self> {
2274        let ed25519: [u8; ED25519_PUBLIC_KEY_BYTES] = ed25519
2275            .try_into()
2276            .map_err(|_| anyhow::anyhow!("deployment CA Ed25519 key must be 32 bytes"))?;
2277        let ed25519 = VerifyingKey::from_bytes(&ed25519)
2278            .map_err(|error| anyhow::anyhow!("deployment CA Ed25519 key is malformed: {error}"))?;
2279        let ml_dsa_65 =
2280            hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(ml_dsa_65).map_err(|error| {
2281                anyhow::anyhow!("deployment CA ML-DSA-65 key is malformed: {error}")
2282            })?;
2283        Ok(Self { ed25519, ml_dsa_65 })
2284    }
2285
2286    fn from_os_pin(bytes: &[u8]) -> Result<Self> {
2287        anyhow::ensure!(
2288            bytes.len() == DEPLOYMENT_CA_ROOT_BYTES,
2289            "deployment CA root must be exactly {DEPLOYMENT_CA_ROOT_BYTES} bytes \
2290             (32-byte Ed25519 followed by 1952-byte ML-DSA-65)"
2291        );
2292        let (ed25519, ml_dsa_65) = bytes.split_at(ED25519_PUBLIC_KEY_BYTES);
2293        Self::from_public_key_bytes(ed25519, ml_dsa_65)
2294    }
2295
2296    fn domain(&self) -> String {
2297        hyprstream_rpc::auth::composite_kid(&self.ml_dsa_65, &self.ed25519)
2298    }
2299
2300    pub(crate) fn ed25519_bytes(&self) -> [u8; ED25519_PUBLIC_KEY_BYTES] {
2301        self.ed25519.to_bytes()
2302    }
2303
2304    pub(crate) fn ml_dsa_65_bytes(&self) -> Vec<u8> {
2305        hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&self.ml_dsa_65)
2306    }
2307}
2308
2309/// Opaque verification-only view of the authenticated deployment registry.
2310/// It exposes neither raw root material nor an authority replacement
2311/// operation. Retaining the authenticated hybrid CA prevents this capability
2312/// from being represented as provenance from a classical-only root.
2313#[derive(Clone)]
2314pub struct RegistryDeploymentVerifier {
2315    verifying_key: VerifyingKey,
2316    // Retained as non-extractable provenance; the leading underscore is
2317    // intentional because verification consumed it before construction.
2318    _deployment_ca: Arc<HybridDeploymentCa>,
2319}
2320
2321impl RegistryDeploymentVerifier {
2322    pub fn verify_strict(
2323        &self,
2324        message: &[u8],
2325        signature: &ed25519_dalek::Signature,
2326    ) -> Result<()> {
2327        self.verifying_key
2328            .verify_strict(message, signature)
2329            .map_err(|error| anyhow::anyhow!("deployment registry signature rejected: {error}"))
2330    }
2331
2332    pub fn matches(&self, key: &VerifyingKey) -> bool {
2333        self.verifying_key == *key
2334    }
2335
2336    /// Hybrid public key authenticated as this deployment's trust root.
2337    ///
2338    /// The operable PLC directory uses this projection to prove that an
2339    /// operator-manual genesis was signed by the same mandatory-Hybrid root
2340    /// installed at process bootstrap. It deliberately exposes no signing
2341    /// material and has no classical-only representation.
2342    pub(crate) fn deployment_did_op_key(&self) -> Result<crate::did_op::HybridRotationKey> {
2343        crate::did_op::HybridRotationKey::new(
2344            self._deployment_ca.ed25519.to_bytes(),
2345            hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&self._deployment_ca.ml_dsa_65),
2346        )
2347    }
2348
2349    #[cfg(test)]
2350    pub(crate) fn for_test_deployment_root(
2351        ed25519: &SigningKey,
2352        ml_dsa_65: &hyprstream_rpc::crypto::pq::MlDsaSigningKey,
2353    ) -> Result<Self> {
2354        let root = HybridDeploymentCa::from_public_key_bytes(
2355            ed25519.verifying_key().as_bytes(),
2356            &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(ml_dsa_65),
2357        )?;
2358        Ok(Self {
2359            verifying_key: ed25519.verifying_key(),
2360            _deployment_ca: Arc::new(root),
2361        })
2362    }
2363
2364    #[cfg(test)]
2365    fn matches_deployment_root(&self, root: &HybridDeploymentCa) -> bool {
2366        self._deployment_ca.domain() == root.domain()
2367    }
2368}
2369
2370/// Read the verification-only identity installed by trusted startup.
2371pub fn deployment_registry_verifier() -> Result<RegistryDeploymentVerifier> {
2372    PROCESS_REGISTRY_VERIFIER
2373        .get()
2374        .cloned()
2375        .ok_or_else(|| anyhow::anyhow!("deployment registry verifier is not installed"))
2376}
2377
2378/// Authenticate the fixed, OS-owned deployment artifacts for an offline
2379/// registry provisioning operation using the same role-specific trusted-file
2380/// loader as startup. Returns verification-only evidence, not a raw-key
2381/// authority constructor, and does not install a process resolver.
2382pub fn authenticate_local_deployment_registry() -> Result<RegistryDeploymentVerifier> {
2383    authenticate_registry_deployment_credentials(load_trusted_registry_deployment_credentials()?)
2384        .map(|identity| identity.verifier)
2385}
2386
2387/// Non-cloneable proof privately minted from the fixed CA/JWT pair.
2388struct AuthenticatedRegistryDeploymentIdentity {
2389    verifier: RegistryDeploymentVerifier,
2390}
2391
2392struct TrustedRegistryDeploymentCredentials {
2393    ca_verifying_key: HybridDeploymentCa,
2394    authority_log: DeploymentAuthorityLog,
2395    authority_checkpoint: DeploymentAuthorityCheckpoint,
2396    registry_credential: String,
2397}
2398
2399enum DeploymentAuthorityState<'a> {
2400    /// Explicit, one-time genesis verification before enrollment. Production
2401    /// loaders never select this state from an absent file.
2402    GenesisOnly,
2403    /// Normal enrolled operation: both independently trusted checkpoint and
2404    /// current root-anchored log are mandatory.
2405    Enrolled {
2406        authority_log: &'a DeploymentAuthorityLog,
2407        authority_checkpoint: &'a DeploymentAuthorityCheckpoint,
2408    },
2409}
2410
2411/// JSON value decoded while preserving the security property that every object
2412/// member has exactly one interpretation. `serde_json::Value` alone accepts a
2413/// duplicate member with last-value-wins semantics, which is unsuitable for a
2414/// credential profile.
2415struct UniqueJson(serde_json::Value);
2416
2417impl<'de> serde::Deserialize<'de> for UniqueJson {
2418    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
2419    where
2420        D: serde::Deserializer<'de>,
2421    {
2422        struct Visitor;
2423
2424        impl<'de> serde::de::Visitor<'de> for Visitor {
2425            type Value = UniqueJson;
2426
2427            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2428                formatter.write_str("JSON without duplicate object members")
2429            }
2430
2431            fn visit_unit<E>(self) -> std::result::Result<Self::Value, E> {
2432                Ok(UniqueJson(serde_json::Value::Null))
2433            }
2434
2435            fn visit_bool<E>(self, value: bool) -> std::result::Result<Self::Value, E> {
2436                Ok(UniqueJson(serde_json::Value::Bool(value)))
2437            }
2438
2439            fn visit_i64<E>(self, value: i64) -> std::result::Result<Self::Value, E> {
2440                Ok(UniqueJson(serde_json::Value::Number(value.into())))
2441            }
2442
2443            fn visit_u64<E>(self, value: u64) -> std::result::Result<Self::Value, E> {
2444                Ok(UniqueJson(serde_json::Value::Number(value.into())))
2445            }
2446
2447            fn visit_f64<E>(self, value: f64) -> std::result::Result<Self::Value, E>
2448            where
2449                E: serde::de::Error,
2450            {
2451                serde_json::Number::from_f64(value)
2452                    .map(serde_json::Value::Number)
2453                    .map(UniqueJson)
2454                    .ok_or_else(|| E::custom("non-finite JSON number"))
2455            }
2456
2457            fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E> {
2458                Ok(UniqueJson(serde_json::Value::String(value.to_owned())))
2459            }
2460
2461            fn visit_string<E>(self, value: String) -> std::result::Result<Self::Value, E> {
2462                Ok(UniqueJson(serde_json::Value::String(value)))
2463            }
2464
2465            fn visit_none<E>(self) -> std::result::Result<Self::Value, E>
2466            where
2467                E: serde::de::Error,
2468            {
2469                self.visit_unit()
2470            }
2471
2472            fn visit_some<D>(self, deserializer: D) -> std::result::Result<Self::Value, D::Error>
2473            where
2474                D: serde::Deserializer<'de>,
2475            {
2476                <UniqueJson as serde::Deserialize>::deserialize(deserializer)
2477            }
2478
2479            fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
2480            where
2481                A: serde::de::SeqAccess<'de>,
2482            {
2483                let mut values = Vec::new();
2484                while let Some(value) = seq.next_element::<UniqueJson>()? {
2485                    values.push(value.0);
2486                }
2487                Ok(UniqueJson(serde_json::Value::Array(values)))
2488            }
2489
2490            fn visit_map<A>(self, mut map: A) -> std::result::Result<Self::Value, A::Error>
2491            where
2492                A: serde::de::MapAccess<'de>,
2493            {
2494                let mut values = serde_json::Map::new();
2495                while let Some((key, value)) = map.next_entry::<String, UniqueJson>()? {
2496                    if values.insert(key.clone(), value.0).is_some() {
2497                        return Err(serde::de::Error::custom(format!(
2498                            "duplicate JSON member {key:?}"
2499                        )));
2500                    }
2501                }
2502                Ok(UniqueJson(serde_json::Value::Object(values)))
2503            }
2504        }
2505
2506        deserializer.deserialize_any(Visitor)
2507    }
2508}
2509
2510fn exact_json_object<'a>(
2511    value: &'a serde_json::Value,
2512    expected: &[&str],
2513    description: &str,
2514) -> Result<&'a serde_json::Map<String, serde_json::Value>> {
2515    let object = value
2516        .as_object()
2517        .ok_or_else(|| anyhow::anyhow!("{description} must be a JSON object"))?;
2518    for member in object.keys() {
2519        anyhow::ensure!(
2520            expected.contains(&member.as_str()),
2521            "{description} contains unexpected member {member:?}"
2522        );
2523    }
2524    for member in expected {
2525        anyhow::ensure!(
2526            object.contains_key(*member),
2527            "{description} is missing member {member:?}"
2528        );
2529    }
2530    Ok(object)
2531}
2532
2533fn exact_string_member<'a>(
2534    object: &'a serde_json::Map<String, serde_json::Value>,
2535    member: &str,
2536    description: &str,
2537) -> Result<&'a str> {
2538    object[member]
2539        .as_str()
2540        .ok_or_else(|| anyhow::anyhow!("{description}.{member} must be a string"))
2541}
2542
2543fn exact_i64_member(
2544    object: &serde_json::Map<String, serde_json::Value>,
2545    member: &str,
2546    description: &str,
2547) -> Result<i64> {
2548    object[member]
2549        .as_i64()
2550        .ok_or_else(|| anyhow::anyhow!("{description}.{member} must be an integer"))
2551}
2552
2553fn decode_canonical_jwt_segment(segment: &str, description: &str, limit: usize) -> Result<Vec<u8>> {
2554    anyhow::ensure!(!segment.is_empty(), "{description} is empty");
2555    anyhow::ensure!(segment.len() <= limit, "{description} is too large");
2556    let decoded = URL_SAFE_NO_PAD
2557        .decode(segment)
2558        .map_err(|error| anyhow::anyhow!("{description} is not unpadded base64url: {error}"))?;
2559    anyhow::ensure!(
2560        URL_SAFE_NO_PAD.encode(&decoded) == segment,
2561        "{description} is not canonical unpadded base64url"
2562    );
2563    Ok(decoded)
2564}
2565
2566fn parse_unique_jwt_json(
2567    segment: &str,
2568    description: &str,
2569    limit: usize,
2570) -> Result<serde_json::Value> {
2571    let bytes = decode_canonical_jwt_segment(segment, description, limit)?;
2572    let mut deserializer = serde_json::Deserializer::from_slice(&bytes);
2573    let value = <UniqueJson as serde::Deserialize>::deserialize(&mut deserializer)
2574        .map_err(|error| anyhow::anyhow!("{description} is malformed: {error}"))?;
2575    deserializer
2576        .end()
2577        .map_err(|error| anyhow::anyhow!("{description} has trailing data: {error}"))?;
2578    Ok(value.0)
2579}
2580
2581fn deployment_domain(ca_verifying_key: &HybridDeploymentCa) -> String {
2582    ca_verifying_key.domain()
2583}
2584
2585fn validate_registry_credential_numeric_dates(
2586    exp: i64,
2587    nbf: i64,
2588    iat: i64,
2589    now: i64,
2590) -> Result<()> {
2591    for (name, value) in [("exp", exp), ("nbf", nbf), ("iat", iat)] {
2592        anyhow::ensure!(
2593            value >= 0,
2594            "credential {name} must not precede the Unix epoch"
2595        );
2596    }
2597
2598    anyhow::ensure!(exp > now, "credential has expired");
2599    let latest_future_time = now
2600        .checked_add(REGISTRY_DEPLOYMENT_CREDENTIAL_CLOCK_SKEW_SECONDS)
2601        .ok_or_else(|| anyhow::anyhow!("credential clock-skew boundary overflow"))?;
2602    anyhow::ensure!(nbf <= latest_future_time, "credential is not yet valid");
2603    anyhow::ensure!(
2604        iat <= latest_future_time,
2605        "credential was issued in the future"
2606    );
2607    anyhow::ensure!(nbf <= exp, "credential nbf is after exp");
2608    anyhow::ensure!(nbf <= iat, "credential nbf is after iat");
2609    anyhow::ensure!(iat < exp, "credential iat is not before exp");
2610    let lifetime = exp
2611        .checked_sub(iat)
2612        .ok_or_else(|| anyhow::anyhow!("credential lifetime arithmetic overflow"))?;
2613    anyhow::ensure!(
2614        lifetime <= REGISTRY_DEPLOYMENT_CREDENTIAL_MAX_TTL_SECONDS,
2615        "credential lifetime exceeds the inclusive one-hour profile limit"
2616    );
2617    Ok(())
2618}
2619
2620fn validate_registry_deployment_credential_profile(
2621    token: &str,
2622    ca_verifying_key: &HybridDeploymentCa,
2623    authority_state: DeploymentAuthorityState<'_>,
2624) -> Result<([u8; 32], i64)> {
2625    anyhow::ensure!(
2626        !token.is_empty() && !token.bytes().any(|byte| byte.is_ascii_whitespace()),
2627        "registry credential must be compact with no whitespace"
2628    );
2629    anyhow::ensure!(
2630        token.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
2631        "registry credential exceeds the 64 KiB cloud-secret contract"
2632    );
2633    let mut segments = token.split('.');
2634    let protected_segment = segments.next().unwrap_or_default();
2635    let claims_segment = segments.next().unwrap_or_default();
2636    let signature_segment = segments.next().unwrap_or_default();
2637    anyhow::ensure!(
2638        segments.next().is_none(),
2639        "credential must contain exactly three segments"
2640    );
2641    let signature = decode_canonical_jwt_segment(signature_segment, "credential signature", 8_192)?;
2642    anyhow::ensure!(
2643        signature.len() == HYBRID_JWT_SIGNATURE_BYTES,
2644        "credential signature must contain both ML-DSA-65 and Ed25519 components"
2645    );
2646
2647    let protected = parse_unique_jwt_json(protected_segment, "protected header", 4_096)?;
2648    let protected = exact_json_object(&protected, &["alg", "typ", "kid"], "protected header")?;
2649    anyhow::ensure!(
2650        exact_string_member(protected, "alg", "protected header")? == "ML-DSA-65-Ed25519",
2651        "protected header alg must be exactly ML-DSA-65-Ed25519"
2652    );
2653    anyhow::ensure!(
2654        exact_string_member(protected, "typ", "protected header")? == "wit+jwt",
2655        "protected header typ must be exactly wit+jwt"
2656    );
2657    let domain = deployment_domain(ca_verifying_key);
2658
2659    let claims = parse_unique_jwt_json(
2660        claims_segment,
2661        "credential claims",
2662        MAX_REGISTRY_DELEGATION_BYTES + 16_384,
2663    )?;
2664    let claims_object = claims
2665        .as_object()
2666        .ok_or_else(|| anyhow::anyhow!("credential claims must be a JSON object"))?;
2667    let delegated = claims_object.contains_key("delegation");
2668    let direct_members = [
2669        "iss",
2670        "sub",
2671        "aud",
2672        "exp",
2673        "nbf",
2674        "iat",
2675        "deployment_domain",
2676        "profile",
2677        "cnf",
2678    ];
2679    let delegated_members = [
2680        "iss",
2681        "sub",
2682        "aud",
2683        "exp",
2684        "nbf",
2685        "iat",
2686        "deployment_domain",
2687        "profile",
2688        "cnf",
2689        "delegation",
2690    ];
2691    let claims = exact_json_object(
2692        &claims,
2693        if delegated {
2694            &delegated_members
2695        } else {
2696            &direct_members
2697        },
2698        "credential claims",
2699    )?;
2700    anyhow::ensure!(
2701        exact_string_member(claims, "sub", "credential claims")? == "service:registry",
2702        "credential subject must be exactly service:registry"
2703    );
2704    let intended_issuer = format!("urn:hyprstream:deployment:{domain}");
2705    anyhow::ensure!(
2706        exact_string_member(claims, "iss", "credential claims")? == intended_issuer,
2707        "credential issuer does not bind the pinned deployment domain"
2708    );
2709    anyhow::ensure!(
2710        exact_string_member(claims, "aud", "credential claims")?
2711            == REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE,
2712        "credential audience must be exactly {REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE}"
2713    );
2714    anyhow::ensure!(
2715        exact_string_member(claims, "deployment_domain", "credential claims")? == domain,
2716        "credential deployment_domain does not bind the pinned deployment CA"
2717    );
2718    anyhow::ensure!(
2719        exact_string_member(claims, "profile", "credential claims")?
2720            == REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE,
2721        "credential profile must be exactly {REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE}"
2722    );
2723
2724    let now = chrono::Utc::now().timestamp();
2725    let exp = exact_i64_member(claims, "exp", "credential claims")?;
2726    let nbf = exact_i64_member(claims, "nbf", "credential claims")?;
2727    let iat = exact_i64_member(claims, "iat", "credential claims")?;
2728    validate_registry_credential_numeric_dates(exp, nbf, iat, now)?;
2729
2730    let cnf = exact_json_object(&claims["cnf"], &["jwk"], "credential claims.cnf")?;
2731    let jwk = exact_json_object(
2732        &cnf["jwk"],
2733        &["kty", "crv", "x"],
2734        "credential claims.cnf.jwk",
2735    )?;
2736    anyhow::ensure!(
2737        exact_string_member(jwk, "kty", "credential claims.cnf.jwk")? == "OKP",
2738        "credential cnf.jwk.kty must be exactly OKP"
2739    );
2740    anyhow::ensure!(
2741        exact_string_member(jwk, "crv", "credential claims.cnf.jwk")? == "Ed25519",
2742        "credential cnf.jwk.crv must be exactly Ed25519"
2743    );
2744    let x = exact_string_member(jwk, "x", "credential claims.cnf.jwk")?;
2745    let key = decode_canonical_jwt_segment(x, "credential cnf.jwk.x", 64)?;
2746    let key: [u8; 32] = key
2747        .try_into()
2748        .map_err(|_| anyhow::anyhow!("credential cnf.jwk.x must decode to exactly 32 bytes"))?;
2749
2750    let enrolled = match authority_state {
2751        DeploymentAuthorityState::GenesisOnly => None,
2752        DeploymentAuthorityState::Enrolled {
2753            authority_log,
2754            authority_checkpoint,
2755        } => Some((
2756            authority_log,
2757            validate_deployment_authority_log(
2758                ca_verifying_key,
2759                authority_log,
2760                authority_checkpoint,
2761            )?,
2762        )),
2763    };
2764    let delegated_ca;
2765    let signing_ca = if delegated {
2766        let (installed_authority_log, active) = enrolled.as_ref().ok_or_else(|| {
2767            anyhow::anyhow!(
2768                "delegated credential is forbidden during one-time genesis verification"
2769            )
2770        })?;
2771        let encoded = exact_string_member(claims, "delegation", "credential claims")?;
2772        let bytes = decode_canonical_jwt_segment(
2773            encoded,
2774            "credential delegation",
2775            MAX_REGISTRY_DELEGATION_BYTES,
2776        )?;
2777        let artifact: RegistryDelegationArtifact = serde_json::from_slice(&bytes)
2778            .map_err(|error| anyhow::anyhow!("credential delegation is malformed: {error}"))?;
2779        delegated_ca = validate_registry_delegation_artifact(
2780            ca_verifying_key,
2781            &artifact,
2782            installed_authority_log,
2783            active,
2784            u64::try_from(now).map_err(|_| anyhow::anyhow!("system clock precedes Unix epoch"))?,
2785        )?;
2786        &delegated_ca.delegated
2787    } else if let Some((_installed_authority_log, active)) = enrolled.as_ref() {
2788        anyhow::ensure!(
2789            active.rotation_keys.iter().any(|key| {
2790                key.ed25519_pub == ca_verifying_key.ed25519.to_bytes()
2791                    && key.mldsa65_pub
2792                        == hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&ca_verifying_key.ml_dsa_65)
2793            }),
2794            "direct root credential issuer is retired at the installed authority-log head"
2795        );
2796        ca_verifying_key
2797    } else {
2798        ca_verifying_key
2799    };
2800    anyhow::ensure!(
2801        exact_string_member(protected, "kid", "protected header")? == signing_ca.domain(),
2802        "protected header kid does not bind the authorized deployment signer"
2803    );
2804
2805    // Signature verification is deliberately last: no parsed material can mint
2806    // authority unless the exact closed profile is authenticated by the fixed CA.
2807    let dispatch = hyprstream_rpc::auth::jwt::parse_composite_dispatch(token, &["wit+jwt"])
2808        .map_err(|error| anyhow::anyhow!("credential composite dispatch rejected: {error}"))?;
2809    let verified = hyprstream_rpc::auth::jwt::decode_composite(
2810        token,
2811        &signing_ca.ml_dsa_65,
2812        &signing_ca.ed25519,
2813        Some(REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE),
2814        &dispatch,
2815    )
2816    .map_err(|error| anyhow::anyhow!("credential hybrid signature rejected: {error}"))?;
2817    anyhow::ensure!(
2818        verified.sub == "service:registry" && verified.cnf_key_bytes() == Some(key),
2819        "verified credential differs from the exact deployment profile"
2820    );
2821    Ok((key, exp))
2822}
2823
2824struct AuthoritySetUcanVerifier<'a> {
2825    keys: &'a [crate::did_op::HybridRotationKey],
2826}
2827
2828impl hyprstream_rpc::auth::ucan::UcanVerifier for AuthoritySetUcanVerifier<'_> {
2829    fn verify(
2830        &self,
2831        _issuer: &hyprstream_rpc::auth::ucan::Did,
2832        ed_key: &[u8; ED25519_PUBLIC_KEY_BYTES],
2833        payload: &[u8],
2834        signature: &[u8],
2835    ) -> std::result::Result<(), hyprstream_rpc::auth::ucan::UcanError> {
2836        use hyprstream_rpc::auth::ucan::UcanError;
2837        let authority = self
2838            .keys
2839            .iter()
2840            .find(|key| &key.ed25519_pub == ed_key)
2841            .ok_or_else(|| {
2842                UcanError::BadSignature("UCAN issuer is not in the active authority set".to_owned())
2843            })?;
2844        let (ed, pq) = authority
2845            .verifying_keys()
2846            .map_err(|error| UcanError::BadSignature(error.to_string()))?;
2847        hyprstream_rpc::crypto::cose_sign::verify_composite(
2848            signature,
2849            &ed,
2850            Some(&pq),
2851            payload,
2852            hyprstream_rpc::auth::ucan::token::UCAN_AAD,
2853            true,
2854        )
2855        .map(|_| ())
2856        .map_err(|error| UcanError::BadSignature(error.to_string()))
2857    }
2858}
2859
2860/// The exact registry-mint capability every delegation must carry.
2861///
2862/// Public so the ceremony/mint CLI constructs delegations with the same code
2863/// the verifier validates them with (hyprstream#1562 H3) — no second copy to
2864/// drift.
2865pub fn registry_mint_capability(
2866    deployment_domain: &str,
2867    delegated_public_key_b64: &str,
2868) -> hyprstream_rpc::auth::ucan::Capability {
2869    use hyprstream_rpc::auth::ucan::{Ability, Capability, CaveatValue, Caveats, Resource};
2870
2871    let mut caveats = std::collections::BTreeMap::new();
2872    caveats.insert(
2873        "audience".to_owned(),
2874        CaveatValue::Text(REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE.to_owned()),
2875    );
2876    caveats.insert(
2877        "deployment_domain".to_owned(),
2878        CaveatValue::Text(deployment_domain.to_owned()),
2879    );
2880    caveats.insert(
2881        "delegated_public_key_b64".to_owned(),
2882        CaveatValue::Text(delegated_public_key_b64.to_owned()),
2883    );
2884    caveats.insert(
2885        "max_ttl_seconds".to_owned(),
2886        CaveatValue::Int(REGISTRY_DEPLOYMENT_CREDENTIAL_MAX_TTL_SECONDS),
2887    );
2888    caveats.insert(
2889        "profile".to_owned(),
2890        CaveatValue::Text(REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE.to_owned()),
2891    );
2892    Capability::with_caveats(
2893        Resource::new(format!(
2894            "hyprstream://deployment/{deployment_domain}/service/registry"
2895        )),
2896        Ability::new(REGISTRY_DELEGATION_ABILITY),
2897        Caveats(caveats),
2898    )
2899}
2900
2901/// The exact service-key-enrollment capability (hyprstream#1562): a fixed
2902/// allowlist, hybrid-only key type, and a one-hour attestation TTL ceiling.
2903///
2904/// Public so the ceremony/mint CLI constructs delegations with the same code
2905/// the verifier validates them with (hyprstream#1562 H3).
2906pub fn service_key_enrollment_capability(
2907    deployment_domain: &str,
2908    delegated_public_key_b64: &str,
2909) -> hyprstream_rpc::auth::ucan::Capability {
2910    use hyprstream_rpc::auth::ucan::{Ability, Capability, CaveatValue, Caveats, Resource};
2911
2912    let mut caveats = std::collections::BTreeMap::new();
2913    caveats.insert(
2914        "deployment_domain".to_owned(),
2915        CaveatValue::Text(deployment_domain.to_owned()),
2916    );
2917    caveats.insert(
2918        "delegated_public_key_b64".to_owned(),
2919        CaveatValue::Text(delegated_public_key_b64.to_owned()),
2920    );
2921    caveats.insert(
2922        "allowed_services".to_owned(),
2923        CaveatValue::List(
2924            SERVICE_KEY_ENROLLMENT_ALLOWED_SERVICES
2925                .iter()
2926                .map(|service| (*service).to_owned())
2927                .collect(),
2928        ),
2929    );
2930    caveats.insert(
2931        "key_type".to_owned(),
2932        CaveatValue::Text(SERVICE_KEY_ENROLLMENT_KEY_TYPE.to_owned()),
2933    );
2934    caveats.insert(
2935        "max_attestation_ttl_seconds".to_owned(),
2936        CaveatValue::Int(SERVICE_KEY_ENROLLMENT_MAX_TTL_SECONDS),
2937    );
2938    Capability::with_caveats(
2939        Resource::new(format!(
2940            "hyprstream://deployment/{deployment_domain}/service-key-enrollment"
2941        )),
2942        Ability::new(SERVICE_KEY_ENROLLMENT_ABILITY),
2943        Caveats(caveats),
2944    )
2945}
2946
2947/// Exact-set capability validation (hyprstream#1562): a delegation is either
2948/// the legacy registry-only scope minted before enrollment existed, or exactly
2949/// the registry + enrollment pair. Anything narrower, wider, or reordered is
2950/// rejected. Returns true when the enrollment capability is present.
2951pub fn delegation_capability_set_grants_enrollment(
2952    capabilities: &[hyprstream_rpc::auth::ucan::Capability],
2953    deployment_domain: &str,
2954    delegated_public_key_b64: &str,
2955) -> Result<bool> {
2956    let registry = registry_mint_capability(deployment_domain, delegated_public_key_b64);
2957    let enrollment = service_key_enrollment_capability(deployment_domain, delegated_public_key_b64);
2958    if capabilities.len() == 1 && capabilities[0] == registry {
2959        return Ok(false);
2960    }
2961    anyhow::ensure!(
2962        capabilities.len() == 2
2963            && capabilities.contains(&registry)
2964            && capabilities.contains(&enrollment),
2965        "delegation capability set is not the exact registry or registry+enrollment scope"
2966    );
2967    Ok(true)
2968}
2969
2970/// Outcome of validating a root-authorized delegation: whether it may enroll
2971/// service keys, and the delegation expiry that caps anything it mints.
2972#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2973pub struct ValidatedDelegation {
2974    pub grants_service_key_enrollment: bool,
2975    pub expires_at: u64,
2976}
2977
2978/// Validate one root-authorized delegation UCAN (hyprstream#1562): a single
2979/// authority-to-signer link (no proofs), signed by an active authority,
2980/// addressed to the delegated signer, carrying exactly the registry-mint or
2981/// the registry + service-key-enrollment capability set, and expiring.
2982///
2983/// Public so the ceremony/mint CLI self-checks freshly minted delegations with
2984/// the same code the production verifier applies (hyprstream#1562 H3).
2985pub fn validate_delegation_ucan(
2986    ucan: &hyprstream_rpc::auth::ucan::Ucan,
2987    deployment_domain: &str,
2988    delegated_public_key_b64: &str,
2989    active_keys: &[crate::did_op::HybridRotationKey],
2990    now: u64,
2991) -> Result<ValidatedDelegation> {
2992    use hyprstream_rpc::auth::ucan::validate as validate_ucan;
2993
2994    anyhow::ensure!(
2995        ucan.proofs.is_empty(),
2996        "registry delegation must be one authority-to-signer link"
2997    );
2998    validate_ucan(ucan, &AuthoritySetUcanVerifier { keys: active_keys }, now)
2999        .context("validating registry delegation UCAN")?;
3000    anyhow::ensure!(
3001        active_keys
3002            .iter()
3003            .any(|key| ucan.issuer().to_ed25519().ok() == Some(key.ed25519_pub)),
3004        "registry delegation issuer is not an active authority"
3005    );
3006    let delegated_public = base64::engine::general_purpose::STANDARD
3007        .decode(delegated_public_key_b64)
3008        .context("decoding delegated registry-signer public key")?;
3009    let delegated_ed: [u8; ED25519_PUBLIC_KEY_BYTES] = delegated_public
3010        .get(..ED25519_PUBLIC_KEY_BYTES)
3011        .ok_or_else(|| anyhow::anyhow!("delegated registry-signer public key is truncated"))?
3012        .try_into()
3013        .map_err(|_| anyhow::anyhow!("delegated Ed25519 key is malformed"))?;
3014    anyhow::ensure!(
3015        ucan.audience().to_ed25519()? == delegated_ed,
3016        "registry delegation audience does not match delegated signer"
3017    );
3018    let grants_service_key_enrollment = delegation_capability_set_grants_enrollment(
3019        ucan.capabilities(),
3020        deployment_domain,
3021        delegated_public_key_b64,
3022    )?;
3023    let expires_at = ucan
3024        .payload
3025        .expiration
3026        .ok_or_else(|| anyhow::anyhow!("registry delegation must expire"))?;
3027    Ok(ValidatedDelegation {
3028        grants_service_key_enrollment,
3029        expires_at,
3030    })
3031}
3032
3033/// Outcome of validating a delegation artifact: the authenticated delegated
3034/// signer, whether it may enroll service keys, and the delegation expiry that
3035/// caps anything it mints.
3036struct ValidatedRegistryDelegation {
3037    delegated: HybridDeploymentCa,
3038    grants_service_key_enrollment: bool,
3039    expires_at: u64,
3040}
3041
3042fn validate_registry_delegation_artifact(
3043    root: &HybridDeploymentCa,
3044    artifact: &RegistryDelegationArtifact,
3045    installed_authority_log: &DeploymentAuthorityLog,
3046    active: &crate::did_op::VerifiedDidOpLog,
3047    now: u64,
3048) -> Result<ValidatedRegistryDelegation> {
3049    use hyprstream_rpc::auth::ucan::Ucan;
3050
3051    anyhow::ensure!(
3052        artifact.schema == REGISTRY_DELEGATION_SCHEMA,
3053        "unsupported registry delegation schema"
3054    );
3055    anyhow::ensure!(
3056        artifact.deployment_domain == root.domain(),
3057        "registry delegation deployment domain does not match pinned root"
3058    );
3059    anyhow::ensure!(
3060        artifact.authority_log_did == installed_authority_log.did,
3061        "registry delegation names a different authority log"
3062    );
3063    let active_keys = &active.rotation_keys;
3064
3065    let delegated_public = base64::engine::general_purpose::STANDARD
3066        .decode(&artifact.delegated_public_key_b64)
3067        .context("decoding delegated registry-signer public key")?;
3068    anyhow::ensure!(
3069        delegated_public.len() == DEPLOYMENT_CA_ROOT_BYTES,
3070        "delegated registry-signer public key has the wrong length"
3071    );
3072    let delegated = HybridDeploymentCa::from_public_key_bytes(
3073        &delegated_public[..ED25519_PUBLIC_KEY_BYTES],
3074        &delegated_public[ED25519_PUBLIC_KEY_BYTES..],
3075    )?;
3076
3077    let ucan_bytes = base64::engine::general_purpose::STANDARD
3078        .decode(&artifact.ucan_b64)
3079        .context("decoding registry delegation UCAN")?;
3080    anyhow::ensure!(
3081        ucan_bytes.len() <= MAX_REGISTRY_DELEGATION_BYTES,
3082        "registry delegation UCAN is too large"
3083    );
3084    let ucan = Ucan::from_cbor(&ucan_bytes).context("decoding registry delegation UCAN")?;
3085    let validated = validate_delegation_ucan(
3086        &ucan,
3087        &artifact.deployment_domain,
3088        &artifact.delegated_public_key_b64,
3089        active_keys,
3090        now,
3091    )?;
3092    Ok(ValidatedRegistryDelegation {
3093        delegated,
3094        grants_service_key_enrollment: validated.grants_service_key_enrollment,
3095        expires_at: validated.expires_at,
3096    })
3097}
3098
3099/// Validate a delegation artifact against the pinned production root and the
3100/// installed/current authority log with its independently trusted head
3101/// (hyprstream#1562 H3).
3102///
3103/// This is the ceremony/mint-side entry point: it runs the exact checks the
3104/// production verifier applies, so tooling cannot bless a delegation shape
3105/// production would reject.
3106pub fn validate_registry_delegation(
3107    public_ca: &[u8],
3108    authority_log: &DeploymentAuthorityLog,
3109    authority_checkpoint: &DeploymentAuthorityCheckpoint,
3110    artifact: &RegistryDelegationArtifact,
3111    now: u64,
3112) -> Result<ValidatedDelegation> {
3113    let root = HybridDeploymentCa::from_os_pin(public_ca)?;
3114    let active = validate_deployment_authority_log(&root, authority_log, authority_checkpoint)?;
3115    let validated =
3116        validate_registry_delegation_artifact(&root, artifact, authority_log, &active, now)?;
3117    Ok(ValidatedDelegation {
3118        grants_service_key_enrollment: validated.grants_service_key_enrollment,
3119        expires_at: validated.expires_at,
3120    })
3121}
3122
3123fn validate_deployment_authority_log(
3124    root: &HybridDeploymentCa,
3125    log: &DeploymentAuthorityLog,
3126    checkpoint: &DeploymentAuthorityCheckpoint,
3127) -> Result<crate::did_op::VerifiedDidOpLog> {
3128    anyhow::ensure!(
3129        log.schema == AUTHORITY_LOG_SCHEMA,
3130        "unsupported deployment authority-log schema"
3131    );
3132    anyhow::ensure!(
3133        log.deployment_domain == root.domain(),
3134        "deployment authority-log domain does not match pinned root"
3135    );
3136    anyhow::ensure!(
3137        !log.operations_b64.is_empty() && log.operations_b64.len() <= MAX_AUTHORITY_LOG_OPERATIONS,
3138        "deployment authority-log operation count is invalid"
3139    );
3140    let operations = log
3141        .operations_b64
3142        .iter()
3143        .map(|encoded| {
3144            let bytes = base64::engine::general_purpose::STANDARD
3145                .decode(encoded)
3146                .context("decoding deployment authority operation")?;
3147            crate::did_op::DidOp::from_dag_cbor(&bytes)
3148        })
3149        .collect::<Result<Vec<_>>>()?;
3150    let active = crate::did_op::verify_did_op_log(&log.did, &operations)
3151        .context("verifying deployment authority log")?;
3152    anyhow::ensure!(
3153        checkpoint.schema == AUTHORITY_CHECKPOINT_SCHEMA,
3154        "unsupported deployment authority-checkpoint schema"
3155    );
3156    anyhow::ensure!(
3157        checkpoint.deployment_domain == root.domain(),
3158        "deployment authority checkpoint domain does not match pinned root"
3159    );
3160    anyhow::ensure!(
3161        checkpoint.did == log.did
3162            && checkpoint.did == active.did
3163            && checkpoint.sequence == active.sequence
3164            && checkpoint.head_cid == active.head_cid,
3165        "deployment authority-log head does not match the independently trusted checkpoint"
3166    );
3167    let genesis = operations
3168        .first()
3169        .ok_or_else(|| anyhow::anyhow!("deployment authority log is empty"))?;
3170    let genesis_composite = hyprstream_rpc::crypto::cose_sign::assemble_composite_nested(
3171        (
3172            root.ed25519.to_bytes().to_vec(),
3173            genesis.signature.ed25519.clone(),
3174        ),
3175        Some((
3176            hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&root.ml_dsa_65),
3177            genesis.signature.mldsa65.clone(),
3178        )),
3179    )
3180    .context("assembling pinned-root authority genesis signature")?;
3181    hyprstream_rpc::crypto::cose_sign::verify_composite(
3182        &genesis_composite,
3183        &root.ed25519,
3184        Some(&root.ml_dsa_65),
3185        &genesis.signable_bytes(),
3186        crate::did_op::DID_OP_SIGNATURE_CONTEXT,
3187        true,
3188    )
3189    .context("authority-log genesis was not signed by the pinned root")?;
3190    anyhow::ensure!(
3191        genesis.rotation_keys.iter().any(|key| {
3192            key.ed25519_pub == root.ed25519.to_bytes()
3193                && key.mldsa65_pub == hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&root.ml_dsa_65)
3194        }),
3195        "deployment authority-log genesis is not anchored by the pinned public root"
3196    );
3197    Ok(active)
3198}
3199
3200/// Result of verifying the enrolled deployment artifact set consumed at
3201/// production bootstrap. This exposes no authority material and cannot install
3202/// or replace process trust.
3203#[derive(Clone, Debug, Eq, PartialEq)]
3204pub struct VerifiedDeploymentArtifacts {
3205    pub deployment_domain: String,
3206    pub registry_public_key: [u8; ED25519_PUBLIC_KEY_BYTES],
3207    pub expires_at: i64,
3208}
3209
3210/// Parse the exact production public-root pin and return its stable deployment
3211/// domain. This is the mint-time half of
3212/// [`verify_deployment_artifacts_with_authority_log`].
3213pub fn verify_deployment_public_ca(public_ca: &[u8]) -> Result<String> {
3214    HybridDeploymentCa::from_os_pin(public_ca).map(|ca| ca.domain())
3215}
3216
3217/// Explicitly verify a direct-root credential during one-time genesis before
3218/// authority-log enrollment. Production startup never calls this function and
3219/// never infers genesis mode from a missing file.
3220///
3221/// Tooling uses this entry point for pre-installation self-checks. Keeping the
3222/// parser and validator here prevents a mint tool from accidentally blessing a
3223/// byte layout or JWT shape that production would reject.
3224pub fn verify_genesis_deployment_artifacts(
3225    public_ca: &[u8],
3226    registry_credential: &str,
3227) -> Result<VerifiedDeploymentArtifacts> {
3228    let ca = HybridDeploymentCa::from_os_pin(public_ca)?;
3229    let (registry_public_key, expires_at) = validate_registry_deployment_credential_profile(
3230        registry_credential,
3231        &ca,
3232        DeploymentAuthorityState::GenesisOnly,
3233    )?;
3234    Ok(VerifiedDeploymentArtifacts {
3235        deployment_domain: ca.domain(),
3236        registry_public_key,
3237        expires_at,
3238    })
3239}
3240
3241/// Verify any enrolled deployment credential against the public root, the
3242/// installed/current authority log, and its independently trusted head.
3243pub fn verify_deployment_artifacts_with_authority_log(
3244    public_ca: &[u8],
3245    authority_log_json: &[u8],
3246    authority_checkpoint_json: &[u8],
3247    registry_credential: &str,
3248) -> Result<VerifiedDeploymentArtifacts> {
3249    anyhow::ensure!(
3250        authority_log_json.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3251        "installed deployment authority log is too large"
3252    );
3253    let ca = HybridDeploymentCa::from_os_pin(public_ca)?;
3254    let authority_log: DeploymentAuthorityLog = serde_json::from_slice(authority_log_json)
3255        .map_err(|error| {
3256            anyhow::anyhow!("installed deployment authority log is malformed: {error}")
3257        })?;
3258    anyhow::ensure!(
3259        authority_checkpoint_json.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3260        "installed deployment authority checkpoint is too large"
3261    );
3262    let authority_checkpoint: DeploymentAuthorityCheckpoint =
3263        serde_json::from_slice(authority_checkpoint_json).map_err(|error| {
3264            anyhow::anyhow!("installed deployment authority checkpoint is malformed: {error}")
3265        })?;
3266    let (registry_public_key, expires_at) = validate_registry_deployment_credential_profile(
3267        registry_credential,
3268        &ca,
3269        DeploymentAuthorityState::Enrolled {
3270            authority_log: &authority_log,
3271            authority_checkpoint: &authority_checkpoint,
3272        },
3273    )?;
3274    Ok(VerifiedDeploymentArtifacts {
3275        deployment_domain: ca.domain(),
3276        registry_public_key,
3277        expires_at,
3278    })
3279}
3280
3281/// Result of verifying a service-key enrollment attestation: the authenticated
3282/// hybrid public key for one allowlisted service. This exposes no authority or
3283/// signing material.
3284#[derive(Clone, Debug, Eq, PartialEq)]
3285pub struct VerifiedServiceKeyEnrollment {
3286    pub deployment_domain: String,
3287    pub service: String,
3288    /// The exact 1984-byte `bootstrap-pubkeys` entry (32-byte Ed25519, then
3289    /// 1952-byte ML-DSA-65).
3290    pub hybrid_public_key: Vec<u8>,
3291    pub expires_at: i64,
3292}
3293
3294/// Verify a `hyprstream.service-key-enrollment.v1` attestation against the
3295/// public root, the installed/current authority log, and its independently
3296/// trusted head (hyprstream#1562).
3297///
3298/// Fail-closed on: wrong schema/domain, a service outside the fixed
3299/// allowlist, a malformed or wrong-length hybrid key, an attestation lifetime
3300/// above one hour or past the delegation expiry, expiry at the current time,
3301/// a delegation that does not carry the exact enrollment capability, and any
3302/// signature half that does not verify against the delegated signer.
3303pub fn verify_service_key_enrollment(
3304    public_ca: &[u8],
3305    authority_log_json: &[u8],
3306    authority_checkpoint_json: &[u8],
3307    attestation_json: &[u8],
3308) -> Result<VerifiedServiceKeyEnrollment> {
3309    anyhow::ensure!(
3310        authority_log_json.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3311        "installed deployment authority log is too large"
3312    );
3313    anyhow::ensure!(
3314        authority_checkpoint_json.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3315        "installed deployment authority checkpoint is too large"
3316    );
3317    anyhow::ensure!(
3318        attestation_json.len() <= MAX_SERVICE_KEY_ENROLLMENT_BYTES,
3319        "service-key enrollment attestation is too large"
3320    );
3321    let ca = HybridDeploymentCa::from_os_pin(public_ca)?;
3322    let authority_log: DeploymentAuthorityLog = serde_json::from_slice(authority_log_json)
3323        .map_err(|error| {
3324            anyhow::anyhow!("installed deployment authority log is malformed: {error}")
3325        })?;
3326    let authority_checkpoint: DeploymentAuthorityCheckpoint =
3327        serde_json::from_slice(authority_checkpoint_json).map_err(|error| {
3328            anyhow::anyhow!("installed deployment authority checkpoint is malformed: {error}")
3329        })?;
3330    let active = validate_deployment_authority_log(&ca, &authority_log, &authority_checkpoint)?;
3331    let attestation: ServiceKeyEnrollmentArtifact = serde_json::from_slice(attestation_json)
3332        .map_err(|error| {
3333            anyhow::anyhow!("service-key enrollment attestation is malformed: {error}")
3334        })?;
3335    anyhow::ensure!(
3336        attestation.schema == SERVICE_KEY_ENROLLMENT_SCHEMA,
3337        "unsupported service-key enrollment schema"
3338    );
3339    anyhow::ensure!(
3340        attestation.deployment_domain == ca.domain(),
3341        "enrollment deployment domain does not match pinned root"
3342    );
3343    anyhow::ensure!(
3344        SERVICE_KEY_ENROLLMENT_ALLOWED_SERVICES.contains(&attestation.service.as_str()),
3345        "enrollment service is outside the fixed discovery/policy allowlist"
3346    );
3347    let hybrid_public_key = base64::engine::general_purpose::STANDARD
3348        .decode(&attestation.hybrid_public_key_b64)
3349        .context("decoding enrolled hybrid public key")?;
3350    anyhow::ensure!(
3351        hybrid_public_key.len() == DEPLOYMENT_CA_ROOT_BYTES,
3352        "enrolled hybrid public key must be exactly {DEPLOYMENT_CA_ROOT_BYTES} bytes \
3353         (32-byte Ed25519 followed by 1952-byte ML-DSA-65)"
3354    );
3355    // Well-formedness of both halves; the mint cannot recompute the ML-DSA-65
3356    // half from a public sidecar, so a truncated or classical-only key must
3357    // fail here rather than at first use.
3358    HybridDeploymentCa::from_public_key_bytes(
3359        &hybrid_public_key[..ED25519_PUBLIC_KEY_BYTES],
3360        &hybrid_public_key[ED25519_PUBLIC_KEY_BYTES..],
3361    )?;
3362
3363    let now = chrono::Utc::now().timestamp();
3364    let latest_future_time = now
3365        .checked_add(REGISTRY_DEPLOYMENT_CREDENTIAL_CLOCK_SKEW_SECONDS)
3366        .ok_or_else(|| anyhow::anyhow!("enrollment clock-skew arithmetic overflow"))?;
3367    anyhow::ensure!(
3368        attestation.not_before >= 0,
3369        "enrollment not_before is negative"
3370    );
3371    anyhow::ensure!(
3372        attestation.not_before <= latest_future_time,
3373        "enrollment is not yet valid"
3374    );
3375    anyhow::ensure!(
3376        attestation.not_before < attestation.expires_at,
3377        "enrollment not_before is not before expires_at"
3378    );
3379    let lifetime = attestation
3380        .expires_at
3381        .checked_sub(attestation.not_before)
3382        .ok_or_else(|| anyhow::anyhow!("enrollment lifetime arithmetic overflow"))?;
3383    anyhow::ensure!(
3384        lifetime <= SERVICE_KEY_ENROLLMENT_MAX_TTL_SECONDS,
3385        "enrollment lifetime exceeds the inclusive one-hour limit"
3386    );
3387    anyhow::ensure!(now < attestation.expires_at, "enrollment has expired");
3388
3389    let validated = validate_registry_delegation_artifact(
3390        &ca,
3391        &attestation.delegation,
3392        &authority_log,
3393        &active,
3394        u64::try_from(now).map_err(|_| anyhow::anyhow!("system clock precedes Unix epoch"))?,
3395    )?;
3396    anyhow::ensure!(
3397        validated.grants_service_key_enrollment,
3398        "delegation does not carry the service-key-enrollment capability"
3399    );
3400    anyhow::ensure!(
3401        attestation.deployment_domain == attestation.delegation.deployment_domain,
3402        "enrollment domain does not match its delegation"
3403    );
3404    anyhow::ensure!(
3405        attestation.expires_at >= 0
3406            && u64::try_from(attestation.expires_at)
3407                .map_err(|_| anyhow::anyhow!("enrollment expiry conversion failed"))?
3408                <= validated.expires_at,
3409        "enrollment expiry exceeds the delegation expiry"
3410    );
3411
3412    let signature = base64::engine::general_purpose::STANDARD
3413        .decode(&attestation.signature_b64)
3414        .context("decoding enrollment signature")?;
3415    // Signature verification is deliberately last: no parsed material becomes a
3416    // trusted key unless the exact artifact is authenticated by the delegated
3417    // signer the root authorized.
3418    hyprstream_rpc::crypto::cose_sign::verify_composite(
3419        &signature,
3420        &validated.delegated.ed25519,
3421        Some(&validated.delegated.ml_dsa_65),
3422        &attestation.signing_bytes()?,
3423        SERVICE_KEY_ENROLLMENT_SIGNATURE_CONTEXT,
3424        true,
3425    )
3426    .context("enrollment hybrid signature rejected")?;
3427    Ok(VerifiedServiceKeyEnrollment {
3428        deployment_domain: attestation.deployment_domain,
3429        service: attestation.service,
3430        hybrid_public_key,
3431        expires_at: attestation.expires_at,
3432    })
3433}
3434
3435/// Verify a `hyprstream.service-key-enrollment.v1` attestation against this
3436/// node's OS-owned deployment trust chain (hyprstream#1562 H3).
3437///
3438/// The root, authority log, and checkpoint are read through the same
3439/// trusted-artifact seam the process bootstrap uses (root-owned,
3440/// symlink-free, not group/world-writable), so the attestation is
3441/// authenticated by the ceremony chain — never by pinned key material. Every
3442/// failure mode of [`verify_service_key_enrollment`] applies; an unreadable
3443/// or untrusted chain artifact is equally fatal.
3444pub fn verify_os_owned_service_key_enrollment(
3445    attestation_json: &[u8],
3446) -> Result<VerifiedServiceKeyEnrollment> {
3447    let paths = resolve_deployment_trust_paths()?;
3448    let public_ca = read_trusted_artifact(&paths.public_ca, "deployment CA root")?;
3449    let authority_log = read_trusted_artifact(&paths.authority_log, "deployment authority log")?;
3450    let authority_checkpoint = read_trusted_artifact(
3451        &paths.authority_checkpoint,
3452        "deployment authority checkpoint",
3453    )?;
3454    verify_service_key_enrollment(
3455        &public_ca,
3456        &authority_log,
3457        &authority_checkpoint,
3458        attestation_json,
3459    )
3460}
3461
3462#[derive(Clone, Debug, Eq, PartialEq)]
3463struct TrustedArtifactPath {
3464    path: std::path::PathBuf,
3465}
3466
3467#[derive(Clone, Debug, Eq, PartialEq)]
3468struct DeploymentTrustPaths {
3469    public_ca: TrustedArtifactPath,
3470    authority_log: TrustedArtifactPath,
3471    authority_checkpoint: TrustedArtifactPath,
3472    registry_credential: TrustedArtifactPath,
3473}
3474
3475fn explicit_absolute_directory(
3476    env_name: &str,
3477    value: std::ffi::OsString,
3478) -> Result<std::path::PathBuf> {
3479    use std::path::Component;
3480
3481    let path = std::path::PathBuf::from(value);
3482    anyhow::ensure!(
3483        !path.as_os_str().is_empty(),
3484        "{env_name} is present but empty"
3485    );
3486    anyhow::ensure!(path.is_absolute(), "{env_name} must be an absolute path");
3487    anyhow::ensure!(
3488        path.components()
3489            .all(|component| matches!(component, Component::RootDir | Component::Normal(_))),
3490        "{env_name} must not contain '..' components"
3491    );
3492    Ok(path)
3493}
3494
3495fn resolve_deployment_trust_paths_from(
3496    trust_directory: Option<std::ffi::OsString>,
3497    credentials_directory: Option<std::ffi::OsString>,
3498) -> Result<DeploymentTrustPaths> {
3499    let (public_ca, authority_log, authority_checkpoint) = match trust_directory {
3500        Some(value) => {
3501            let directory = explicit_absolute_directory(DEPLOYMENT_TRUST_DIR_ENV, value)?;
3502            (
3503                directory.join(DEPLOYMENT_CA_ROOT_FILE),
3504                directory.join(DEPLOYMENT_AUTHORITY_LOG_FILE),
3505                directory.join(DEPLOYMENT_AUTHORITY_CHECKPOINT_FILE),
3506            )
3507        }
3508        None => (
3509            std::path::PathBuf::from(DEPLOYMENT_CA_ROOT_PATH),
3510            std::path::PathBuf::from(DEPLOYMENT_AUTHORITY_LOG_PATH),
3511            std::path::PathBuf::from(DEPLOYMENT_AUTHORITY_CHECKPOINT_PATH),
3512        ),
3513    };
3514    let registry_credential = match credentials_directory {
3515        Some(value) => {
3516            let directory = explicit_absolute_directory(SYSTEMD_CREDENTIALS_DIRECTORY_ENV, value)?;
3517            directory.join(REGISTRY_DEPLOYMENT_CREDENTIAL_FILE)
3518        }
3519        None => std::path::PathBuf::from(REGISTRY_DEPLOYMENT_CREDENTIAL_PATH),
3520    };
3521    Ok(DeploymentTrustPaths {
3522        public_ca: TrustedArtifactPath { path: public_ca },
3523        authority_log: TrustedArtifactPath {
3524            path: authority_log,
3525        },
3526        authority_checkpoint: TrustedArtifactPath {
3527            path: authority_checkpoint,
3528        },
3529        registry_credential: TrustedArtifactPath {
3530            path: registry_credential,
3531        },
3532    })
3533}
3534
3535fn resolve_deployment_trust_paths() -> Result<DeploymentTrustPaths> {
3536    resolve_deployment_trust_paths_from(
3537        std::env::var_os(DEPLOYMENT_TRUST_DIR_ENV),
3538        std::env::var_os(SYSTEMD_CREDENTIALS_DIRECTORY_ENV),
3539    )
3540}
3541
3542#[cfg(unix)]
3543fn validate_trusted_artifact_metadata(
3544    owner_uid: u32,
3545    mode: u32,
3546    effective_uid: u32,
3547    subject: &str,
3548) -> Result<()> {
3549    anyhow::ensure!(
3550        owner_uid == 0 || owner_uid == effective_uid,
3551        "{subject} has an untrusted owner"
3552    );
3553    anyhow::ensure!(mode & 0o022 == 0, "{subject} is group/world writable");
3554    Ok(())
3555}
3556
3557#[cfg(unix)]
3558fn read_trusted_artifact(artifact: &TrustedArtifactPath, description: &str) -> Result<Vec<u8>> {
3559    use std::io::Read as _;
3560    use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _};
3561
3562    anyhow::ensure!(
3563        artifact.path.is_absolute(),
3564        "{description} path must be absolute"
3565    );
3566    let effective_uid = {
3567        // SAFETY: geteuid has no preconditions and returns process-local state.
3568        unsafe { libc::geteuid() }
3569    };
3570    for parent in artifact.path.ancestors().skip(1) {
3571        let parent_metadata = std::fs::symlink_metadata(parent).map_err(|error| {
3572            anyhow::anyhow!("{description} parent {parent:?} is unavailable: {error}")
3573        })?;
3574        anyhow::ensure!(
3575            parent_metadata.file_type().is_dir(),
3576            "{description} parent {parent:?} is not a real directory"
3577        );
3578        validate_trusted_artifact_metadata(
3579            parent_metadata.uid(),
3580            parent_metadata.mode(),
3581            effective_uid,
3582            &format!("{description} parent {parent:?}"),
3583        )?;
3584    }
3585    let mut file = std::fs::OpenOptions::new()
3586        .read(true)
3587        .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW)
3588        .open(&artifact.path)
3589        .map_err(|error| {
3590            anyhow::anyhow!(
3591                "{description} is unavailable at {:?}: {error}",
3592                artifact.path
3593            )
3594        })?;
3595    let metadata = file.metadata().map_err(|error| {
3596        anyhow::anyhow!(
3597            "failed to inspect {description} at {:?}: {error}",
3598            artifact.path
3599        )
3600    })?;
3601    anyhow::ensure!(
3602        metadata.file_type().is_file(),
3603        "{description} is not a regular file"
3604    );
3605    validate_trusted_artifact_metadata(
3606        metadata.uid(),
3607        metadata.mode(),
3608        effective_uid,
3609        description,
3610    )?;
3611    let mut bytes = Vec::new();
3612    file.read_to_end(&mut bytes).map_err(|error| {
3613        anyhow::anyhow!(
3614            "failed to read {description} at {:?}: {error}",
3615            artifact.path
3616        )
3617    })?;
3618    Ok(bytes)
3619}
3620
3621#[cfg(not(unix))]
3622fn read_trusted_artifact(_artifact: &TrustedArtifactPath, description: &str) -> Result<Vec<u8>> {
3623    anyhow::bail!("{description} requires the trusted Unix deployment seam")
3624}
3625
3626fn load_registry_deployment_credential(paths: &DeploymentTrustPaths) -> Result<String> {
3627    String::from_utf8(read_trusted_artifact(
3628        &paths.registry_credential,
3629        "registry deployment credential",
3630    )?)
3631    .map_err(|error| anyhow::anyhow!("registry deployment credential is not UTF-8: {error}"))
3632}
3633
3634fn load_trusted_registry_deployment_credentials_from(
3635    paths: &DeploymentTrustPaths,
3636) -> Result<TrustedRegistryDeploymentCredentials> {
3637    let ca_bytes = read_trusted_artifact(&paths.public_ca, "deployment CA root")?;
3638    let ca_verifying_key = HybridDeploymentCa::from_os_pin(&ca_bytes)?;
3639    let authority_log_bytes =
3640        read_trusted_artifact(&paths.authority_log, "deployment authority log")?;
3641    anyhow::ensure!(
3642        authority_log_bytes.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3643        "deployment authority log exceeds the 64 KiB cloud-secret contract"
3644    );
3645    let authority_log = serde_json::from_slice(&authority_log_bytes)
3646        .map_err(|error| anyhow::anyhow!("deployment authority log is malformed: {error}"))?;
3647    let authority_checkpoint = load_deployment_authority_checkpoint_from(paths)?;
3648    let registry_credential = load_registry_deployment_credential(paths)?;
3649    Ok(TrustedRegistryDeploymentCredentials {
3650        ca_verifying_key,
3651        authority_log,
3652        authority_checkpoint,
3653        registry_credential,
3654    })
3655}
3656
3657fn load_trusted_registry_deployment_credentials() -> Result<TrustedRegistryDeploymentCredentials> {
3658    let paths = resolve_deployment_trust_paths()?;
3659    load_trusted_registry_deployment_credentials_from(&paths)
3660}
3661
3662fn load_deployment_authority_checkpoint_from(
3663    paths: &DeploymentTrustPaths,
3664) -> Result<DeploymentAuthorityCheckpoint> {
3665    let bytes = read_trusted_artifact(
3666        &paths.authority_checkpoint,
3667        "deployment authority checkpoint",
3668    )?;
3669    anyhow::ensure!(
3670        bytes.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3671        "deployment authority checkpoint exceeds the 64 KiB cloud-secret contract"
3672    );
3673    serde_json::from_slice(&bytes)
3674        .map_err(|error| anyhow::anyhow!("deployment authority checkpoint is malformed: {error}"))
3675}
3676
3677fn load_deployment_authority_checkpoint() -> Result<DeploymentAuthorityCheckpoint> {
3678    let paths = resolve_deployment_trust_paths()?;
3679    load_deployment_authority_checkpoint_from(&paths)
3680}
3681
3682fn authenticate_registry_deployment_credentials(
3683    credentials: TrustedRegistryDeploymentCredentials,
3684) -> Result<AuthenticatedRegistryDeploymentIdentity> {
3685    let (key_bytes, _expires_at) = validate_registry_deployment_credential_profile(
3686        &credentials.registry_credential,
3687        &credentials.ca_verifying_key,
3688        DeploymentAuthorityState::Enrolled {
3689            authority_log: &credentials.authority_log,
3690            authority_checkpoint: &credentials.authority_checkpoint,
3691        },
3692    )
3693    .map_err(|error| anyhow::anyhow!("registry deployment credential rejected: {error}"))?;
3694    let verifying_key = VerifyingKey::from_bytes(&key_bytes)
3695        .map_err(|error| anyhow::anyhow!("registry credential key is malformed: {error}"))?;
3696    Ok(AuthenticatedRegistryDeploymentIdentity {
3697        verifier: RegistryDeploymentVerifier {
3698            verifying_key,
3699            _deployment_ca: Arc::new(credentials.ca_verifying_key),
3700        },
3701    })
3702}
3703
3704// `store_path` is consumed by the checkpointed-store constructors, which are
3705// gated on `rocksdb`; without that feature the field is intentionally unread.
3706#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
3707struct ProcessBootstrapAuthority {
3708    store_path: std::path::PathBuf,
3709    acceptance_identity: ProcessAcceptanceIdentity,
3710    network_required: bool,
3711}
3712
3713enum ProcessAcceptanceIdentity {
3714    Deployment(RegistryDeploymentVerifier),
3715    #[cfg(test)]
3716    #[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
3717    Test(VerifyingKey),
3718}
3719
3720enum ProcessBootstrapAuthorityState {
3721    Unsealed,
3722    Sealed,
3723    Consumed,
3724}
3725
3726static PROCESS_BOOTSTRAP_AUTHORITY: parking_lot::Mutex<ProcessBootstrapAuthorityState> =
3727    parking_lot::Mutex::new(ProcessBootstrapAuthorityState::Unsealed);
3728
3729/// Resolve the lazy local discovery transport for the OS-owned-files
3730/// bootstrap path.
3731///
3732/// `try_endpoint` falls back to the default discovery socket when nothing is
3733/// registered yet, and the transport returned by [`dial`](hyprstream_rpc::dial)
3734/// connects on first use — so Discovery may start *after* the process resolver
3735/// is installed (first-boot / standalone / `--ipc` containers sharing the IPC
3736/// volume). This mirrors the same-node DID-anchored fabric in
3737/// [`bootstrap_deployment_process`] and replaces the eager
3738/// `DiscoveryClient::for_local_bootstrap`, which required
3739/// `registered_endpoint("discovery")` to be `Some` — impossible at first-boot
3740/// before any service binds, causing every `service start` to abort with
3741/// "local bootstrap requires an explicitly registered service endpoint".
3742///
3743/// Trust is not weakened: the checkpoint was already verified by
3744/// `authenticate_deployment_bootstrap`, and the caller still authenticates
3745/// every lazy response with the pinned `discovery_vk`.
3746fn resolve_local_discovery_transport() -> anyhow::Result<hyprstream_rpc::transport::TransportConfig>
3747{
3748    hyprstream_rpc::registry::try_global()
3749        .ok_or_else(|| {
3750            anyhow::anyhow!("EndpointRegistry not initialized — local discovery fabric unavailable")
3751        })?
3752        .try_endpoint("discovery", hyprstream_rpc::registry::SocketKind::Rep)
3753}
3754
3755/// Build the lazy local discovery client used by the OS-owned-files bootstrap
3756/// arm of [`bootstrap_deployment_process`].
3757///
3758/// This is the resolver-install half of that arm, extracted so the production
3759/// code path — `resolve_local_discovery_transport` → `dial` →
3760/// `DiscoveryClient::new` — is the single source of truth AND directly
3761/// testable without the deployment-credential chain (which is orthogonal to
3762/// the endpoint-resolution regression). `dial` connects on first use, so
3763/// Discovery may start *after* this install inside the same process. Trust is
3764/// not weakened: the checkpoint was already verified by the caller, and
3765/// `discovery_vk` authenticates every lazy response.
3766fn install_local_discovery_client(
3767    signing_key: SigningKey,
3768    discovery_vk: VerifyingKey,
3769    token: Option<String>,
3770) -> Result<hyprstream_rpc_std::discovery_client::DiscoveryClient> {
3771    let transport = resolve_local_discovery_transport()?;
3772    let token_provider = renewing_service_token_provider(&signing_key, token);
3773    let signer = hyprstream_rpc::signer::LocalSigner::new(signing_key);
3774    let rpc = hyprstream_rpc::dial::dial_with_token_provider(
3775        &transport,
3776        signer,
3777        Some(discovery_vk),
3778        token_provider,
3779    )?;
3780    Ok(hyprstream_rpc_std::discovery_client::DiscoveryClient::new(
3781        rpc,
3782    ))
3783}
3784
3785/// Only a caller that supplied its own enrolled credential can obtain a
3786/// default JWT. Renewal replaces that same signer's attestation in the process
3787/// trust store; the initial credential covers the interval before registration.
3788fn current_service_token(signing_key: &SigningKey, initial: &Option<String>) -> Option<String> {
3789    initial.as_ref()?;
3790    hyprstream_service::global_trust_store()
3791        .get(&signing_key.verifying_key())
3792        .and_then(|att| att.jwt)
3793        .or_else(|| initial.clone())
3794}
3795
3796fn renewing_service_token_provider(
3797    signing_key: &SigningKey,
3798    initial: Option<String>,
3799) -> Option<Arc<dyn Fn() -> Option<String> + Send + Sync>> {
3800    initial.as_ref()?;
3801    let signer = signing_key.clone();
3802    Some(Arc::new(move || current_service_token(&signer, &initial)))
3803}
3804
3805fn bootstrap_client_transport_key(
3806    service_key: &SigningKey,
3807    ephemeral_override: Option<SigningKey>,
3808) -> SigningKey {
3809    ephemeral_override.unwrap_or_else(|| {
3810        hyprstream_rpc::node_identity::derive_purpose_key(
3811            service_key,
3812            "hyprstream-bootstrap-client-transport-v1",
3813        )
3814    })
3815}
3816
3817/// Atomically consume the explicitly selected deployment witness and install
3818/// the process Discovery resolver. Selection never falls back between the
3819/// OS-owned and DID-anchored providers.
3820///
3821/// `remote_node` selects how the DID-anchored provider reaches Discovery:
3822/// - `false` (same-node fabric, e.g. the metal stack's `service start --ipc`
3823///   containers sharing the IPC volume): a lazy local discovery client —
3824///   identical posture to the OS-owned path, which also never pings. This is
3825///   required on nodes that HOST Discovery (possibly as an in-process
3826///   dependency started after this install), where a pre-install network
3827///   liveness check would self-deadlock.
3828/// - `true` (remote worker): dial the DID-advertised network transport and
3829///   REQUIRE a successful signed liveness ping before installing — a dead or
3830///   wrong-key endpoint refuses the boot.
3831///
3832/// `service_token` is the caller's own enrolled, signer-bound credential.
3833/// Discovery must not borrow another service's credential or infer MAC
3834/// clearance from a key.
3835#[cfg(not(target_arch = "wasm32"))]
3836// Without `rocksdb` the accepted-state authority construction fails closed,
3837// leaving the verifier binding unused in that configuration.
3838#[cfg_attr(
3839    not(any(feature = "rocksdb", feature = "postgres")),
3840    allow(unused_variables)
3841)]
3842pub async fn bootstrap_deployment_process(
3843    signing_key: SigningKey,
3844    service_token: Option<String>,
3845    trust_source: crate::DeploymentTrustSource,
3846    remote_node: bool,
3847    network_required: bool,
3848    records: &hyprstream_pds::rds::RdsConfig,
3849) -> Result<()> {
3850    bootstrap_deployment_process_with_carrier_identity(
3851        signing_key,
3852        service_token,
3853        trust_source,
3854        remote_node,
3855        network_required,
3856        records,
3857        None,
3858    )
3859    .await
3860}
3861
3862/// Bootstrap a one-shot client using the same authenticated service identity
3863/// but a fresh outbound Iroh carrier identity. N0 may publish transient reach
3864/// for this random carrier, but it cannot overwrite the live service's stable
3865/// carrier record. Intended for short-lived diagnostics, never normal services.
3866#[cfg(not(target_arch = "wasm32"))]
3867pub async fn bootstrap_deployment_process_with_ephemeral_client_transport(
3868    signing_key: SigningKey,
3869    service_token: Option<String>,
3870    trust_source: crate::DeploymentTrustSource,
3871    remote_node: bool,
3872    network_required: bool,
3873    records: &hyprstream_pds::rds::RdsConfig,
3874) -> Result<()> {
3875    let ephemeral_transport_key = SigningKey::generate(&mut rand::rngs::OsRng);
3876    bootstrap_deployment_process_with_carrier_identity(
3877        signing_key,
3878        service_token,
3879        trust_source,
3880        remote_node,
3881        network_required,
3882        records,
3883        Some(ephemeral_transport_key),
3884    )
3885    .await
3886}
3887
3888#[cfg(not(target_arch = "wasm32"))]
3889#[cfg_attr(
3890    not(any(feature = "rocksdb", feature = "postgres")),
3891    allow(unused_variables)
3892)]
3893async fn bootstrap_deployment_process_with_carrier_identity(
3894    signing_key: SigningKey,
3895    service_token: Option<String>,
3896    trust_source: crate::DeploymentTrustSource,
3897    remote_node: bool,
3898    network_required: bool,
3899    records: &hyprstream_pds::rds::RdsConfig,
3900    client_transport_key_override: Option<SigningKey>,
3901) -> Result<()> {
3902    let discovery_vk = hyprstream_service::global_trust_store()
3903        .resolve_one("discovery")
3904        .ok_or_else(|| anyhow::anyhow!("trust store has no authenticated discovery key"))?;
3905    let (authority, discovery_client) = match trust_source {
3906        crate::DeploymentTrustSource::OsOwnedFiles => {
3907            let mut authority = authenticate_deployment_bootstrap()?;
3908            authority.network_required = network_required;
3909            // Lazy local discovery client — see `install_local_discovery_client`:
3910            // the default discovery socket is resolved via `try_endpoint` (not
3911            // the eager registered_endpoint path), and `dial` connects on first use,
3912            // so Discovery may start *after* this install inside the same
3913            // process. Trust is not weakened — the checkpoint was already
3914            // verified by `authenticate_deployment_bootstrap()`, and
3915            // `discovery_vk` authenticates every lazy response.
3916            let discovery_client = if network_required {
3917                #[cfg(not(test))]
3918                let ProcessAcceptanceIdentity::Deployment(verifier) =
3919                    &authority.acceptance_identity;
3920                #[cfg(test)]
3921                let verifier = match &authority.acceptance_identity {
3922                    ProcessAcceptanceIdentity::Deployment(verifier) => verifier,
3923                    #[cfg(test)]
3924                    ProcessAcceptanceIdentity::Test(_) => {
3925                        anyhow::bail!("native bootstrap requires deployment authority")
3926                    }
3927                };
3928                #[cfg(any(feature = "rocksdb", feature = "postgres"))]
3929                let source = crate::checkpointed_pds::open_deployment_accepted_state_source(
3930                    &authority.store_path,
3931                    verifier.clone(),
3932                    records,
3933                    true,
3934                )?;
3935                // Fail closed: without a concrete store backend there is no
3936                // accepted-state authority to bootstrap against.
3937                #[cfg(not(any(feature = "rocksdb", feature = "postgres")))]
3938                let source: Arc<dyn AcceptedStateSource> =
3939                    Err(anyhow::anyhow!(
3940                        "checkpointed PDS accepted-state authority requires the `rocksdb` or `postgres` feature"
3941                    ))?;
3942                // Authenticate the fixed bootstrap roles now, but defer I/O:
3943                // Discovery and Policy have not necessarily bound yet.
3944                source.bootstrap_endpoints("discovery")?;
3945                source.bootstrap_endpoints("policy")?;
3946                // CLI callers also need an outbound carrier, before any service
3947                // binds. Keep this distinct from the service's inbound address.
3948                use hyprstream_rpc::transport::iroh_substrate::{IrohSubstrate, RefuseHandler};
3949                let transport_key =
3950                    bootstrap_client_transport_key(&signing_key, client_transport_key_override);
3951                let carrier = IrohSubstrate::new(
3952                    transport_key.to_bytes(),
3953                    RefuseHandler::new("outbound bootstrap only"),
3954                    RefuseHandler::new("outbound bootstrap only"),
3955                )
3956                .await?;
3957                let _ = hyprstream_rpc::transport::lazy_iroh::install_iroh_client_endpoint(
3958                    carrier.owned_client_endpoint(),
3959                );
3960                PROCESS_BOOTSTRAP_CARRIER
3961                    .set(carrier)
3962                    .map_err(|_| anyhow::anyhow!("bootstrap carrier already installed"))?;
3963                let resolver = Arc::new(DiscoveryServiceResolver {
3964                    state_store: MemoryStateStore::production_default(),
3965                    accepted_state_source: source,
3966                    discovery_client: None,
3967                });
3968                hyprstream_rpc_std::discovery_client::DiscoveryClient::new(Arc::new(
3969                    ProductionRpcClient::new(
3970                        "discovery",
3971                        "discovery",
3972                        None,
3973                        signing_key,
3974                        service_token,
3975                        resolver,
3976                    )?,
3977                ))
3978            } else {
3979                install_local_discovery_client(signing_key, discovery_vk, service_token)?
3980            };
3981            (authority, discovery_client)
3982        }
3983        crate::DeploymentTrustSource::DidAnchored(anchors) => {
3984            anyhow::ensure!(
3985                !network_required,
3986                "required native bootstrap needs provisioned OS-owned accepted service state"
3987            );
3988            let (authority, discovery_transport, mesh_kem_recipient, ml_dsa_65_keys) =
3989                authenticate_did_anchored_bootstrap(&anchors).await?;
3990            let token_provider = renewing_service_token_provider(&signing_key, service_token);
3991            let signer = hyprstream_rpc::signer::LocalSigner::new(signing_key);
3992            if remote_node {
3993                // Remote worker: the DID-advertised transport is the only
3994                // reach. QUIC forbids cleartext envelopes, so the document's
3995                // #mesh-kem recipient (anchored by the mutual-alias + CA-bound
3996                // trust decision above) is REQUIRED — no recipient, no boot.
3997                let recipient = mesh_kem_recipient.ok_or_else(|| {
3998                    anyhow::anyhow!(
3999                        "did:web deployment document publishes no #mesh-kem keyAgreement; \
4000                         refusing cleartext downgrade for the remote-node bootstrap"
4001                    )
4002                })?;
4003                // Response authentication likewise requires the document's
4004                // single #mesh-pq (ML-DSA-65) anchor for the discovery key.
4005                let pq_key = match ml_dsa_65_keys.as_slice() {
4006                    [key] => key.clone(),
4007                    keys => anyhow::bail!(
4008                        "did:web deployment document must publish exactly one ML-DSA-65 \
4009                         verification method for the remote-node bootstrap (found {})",
4010                        keys.len()
4011                    ),
4012                };
4013                let mut kem_store = hyprstream_rpc::crypto::hybrid_kem::KeyedKemTrustStore::new();
4014                kem_store.bind(discovery_vk.to_bytes(), recipient);
4015                let mut pq_store = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
4016                pq_store.bind(discovery_vk.to_bytes(), &pq_key);
4017                // Dial it and require liveness — a reachable-but-wrong
4018                // endpoint (no discovery key) or a dead one fails closed here.
4019                let rpc = hyprstream_rpc::dial::dial_with_crypto_stores_and_token_provider(
4020                    &discovery_transport,
4021                    signer,
4022                    Some(discovery_vk),
4023                    token_provider,
4024                    Some(Arc::new(kem_store)),
4025                    Some(Arc::new(pq_store)),
4026                )?;
4027                let client = hyprstream_rpc_std::discovery_client::DiscoveryClient::new(rpc);
4028                let health = client
4029                    .ping()
4030                    .await
4031                    .context("DID-anchored Discovery reach failed liveness check")?;
4032                anyhow::ensure!(
4033                    health.status == "ok",
4034                    "DID-anchored Discovery liveness returned status {:?}",
4035                    health.status
4036                );
4037                (authority, client)
4038            } else {
4039                // Same-node fabric: the local default discovery endpoint is a
4040                // lazy client (connects on first use), so Discovery may start
4041                // after this install inside the same process. Trust decisions
4042                // were already made above (GATE + mutual alias + CA-bound
4043                // credential) — no liveness check is meaningful or required
4044                // for a lazy local client, matching the OS-owned path.
4045                let transport = hyprstream_rpc::registry::try_global()
4046                    .ok_or_else(|| {
4047                        anyhow::anyhow!(
4048                        "EndpointRegistry not initialized — same-node discovery fabric unavailable"
4049                    )
4050                    })?
4051                    .try_endpoint("discovery", hyprstream_rpc::registry::SocketKind::Rep)?;
4052                let rpc = hyprstream_rpc::dial::dial_with_token_provider(
4053                    &transport,
4054                    signer,
4055                    Some(discovery_vk),
4056                    token_provider,
4057                )?;
4058                (
4059                    authority,
4060                    hyprstream_rpc_std::discovery_client::DiscoveryClient::new(rpc),
4061                )
4062            }
4063        }
4064    };
4065    DiscoveryService::bootstrap_authenticated_process(authority, discovery_client, records)
4066}
4067
4068#[cfg(not(target_arch = "wasm32"))]
4069fn seal_process_bootstrap_authority() -> Result<()> {
4070    let mut state = PROCESS_BOOTSTRAP_AUTHORITY.lock();
4071    anyhow::ensure!(
4072        matches!(*state, ProcessBootstrapAuthorityState::Unsealed),
4073        "Discovery bootstrap authority is already sealed or consumed"
4074    );
4075    *state = ProcessBootstrapAuthorityState::Sealed;
4076    Ok(())
4077}
4078
4079#[cfg(not(target_arch = "wasm32"))]
4080fn authenticate_deployment_bootstrap() -> Result<ProcessBootstrapAuthority> {
4081    seal_process_bootstrap_authority()?;
4082    let verifier = authenticate_local_deployment_registry()?;
4083    let store_path = hyprstream_service::deployment_data_dir()?.join("pds-store");
4084    Ok(ProcessBootstrapAuthority {
4085        store_path,
4086        acceptance_identity: ProcessAcceptanceIdentity::Deployment(verifier),
4087        network_required: false,
4088    })
4089}
4090
4091#[cfg(not(target_arch = "wasm32"))]
4092async fn authenticate_did_anchored_bootstrap(
4093    anchors: &crate::DidAnchors,
4094) -> Result<(
4095    ProcessBootstrapAuthority,
4096    TransportConfig,
4097    Option<hyprstream_rpc::crypto::hybrid_kem::RecipientPublic>,
4098    Vec<hyprstream_rpc::crypto::pq::MlDsaVerifyingKey>,
4099)> {
4100    seal_process_bootstrap_authority()?;
4101    let trust = crate::did_anchored::resolve_did_anchored_trust(anchors).await?;
4102    let authority_checkpoint = load_deployment_authority_checkpoint()?;
4103    let discovery_transport = trust.discovery_transport.clone();
4104    let mesh_kem_recipient = trust.mesh_kem_recipient.clone();
4105    let ml_dsa_65_keys = trust.ml_dsa_65_keys.clone();
4106    let verifier = authenticate_resolved_did_trust(trust, authority_checkpoint)?;
4107    let authority = ProcessBootstrapAuthority {
4108        store_path: hyprstream_service::deployment_data_dir()?.join("pds-store"),
4109        acceptance_identity: ProcessAcceptanceIdentity::Deployment(verifier),
4110        network_required: false,
4111    };
4112    Ok((
4113        authority,
4114        discovery_transport,
4115        mesh_kem_recipient,
4116        ml_dsa_65_keys,
4117    ))
4118}
4119
4120pub(crate) fn authenticate_resolved_did_trust(
4121    trust: crate::did_anchored::DidAnchoredTrust,
4122    authority_checkpoint: DeploymentAuthorityCheckpoint,
4123) -> Result<RegistryDeploymentVerifier> {
4124    authenticate_registry_deployment_credentials(TrustedRegistryDeploymentCredentials {
4125        ca_verifying_key: trust.ca_verifying_key,
4126        authority_log: trust.authority_log,
4127        authority_checkpoint,
4128        registry_credential: trust.registry_credential,
4129    })
4130    .map(|witness| witness.verifier)
4131}
4132
4133/// Resolve the configured DID anchors and authenticate the fetched registry
4134/// deployment credential against the DID-derived deployment CA. This is the
4135/// full DID-anchored trust decision — GATE (canon → hash → sig), mutual
4136/// alsoKnownAs attestation, credential fetch, and the exact credential profile
4137/// (CA-bound kid/iss/domain, one-hour freshness) — WITHOUT the process-global
4138/// one-shot seal and resolver install performed by
4139/// [`bootstrap_deployment_process`]. Exposed so integration tests can exercise
4140/// the trust decision (and its failure modes) more than once per process.
4141/// Any failure is terminal for the caller; nothing here retries or downgrades.
4142#[cfg(not(target_arch = "wasm32"))]
4143pub async fn resolve_and_authenticate_did_anchors(
4144    anchors: &crate::DidAnchors,
4145) -> Result<(RegistryDeploymentVerifier, TransportConfig)> {
4146    let trust = crate::did_anchored::resolve_did_anchored_trust(anchors).await?;
4147    let authority_checkpoint = load_deployment_authority_checkpoint()?;
4148    let discovery_transport = trust.discovery_transport.clone();
4149    let verifier = authenticate_resolved_did_trust(trust, authority_checkpoint)?;
4150    Ok((verifier, discovery_transport))
4151}
4152
4153#[cfg(test)]
4154#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
4155fn authenticate_discovery_bootstrap_identity(
4156    acceptance_identity: ed25519_dalek::VerifyingKey,
4157) -> Result<ProcessBootstrapAuthority> {
4158    let mut state = PROCESS_BOOTSTRAP_AUTHORITY.lock();
4159    anyhow::ensure!(
4160        matches!(*state, ProcessBootstrapAuthorityState::Unsealed),
4161        "Discovery bootstrap authority is already sealed or consumed"
4162    );
4163    *state = ProcessBootstrapAuthorityState::Sealed;
4164    Ok(ProcessBootstrapAuthority {
4165        store_path: hyprstream_service::deployment_data_dir()?.join("pds-store"),
4166        acceptance_identity: ProcessAcceptanceIdentity::Test(acceptance_identity),
4167        network_required: false,
4168    })
4169}
4170
4171/// Construct an ordinary production RPC client. Production authority cannot
4172/// be implemented, injected, or replaced by a downstream caller.
4173///
4174/// ```compile_fail
4175/// use hyprstream_rpc::{ResolvedRpcClient, ServiceResolver};
4176/// ```
4177///
4178/// ```compile_fail
4179/// use hyprstream_discovery::{DiscoveryServiceResolver, ResolvedService};
4180/// ```
4181///
4182/// Raw accepted evidence, candidates, selection, and the selected transport/key
4183/// bundle are private to Discovery's checkpoint-verifying implementation.
4184/// ```compile_fail
4185/// use hyprstream_rpc::{
4186///     select_service_candidate, AcceptedStateEvidence, AnchoredKemRecipient,
4187///     SelectedService, ServiceCandidate,
4188/// };
4189/// ```
4190pub fn production_rpc_client(
4191    service_name: &str,
4192    signing_key: SigningKey,
4193    token: Option<String>,
4194) -> Result<Arc<dyn RpcClient>> {
4195    let resolver = PRODUCTION_RESOLVER
4196        .get()
4197        .cloned()
4198        .ok_or_else(|| anyhow::anyhow!("checkpoint-backed production resolver is not installed"))?;
4199    Ok(Arc::new(ProductionRpcClient::new(
4200        service_name,
4201        service_name,
4202        None,
4203        signing_key,
4204        token,
4205        resolver,
4206    )?))
4207}
4208
4209/// The authenticated Discovery client this process's own bootstrap installed,
4210/// when bootstrap installed one.
4211///
4212/// Present after required-native bootstrap (checkpoint-backed resolver) and
4213/// after both DID-anchored arms — the remote-node network client
4214/// (KEM/PQ-bound, liveness-verified at install) and the same-node lazy local
4215/// client. `None` when no bootstrap ran, so callers fall back to their own
4216/// profile selection.
4217///
4218/// This — not the native-network profile — carries installed-bootstrap reach:
4219/// DID-anchored `remote_node = true` explicitly boots with
4220/// `network_required = false`, and the installed network client is the only
4221/// Discovery reach a remote node has. The client is authenticated per its
4222/// installing arm (pinned discovery key; remote-node additionally KEM/PQ-bound
4223/// and liveness-verified before install).
4224pub fn installed_bootstrap_discovery_client(
4225) -> Option<hyprstream_rpc_std::discovery_client::DiscoveryClient> {
4226    PRODUCTION_RESOLVER.get().cloned()?.discovery_client.clone()
4227}
4228
4229/// Construct a production RPC client pinned to one router-selected advertised
4230/// reach while retaining Discovery's opaque identity/currentness authority.
4231///
4232/// `resolution_service_name` selects the accepted dynamic inference entry.
4233/// The generated RPC schema domain remains hard-bound to canonical
4234/// `inference`; callers cannot choose a different application authority.
4235///
4236/// The selected reach is only a selector. It grants no authority: every call
4237/// must still find that exact reach in the checkpoint-current, same-authority
4238/// candidate set before Discovery constructs the crypto-bound dial.
4239pub fn production_inference_rpc_client_at_transport(
4240    resolution_service_name: &str,
4241    selected_transport: &TransportConfig,
4242    signing_key: SigningKey,
4243    token: Option<String>,
4244) -> Result<Arc<dyn RpcClient>> {
4245    let resolver = PRODUCTION_RESOLVER
4246        .get()
4247        .cloned()
4248        .ok_or_else(|| anyhow::anyhow!("checkpoint-backed production resolver is not installed"))?;
4249    Ok(Arc::new(ProductionRpcClient::new(
4250        resolution_service_name,
4251        "inference",
4252        Some(selected_transport.clone()),
4253        signing_key,
4254        token,
4255        resolver,
4256    )?))
4257}
4258
4259/// Project a browser/WebTransport provisioning document from the installed
4260/// checkpoint-backed production resolver. No caller can inject candidates,
4261/// accepted state, keys, reach, or a fallback resolver through this seam.
4262pub async fn production_browser_provisioning(
4263    request: BrowserProvisioningRequest,
4264) -> Result<BrowserProvisioningDocument> {
4265    let resolver = PRODUCTION_RESOLVER
4266        .get()
4267        .cloned()
4268        .ok_or_else(|| anyhow::anyhow!("checkpoint-backed production resolver is not installed"))?;
4269    resolver.browser_provisioning(request).await
4270}
4271
4272/// Return the verifier backed by the same opaque production resolver used for
4273/// projection. It is installed at the server dispatch boundary; no caller can
4274/// inject accepted state or a fallback trust source.
4275pub fn production_browser_currentness_verifier() -> Result<Arc<dyn BrowserCurrentnessVerifier>> {
4276    let resolver = PRODUCTION_RESOLVER
4277        .get()
4278        .cloned()
4279        .ok_or_else(|| anyhow::anyhow!("checkpoint-backed production resolver is not installed"))?;
4280    Ok(Arc::new(ProductionBrowserCurrentnessVerifier { resolver }))
4281}
4282
4283struct ProductionRpcClient {
4284    resolution_service_name: String,
4285    service_domain: String,
4286    selected_transport: Option<TransportConfig>,
4287    signing_key: SigningKey,
4288    token: Option<String>,
4289    resolver: Arc<DiscoveryServiceResolver>,
4290    request_id: std::sync::atomic::AtomicU64,
4291}
4292
4293fn production_service_query(service_name: &str, required: bool) -> Result<ServiceQuery> {
4294    ServiceQuery::new(
4295        service_name,
4296        ["hyprstream-rpc/1".to_owned()],
4297        if required {
4298            ResolverProfile::NativeIrohRequired
4299        } else {
4300            ResolverProfile::NetworkDiscovery
4301        },
4302        3,
4303    )
4304}
4305
4306impl ProductionRpcClient {
4307    fn new(
4308        resolution_service_name: &str,
4309        service_domain: &str,
4310        selected_transport: Option<TransportConfig>,
4311        signing_key: SigningKey,
4312        token: Option<String>,
4313        resolver: Arc<DiscoveryServiceResolver>,
4314    ) -> Result<Self> {
4315        for (label, name) in [
4316            ("resolution service name", resolution_service_name),
4317            ("service domain", service_domain),
4318        ] {
4319            anyhow::ensure!(
4320                !name.is_empty()
4321                    && name
4322                        .bytes()
4323                        .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'),
4324                "{label} is not canonical"
4325            );
4326        }
4327        Ok(Self {
4328            resolution_service_name: resolution_service_name.to_owned(),
4329            service_domain: service_domain.to_owned(),
4330            selected_transport,
4331            signing_key,
4332            token,
4333            resolver,
4334            request_id: std::sync::atomic::AtomicU64::new(1),
4335        })
4336    }
4337    async fn snapshots(&self) -> Result<Vec<ResolvedService>> {
4338        let query =
4339            production_service_query(&self.resolution_service_name, native_network_required())?;
4340        let max_attempts = query.max_attempts;
4341        let mut snapshots = self.resolver.resolve_service_candidates(query).await?;
4342        let authority = snapshots
4343            .first()
4344            .ok_or_else(|| anyhow::anyhow!("resolver returned no validated alternatives"))?;
4345        anyhow::ensure!(
4346            snapshots.iter().all(|item| item.same_authority(authority)),
4347            "resolver retry set crosses service authority"
4348        );
4349        if let Some(selected) = &self.selected_transport {
4350            snapshots.retain(|snapshot| snapshot.transport() == selected);
4351            anyhow::ensure!(
4352                !snapshots.is_empty(),
4353                "router-selected reach is not a current authorized candidate for service '{}'",
4354                self.resolution_service_name
4355            );
4356        }
4357        Ok(snapshots.into_iter().take(max_attempts).collect())
4358    }
4359    fn client_for(&self, snapshot: &ResolvedService) -> Result<Arc<dyn RpcClient>> {
4360        let (kem, pq) = snapshot.crypto_stores()?;
4361        #[cfg(feature = "test-fixtures")]
4362        if let Some(client) = test_fixtures::dial_override(snapshot.transport())? {
4363            return Ok(client);
4364        }
4365        let signer = hyprstream_rpc::signer::LocalSigner::new(self.signing_key.clone());
4366        hyprstream_rpc::dial::dial_with_crypto_stores(
4367            snapshot.transport(),
4368            signer,
4369            Some(snapshot.response_verifying_key()),
4370            current_service_token(&self.signing_key, &self.token),
4371            Some(kem),
4372            Some(pq),
4373        )
4374    }
4375    async fn attempt<T, F, Fut>(&self, mut call: F) -> Result<(T, ResolvedService)>
4376    where
4377        F: FnMut(Arc<dyn RpcClient>) -> Fut,
4378        Fut: std::future::Future<Output = Result<T>>,
4379    {
4380        for refresh in 0..2 {
4381            let mut last_transport_error = None;
4382            let mut invalidated = None;
4383            for snapshot in self.snapshots().await? {
4384                if let Err(error) = self.resolver.ensure_current(&snapshot).await {
4385                    invalidated = Some(error);
4386                    break;
4387                }
4388                match call(self.client_for(&snapshot)?).await {
4389                    Ok(value) => return Ok((value, snapshot)),
4390                    Err(error)
4391                        if hyprstream_rpc::transport_traits::is_pre_dispatch_transport_error(
4392                            &error,
4393                        ) =>
4394                    {
4395                        last_transport_error = Some(error);
4396                    }
4397                    Err(error) => return Err(error),
4398                }
4399            }
4400            if let Some(error) = invalidated {
4401                if refresh == 0 {
4402                    continue;
4403                }
4404                return Err(anyhow::anyhow!(
4405                    "resolved service remained invalid after re-resolution: {error}"
4406                ));
4407            }
4408            return Err(last_transport_error.unwrap_or_else(|| {
4409                anyhow::anyhow!("validated same-authority alternatives exhausted")
4410            }));
4411        }
4412        unreachable!("bounded re-resolution loop always returns")
4413    }
4414    fn checked_stream(
4415        &self,
4416        inner: Box<dyn StreamHandle>,
4417        snapshot: ResolvedService,
4418    ) -> Box<dyn StreamHandle> {
4419        Box::new(CurrentStreamHandle {
4420            inner,
4421            resolver: Arc::clone(&self.resolver),
4422            snapshot,
4423            invalidated: false,
4424        })
4425    }
4426}
4427
4428#[cfg(feature = "test-fixtures")]
4429#[doc(hidden)]
4430pub mod test_fixtures {
4431    use super::*;
4432    use crate::state_store::PutResult;
4433    use hyprstream_crypto::pq::ml_dsa_sk_to_vk_bytes;
4434    use hyprstream_pds::at9p::{
4435        CapsuleBody, HybridKeyPair, ServiceEndpoint, ServiceEntry, ServiceType,
4436        Transport as At9pTransport,
4437    };
4438    use hyprstream_pds::at9p_duplicity::AcceptedAt9pState;
4439    use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
4440
4441    type DialOverride = dyn Fn(&TransportConfig) -> Result<Arc<dyn RpcClient>> + Send + Sync;
4442
4443    static DIAL_OVERRIDE: std::sync::OnceLock<Arc<DialOverride>> = std::sync::OnceLock::new();
4444
4445    struct FixtureAcceptedStates(parking_lot::Mutex<HashMap<String, AcceptedAt9pState>>);
4446
4447    impl AcceptedStateSource for FixtureAcceptedStates {
4448        fn accepted_state(&self, did: &str) -> Result<Option<AcceptedAt9pState>> {
4449            Ok(self.0.lock().get(did).cloned())
4450        }
4451        fn accepted_states(&self) -> Result<Vec<AcceptedAt9pState>> {
4452            Ok(self.0.lock().values().cloned().collect())
4453        }
4454    }
4455
4456    #[derive(Clone)]
4457    struct FixtureAuthority {
4458        state: AcceptedAt9pState,
4459        signing: SigningKey,
4460        request_kem_recipient: hyprstream_rpc::crypto::hybrid_kem::RecipientPublic,
4461    }
4462
4463    /// Multi-endpoint announcement backend for the fixture. The production
4464    /// [`MemoryStateStore`] deliberately keeps one live announcement per
4465    /// (service, socket kind) — the single-replica lease model — while
4466    /// production retry sets with several same-authority reaches are served
4467    /// from the remote Discovery `get_endpoints` fan-out. A resolver fixture
4468    /// runs client-less, so it needs a backend that keeps every announced
4469    /// endpoint to express those sets.
4470    #[derive(Default)]
4471    struct FixtureAnnouncementStore {
4472        services: parking_lot::Mutex<HashMap<String, Vec<AnnouncedEndpoint>>>,
4473    }
4474
4475    impl FixtureAnnouncementStore {
4476        fn put_announcement_sync(&self, service_name: &str, endpoint: AnnouncedEndpoint) {
4477            let mut services = self.services.lock();
4478            let endpoints = services.entry(service_name.to_owned()).or_default();
4479            endpoints.retain(|existing| existing.endpoint != endpoint.endpoint);
4480            endpoints.push(endpoint);
4481        }
4482
4483        fn announcements_for_sync(
4484            &self,
4485            service_name: &str,
4486            now_unix_ms: i64,
4487        ) -> Vec<AnnouncedEndpoint> {
4488            self.services
4489                .lock()
4490                .get(service_name)
4491                .into_iter()
4492                .flatten()
4493                .filter(|entry| entry.is_live_at(now_unix_ms))
4494                .cloned()
4495                .collect()
4496        }
4497
4498        fn clear_announcements_sync(&self, service_name: &str) {
4499            self.services.lock().remove(service_name);
4500        }
4501    }
4502
4503    #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
4504    #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
4505    impl DiscoveryStateStore for FixtureAnnouncementStore {
4506        async fn put_announcement(
4507            &self,
4508            service_name: &str,
4509            endpoint: AnnouncedEndpoint,
4510        ) -> Result<PutResult> {
4511            self.put_announcement_sync(service_name, endpoint);
4512            Ok(PutResult::Stored)
4513        }
4514
4515        async fn announcements_for(
4516            &self,
4517            service_name: &str,
4518            now_unix_ms: i64,
4519        ) -> Result<Vec<AnnouncedEndpoint>> {
4520            Ok(self.announcements_for_sync(service_name, now_unix_ms))
4521        }
4522
4523        async fn all_announcements(
4524            &self,
4525            now_unix_ms: i64,
4526        ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>> {
4527            Ok(self
4528                .services
4529                .lock()
4530                .iter()
4531                .map(|(service_name, endpoints)| {
4532                    (
4533                        service_name.clone(),
4534                        endpoints
4535                            .iter()
4536                            .filter(|entry| entry.is_live_at(now_unix_ms))
4537                            .cloned()
4538                            .collect(),
4539                    )
4540                })
4541                .collect())
4542        }
4543
4544        // The resolver fixture exercises only the announcement plane; the
4545        // remaining store surface belongs to the Discovery daemon backends and
4546        // fails closed here rather than pretending to track it.
4547        async fn put_liveness(&self, _node: &Did, _value: LiveAllocatable) -> Result<PutResult> {
4548            anyhow::bail!("fixture announcement store does not track liveness")
4549        }
4550
4551        #[cfg(test)]
4552        async fn liveness(
4553            &self,
4554            _node: &Did,
4555            _now_unix_ms: i64,
4556        ) -> Result<Option<LiveAllocatable>> {
4557            anyhow::bail!("fixture announcement store does not track liveness")
4558        }
4559
4560        async fn all_liveness(&self, _now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>> {
4561            anyhow::bail!("fixture announcement store does not track liveness")
4562        }
4563
4564        async fn put_entity_statement(
4565            &self,
4566            _issuer: &str,
4567            _value: CachedEntityStatement,
4568        ) -> Result<()> {
4569            anyhow::bail!("fixture announcement store does not track entity statements")
4570        }
4571
4572        async fn entity_statement(&self, _issuer: &str) -> Result<Option<CachedEntityStatement>> {
4573            anyhow::bail!("fixture announcement store does not track entity statements")
4574        }
4575
4576        async fn known_issuers(&self) -> Result<Vec<String>> {
4577            anyhow::bail!("fixture announcement store does not track entity statements")
4578        }
4579
4580        async fn known_issuer_count(&self) -> Result<usize> {
4581            anyhow::bail!("fixture announcement store does not track entity statements")
4582        }
4583
4584        async fn put_envelope_keyset(
4585            &self,
4586            _service_did: &str,
4587            _value: CachedEnvelopeKeyset,
4588        ) -> Result<()> {
4589            anyhow::bail!("fixture announcement store does not track envelope keysets")
4590        }
4591
4592        async fn envelope_keyset(
4593            &self,
4594            _service_did: &str,
4595        ) -> Result<Option<CachedEnvelopeKeyset>> {
4596            anyhow::bail!("fixture announcement store does not track envelope keysets")
4597        }
4598    }
4599
4600    /// Mutable handle for one process-global, model-free production resolver
4601    /// fixture. The resolver and dial hook are installed once; individual tests
4602    /// reset only the accepted-state and announcement data behind that fixed
4603    /// authority boundary.
4604    #[derive(Clone)]
4605    pub struct ProductionInferenceFixture {
4606        service_name: String,
4607        announced: Arc<FixtureAnnouncementStore>,
4608        states: Arc<FixtureAcceptedStates>,
4609        primary: FixtureAuthority,
4610        foreign: FixtureAuthority,
4611    }
4612
4613    fn authority(service_name: &str, tag: u8) -> Result<FixtureAuthority> {
4614        let signing = SigningKey::from_bytes(&[tag; 32]);
4615        let pq_signing = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signing);
4616        let keys = HybridKeyPair::new(
4617            signing.verifying_key().to_bytes().to_vec(),
4618            ml_dsa_sk_to_vk_bytes(&pq_signing),
4619        )?;
4620        let endpoint = ServiceEndpoint::new(At9pTransport::Iroh, "iroh://fixture")?;
4621        let service = ServiceEntry::new(
4622            format!("#{service_name}"),
4623            ServiceType::NinePExport,
4624            endpoint,
4625        )?;
4626        let body = CapsuleBody::new(vec![keys], vec![service])?;
4627        let genesis = sign_capsule(body.clone(), &signing, &pq_signing)?;
4628        let subject = genesis.cid512()?;
4629        let update = sign_update_record(
4630            subject,
4631            1,
4632            [1; 64],
4633            body,
4634            "2099-01-01T00:00:00Z".to_owned(),
4635            &signing,
4636            &pq_signing,
4637        )?;
4638        let state = AcceptedAt9pState::from_persisted_update(&update.to_dag_cbor()?)?;
4639        let request_kem_recipient = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
4640            hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
4641        )?
4642        .public()
4643        .clone();
4644        Ok(FixtureAuthority {
4645            state,
4646            signing,
4647            request_kem_recipient,
4648        })
4649    }
4650
4651    fn announcement(
4652        authority: &FixtureAuthority,
4653        transport: &TransportConfig,
4654        last_heartbeat: Instant,
4655    ) -> Result<AnnouncedEndpoint> {
4656        let socket_kind = match &transport.endpoint {
4657            EndpointType::Quic { .. } => "quic",
4658            EndpointType::Iroh { .. } => "iroh",
4659            _ => anyhow::bail!("production inference fixture advertises only network candidates"),
4660        };
4661        Ok(AnnouncedEndpoint {
4662            socket_kind: socket_kind.to_owned(),
4663            endpoint: transport.endpoint_string(),
4664            service_did: Did::from(authority.state.did.clone()),
4665            capabilities: ["hyprstream-rpc/1".to_owned(), "hyprstream-moq/1".to_owned()]
4666                .into_iter()
4667                .collect(),
4668            accepted_state_digest: authority.state.head_digest.to_vec(),
4669            accepted_state_epoch: authority.state.epoch,
4670            response_key_id: format!("{}#response-current", authority.state.did),
4671            request_kem_key_id: format!("{}#kem-current", authority.state.did),
4672            request_kem_recipient: authority.request_kem_recipient.encode(),
4673            expires_at_unix_ms: 4_070_908_800_000,
4674            source_signer: authority.signing.verifying_key().to_bytes(),
4675            live_until_unix_ms: unix_millis_now()
4676                .saturating_add(ANNOUNCED_ENDPOINT_TTL.as_millis() as i64)
4677                .saturating_sub(
4678                    Instant::now()
4679                        .saturating_duration_since(last_heartbeat)
4680                        .as_millis() as i64,
4681                ),
4682        })
4683    }
4684
4685    impl ProductionInferenceFixture {
4686        /// Restore one same-authority set of fresh, checkpoint-current reaches.
4687        pub fn reset(&self, transports: &[TransportConfig]) -> Result<()> {
4688            {
4689                let mut states = self.states.0.lock();
4690                states.clear();
4691                states.insert(self.primary.state.did.clone(), self.primary.state.clone());
4692            }
4693            self.announced.clear_announcements_sync(&self.service_name);
4694            for transport in transports {
4695                self.announced.put_announcement_sync(
4696                    &self.service_name,
4697                    announcement(&self.primary, transport, Instant::now())?,
4698                );
4699            }
4700            Ok(())
4701        }
4702
4703        /// Age every announcement beyond the production freshness bound.
4704        pub fn mark_stale(&self) {
4705            for mut endpoint in self
4706                .announced
4707                .announcements_for_sync(&self.service_name, unix_millis_now())
4708            {
4709                endpoint.live_until_unix_ms = unix_millis_now() - 1;
4710                self.announced
4711                    .put_announcement_sync(&self.service_name, endpoint);
4712            }
4713        }
4714
4715        /// Add a fresh candidate backed by a different accepted DID authority.
4716        pub fn add_foreign_authority(&self, transport: &TransportConfig) -> Result<()> {
4717            self.states
4718                .0
4719                .lock()
4720                .insert(self.foreign.state.did.clone(), self.foreign.state.clone());
4721            self.announced.put_announcement_sync(
4722                &self.service_name,
4723                announcement(&self.foreign, transport, Instant::now())?,
4724            );
4725            Ok(())
4726        }
4727    }
4728
4729    /// Install the sole process-global resolver fixture used by downstream
4730    /// model-selector unit tests. This API exists only under `test-fixtures`.
4731    pub fn install_production_inference_fixture(
4732        service_name: &str,
4733        transports: &[TransportConfig],
4734        dial_override: Arc<dyn Fn(&TransportConfig) -> Result<Arc<dyn RpcClient>> + Send + Sync>,
4735    ) -> Result<ProductionInferenceFixture> {
4736        anyhow::ensure!(
4737            PRODUCTION_RESOLVER.get().is_none() && DIAL_OVERRIDE.get().is_none(),
4738            "production inference fixture is already installed"
4739        );
4740        let primary = authority(service_name, 0x61)?;
4741        let foreign = authority(service_name, 0x62)?;
4742        let states = Arc::new(FixtureAcceptedStates(parking_lot::Mutex::new(
4743            HashMap::new(),
4744        )));
4745        let announced = Arc::new(FixtureAnnouncementStore::default());
4746        let fixture = ProductionInferenceFixture {
4747            service_name: service_name.to_owned(),
4748            announced: Arc::clone(&announced),
4749            states: Arc::clone(&states),
4750            primary,
4751            foreign,
4752        };
4753        fixture.reset(transports)?;
4754        DIAL_OVERRIDE.set(dial_override).map_err(|_| {
4755            anyhow::anyhow!("production inference fixture dial is already installed")
4756        })?;
4757        PRODUCTION_RESOLVER
4758            .set(Arc::new(DiscoveryServiceResolver {
4759                state_store: announced,
4760                accepted_state_source: states,
4761                discovery_client: None,
4762            }))
4763            .map_err(|_| {
4764                anyhow::anyhow!("production inference fixture resolver is already installed")
4765            })?;
4766        Ok(fixture)
4767    }
4768
4769    /// Install a bootstrap-carried Discovery client as the process resolver,
4770    /// mirroring the post-`bootstrap_authenticated_process` state: the
4771    /// resolver holds the authenticated client bootstrap installed.
4772    ///
4773    /// Bounds: proves downstream selection/reach wiring only — that callers
4774    /// use the installed client rather than re-resolving — and proves nothing
4775    /// about deployed Policy/MAC acceptance or the deployment-credential
4776    /// chain (candidates are served by the installed client itself).
4777    pub fn install_bootstrap_discovery_client_fixture(
4778        discovery_client: hyprstream_rpc_std::discovery_client::DiscoveryClient,
4779    ) -> Result<()> {
4780        anyhow::ensure!(
4781            PRODUCTION_RESOLVER.get().is_none(),
4782            "production resolver fixture is already installed"
4783        );
4784        PRODUCTION_RESOLVER
4785            .set(Arc::new(DiscoveryServiceResolver {
4786                state_store: MemoryStateStore::production_default(),
4787                accepted_state_source: Arc::new(FixtureAcceptedStates(parking_lot::Mutex::new(
4788                    HashMap::new(),
4789                ))),
4790                discovery_client: Some(discovery_client),
4791            }))
4792            .map_err(|_| anyhow::anyhow!("production resolver is already installed"))
4793    }
4794
4795    pub(super) fn dial_override(transport: &TransportConfig) -> Result<Option<Arc<dyn RpcClient>>> {
4796        DIAL_OVERRIDE.get().map(|dial| dial(transport)).transpose()
4797    }
4798}
4799
4800#[async_trait]
4801impl RpcClient for ProductionRpcClient {
4802    async fn call(&self, payload: Vec<u8>) -> Result<Vec<u8>> {
4803        Ok(self
4804            .attempt(|c| {
4805                let p = payload.clone();
4806                async move { c.call(p).await }
4807            })
4808            .await?
4809            .0)
4810    }
4811    async fn call_for_service(&self, service: &str, payload: Vec<u8>) -> Result<Vec<u8>> {
4812        anyhow::ensure!(
4813            service == self.service_domain,
4814            "generated service authority mismatch"
4815        );
4816        let service = service.to_owned();
4817        Ok(self
4818            .attempt(|c| {
4819                let p = payload.clone();
4820                let s = service.clone();
4821                async move { c.call_for_service(&s, p).await }
4822            })
4823            .await?
4824            .0)
4825    }
4826    async fn call_for_service_with_method(
4827        &self,
4828        service: &str,
4829        method_discriminator: u16,
4830        payload: Vec<u8>,
4831    ) -> Result<Vec<u8>> {
4832        anyhow::ensure!(
4833            service == self.service_domain,
4834            "generated service authority mismatch"
4835        );
4836        let service = service.to_owned();
4837        Ok(self
4838            .attempt(|c| {
4839                let p = payload.clone();
4840                let s = service.clone();
4841                async move {
4842                    c.call_for_service_with_method(&s, method_discriminator, p)
4843                        .await
4844                }
4845            })
4846            .await?
4847            .0)
4848    }
4849
4850    async fn call_with_options(&self, payload: Vec<u8>, options: CallOptions) -> Result<Vec<u8>> {
4851        Ok(self
4852            .attempt(|c| {
4853                let p = payload.clone();
4854                let o = options.clone();
4855                async move { c.call_with_options(p, o).await }
4856            })
4857            .await?
4858            .0)
4859    }
4860    async fn call_with_options_for_service(
4861        &self,
4862        service: &str,
4863        payload: Vec<u8>,
4864        options: CallOptions,
4865    ) -> Result<Vec<u8>> {
4866        anyhow::ensure!(
4867            service == self.service_domain,
4868            "generated service authority mismatch"
4869        );
4870        let service = service.to_owned();
4871        Ok(self
4872            .attempt(|c| {
4873                let p = payload.clone();
4874                let o = options.clone();
4875                let s = service.clone();
4876                async move { c.call_with_options_for_service(&s, p, o).await }
4877            })
4878            .await?
4879            .0)
4880    }
4881    async fn call_with_options_for_service_with_method(
4882        &self,
4883        service: &str,
4884        method_discriminator: u16,
4885        payload: Vec<u8>,
4886        options: CallOptions,
4887    ) -> Result<Vec<u8>> {
4888        anyhow::ensure!(
4889            service == self.service_domain,
4890            "generated service authority mismatch"
4891        );
4892        let service = service.to_owned();
4893        Ok(self
4894            .attempt(|c| {
4895                let p = payload.clone();
4896                let o = options.clone();
4897                let s = service.clone();
4898                async move {
4899                    c.call_with_options_for_service_with_method(&s, method_discriminator, p, o)
4900                        .await
4901                }
4902            })
4903            .await?
4904            .0)
4905    }
4906    async fn call_streaming(&self, payload: Vec<u8>, ephemeral: [u8; 32]) -> Result<Vec<u8>> {
4907        Ok(self
4908            .attempt(|c| {
4909                let p = payload.clone();
4910                async move { c.call_streaming(p, ephemeral).await }
4911            })
4912            .await?
4913            .0)
4914    }
4915    async fn call_streaming_for_service(
4916        &self,
4917        service: &str,
4918        payload: Vec<u8>,
4919        ephemeral: [u8; 32],
4920    ) -> Result<Vec<u8>> {
4921        anyhow::ensure!(
4922            service == self.service_domain,
4923            "generated service authority mismatch"
4924        );
4925        let service = service.to_owned();
4926        Ok(self
4927            .attempt(|c| {
4928                let p = payload.clone();
4929                let s = service.clone();
4930                async move { c.call_streaming_for_service(&s, p, ephemeral).await }
4931            })
4932            .await?
4933            .0)
4934    }
4935    async fn call_streaming_with_options_for_service(
4936        &self,
4937        service: &str,
4938        payload: Vec<u8>,
4939        ephemeral: [u8; 32],
4940        options: CallOptions,
4941    ) -> Result<Vec<u8>> {
4942        anyhow::ensure!(
4943            service == self.service_domain,
4944            "generated service authority mismatch"
4945        );
4946        let service = service.to_owned();
4947        Ok(self
4948            .attempt(|c| {
4949                let p = payload.clone();
4950                let o = options.clone();
4951                let s = service.clone();
4952                async move {
4953                    c.call_streaming_with_options_for_service(&s, p, ephemeral, o)
4954                        .await
4955                }
4956            })
4957            .await?
4958            .0)
4959    }
4960    async fn call_streaming_for_service_with_method(
4961        &self,
4962        service: &str,
4963        method_discriminator: u16,
4964        payload: Vec<u8>,
4965        ephemeral: [u8; 32],
4966    ) -> Result<Vec<u8>> {
4967        anyhow::ensure!(
4968            service == self.service_domain,
4969            "generated service authority mismatch"
4970        );
4971        let service = service.to_owned();
4972        Ok(self
4973            .attempt(|c| {
4974                let p = payload.clone();
4975                let s = service.clone();
4976                async move {
4977                    c.call_streaming_for_service_with_method(&s, method_discriminator, p, ephemeral)
4978                        .await
4979                }
4980            })
4981            .await?
4982            .0)
4983    }
4984    async fn call_streaming_with_options_for_service_with_method(
4985        &self,
4986        service: &str,
4987        method_discriminator: u16,
4988        payload: Vec<u8>,
4989        ephemeral: [u8; 32],
4990        options: CallOptions,
4991    ) -> Result<Vec<u8>> {
4992        anyhow::ensure!(
4993            service == self.service_domain,
4994            "generated service authority mismatch"
4995        );
4996        let service = service.to_owned();
4997        Ok(self
4998            .attempt(|c| {
4999                let p = payload.clone();
5000                let o = options.clone();
5001                let s = service.clone();
5002                async move {
5003                    c.call_streaming_with_options_for_service_with_method(
5004                        &s,
5005                        method_discriminator,
5006                        p,
5007                        ephemeral,
5008                        o,
5009                    )
5010                    .await
5011                }
5012            })
5013            .await?
5014            .0)
5015    }
5016    async fn open_stream(&self, payload: Vec<u8>) -> Result<Box<dyn StreamHandle>> {
5017        let (inner, snapshot) = self
5018            .attempt(|c| {
5019                let p = payload.clone();
5020                async move { c.open_stream(p).await }
5021            })
5022            .await?;
5023        Ok(self.checked_stream(inner, snapshot))
5024    }
5025    async fn open_stream_from_info(
5026        &self,
5027        info: hyprstream_rpc::stream_info::StreamInfo,
5028        secret: [u8; 32],
5029        public: [u8; 32],
5030    ) -> Result<Box<dyn StreamHandle>> {
5031        let (inner, snapshot) = self
5032            .attempt(|c| {
5033                let i = info.clone();
5034                async move { c.open_stream_from_info(i, secret, public).await }
5035            })
5036            .await?;
5037        Ok(self.checked_stream(inner, snapshot))
5038    }
5039    fn next_id(&self) -> u64 {
5040        self.request_id
5041            .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
5042    }
5043}
5044
5045impl DiscoveryService {
5046    async fn resolve_announced_endpoint(
5047        &self,
5048        name: &str,
5049        kind: SocketKind,
5050    ) -> anyhow::Result<Option<TransportConfig>> {
5051        let wanted = socket_kind_to_string(kind);
5052        let endpoints = self
5053            .state_store
5054            .announcements_for(name, unix_millis_now())
5055            .await?;
5056        let Some(endpoint) = endpoints.iter().find(|ep| ep.socket_kind == wanted) else {
5057            return Ok(None);
5058        };
5059
5060        Ok(Some(announced_endpoint_to_transport(endpoint)?))
5061    }
5062}
5063
5064fn announced_endpoint_to_transport(
5065    endpoint: &AnnouncedEndpoint,
5066) -> anyhow::Result<TransportConfig> {
5067    match endpoint.socket_kind.as_str() {
5068        "quic" => parse_announced_quic(&endpoint.endpoint),
5069        "iroh" => parse_announced_iroh(&endpoint.endpoint),
5070        _ => Ok(TransportConfig::from_endpoint(&endpoint.endpoint)),
5071    }
5072}
5073
5074fn parse_announced_quic(endpoint: &str) -> anyhow::Result<TransportConfig> {
5075    let rest = endpoint
5076        .strip_prefix("quic://")
5077        .ok_or_else(|| anyhow::anyhow!("announced QUIC endpoint must start with quic://"))?;
5078    let (reach, encoded_hashes) = rest
5079        .split_once('#')
5080        .map_or((rest, None), |(reach, hashes)| (reach, Some(hashes)));
5081    let (server_name, addr) = reach.split_once(':').ok_or_else(|| {
5082        anyhow::anyhow!("announced QUIC endpoint must be quic://<server-name>:<socket-addr>")
5083    })?;
5084    anyhow::ensure!(
5085        !server_name.is_empty(),
5086        "announced QUIC endpoint is missing server name"
5087    );
5088    let addr = SocketAddr::from_str(addr)
5089        .map_err(|e| anyhow::anyhow!("invalid announced QUIC socket address '{addr}': {e}"))?;
5090    anyhow::ensure!(
5091        addr.port() != 0,
5092        "announced QUIC endpoint must not use port 0"
5093    );
5094    let transport = if let Some(encoded_hashes) = encoded_hashes {
5095        let hashes = encoded_hashes
5096            .split(',')
5097            .map(|value| {
5098                URL_SAFE_NO_PAD
5099                    .decode(value)
5100                    .context("invalid announced QUIC certificate hash")?
5101                    .try_into()
5102                    .map_err(|_| {
5103                        anyhow::anyhow!("announced QUIC certificate hash must be 32 bytes")
5104                    })
5105            })
5106            .collect::<Result<Vec<[u8; 32]>>>()?;
5107        anyhow::ensure!(!hashes.is_empty(), "announced QUIC pin set is empty");
5108        TransportConfig::quic_with_auth(
5109            addr,
5110            server_name,
5111            hyprstream_rpc::transport::QuicServerAuth::pinned(hashes)?,
5112        )
5113    } else {
5114        TransportConfig::quic(addr, server_name).with_connect_mode()
5115    };
5116    Ok(transport)
5117}
5118
5119fn parse_announced_iroh(endpoint: &str) -> anyhow::Result<TransportConfig> {
5120    let hex = endpoint
5121        .strip_prefix("iroh://")
5122        .ok_or_else(|| anyhow::anyhow!("announced iroh endpoint must start with iroh://"))?;
5123    anyhow::ensure!(
5124        hex.len() == 64,
5125        "announced iroh node id must be 32 bytes of hex"
5126    );
5127    let mut node_id = [0u8; 32];
5128    for (idx, byte) in node_id.iter_mut().enumerate() {
5129        let start = idx * 2;
5130        *byte = u8::from_str_radix(&hex[start..start + 2], 16)
5131            .map_err(|e| anyhow::anyhow!("invalid announced iroh node id hex: {e}"))?;
5132    }
5133    Ok(TransportConfig::iroh(node_id, Vec::new(), None))
5134}
5135
5136#[cfg(test)]
5137#[allow(clippy::expect_used, clippy::unwrap_used)]
5138mod resolver_tests {
5139    use super::*;
5140    use ed25519_dalek::Signer as _;
5141    use hyprstream_crypto::pq::ml_dsa_sk_to_vk_bytes;
5142    use hyprstream_pds::at9p::{
5143        CapsuleBody, HybridKeyPair, ServiceEndpoint, ServiceEntry, ServiceType,
5144        Transport as At9pTransport,
5145    };
5146    use hyprstream_pds::at9p_duplicity::AcceptedAt9pState;
5147    use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
5148    use hyprstream_rpc::transport::{BindMode, EndpointType};
5149
5150    fn checked_test_time(now: i64, offset: i64) -> i64 {
5151        now.checked_add(offset).expect("test NumericDate offset")
5152    }
5153
5154    struct TestDeploymentCa {
5155        ed25519: SigningKey,
5156        ml_dsa_65: hyprstream_rpc::crypto::pq::MlDsaSigningKey,
5157        verifying_keys: HybridDeploymentCa,
5158    }
5159
5160    fn test_deployment_ca(tag: u8) -> TestDeploymentCa {
5161        let ed25519 = SigningKey::from_bytes(&[tag; 32]);
5162        let (ml_dsa_65, ml_dsa_65_vk) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
5163        let verifying_keys = HybridDeploymentCa::from_public_key_bytes(
5164            ed25519.verifying_key().as_bytes(),
5165            &hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&ml_dsa_65_vk),
5166        )
5167        .expect("test hybrid deployment CA");
5168        TestDeploymentCa {
5169            ed25519,
5170            ml_dsa_65,
5171            verifying_keys,
5172        }
5173    }
5174
5175    fn sign_registry_credential_json(
5176        ca: &TestDeploymentCa,
5177        protected_json: &str,
5178        claims_json: &str,
5179    ) -> String {
5180        let protected = URL_SAFE_NO_PAD.encode(protected_json.as_bytes());
5181        let claims = URL_SAFE_NO_PAD.encode(claims_json.as_bytes());
5182        let signing_input = format!("{protected}.{claims}");
5183        let ml_dsa_signature =
5184            hyprstream_rpc::crypto::pq::ml_dsa_sign(&ca.ml_dsa_65, signing_input.as_bytes());
5185        let ed25519_signature = ca.ed25519.sign(signing_input.as_bytes());
5186        let mut signature = Vec::with_capacity(HYBRID_JWT_SIGNATURE_BYTES);
5187        signature.extend_from_slice(&ml_dsa_signature);
5188        signature.extend_from_slice(&ed25519_signature.to_bytes());
5189        format!("{signing_input}.{}", URL_SAFE_NO_PAD.encode(signature))
5190    }
5191
5192    fn exact_registry_credential_values(
5193        ca: &TestDeploymentCa,
5194        registry: &SigningKey,
5195    ) -> (serde_json::Value, serde_json::Value) {
5196        let domain = deployment_domain(&ca.verifying_keys);
5197        let now = chrono::Utc::now().timestamp();
5198        (
5199            serde_json::json!({
5200                "alg":"ML-DSA-65-Ed25519",
5201                "typ":"wit+jwt",
5202                "kid":domain
5203            }),
5204            serde_json::json!({
5205                "iss": format!("urn:hyprstream:deployment:{domain}"),
5206                "sub": "service:registry",
5207                "aud": REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE,
5208                "exp": checked_test_time(now, 3600),
5209                "nbf": now,
5210                "iat": now,
5211                "deployment_domain": domain,
5212                "profile": REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE,
5213                "cnf": {"jwk": {
5214                    "kty": "OKP",
5215                    "crv": "Ed25519",
5216                    "x": URL_SAFE_NO_PAD.encode(registry.verifying_key().as_bytes()),
5217                }},
5218            }),
5219        )
5220    }
5221
5222    fn exact_registry_credential(ca: &TestDeploymentCa, registry: &SigningKey) -> String {
5223        let (protected, claims) = exact_registry_credential_values(ca, registry);
5224        sign_registry_credential_json(ca, &protected.to_string(), &claims.to_string())
5225    }
5226
5227    fn authenticate_test_registry_credential(
5228        ca: &TestDeploymentCa,
5229        credential: String,
5230    ) -> Result<AuthenticatedRegistryDeploymentIdentity> {
5231        let (key_bytes, _expires_at) = validate_registry_deployment_credential_profile(
5232            &credential,
5233            &ca.verifying_keys,
5234            DeploymentAuthorityState::GenesisOnly,
5235        )?;
5236        let verifying_key = VerifyingKey::from_bytes(&key_bytes)
5237            .map_err(|error| anyhow::anyhow!("registry credential key is malformed: {error}"))?;
5238        Ok(AuthenticatedRegistryDeploymentIdentity {
5239            verifier: RegistryDeploymentVerifier {
5240                verifying_key,
5241                _deployment_ca: Arc::new(ca.verifying_keys.clone()),
5242            },
5243        })
5244    }
5245
5246    fn assert_registry_credential_rejected(
5247        ca: &TestDeploymentCa,
5248        protected: &serde_json::Value,
5249        claims: &serde_json::Value,
5250        case: &str,
5251    ) {
5252        let credential =
5253            sign_registry_credential_json(ca, &protected.to_string(), &claims.to_string());
5254        assert!(
5255            authenticate_test_registry_credential(ca, credential).is_err(),
5256            "invalid deployment credential accepted: {case}"
5257        );
5258    }
5259
5260    #[test]
5261    fn exact_registry_deployment_credential_installs_the_exact_bound_jwk() {
5262        let ca = test_deployment_ca(0x31);
5263        let registry = SigningKey::from_bytes(&[0x32; 32]);
5264        let witness =
5265            authenticate_test_registry_credential(&ca, exact_registry_credential(&ca, &registry))
5266                .expect("exact deployment credential profile");
5267        assert!(witness.verifier.matches(&registry.verifying_key()));
5268        assert!(
5269            witness.verifier.matches_deployment_root(&ca.verifying_keys),
5270            "registry verifier lost its hybrid deployment-root provenance"
5271        );
5272    }
5273
5274    #[test]
5275    fn classical_only_deployment_root_and_credential_fail_closed() {
5276        let classical_ca = SigningKey::from_bytes(&[0x41; 32]);
5277        assert!(
5278            HybridDeploymentCa::from_public_key_bytes(
5279                classical_ca.verifying_key().as_bytes(),
5280                &[],
5281            )
5282            .is_err(),
5283            "classical-only deployment CA was accepted"
5284        );
5285        assert!(
5286            HybridDeploymentCa::from_os_pin(classical_ca.verifying_key().as_bytes()).is_err(),
5287            "legacy 32-byte classical root pin was accepted"
5288        );
5289
5290        let ca = test_deployment_ca(0x42);
5291        let registry = SigningKey::from_bytes(&[0x43; 32]);
5292        let (protected, claims) = exact_registry_credential_values(&ca, &registry);
5293        let protected = URL_SAFE_NO_PAD.encode(protected.to_string());
5294        let claims = URL_SAFE_NO_PAD.encode(claims.to_string());
5295        let signing_input = format!("{protected}.{claims}");
5296        let classical_signature = ca.ed25519.sign(signing_input.as_bytes());
5297        let classical_token = format!(
5298            "{signing_input}.{}",
5299            URL_SAFE_NO_PAD.encode(classical_signature.to_bytes())
5300        );
5301        assert!(
5302            authenticate_test_registry_credential(&ca, classical_token).is_err(),
5303            "classical-only credential was accepted beneath the hybrid root"
5304        );
5305    }
5306
5307    #[test]
5308    fn hybrid_deployment_credential_requires_each_signature_half() {
5309        let ca = test_deployment_ca(0x44);
5310        let registry = SigningKey::from_bytes(&[0x45; 32]);
5311        let credential = exact_registry_credential(&ca, &registry);
5312        authenticate_test_registry_credential(&ca, credential.clone())
5313            .expect("intact hybrid deployment credential");
5314
5315        let (signing_input, signature_segment) =
5316            credential.rsplit_once('.').expect("credential signature");
5317        let signature = URL_SAFE_NO_PAD
5318            .decode(signature_segment)
5319            .expect("hybrid signature");
5320        for (case, index) in [("ML-DSA-65", 0usize), ("Ed25519", 3_309usize)] {
5321            let mut corrupted = signature.clone();
5322            corrupted[index] ^= 0x01;
5323            let token = format!("{signing_input}.{}", URL_SAFE_NO_PAD.encode(corrupted));
5324            assert!(
5325                authenticate_test_registry_credential(&ca, token).is_err(),
5326                "credential with invalid {case} half was accepted"
5327            );
5328        }
5329    }
5330
5331    #[test]
5332    fn deployment_credential_numeric_dates_enforce_all_boundaries_and_orderings() {
5333        let now = 1_000_000;
5334
5335        for (case, exp, nbf, iat) in [
5336            (
5337                "one-hour lifetime endpoint",
5338                checked_test_time(now, 3600),
5339                now,
5340                now,
5341            ),
5342            (
5343                "inside lifetime endpoint",
5344                checked_test_time(now, 3599),
5345                now,
5346                now,
5347            ),
5348            (
5349                "future-skew and lifetime endpoints",
5350                checked_test_time(now, 3660),
5351                checked_test_time(now, 60),
5352                checked_test_time(now, 60),
5353            ),
5354        ] {
5355            validate_registry_credential_numeric_dates(exp, nbf, iat, now)
5356                .unwrap_or_else(|error| panic!("valid {case} rejected: {error}"));
5357        }
5358
5359        for (case, exp, nbf, iat, validation_now) in [
5360            (
5361                "expired endpoint",
5362                now,
5363                now,
5364                checked_test_time(now, -1),
5365                now,
5366            ),
5367            (
5368                "outside lifetime endpoint",
5369                checked_test_time(now, 3601),
5370                now,
5371                now,
5372                now,
5373            ),
5374            (
5375                "future nbf outside skew",
5376                checked_test_time(now, 3600),
5377                checked_test_time(now, 61),
5378                now,
5379                now,
5380            ),
5381            (
5382                "future iat outside skew",
5383                checked_test_time(now, 3661),
5384                now,
5385                checked_test_time(now, 61),
5386                now,
5387            ),
5388            (
5389                "exp equal to iat",
5390                checked_test_time(now, 1),
5391                checked_test_time(now, 1),
5392                checked_test_time(now, 1),
5393                now,
5394            ),
5395            (
5396                "exp before iat",
5397                checked_test_time(now, 1),
5398                now,
5399                checked_test_time(now, 2),
5400                now,
5401            ),
5402            (
5403                "nbf after exp",
5404                checked_test_time(now, 2),
5405                checked_test_time(now, 3),
5406                now,
5407                now,
5408            ),
5409            (
5410                "nbf after iat",
5411                checked_test_time(now, 2),
5412                checked_test_time(now, 1),
5413                now,
5414                now,
5415            ),
5416            ("negative exp", -1, 0, 0, -2),
5417            ("negative nbf", checked_test_time(now, 1), -1, 0, now),
5418            ("negative iat", checked_test_time(now, 1), 0, -1, now),
5419            ("minimum and maximum extremes", i64::MAX, 0, i64::MIN, now),
5420            ("all minimum extremes", i64::MIN, i64::MIN, i64::MIN, now),
5421            ("all maximum extremes", i64::MAX, i64::MAX, i64::MAX, now),
5422            (
5423                "clock-skew boundary overflow",
5424                i64::MAX,
5425                i64::MAX - 1,
5426                i64::MAX - 1,
5427                i64::MAX - 1,
5428            ),
5429        ] {
5430            assert!(
5431                validate_registry_credential_numeric_dates(exp, nbf, iat, validation_now).is_err(),
5432                "invalid NumericDate profile accepted: {case}"
5433            );
5434        }
5435    }
5436
5437    #[test]
5438    fn deployment_credential_rejects_signed_wraparound_lifetime_in_all_profiles() {
5439        let ca = test_deployment_ca(0x6a);
5440        let registry = SigningKey::from_bytes(&[0x6b; 32]);
5441        let (protected, mut claims) = exact_registry_credential_values(&ca, &registry);
5442        claims["exp"] = serde_json::json!(checked_test_time(
5443            chrono::Utc::now().timestamp(),
5444            REGISTRY_DEPLOYMENT_CREDENTIAL_MAX_TTL_SECONDS,
5445        ));
5446        claims["nbf"] = serde_json::json!(i64::MIN);
5447        claims["iat"] = serde_json::json!(i64::MIN);
5448
5449        assert_registry_credential_rejected(
5450            &ca,
5451            &protected,
5452            &claims,
5453            "signed exp-minus-iat wraparound",
5454        );
5455    }
5456
5457    #[test]
5458    fn deployment_credential_rejects_every_protected_and_trust_dimension() {
5459        let ca = test_deployment_ca(0x33);
5460        let other_ca = test_deployment_ca(0x34);
5461        let registry = SigningKey::from_bytes(&[0x35; 32]);
5462        let (protected, claims) = exact_registry_credential_values(&ca, &registry);
5463
5464        for (case, member, value) in [
5465            ("access token type", "typ", serde_json::json!("at+jwt")),
5466            (
5467                "application access token type",
5468                "typ",
5469                serde_json::json!("application/at+jwt"),
5470            ),
5471            ("generic JWT type", "typ", serde_json::json!("JWT")),
5472            ("wrong-case type", "typ", serde_json::json!("WIT+JWT")),
5473            ("whitespace type", "typ", serde_json::json!("wit+jwt ")),
5474            ("alternate algorithm", "alg", serde_json::json!("ES256")),
5475            ("wrong key id", "kid", serde_json::json!("attacker")),
5476        ] {
5477            let mut changed = protected.clone();
5478            changed[member] = value;
5479            assert_registry_credential_rejected(&ca, &changed, &claims, case);
5480        }
5481        for missing in ["typ", "alg", "kid"] {
5482            let mut changed = protected.clone();
5483            changed
5484                .as_object_mut()
5485                .expect("header object")
5486                .remove(missing);
5487            assert_registry_credential_rejected(&ca, &changed, &claims, missing);
5488        }
5489        let mut unknown = protected.clone();
5490        unknown["crit"] = serde_json::json!([]);
5491        assert_registry_credential_rejected(&ca, &unknown, &claims, "unknown header member");
5492
5493        let credential = exact_registry_credential(&ca, &registry);
5494        assert!(
5495            authenticate_test_registry_credential(&other_ca, credential).is_err(),
5496            "CA substitution accepted"
5497        );
5498        let signed_by_other = {
5499            let (other_header, _) = exact_registry_credential_values(&other_ca, &registry);
5500            sign_registry_credential_json(&other_ca, &other_header.to_string(), &claims.to_string())
5501        };
5502        assert!(
5503            authenticate_test_registry_credential(&ca, signed_by_other).is_err(),
5504            "signature substitution accepted"
5505        );
5506    }
5507
5508    #[test]
5509    fn deployment_credential_rejects_every_claim_and_time_dimension() {
5510        let ca = test_deployment_ca(0x36);
5511        let registry = SigningKey::from_bytes(&[0x37; 32]);
5512        let (protected, claims) = exact_registry_credential_values(&ca, &registry);
5513        let now = chrono::Utc::now().timestamp();
5514
5515        for (case, member, value) in [
5516            ("wrong subject", "sub", serde_json::json!("service:model")),
5517            (
5518                "wrong issuer",
5519                "iss",
5520                serde_json::json!("https://attacker.invalid"),
5521            ),
5522            (
5523                "wrong audience",
5524                "aud",
5525                serde_json::json!("service:registry"),
5526            ),
5527            (
5528                "audience array",
5529                "aud",
5530                serde_json::json!([REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE]),
5531            ),
5532            (
5533                "wrong deployment domain",
5534                "deployment_domain",
5535                serde_json::json!("attacker"),
5536            ),
5537            (
5538                "wrong profile",
5539                "profile",
5540                serde_json::json!("hyprstream.registry-deployment.v2"),
5541            ),
5542            (
5543                "expired",
5544                "exp",
5545                serde_json::json!(checked_test_time(now, -1)),
5546            ),
5547            (
5548                "future nbf",
5549                "nbf",
5550                serde_json::json!(checked_test_time(now, 3600)),
5551            ),
5552            (
5553                "future iat",
5554                "iat",
5555                serde_json::json!(checked_test_time(now, 3600)),
5556            ),
5557            (
5558                "nbf after iat",
5559                "nbf",
5560                serde_json::json!(checked_test_time(now, 1)),
5561            ),
5562            (
5563                "excessive lifetime",
5564                "exp",
5565                serde_json::json!(checked_test_time(now, 3601)),
5566            ),
5567        ] {
5568            let mut changed = claims.clone();
5569            changed[member] = value;
5570            assert_registry_credential_rejected(&ca, &protected, &changed, case);
5571        }
5572        for missing in [
5573            "iss",
5574            "sub",
5575            "aud",
5576            "exp",
5577            "nbf",
5578            "iat",
5579            "deployment_domain",
5580            "profile",
5581            "cnf",
5582        ] {
5583            let mut changed = claims.clone();
5584            changed
5585                .as_object_mut()
5586                .expect("claims object")
5587                .remove(missing);
5588            assert_registry_credential_rejected(&ca, &protected, &changed, missing);
5589        }
5590        let mut unknown = claims.clone();
5591        unknown["scope"] = serde_json::json!("registry");
5592        assert_registry_credential_rejected(&ca, &protected, &unknown, "unknown claim");
5593    }
5594
5595    #[test]
5596    fn deployment_credential_rejects_every_confirmation_and_jwk_dimension() {
5597        let ca = test_deployment_ca(0x38);
5598        let registry = SigningKey::from_bytes(&[0x39; 32]);
5599        let (protected, claims) = exact_registry_credential_values(&ca, &registry);
5600
5601        for (case, member, value) in [
5602            ("RSA declaration", "kty", serde_json::json!("RSA")),
5603            ("EC declaration", "kty", serde_json::json!("EC")),
5604            ("P-256 curve", "crv", serde_json::json!("P-256")),
5605            ("alternate OKP curve", "crv", serde_json::json!("X25519")),
5606            ("malformed x", "x", serde_json::json!("%%%")),
5607            (
5608                "padded x",
5609                "x",
5610                serde_json::json!(format!(
5611                    "{}=",
5612                    URL_SAFE_NO_PAD.encode(registry.verifying_key().as_bytes())
5613                )),
5614            ),
5615            (
5616                "short x",
5617                "x",
5618                serde_json::json!(URL_SAFE_NO_PAD.encode([0u8; 31])),
5619            ),
5620        ] {
5621            let mut changed = claims.clone();
5622            changed["cnf"]["jwk"][member] = value;
5623            assert_registry_credential_rejected(&ca, &protected, &changed, case);
5624        }
5625        for missing in ["kty", "crv", "x"] {
5626            let mut changed = claims.clone();
5627            changed["cnf"]["jwk"]
5628                .as_object_mut()
5629                .expect("jwk object")
5630                .remove(missing);
5631            assert_registry_credential_rejected(&ca, &protected, &changed, missing);
5632        }
5633        for incompatible in ["alg", "use", "key_ops", "kid"] {
5634            let mut changed = claims.clone();
5635            changed["cnf"]["jwk"][incompatible] = serde_json::json!("unexpected");
5636            assert_registry_credential_rejected(&ca, &protected, &changed, incompatible);
5637        }
5638        let mut ambiguous = claims.clone();
5639        ambiguous["cnf"]["jkt"] = serde_json::json!("thumbprint");
5640        assert_registry_credential_rejected(&ca, &protected, &ambiguous, "jwk plus jkt");
5641        let mut alternate = claims.clone();
5642        alternate["cnf"] = serde_json::json!({"jkt":"thumbprint"});
5643        assert_registry_credential_rejected(&ca, &protected, &alternate, "jkt only");
5644    }
5645
5646    #[test]
5647    fn deployment_credential_duplicate_members_fail_closed_at_every_level() {
5648        let ca = test_deployment_ca(0x3a);
5649        let registry = SigningKey::from_bytes(&[0x3b; 32]);
5650        let domain = deployment_domain(&ca.verifying_keys);
5651        let (_, claims) = exact_registry_credential_values(&ca, &registry);
5652        let duplicate_headers = [
5653            format!(
5654                r#"{{"alg":"ML-DSA-65-Ed25519","alg":"ES256","typ":"wit+jwt","kid":"{domain}"}}"#
5655            ),
5656            format!(
5657                r#"{{"alg":"ML-DSA-65-Ed25519","typ":"wit+jwt","typ":"at+jwt","kid":"{domain}"}}"#
5658            ),
5659            format!(
5660                r#"{{"alg":"ML-DSA-65-Ed25519","typ":"wit+jwt","kid":"{domain}","kid":"attacker"}}"#
5661            ),
5662        ];
5663        for header in duplicate_headers {
5664            let token = sign_registry_credential_json(&ca, &header, &claims.to_string());
5665            assert!(authenticate_test_registry_credential(&ca, token).is_err());
5666        }
5667
5668        let now = chrono::Utc::now().timestamp();
5669        let exp = checked_test_time(now, 3600);
5670        let x = URL_SAFE_NO_PAD.encode(registry.verifying_key().as_bytes());
5671        let protected = serde_json::json!({
5672            "alg":"ML-DSA-65-Ed25519",
5673            "typ":"wit+jwt",
5674            "kid":domain
5675        });
5676        for duplicate_claims in [
5677            format!(
5678                r#"{{"iss":"urn:hyprstream:deployment:{domain}","sub":"service:registry","sub":"service:model","aud":"{REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE}","exp":{},"nbf":{now},"iat":{now},"deployment_domain":"{domain}","profile":"{REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE}","cnf":{{"jwk":{{"kty":"OKP","crv":"Ed25519","x":"{x}"}}}}}}"#,
5679                exp
5680            ),
5681            format!(
5682                r#"{{"iss":"urn:hyprstream:deployment:{domain}","sub":"service:registry","aud":"{REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE}","exp":{},"nbf":{now},"iat":{now},"deployment_domain":"{domain}","profile":"{REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE}","cnf":{{"jwk":{{"kty":"OKP","crv":"Ed25519","x":"{x}"}},"jwk":{{"kty":"RSA","crv":"P-256","x":"{x}"}}}}}}"#,
5683                exp
5684            ),
5685            format!(
5686                r#"{{"iss":"urn:hyprstream:deployment:{domain}","sub":"service:registry","aud":"{REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE}","exp":{},"nbf":{now},"iat":{now},"deployment_domain":"{domain}","profile":"{REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE}","cnf":{{"jwk":{{"kty":"OKP","kty":"RSA","crv":"Ed25519","x":"{x}"}}}}}}"#,
5687                exp
5688            ),
5689        ] {
5690            let token =
5691                sign_registry_credential_json(&ca, &protected.to_string(), &duplicate_claims);
5692            assert!(authenticate_test_registry_credential(&ca, token).is_err());
5693        }
5694    }
5695
5696    #[test]
5697    fn production_authority_api_inventory_has_no_public_composition_chain() {
5698        let service_factory = include_str!(concat!(
5699            env!("CARGO_MANIFEST_DIR"),
5700            "/../hyprstream-service/src/service/factory.rs"
5701        ));
5702        for forbidden in [
5703            "pub fn seal_discovery_bootstrap_authority",
5704            "pub fn consume_discovery_bootstrap_authority",
5705            "struct DiscoveryBootstrapAuthority",
5706            "discovery_authority:",
5707            "pub fn authenticate_registry_deployment_credential",
5708            "pub fn take_authenticated_registry_identity",
5709            "pub fn authenticated_registry_identity",
5710            "pub struct AuthenticatedRegistryDeploymentIdentity",
5711            "into_verifying_key",
5712        ] {
5713            assert!(
5714                !service_factory.contains(forbidden),
5715                "authority seam returned: {forbidden}"
5716            );
5717        }
5718        let discovery = include_str!("service.rs");
5719        let production = discovery
5720            .split("mod resolver_tests {")
5721            .next()
5722            .expect("production Discovery source");
5723        assert!(!production.contains("\n    pub fn install_process_bootstrap("));
5724        assert!(production.contains("struct ProcessBootstrapAuthority {"));
5725        assert!(production.contains("enum ProcessBootstrapAuthorityState {"));
5726        let authentication = production
5727            .split("fn authenticate_deployment_bootstrap()")
5728            .nth(1)
5729            .expect("private deployment authentication boundary")
5730            .split("#[cfg(test)]\nfn authenticate_discovery_bootstrap_identity")
5731            .next()
5732            .expect("authentication body");
5733        assert!(authentication.contains("hyprstream_service::deployment_data_dir()"));
5734        assert!(authentication.contains("authenticate_local_deployment_registry()"));
5735        assert!(authentication.contains("authenticate_registry_deployment_credentials("));
5736        assert!(!authentication.contains("global_trust_store()"));
5737        assert!(!authentication.contains("resolve_one("));
5738        assert!(!authentication.contains("FnOnce"));
5739        assert!(!production.contains("pub fn authenticate_discovery_bootstrap("));
5740        assert!(!production.contains("pub struct AuthenticatedDiscoveryBootstrap"));
5741        assert!(!production.contains("pub fn bootstrap_authenticated_process("));
5742        assert!(production.contains("/etc/hyprstream/trust/deployment-ca.hybrid"));
5743        assert!(production.contains("/run/hyprstream/credentials/registry-service.jwt"));
5744        assert!(production.contains("HYPRSTREAM_DEPLOYMENT_TRUST_DIR"));
5745        assert!(production.contains("CREDENTIALS_DIRECTORY"));
5746        let loader = production
5747            .split("fn load_trusted_registry_deployment_credentials()")
5748            .nth(1)
5749            .expect("deployment trust loader")
5750            .split("fn authenticate_registry_deployment_credentials(")
5751            .next()
5752            .expect("loader body");
5753        assert!(loader.contains("resolve_deployment_trust_paths()"));
5754        assert!(loader.contains("load_trusted_registry_deployment_credentials_from(&paths)"));
5755        assert!(!loader.contains("dirs::config_dir"));
5756        assert!(!loader.contains("HYPRSTREAM__SECRETS__PATH"));
5757    }
5758
5759    fn service() -> DiscoveryService {
5760        let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
5761        DiscoveryService::new(Arc::new(sk), vk, TransportConfig::inproc("resolver-test"))
5762    }
5763
5764    fn legacy_endpoint(
5765        socket_kind: &str,
5766        endpoint: &str,
5767        last_heartbeat: Instant,
5768    ) -> AnnouncedEndpoint {
5769        let age_ms = Instant::now()
5770            .saturating_duration_since(last_heartbeat)
5771            .as_millis() as i64;
5772        let live_until_unix_ms = unix_millis_now()
5773            .saturating_add(ANNOUNCED_ENDPOINT_TTL.as_millis() as i64)
5774            .saturating_sub(age_ms);
5775        AnnouncedEndpoint {
5776            socket_kind: socket_kind.to_owned(),
5777            endpoint: endpoint.to_owned(),
5778            service_did: Did::default(),
5779            capabilities: BTreeSet::new(),
5780            accepted_state_digest: Vec::new(),
5781            accepted_state_epoch: 0,
5782            response_key_id: String::new(),
5783            request_kem_key_id: String::new(),
5784            request_kem_recipient: Vec::new(),
5785            expires_at_unix_ms: i64::MAX,
5786            source_signer: [0; 32],
5787            live_until_unix_ms,
5788        }
5789    }
5790
5791    fn accepted_state(tag: u8) -> (AcceptedAt9pState, SigningKey) {
5792        accepted_state_with_expiry(tag, "2099-01-01T00:00:00Z")
5793    }
5794
5795    fn accepted_state_with_expiry(tag: u8, expiry: &str) -> (AcceptedAt9pState, SigningKey) {
5796        let signing = SigningKey::from_bytes(&[tag; 32]);
5797        let pq_signing = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signing);
5798        let keys = HybridKeyPair::new(
5799            signing.verifying_key().to_bytes().to_vec(),
5800            ml_dsa_sk_to_vk_bytes(&pq_signing),
5801        )
5802        .unwrap_or_else(|e| panic!("test hybrid keys invalid: {e}"));
5803        let endpoint = ServiceEndpoint::new(At9pTransport::Iroh, "iroh://reach")
5804            .unwrap_or_else(|e| panic!("test endpoint invalid: {e}"));
5805        let service = ServiceEntry::new("#model", ServiceType::NinePExport, endpoint)
5806            .unwrap_or_else(|e| panic!("test service invalid: {e}"));
5807        let body = CapsuleBody::new(vec![keys], vec![service])
5808            .unwrap_or_else(|e| panic!("test body invalid: {e}"));
5809        let genesis = sign_capsule(body.clone(), &signing, &pq_signing)
5810            .unwrap_or_else(|e| panic!("test genesis signing failed: {e}"));
5811        let subject = genesis
5812            .cid512()
5813            .unwrap_or_else(|e| panic!("test genesis CID failed: {e}"));
5814        let update = sign_update_record(
5815            subject,
5816            1,
5817            [1; 64],
5818            body,
5819            expiry.to_owned(),
5820            &signing,
5821            &pq_signing,
5822        )
5823        .unwrap_or_else(|e| panic!("test update signing failed: {e}"));
5824        let bytes = update
5825            .to_dag_cbor()
5826            .unwrap_or_else(|e| panic!("test update encoding failed: {e}"));
5827        let state = AcceptedAt9pState::from_persisted_update(&bytes)
5828            .unwrap_or_else(|e| panic!("test accepted state invalid: {e}"));
5829        (state, signing)
5830    }
5831
5832    /// Accepted `#model` capsule with an explicit signed Iroh address and an
5833    /// optional signed request KEM (`None` models a legacy identity).
5834    fn accepted_service_state(
5835        tag: u8,
5836        address: &str,
5837        request_kem: Option<&hyprstream_rpc::crypto::hybrid_kem::RecipientPublic>,
5838    ) -> (AcceptedAt9pState, SigningKey) {
5839        let signing = SigningKey::from_bytes(&[tag; 32]);
5840        let pq_signing = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signing);
5841        let keys = HybridKeyPair::new(
5842            signing.verifying_key().to_bytes().to_vec(),
5843            ml_dsa_sk_to_vk_bytes(&pq_signing),
5844        )
5845        .unwrap_or_else(|e| panic!("test hybrid keys invalid: {e}"));
5846        let mut endpoint = ServiceEndpoint::new(At9pTransport::Iroh, address)
5847            .unwrap_or_else(|e| panic!("test endpoint invalid: {e}"));
5848        endpoint.request_kem =
5849            request_kem.map(hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::encode);
5850        let service = ServiceEntry::new("#model", ServiceType::NinePExport, endpoint)
5851            .unwrap_or_else(|e| panic!("test service invalid: {e}"));
5852        let body = CapsuleBody::new(vec![keys], vec![service])
5853            .unwrap_or_else(|e| panic!("test body invalid: {e}"));
5854        let genesis = sign_capsule(body.clone(), &signing, &pq_signing)
5855            .unwrap_or_else(|e| panic!("test genesis signing failed: {e}"));
5856        let subject = genesis
5857            .cid512()
5858            .unwrap_or_else(|e| panic!("test genesis CID failed: {e}"));
5859        let update = sign_update_record(
5860            subject,
5861            1,
5862            [1; 64],
5863            body,
5864            "2099-01-01T00:00:00Z".to_owned(),
5865            &signing,
5866            &pq_signing,
5867        )
5868        .unwrap_or_else(|e| panic!("test update signing failed: {e}"));
5869        let bytes = update
5870            .to_dag_cbor()
5871            .unwrap_or_else(|e| panic!("test update encoding failed: {e}"));
5872        let state = AcceptedAt9pState::from_persisted_update(&bytes)
5873            .unwrap_or_else(|e| panic!("test accepted state invalid: {e}"));
5874        (state, signing)
5875    }
5876
5877    struct MutableAcceptedState(parking_lot::Mutex<Option<AcceptedAt9pState>>);
5878
5879    impl AcceptedStateSource for MutableAcceptedState {
5880        fn accepted_state(&self, _did: &str) -> Result<Option<AcceptedAt9pState>> {
5881            Ok(self.0.lock().clone())
5882        }
5883        fn accepted_states(&self) -> Result<Vec<AcceptedAt9pState>> {
5884            Ok(self.0.lock().clone().into_iter().collect())
5885        }
5886    }
5887
5888    fn production_fixture(
5889        local_reach: bool,
5890    ) -> (DiscoveryServiceResolver, Arc<MutableAcceptedState>) {
5891        production_fixture_with_transport(local_reach, false)
5892    }
5893
5894    fn native_production_fixture(
5895        local_reach: bool,
5896    ) -> (DiscoveryServiceResolver, Arc<MutableAcceptedState>) {
5897        production_fixture_with_transport(local_reach, true)
5898    }
5899
5900    fn production_fixture_with_transport(
5901        local_reach: bool,
5902        native_iroh: bool,
5903    ) -> (DiscoveryServiceResolver, Arc<MutableAcceptedState>) {
5904        let (state, signing) = accepted_state(11);
5905        let kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
5906            hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
5907        )
5908        .unwrap_or_else(|e| panic!("test KEM generation failed: {e}"));
5909        let endpoint = if local_reach {
5910            "inproc://hyprstream/model".to_owned()
5911        } else if native_iroh {
5912            format!("iroh://{}", hex::encode([0x51; 32]))
5913        } else {
5914            "quic://localhost:127.0.0.1:9".to_owned()
5915        };
5916        let state_store = Arc::new(MemoryStateStore::default());
5917        state_store
5918            .put_announcement_sync(
5919                "model",
5920                AnnouncedEndpoint {
5921                    socket_kind: if local_reach {
5922                        "rep"
5923                    } else if native_iroh {
5924                        "iroh"
5925                    } else {
5926                        "quic"
5927                    }
5928                    .to_owned(),
5929                    endpoint,
5930                    service_did: Did::from(state.did.clone()),
5931                    capabilities: ["hyprstream-rpc/1".to_owned(), "hyprstream-moq/1".to_owned()]
5932                        .into_iter()
5933                        .collect(),
5934                    accepted_state_digest: state.head_digest.to_vec(),
5935                    accepted_state_epoch: state.epoch,
5936                    response_key_id: format!("{}#response-current", state.did),
5937                    request_kem_key_id: format!("{}#kem-current", state.did),
5938                    request_kem_recipient: kem.public().encode(),
5939                    expires_at_unix_ms: 4_070_908_800_000,
5940                    source_signer: signing.verifying_key().to_bytes(),
5941                    live_until_unix_ms: unix_millis_now()
5942                        + ANNOUNCED_ENDPOINT_TTL.as_millis() as i64,
5943                },
5944            )
5945            .expect("seed announcement");
5946        let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(state))));
5947        (
5948            DiscoveryServiceResolver {
5949                state_store,
5950                accepted_state_source: Arc::clone(&source) as Arc<dyn AcceptedStateSource>,
5951                discovery_client: None,
5952            },
5953            source,
5954        )
5955    }
5956
5957    async fn mutate_endpoint(
5958        resolver: &DiscoveryServiceResolver,
5959        service_name: &str,
5960        socket_kind: &str,
5961        mutate: impl FnOnce(&mut AnnouncedEndpoint),
5962    ) {
5963        let mut endpoint = resolver
5964            .state_store
5965            .announcements_for(service_name, unix_millis_now())
5966            .await
5967            .expect("read fixture announcement")
5968            .into_iter()
5969            .find(|entry| entry.socket_kind == socket_kind)
5970            .expect("fixture announcement");
5971        mutate(&mut endpoint);
5972        resolver
5973            .state_store
5974            .put_announcement(service_name, endpoint)
5975            .await
5976            .expect("replace fixture announcement");
5977    }
5978
5979    #[tokio::test]
5980    async fn production_resolver_joins_announcement_to_current_pds_state() {
5981        let (resolver, _) = native_production_fixture(false);
5982        let resolved = resolver
5983            .resolve_service(ServiceQuery::network("model").expect("query"))
5984            .await
5985            .unwrap_or_else(|e| panic!("production candidate rejected: {e}"));
5986        assert_eq!(resolved.service_name(), "model");
5987        assert_eq!(resolved.evidence().accepted_state_epoch, 1);
5988        assert!(resolved.service_did().is_did_at9p());
5989        assert!(!resolved.request_kem_recipient().recipient.eks.is_empty());
5990        resolver
5991            .ensure_current(&resolved)
5992            .await
5993            .unwrap_or_else(|e| panic!("unchanged accepted state rejected: {e}"));
5994    }
5995
5996    #[tokio::test]
5997    async fn production_profile_preserves_compatibility_quic_without_local_fallback() {
5998        let (resolver, _) = production_fixture(false);
5999        assert!(resolver
6000            .resolve_service(production_service_query("model", false).expect("query"))
6001            .await
6002            .is_ok());
6003        assert!(resolver
6004            .resolve_service(production_service_query("model", true).expect("query"))
6005            .await
6006            .is_err());
6007        let (resolver, _) = native_production_fixture(false);
6008        for required in [false, true] {
6009            assert!(resolver
6010                .resolve_service(production_service_query("model", required).expect("query"))
6011                .await
6012                .is_ok());
6013        }
6014        let (resolver, _) = production_fixture(true);
6015        for required in [false, true] {
6016            assert!(resolver
6017                .resolve_service(production_service_query("model", required).expect("query"))
6018                .await
6019                .is_err());
6020        }
6021    }
6022
6023    fn owned_browser_request() -> BrowserProvisioningRequest {
6024        BrowserProvisioningRequest::new(
6025            "model",
6026            "hyprstream-rpc/1",
6027            "model",
6028            BrowserCarrierProfile::OwnedHybridWebTransport,
6029        )
6030        .expect("browser request")
6031    }
6032
6033    #[tokio::test]
6034    async fn production_browser_projection_is_current_and_resolution_bound() {
6035        let (resolver, _) = production_fixture(false);
6036        let document = resolver
6037            .browser_provisioning(owned_browser_request())
6038            .await
6039            .unwrap_or_else(|error| panic!("browser projection failed: {error}"))
6040            .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6041            .expect("sign accepted projection");
6042        let json = serde_json::to_vec(&document).expect("serialize provisioning");
6043        let validated = hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6044            &json,
6045            &owned_browser_request(),
6046            unix_millis_now(),
6047        )
6048        .unwrap_or_else(|error| panic!("browser projection did not validate: {error}"));
6049        assert_eq!(validated.service_name(), "model");
6050        assert_eq!(validated.accepted_state_epoch(), 1);
6051        assert!(validated.service_did().starts_with("did:at9p:"));
6052    }
6053
6054    #[tokio::test]
6055    async fn production_browser_projection_normalizes_default_https_port() {
6056        let (resolver, _) = production_fixture(false);
6057        mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6058            endpoint.endpoint = "quic://localhost:127.0.0.1:443".to_owned();
6059        })
6060        .await;
6061        let document = resolver
6062            .browser_provisioning(owned_browser_request())
6063            .await
6064            .expect("project default HTTPS port")
6065            .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6066            .expect("sign projection");
6067        assert_eq!(document.service_origin, "https://localhost/");
6068        assert_eq!(
6069            document.webtransport_url,
6070            "https://localhost/hyprstream/browser-rpc"
6071        );
6072        let json = serde_json::to_vec(&document).expect("serialize provisioning");
6073        hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6074            &json,
6075            &owned_browser_request(),
6076            unix_millis_now(),
6077        )
6078        .expect("default-port projection must validate in the browser");
6079    }
6080
6081    #[tokio::test]
6082    async fn network_iroh_profile_rejects_quic_and_browser_profile_rejects_iroh() {
6083        let (resolver, _) = production_fixture(false);
6084        let native = match resolver
6085            .resolve_service(ServiceQuery::network("model").expect("native query"))
6086            .await
6087        {
6088            Ok(_) => panic!("native Iroh profile accepted a valid QUIC candidate"),
6089            Err(error) => error,
6090        };
6091        assert!(native.to_string().contains("no validated"));
6092
6093        let (resolver, _) = native_production_fixture(false);
6094        let browser = resolver
6095            .browser_provisioning(owned_browser_request())
6096            .await
6097            .expect_err("browser provisioning must reject Iroh reach");
6098        assert!(browser.to_string().contains("no validated"));
6099    }
6100
6101    #[tokio::test]
6102    async fn browser_projection_rejects_state_advance_revocation_and_cross_service() {
6103        let (resolver, source) = production_fixture(false);
6104        source.0.lock().as_mut().expect("fixture state").epoch += 1;
6105        assert!(resolver
6106            .browser_provisioning(owned_browser_request())
6107            .await
6108            .is_err());
6109
6110        let (resolver, _) = production_fixture(false);
6111        mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6112            endpoint.request_kem_recipient = vec![0x01];
6113        })
6114        .await;
6115        assert!(resolver
6116            .browser_provisioning(owned_browser_request())
6117            .await
6118            .is_err());
6119
6120        let (resolver, source) = production_fixture(false);
6121        source
6122            .0
6123            .lock()
6124            .as_mut()
6125            .expect("fixture state")
6126            .current
6127            .services[0]
6128            .id = "#policy".to_owned();
6129        assert!(resolver
6130            .browser_provisioning(owned_browser_request())
6131            .await
6132            .is_err());
6133    }
6134
6135    #[tokio::test]
6136    async fn public_relay_requires_signed_moq_route_and_application_object_profile() {
6137        let (resolver, source) = production_fixture(false);
6138        let request = BrowserProvisioningRequest::new(
6139            "model",
6140            "hyprstream-moq/1",
6141            "tenant-a/track-a",
6142            BrowserCarrierProfile::StandardPublicRelay,
6143        )
6144        .expect("relay request");
6145        assert!(resolver
6146            .browser_provisioning(request.clone())
6147            .await
6148            .is_err());
6149
6150        {
6151            let mut state = source.0.lock();
6152            let endpoint = &mut state.as_mut().expect("fixture state").current.services[0].endpoint;
6153            endpoint.transport = At9pTransport::Moq;
6154            endpoint.address = "https://model.example/".to_owned();
6155            endpoint.relay = Some("https://relay.example/moq".to_owned());
6156        }
6157        let document = resolver
6158            .browser_provisioning(request.clone())
6159            .await
6160            .unwrap_or_else(|error| panic!("signed relay projection failed: {error}"))
6161            .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6162            .expect("sign accepted relay projection");
6163        assert_eq!(
6164            document.transport_security,
6165            BrowserTransportSecurity::ClassicalUntrusted
6166        );
6167        assert!(document.application_hybrid_required);
6168        assert!(document.encrypted_objects_required);
6169        let json = serde_json::to_vec(&document).expect("serialize relay provisioning");
6170        hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6171            &json,
6172            &request,
6173            unix_millis_now(),
6174        )
6175        .expect("validate relay provisioning");
6176    }
6177
6178    #[tokio::test]
6179    async fn post_refetch_pre_dispatch_state_advance_rejects_pinned_binding() {
6180        let (resolver, source) = production_fixture(false);
6181        let document = resolver
6182            .browser_provisioning(owned_browser_request())
6183            .await
6184            .expect("pre-seal refetch")
6185            .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6186            .expect("sign projection");
6187        let json = serde_json::to_vec(&document).expect("serialize projection");
6188        let binding = hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6189            &json,
6190            &owned_browser_request(),
6191            unix_millis_now(),
6192        )
6193        .expect("validate projection")
6194        .request_binding()
6195        .expect("bind accepted evidence");
6196
6197        source.0.lock().as_mut().expect("fixture state").epoch += 1;
6198        assert!(resolver.verify_browser_binding(&binding).await.is_err());
6199    }
6200
6201    #[tokio::test]
6202    async fn post_refetch_route_and_pin_rotation_rejects_dial_evidence() {
6203        async fn binding_for(
6204            resolver: &DiscoveryServiceResolver,
6205        ) -> hyprstream_rpc::browser_provisioning::BrowserRequestBinding {
6206            let document = resolver
6207                .browser_provisioning(owned_browser_request())
6208                .await
6209                .expect("pre-seal refetch")
6210                .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6211                .expect("sign projection");
6212            let json = serde_json::to_vec(&document).expect("serialize projection");
6213            hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6214                &json,
6215                &owned_browser_request(),
6216                unix_millis_now(),
6217            )
6218            .expect("validate projection")
6219            .request_binding()
6220            .expect("bind dial evidence")
6221        }
6222
6223        let (resolver, _) = production_fixture(false);
6224        let route_binding = binding_for(&resolver).await;
6225        mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6226            endpoint.endpoint = "quic://localhost:127.0.0.1:10".to_owned();
6227        })
6228        .await;
6229        assert!(resolver
6230            .verify_browser_binding(&route_binding)
6231            .await
6232            .is_err());
6233
6234        let (resolver, _) = production_fixture(false);
6235        mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6236            endpoint.endpoint = format!(
6237                "quic://localhost:127.0.0.1:9#{}",
6238                URL_SAFE_NO_PAD.encode([0x51; 32])
6239            );
6240        })
6241        .await;
6242        let pin_binding = binding_for(&resolver).await;
6243        mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6244            endpoint.endpoint = format!(
6245                "quic://localhost:127.0.0.1:9#{}",
6246                URL_SAFE_NO_PAD.encode([0x52; 32])
6247            );
6248        })
6249        .await;
6250        assert!(resolver.verify_browser_binding(&pin_binding).await.is_err());
6251    }
6252
6253    #[tokio::test]
6254    async fn ordinary_iroh_announcement_handler_redacts_bearer_before_storage_and_discovery() {
6255        hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6256        let kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6257            hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6258        )
6259        .expect("test KEM");
6260        let reach = format!("iroh://{}", hex::encode([0x7a; 32]));
6261        let (state, service_signing) = accepted_service_state(12, &reach, Some(&kem.public()));
6262        let root = SigningKey::from_bytes(&[0x61; 32]);
6263        let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6264            state.clone(),
6265        ))));
6266        let service = DiscoveryService::new(
6267            Arc::new(root.clone()),
6268            root.verifying_key(),
6269            TransportConfig::inproc("announcement-rpc-test"),
6270        )
6271        .with_accepted_state_source(source);
6272        let claims = hyprstream_rpc::auth::Claims::new(
6273            "service:model".to_owned(),
6274            chrono::Utc::now().timestamp(),
6275            chrono::Utc::now().timestamp() + 3_600,
6276        )
6277        .with_cnf_jwk(service_signing.verifying_key().as_bytes());
6278        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &root);
6279        let service_pq_signing =
6280            hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_signing);
6281        let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6282            hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6283            &service_signing,
6284            &service_pq_signing,
6285        );
6286        let ctx = EnvelopeContext::from_verified_as_system(&signed);
6287        let response = service
6288            .handle_announce(
6289                &ctx,
6290                1,
6291                &ServiceAnnouncement {
6292                    service_name: "model".to_owned(),
6293                    socket_kind: "iroh".to_owned(),
6294                    endpoint: reach,
6295                    service_jwt: Some(jwt.clone()),
6296                    service_did: Did::from(state.did.clone()),
6297                    capabilities: vec!["hyprstream-rpc/1".to_owned()],
6298                    accepted_state_digest: state.head_digest.to_vec(),
6299                    accepted_state_epoch: state.epoch,
6300                    response_key_id: format!("{}#response-current", state.did),
6301                    request_kem_key_id: format!("{}#kem-current", state.did),
6302                    request_kem_recipient: kem.public().encode(),
6303                    expires_at_unix_ms: 4_070_908_800_000,
6304                },
6305            )
6306            .await
6307            .expect("ordinary announcement handler");
6308        assert!(matches!(response, DiscoveryResponseVariant::AnnounceResult));
6309        let stored = service
6310            .state_store
6311            .announcements_for("model", unix_millis_now())
6312            .await
6313            .expect("read stored announcement");
6314        assert_eq!(stored.len(), 1);
6315        let stored_json = serde_json::to_string(&stored[0]).expect("serialize stored announcement");
6316        assert!(!stored_json.contains(&jwt), "bearer JWT must not persist");
6317
6318        let endpoints = service
6319            .handle_get_endpoints(&ctx, 2, "model")
6320            .await
6321            .expect("get endpoints after announcement");
6322        let DiscoveryResponseVariant::GetEndpointsResult(endpoints) = endpoints else {
6323            panic!("announcement must provide a discovery endpoint");
6324        };
6325        assert!(!endpoints.endpoints.is_empty());
6326        assert!(
6327            endpoints
6328                .endpoints
6329                .iter()
6330                .all(|endpoint| endpoint.service_jwt.is_empty()),
6331            "discovery must never project a stored bearer JWT"
6332        );
6333
6334        let resolver = service.production_resolver().expect("production resolver");
6335        let resolved = resolver
6336            .resolve_service(ServiceQuery::network("model").expect("query"))
6337            .await
6338            .expect("ordinary announcement must resolve");
6339        assert_eq!(resolved.evidence().accepted_state_digest, state.head_digest);
6340        assert!(matches!(
6341            &resolved.transport().endpoint,
6342            EndpointType::Iroh {
6343                node_id,
6344                direct_addrs,
6345                relay_url: None,
6346            } if *node_id == [0x7a; 32] && direct_addrs.is_empty()
6347        ));
6348    }
6349
6350    /// A legacy identity whose accepted entry carries no signed request KEM
6351    /// cannot authorize one: an identity-bound announcement presenting an
6352    /// arbitrary encryption recipient and Iroh reach over the genuine
6353    /// accepted-state digest must be refused, never minted as
6354    /// checkpoint-authorized authority.
6355    #[tokio::test]
6356    async fn identity_bound_announcement_refuses_legacy_identity_without_signed_kem() {
6357        let legacy_reach = format!("iroh://{}", hex::encode([0x7c; 32]));
6358        let (state, service_signing) = accepted_service_state(13, &legacy_reach, None);
6359        let root = SigningKey::from_bytes(&[0x62; 32]);
6360        let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6361            state.clone(),
6362        ))));
6363        let service = DiscoveryService::new(
6364            Arc::new(root.clone()),
6365            root.verifying_key(),
6366            TransportConfig::inproc("legacy-kem-announce-test"),
6367        )
6368        .with_accepted_state_source(source);
6369        let claims = hyprstream_rpc::auth::Claims::new(
6370            "service:model".to_owned(),
6371            chrono::Utc::now().timestamp(),
6372            chrono::Utc::now().timestamp() + 3_600,
6373        )
6374        .with_cnf_jwk(service_signing.verifying_key().as_bytes());
6375        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &root);
6376        let service_pq_signing =
6377            hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_signing);
6378        let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6379            hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6380            &service_signing,
6381            &service_pq_signing,
6382        );
6383        let ctx = EnvelopeContext::from_verified_as_system(&signed);
6384        let kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6385            hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6386        )
6387        .expect("test KEM");
6388        let error = service
6389            .handle_announce(
6390                &ctx,
6391                1,
6392                &ServiceAnnouncement {
6393                    service_name: "model".to_owned(),
6394                    socket_kind: "iroh".to_owned(),
6395                    endpoint: format!("iroh://{}", hex::encode([0x7b; 32])),
6396                    service_jwt: Some(jwt),
6397                    service_did: Did::from(state.did.clone()),
6398                    capabilities: vec!["hyprstream-rpc/1".to_owned()],
6399                    accepted_state_digest: state.head_digest.to_vec(),
6400                    accepted_state_epoch: state.epoch,
6401                    response_key_id: format!("{}#response-current", state.did),
6402                    request_kem_key_id: format!("{}#kem-current", state.did),
6403                    request_kem_recipient: kem.public().encode(),
6404                    expires_at_unix_ms: 4_070_908_800_000,
6405                },
6406            )
6407            .await
6408            .expect_err(
6409                "identity-bound announcement over a KEM-less legacy identity must be refused",
6410            );
6411        assert!(error.to_string().contains("signed request KEM"));
6412    }
6413
6414    /// A capsule-bound request KEM must be presented verbatim: a signed
6415    /// announcement substituting its own recipient is refused.
6416    #[tokio::test]
6417    async fn identity_bound_announcement_refuses_kem_outside_accepted_service() {
6418        let accepted_kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6419            hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6420        )
6421        .expect("test accepted KEM");
6422        let reach = format!("iroh://{}", hex::encode([0x7d; 32]));
6423        let (state, service_signing) =
6424            accepted_service_state(14, &reach, Some(&accepted_kem.public()));
6425        let root = SigningKey::from_bytes(&[0x63; 32]);
6426        let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6427            state.clone(),
6428        ))));
6429        let service = DiscoveryService::new(
6430            Arc::new(root.clone()),
6431            root.verifying_key(),
6432            TransportConfig::inproc("foreign-kem-announce-test"),
6433        )
6434        .with_accepted_state_source(source);
6435        let claims = hyprstream_rpc::auth::Claims::new(
6436            "service:model".to_owned(),
6437            chrono::Utc::now().timestamp(),
6438            chrono::Utc::now().timestamp() + 3_600,
6439        )
6440        .with_cnf_jwk(service_signing.verifying_key().as_bytes());
6441        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &root);
6442        let service_pq_signing =
6443            hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_signing);
6444        let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6445            hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6446            &service_signing,
6447            &service_pq_signing,
6448        );
6449        let ctx = EnvelopeContext::from_verified_as_system(&signed);
6450        let foreign_kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6451            hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6452        )
6453        .expect("test foreign KEM");
6454        let error = service
6455            .handle_announce(
6456                &ctx,
6457                1,
6458                &ServiceAnnouncement {
6459                    service_name: "model".to_owned(),
6460                    socket_kind: "iroh".to_owned(),
6461                    endpoint: reach,
6462                    service_jwt: Some(jwt),
6463                    service_did: Did::from(state.did.clone()),
6464                    capabilities: vec!["hyprstream-rpc/1".to_owned()],
6465                    accepted_state_digest: state.head_digest.to_vec(),
6466                    accepted_state_epoch: state.epoch,
6467                    response_key_id: format!("{}#response-current", state.did),
6468                    request_kem_key_id: format!("{}#kem-current", state.did),
6469                    request_kem_recipient: foreign_kem.public().encode(),
6470                    expires_at_unix_ms: 4_070_908_800_000,
6471                },
6472            )
6473            .await
6474            .expect_err("announcement KEM outside the accepted service must be refused");
6475        assert!(error.to_string().contains("announcement KEM differs"));
6476    }
6477
6478    /// Iroh reach is validated against the signed capsule on its own terms:
6479    /// an announcement whose KEM matches but whose reach points elsewhere is
6480    /// refused even though the KEM branch succeeded.
6481    #[tokio::test]
6482    async fn identity_bound_announcement_refuses_iroh_reach_outside_accepted_service() {
6483        let kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6484            hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6485        )
6486        .expect("test KEM");
6487        let accepted_reach = format!("iroh://{}", hex::encode([0x7e; 32]));
6488        let (state, service_signing) =
6489            accepted_service_state(15, &accepted_reach, Some(&kem.public()));
6490        let root = SigningKey::from_bytes(&[0x64; 32]);
6491        let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6492            state.clone(),
6493        ))));
6494        let service = DiscoveryService::new(
6495            Arc::new(root.clone()),
6496            root.verifying_key(),
6497            TransportConfig::inproc("foreign-reach-announce-test"),
6498        )
6499        .with_accepted_state_source(source);
6500        let claims = hyprstream_rpc::auth::Claims::new(
6501            "service:model".to_owned(),
6502            chrono::Utc::now().timestamp(),
6503            chrono::Utc::now().timestamp() + 3_600,
6504        )
6505        .with_cnf_jwk(service_signing.verifying_key().as_bytes());
6506        let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &root);
6507        let service_pq_signing =
6508            hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_signing);
6509        let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6510            hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6511            &service_signing,
6512            &service_pq_signing,
6513        );
6514        let ctx = EnvelopeContext::from_verified_as_system(&signed);
6515        let error = service
6516            .handle_announce(
6517                &ctx,
6518                1,
6519                &ServiceAnnouncement {
6520                    service_name: "model".to_owned(),
6521                    socket_kind: "iroh".to_owned(),
6522                    endpoint: format!("iroh://{}", hex::encode([0x7f; 32])),
6523                    service_jwt: Some(jwt),
6524                    service_did: Did::from(state.did.clone()),
6525                    capabilities: vec!["hyprstream-rpc/1".to_owned()],
6526                    accepted_state_digest: state.head_digest.to_vec(),
6527                    accepted_state_epoch: state.epoch,
6528                    response_key_id: format!("{}#response-current", state.did),
6529                    request_kem_key_id: format!("{}#kem-current", state.did),
6530                    request_kem_recipient: kem.public().encode(),
6531                    expires_at_unix_ms: 4_070_908_800_000,
6532                },
6533            )
6534            .await
6535            .expect_err("announcement reach outside the accepted service must be refused");
6536        assert!(error.to_string().contains("announcement reach differs"));
6537    }
6538
6539    #[tokio::test]
6540    async fn identity_bound_announcement_shadows_identity_less_local_entry() {
6541        hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6542        let service = service();
6543        let name = "shadowed-reach-fixture";
6544        // A co-located service registers its identity-less local endpoint.
6545        hyprstream_rpc::registry::global().register(
6546            name,
6547            hyprstream_rpc::registry::SocketKind::Quic,
6548            hyprstream_rpc::transport::TransportConfig::inproc("local-shadowed-quic"),
6549            None,
6550        );
6551        // The same reach arrives announced, identity-bound, from the mesh.
6552        let mut announced = legacy_endpoint(
6553            "quic",
6554            "quic://localhost:127.0.0.1:4433#announcement",
6555            Instant::now(),
6556        );
6557        announced.service_did = Did::from("did:at9p:shadow-fixture");
6558        service
6559            .state_store
6560            .put_announcement(name, announced)
6561            .await
6562            .expect("install identity-bound announcement fixture");
6563
6564        let signer = SigningKey::from_bytes(&[0x75; 32]);
6565        let pq_signer = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signer);
6566        let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6567            hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6568            &signer,
6569            &pq_signer,
6570        );
6571        let ctx = EnvelopeContext::from_verified_as_system(&signed);
6572        let response = service
6573            .handle_get_endpoints(&ctx, 1, name)
6574            .await
6575            .expect("get endpoints");
6576        let DiscoveryResponseVariant::GetEndpointsResult(endpoints) = response else {
6577            panic!("expected endpoints");
6578        };
6579        assert!(
6580            endpoints
6581                .endpoints
6582                .iter()
6583                .any(|e| e.endpoint == "quic://localhost:127.0.0.1:4433#announcement"
6584                    && !e.service_did.as_str().is_empty()),
6585            "identity-bound announced endpoint must not be shadowed by the identity-less local entry: {:?}",
6586            endpoints.endpoints.iter().map(|e| e.endpoint.clone()).collect::<Vec<_>>(),
6587        );
6588        assert!(
6589            endpoints
6590                .endpoints
6591                .iter()
6592                .all(|e| !e.endpoint.contains("local-shadowed-quic")),
6593            "the identity-less local entry of the same socket kind must yield",
6594        );
6595    }
6596
6597    #[tokio::test]
6598    async fn identity_less_announcement_still_yields_to_local_entry() {
6599        hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6600        let service = service();
6601        let name = "legacy-yield-fixture";
6602        hyprstream_rpc::registry::global().register(
6603            name,
6604            hyprstream_rpc::registry::SocketKind::Quic,
6605            hyprstream_rpc::transport::TransportConfig::inproc("local-legacy-quic"),
6606            None,
6607        );
6608        // An identity-less (legacy) announcement must NOT displace the local entry.
6609        service
6610            .state_store
6611            .put_announcement(
6612                name,
6613                legacy_endpoint(
6614                    "quic",
6615                    "quic://localhost:127.0.0.1:4499#legacy",
6616                    Instant::now(),
6617                ),
6618            )
6619            .await
6620            .expect("install legacy announcement fixture");
6621        let signer = SigningKey::from_bytes(&[0x76; 32]);
6622        let pq_signer = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signer);
6623        let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6624            hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6625            &signer,
6626            &pq_signer,
6627        );
6628        let ctx = EnvelopeContext::from_verified_as_system(&signed);
6629        let response = service
6630            .handle_get_endpoints(&ctx, 1, name)
6631            .await
6632            .expect("get endpoints");
6633        let DiscoveryResponseVariant::GetEndpointsResult(endpoints) = response else {
6634            panic!("expected endpoints");
6635        };
6636        assert!(
6637            endpoints
6638                .endpoints
6639                .iter()
6640                .any(|e| e.endpoint.contains("local-legacy-quic")),
6641            "identity-less announced entry must still yield to the local entry",
6642        );
6643        assert!(
6644            endpoints
6645                .endpoints
6646                .iter()
6647                .all(|e| !e.endpoint.contains("4499")),
6648            "identity-less announced entry must not duplicate the local kind",
6649        );
6650    }
6651
6652    #[tokio::test]
6653    async fn remote_get_endpoints_omits_stale_and_fetch_cannot_refresh_it() {
6654        hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6655        let service = service();
6656        let name = "stale-remote-reach";
6657        service
6658            .state_store
6659            .put_announcement(
6660                name,
6661                legacy_endpoint(
6662                    "iroh",
6663                    &format!("iroh://{}", hex::encode([0x73; 32])),
6664                    Instant::now() - ANNOUNCED_ENDPOINT_TTL - Duration::from_secs(1),
6665                ),
6666            )
6667            .await
6668            .expect("install stale endpoint fixture");
6669        let signer = SigningKey::from_bytes(&[0x74; 32]);
6670        let pq_signer = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signer);
6671        let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6672            hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6673            &signer,
6674            &pq_signer,
6675        );
6676        let ctx = EnvelopeContext::from_verified_as_system(&signed);
6677
6678        let response = service
6679            .handle_get_endpoints(&ctx, 1, name)
6680            .await
6681            .expect("remote getEndpoints handler");
6682        assert!(matches!(response, DiscoveryResponseVariant::Error(_)));
6683        assert!(service
6684            .state_store
6685            .announcements_for(name, unix_millis_now())
6686            .await
6687            .expect("inspect state after fetch")
6688            .is_empty());
6689
6690        let listed = service
6691            .handle_list_services(&ctx, 2)
6692            .await
6693            .expect("remote listServices handler");
6694        let DiscoveryResponseVariant::ListServicesResult(services) = listed else {
6695            panic!("listServices must return service summaries");
6696        };
6697        assert!(services.services.iter().all(|summary| summary.name != name));
6698    }
6699
6700    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
6701    #[tokio::test]
6702    async fn valkey_announcement_handler_lease_and_result_follow_shared_time() {
6703        use crate::state_store::tests::ReplicaClock;
6704        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
6705            return;
6706        };
6707        for backend in [
6708            crate::DiscoveryStateBackend::Valkey,
6709            crate::DiscoveryStateBackend::Tiered,
6710        ] {
6711            let config = crate::DiscoveryStateConfig {
6712                backend,
6713                active_active: true,
6714                valkey: crate::ValkeyStateConfig {
6715                    url: url.clone(),
6716                    key_prefix: format!(
6717                        "hs-handler-lease-{}-{}-{backend:?}",
6718                        std::process::id(),
6719                        unix_millis_now()
6720                    ),
6721                    ..Default::default()
6722                },
6723                ..Default::default()
6724            };
6725            let (state, signing) = accepted_state(12);
6726            let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6727                state.clone(),
6728            ))));
6729            let root = SigningKey::from_bytes(&[0x61; 32]);
6730            let service = DiscoveryService::new(
6731                Arc::new(root.clone()),
6732                root.verifying_key(),
6733                TransportConfig::inproc("lease-handler-test"),
6734            )
6735            .with_accepted_state_source(source.clone())
6736            .with_state(DiscoveryState::connect(&config).await.unwrap());
6737            let claims = hyprstream_rpc::auth::Claims::new(
6738                "service:model".to_owned(),
6739                chrono::Utc::now().timestamp(),
6740                chrono::Utc::now().timestamp() + 7_200,
6741            )
6742            .with_cnf_jwk(signing.verifying_key().as_bytes());
6743            let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signing);
6744            let envelope = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6745                hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6746                &signing,
6747                &pq,
6748            );
6749            let ctx = EnvelopeContext::from_verified_as_system(&envelope);
6750            let kem = hyprstream_rpc::node_identity::derive_mesh_kem_recipient(&signing).unwrap();
6751            let now = unix_millis_now();
6752            let mut request = ServiceAnnouncement {
6753                service_name: "model".to_owned(),
6754                socket_kind: "quic".to_owned(),
6755                endpoint: "quic://localhost:127.0.0.1:9".to_owned(),
6756                service_jwt: Some(hyprstream_rpc::auth::jwt::encode_service_jwt(
6757                    &claims, &root,
6758                )),
6759                service_did: Did::from(state.did.clone()),
6760                capabilities: vec!["hyprstream-rpc/1".to_owned()],
6761                accepted_state_digest: state.head_digest.to_vec(),
6762                accepted_state_epoch: state.epoch,
6763                response_key_id: format!("{}#response-current", state.did),
6764                request_kem_key_id: format!("{}#kem-current", state.did),
6765                request_kem_recipient: kem.public().encode(),
6766                expires_at_unix_ms: now + 7_200_000,
6767            };
6768            for skew in [3_600_000, -3_600_000] {
6769                let _clock = ReplicaClock::at(now + skew);
6770                assert!(matches!(
6771                    service.handle_announce(&ctx, 1, &request).await.unwrap(),
6772                    DiscoveryResponseVariant::AnnounceResult
6773                ));
6774                let rows = service
6775                    .state_store
6776                    .announcements_for("model", now + skew)
6777                    .await
6778                    .unwrap();
6779                assert_eq!(
6780                    rows.len(),
6781                    1,
6782                    "successful response requires a stored publication"
6783                );
6784                assert!((now + 89_000..=now + 91_000).contains(&rows[0].live_until_unix_ms));
6785                assert_eq!(rows[0].expires_at_unix_ms, request.expires_at_unix_ms);
6786                assert_eq!(rows[0].request_kem_recipient, request.request_kem_recipient);
6787                let stored_json = serde_json::to_string(&rows[0]).unwrap();
6788                assert!(!stored_json.contains(request.service_jwt.as_deref().unwrap()));
6789                // Repeated reads exercise populated L1, not only its first fill.
6790                assert!(service
6791                    .resolve_announced_endpoint("model", SocketKind::Quic)
6792                    .await
6793                    .unwrap()
6794                    .is_some());
6795                assert_eq!(
6796                    service
6797                        .state_store
6798                        .all_announcements(now + skew)
6799                        .await
6800                        .unwrap()
6801                        .len(),
6802                    1
6803                );
6804                service
6805                    .production_resolver()
6806                    .unwrap()
6807                    .resolve_service(ServiceQuery::network("model").unwrap())
6808                    .await
6809                    .unwrap();
6810            }
6811            // Legacy publication has no signed expiry, but receives the same
6812            // bounded backend lease on both initial publication and refresh.
6813            let mut legacy = request.clone();
6814            legacy.service_name = "legacy".to_owned();
6815            legacy.service_jwt = None;
6816            legacy.service_did = Did::default();
6817            legacy.capabilities.clear();
6818            legacy.accepted_state_digest.clear();
6819            legacy.accepted_state_epoch = 0;
6820            legacy.response_key_id.clear();
6821            legacy.request_kem_key_id.clear();
6822            legacy.request_kem_recipient.clear();
6823            legacy.expires_at_unix_ms = 0;
6824            for skew in [3_600_000, -3_600_000] {
6825                let _clock = ReplicaClock::at(now + skew);
6826                assert!(matches!(
6827                    service.handle_announce(&ctx, 1, &legacy).await.unwrap(),
6828                    DiscoveryResponseVariant::AnnounceResult
6829                ));
6830                let rows = service
6831                    .state_store
6832                    .announcements_for("legacy", now + skew)
6833                    .await
6834                    .unwrap();
6835                assert_eq!(rows.len(), 1);
6836                assert!((now + 89_000..=now + 91_000).contains(&rows[0].live_until_unix_ms));
6837                assert_eq!(rows[0].expires_at_unix_ms, rows[0].live_until_unix_ms);
6838            }
6839
6840            // Signed expiry is still an independent ceiling. A shorter valid
6841            // signed deadline with the same epoch is an ignored older write;
6842            // the handler must not claim it was published.
6843            request.expires_at_unix_ms = now + 20_000;
6844            assert!(service
6845                .handle_announce(&ctx, 2, &request)
6846                .await
6847                .unwrap_err()
6848                .to_string()
6849                .contains("not stored"));
6850
6851            // Updated signed accepted-state evidence supplies a shorter current-state
6852            // ceiling; the backend must not extend it to the receipt TTL.
6853            let accepted_limit = (now / 1_000) * 1_000 + 20_000;
6854            let accepted_expiry = chrono::DateTime::from_timestamp_millis(accepted_limit)
6855                .unwrap()
6856                .to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
6857            let (bounded, _) = accepted_state_with_expiry(12, &accepted_expiry);
6858            *source.0.lock() = Some(bounded.clone());
6859            request.expires_at_unix_ms = now + 7_200_000;
6860            request.accepted_state_digest = bounded.head_digest.to_vec();
6861            assert!(matches!(
6862                service.handle_announce(&ctx, 3, &request).await.unwrap(),
6863                DiscoveryResponseVariant::AnnounceResult
6864            ));
6865            let values = service
6866                .state_store
6867                .announcements_for("model", now)
6868                .await
6869                .unwrap();
6870            assert_eq!(values[0].live_until_unix_ms, accepted_limit);
6871
6872            // Both signed and accepted expiry reject under a behind clock, and
6873            // neither rejection is reported as AnnounceResult.
6874            let _clock = ReplicaClock::at(now - 3_600_000);
6875            request.expires_at_unix_ms = now - 1;
6876            assert!(service
6877                .handle_announce(&ctx, 4, &request)
6878                .await
6879                .unwrap_err()
6880                .to_string()
6881                .contains("not stored"));
6882            let expired_text = chrono::DateTime::from_timestamp_millis(now - 1)
6883                .unwrap()
6884                .to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
6885            let (expired, _) = accepted_state_with_expiry(12, &expired_text);
6886            *source.0.lock() = Some(expired.clone());
6887            request.expires_at_unix_ms = now + 7_200_000;
6888            request.accepted_state_digest = expired.head_digest.to_vec();
6889            assert!(service
6890                .handle_announce(&ctx, 5, &request)
6891                .await
6892                .unwrap_err()
6893                .to_string()
6894                .contains("not stored"));
6895        }
6896    }
6897
6898    #[tokio::test]
6899    async fn accepted_state_advance_between_selection_and_dial_refuses() {
6900        let (resolver, source) = native_production_fixture(false);
6901        let resolved = resolver
6902            .resolve_service(ServiceQuery::network("model").expect("query"))
6903            .await
6904            .unwrap_or_else(|e| panic!("production candidate rejected: {e}"));
6905        {
6906            let mut guard = source.0.lock();
6907            let state = guard.as_mut().expect("fixture state");
6908            state.epoch += 1;
6909            state.head_digest = [0x55; 64];
6910        }
6911        let error = resolver
6912            .ensure_current(&resolved)
6913            .await
6914            .expect_err("advanced state accepted");
6915        assert!(error.to_string().contains("advanced or forked"));
6916    }
6917
6918    struct CountingStream(Arc<std::sync::atomic::AtomicUsize>);
6919
6920    #[async_trait]
6921    impl StreamHandle for CountingStream {
6922        async fn next_payload(
6923            &mut self,
6924        ) -> Result<Option<hyprstream_rpc::streaming::StreamPayload>> {
6925            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6926            Ok(None)
6927        }
6928        async fn cancel(&mut self) -> Result<()> {
6929            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6930            Ok(())
6931        }
6932        fn stream_id(&self) -> &str {
6933            "counting"
6934        }
6935        fn is_completed(&self) -> bool {
6936            false
6937        }
6938    }
6939
6940    #[tokio::test]
6941    async fn live_stream_continuation_fails_closed_after_snapshot_advance() {
6942        let (resolver, source) = native_production_fixture(false);
6943        let resolver = Arc::new(resolver);
6944        let snapshot = resolver
6945            .resolve_service_candidates(ServiceQuery::network("model").expect("query"))
6946            .await
6947            .expect("snapshot")
6948            .remove(0);
6949        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
6950        let mut stream = CurrentStreamHandle {
6951            inner: Box::new(CountingStream(Arc::clone(&calls))),
6952            resolver,
6953            snapshot,
6954            invalidated: false,
6955        };
6956        {
6957            let mut guard = source.0.lock();
6958            let state = guard.as_mut().expect("fixture state");
6959            state.epoch += 1;
6960            state.head_digest = [0x77; 64];
6961        }
6962        assert!(stream.next_payload().await.is_err());
6963        assert!(stream.cancel().await.is_err());
6964        assert_eq!(
6965            calls.load(std::sync::atomic::Ordering::SeqCst),
6966            0,
6967            "invalid continuation dispatched an operation"
6968        );
6969    }
6970
6971    #[tokio::test]
6972    async fn production_network_resolver_never_falls_back_to_local_reach() {
6973        let (resolver, _) = production_fixture(true);
6974        assert!(resolver
6975            .resolve_service(ServiceQuery::network("model").expect("query"))
6976            .await
6977            .is_err());
6978    }
6979
6980    #[tokio::test]
6981    async fn router_selected_reach_must_match_current_authorized_candidate() {
6982        let (resolver, _) = native_production_fixture(false);
6983        let expected = resolver
6984            .resolve_service_candidates(ServiceQuery::network("model").expect("query"))
6985            .await
6986            .expect("candidate")
6987            .remove(0)
6988            .transport()
6989            .clone();
6990        let client = ProductionRpcClient::new(
6991            "model",
6992            "inference",
6993            Some(expected),
6994            SigningKey::from_bytes(&[0x71; 32]),
6995            None,
6996            Arc::new(resolver),
6997        )
6998        .expect("selected client");
6999        assert_eq!(
7000            client.snapshots().await.expect("selected snapshot").len(),
7001            1
7002        );
7003    }
7004
7005    #[tokio::test]
7006    async fn router_selected_reach_rejects_unadvertised_transport_and_wrong_domain() {
7007        let (resolver, _) = native_production_fixture(false);
7008        let client = ProductionRpcClient::new(
7009            "model",
7010            "inference",
7011            Some(TransportConfig::inproc("attacker-selected")),
7012            SigningKey::from_bytes(&[0x72; 32]),
7013            None,
7014            Arc::new(resolver),
7015        )
7016        .expect("selected client");
7017        let reach_error = match client.snapshots().await {
7018            Ok(_) => panic!("unadvertised transport was authorized"),
7019            Err(error) => error,
7020        };
7021        assert!(reach_error
7022            .to_string()
7023            .contains("not a current authorized candidate"));
7024        let domain_error = client
7025            .call_for_service("model", Vec::new())
7026            .await
7027            .expect_err("resolution name was accepted as generated domain");
7028        assert!(domain_error.to_string().contains("authority mismatch"));
7029    }
7030
7031    #[tokio::test]
7032    async fn generated_client_uses_ordinary_identity_bound_resolver_path() {
7033        let (resolver, _) = production_fixture(false);
7034        let entries = resolver
7035            .state_store
7036            .announcements_for("model", unix_millis_now())
7037            .await
7038            .expect("fixture announcement");
7039        for endpoint in entries {
7040            resolver
7041                .state_store
7042                .put_announcement("discovery", endpoint)
7043                .await
7044                .expect("seed discovery announcement");
7045        }
7046        let resolver = Arc::new(resolver);
7047        let _ = PRODUCTION_RESOLVER.set(resolver);
7048        let client_signing = SigningKey::from_bytes(&[0x44; 32]);
7049        let _client = hyprstream_rpc_std::discovery_client::DiscoveryClient::from_provider(
7050            &crate::ProductionRpcClientProvider,
7051            client_signing,
7052            None,
7053        )
7054        .unwrap_or_else(|e| panic!("generated resolver path failed: {e}"));
7055    }
7056
7057    #[test]
7058    fn isolated_process_without_bootstrap_fails_closed() {
7059        const CHILD: &str = "HYPRSTREAM_TEST_RESOLVER_ABSENT_CHILD";
7060        if std::env::var_os(CHILD).is_some() {
7061            let signing = SigningKey::from_bytes(&[0x46; 32]);
7062            let error = match production_rpc_client("model", signing, None) {
7063                Ok(_) => panic!("isolated process unexpectedly inherited another resolver"),
7064                Err(error) => error,
7065            };
7066            assert!(error.to_string().contains("not installed"));
7067            return;
7068        }
7069        let status =
7070            std::process::Command::new(std::env::current_exe().expect("discovery test executable"))
7071                .arg("--exact")
7072                .arg("service::resolver_tests::isolated_process_without_bootstrap_fails_closed")
7073                .arg("--nocapture")
7074                .env(CHILD, "1")
7075                .status()
7076                .expect("isolated resolver subprocess");
7077        assert!(status.success(), "isolated resolver subprocess failed");
7078    }
7079
7080    #[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
7081    struct NoopBootstrapClient;
7082
7083    #[async_trait::async_trait]
7084    impl RpcClient for NoopBootstrapClient {
7085        async fn call(&self, _: Vec<u8>) -> Result<Vec<u8>> {
7086            anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7087        }
7088        async fn call_for_service(&self, _: &str, _: Vec<u8>) -> Result<Vec<u8>> {
7089            anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7090        }
7091        async fn call_for_service_with_method(
7092            &self,
7093            _: &str,
7094            _: u16,
7095            _: Vec<u8>,
7096        ) -> Result<Vec<u8>> {
7097            anyhow::bail!("noop bootstrap client")
7098        }
7099        async fn call_with_options(&self, _: Vec<u8>, _: CallOptions) -> Result<Vec<u8>> {
7100            anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7101        }
7102        async fn call_with_options_for_service(
7103            &self,
7104            _: &str,
7105            _: Vec<u8>,
7106            _: CallOptions,
7107        ) -> Result<Vec<u8>> {
7108            anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7109        }
7110        async fn call_streaming(&self, _: Vec<u8>, _: [u8; 32]) -> Result<Vec<u8>> {
7111            anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7112        }
7113        async fn call_streaming_for_service(
7114            &self,
7115            _: &str,
7116            _: Vec<u8>,
7117            _: [u8; 32],
7118        ) -> Result<Vec<u8>> {
7119            anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7120        }
7121        async fn call_streaming_for_service_with_method(
7122            &self,
7123            _: &str,
7124            _: u16,
7125            _: Vec<u8>,
7126            _: [u8; 32],
7127        ) -> Result<Vec<u8>> {
7128            anyhow::bail!("noop bootstrap client")
7129        }
7130        async fn open_stream(&self, _: Vec<u8>) -> Result<Box<dyn StreamHandle>> {
7131            anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7132        }
7133        async fn open_stream_from_info(
7134            &self,
7135            _: hyprstream_rpc::stream_info::StreamInfo,
7136            _: [u8; 32],
7137            _: [u8; 32],
7138        ) -> Result<Box<dyn StreamHandle>> {
7139            anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7140        }
7141        fn next_id(&self) -> u64 {
7142            1
7143        }
7144    }
7145
7146    #[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
7147    fn authenticated_registry_identity(tag: u8) -> ed25519_dalek::VerifyingKey {
7148        SigningKey::from_bytes(&[tag; 32]).verifying_key()
7149    }
7150
7151    #[test]
7152    fn deployment_trust_path_resolution_is_explicit_and_split() {
7153        let fixed = resolve_deployment_trust_paths_from(None, None).expect("fixed paths");
7154        assert_eq!(
7155            fixed.public_ca.path,
7156            std::path::Path::new(DEPLOYMENT_CA_ROOT_PATH)
7157        );
7158        assert_eq!(
7159            fixed.authority_log.path,
7160            std::path::Path::new(DEPLOYMENT_AUTHORITY_LOG_PATH)
7161        );
7162        assert_eq!(
7163            fixed.authority_checkpoint.path,
7164            std::path::Path::new(DEPLOYMENT_AUTHORITY_CHECKPOINT_PATH)
7165        );
7166        assert_eq!(
7167            fixed.registry_credential.path,
7168            std::path::Path::new(REGISTRY_DEPLOYMENT_CREDENTIAL_PATH)
7169        );
7170
7171        let trust_dir = std::path::Path::new("/run/user/1000/hyprstream/trust");
7172        let credentials_dir =
7173            std::path::Path::new("/run/user/1000/systemd/credentials/hyprstream.service");
7174        let overridden = resolve_deployment_trust_paths_from(
7175            Some(trust_dir.as_os_str().to_owned()),
7176            Some(credentials_dir.as_os_str().to_owned()),
7177        )
7178        .expect("explicit user-service paths");
7179        assert_eq!(
7180            overridden.public_ca.path,
7181            trust_dir.join(DEPLOYMENT_CA_ROOT_FILE)
7182        );
7183        assert_eq!(
7184            overridden.authority_log.path,
7185            trust_dir.join(DEPLOYMENT_AUTHORITY_LOG_FILE)
7186        );
7187        assert_eq!(
7188            overridden.authority_checkpoint.path,
7189            trust_dir.join(DEPLOYMENT_AUTHORITY_CHECKPOINT_FILE)
7190        );
7191        assert_eq!(
7192            overridden.registry_credential.path,
7193            credentials_dir.join(REGISTRY_DEPLOYMENT_CREDENTIAL_FILE)
7194        );
7195
7196        let trust_only =
7197            resolve_deployment_trust_paths_from(Some(trust_dir.as_os_str().to_owned()), None)
7198                .expect("trust-only override");
7199        assert_eq!(
7200            trust_only.public_ca.path,
7201            trust_dir.join(DEPLOYMENT_CA_ROOT_FILE)
7202        );
7203        assert_eq!(
7204            trust_only.registry_credential.path,
7205            std::path::Path::new(REGISTRY_DEPLOYMENT_CREDENTIAL_PATH)
7206        );
7207
7208        let credentials_only =
7209            resolve_deployment_trust_paths_from(None, Some(credentials_dir.as_os_str().to_owned()))
7210                .expect("credentials-only override");
7211        assert_eq!(
7212            credentials_only.public_ca.path,
7213            std::path::Path::new(DEPLOYMENT_CA_ROOT_PATH)
7214        );
7215        assert_eq!(
7216            credentials_only.registry_credential.path,
7217            credentials_dir.join(REGISTRY_DEPLOYMENT_CREDENTIAL_FILE)
7218        );
7219    }
7220
7221    #[test]
7222    fn invalid_present_trust_path_never_falls_back() {
7223        for (name, trust_dir, credentials_dir) in [
7224            (
7225                DEPLOYMENT_TRUST_DIR_ENV,
7226                Some(std::ffi::OsString::new()),
7227                None,
7228            ),
7229            (
7230                DEPLOYMENT_TRUST_DIR_ENV,
7231                Some(std::ffi::OsString::from("relative/trust")),
7232                None,
7233            ),
7234            (
7235                DEPLOYMENT_TRUST_DIR_ENV,
7236                Some(std::ffi::OsString::from("/secure/../other")),
7237                None,
7238            ),
7239            (
7240                SYSTEMD_CREDENTIALS_DIRECTORY_ENV,
7241                None,
7242                Some(std::ffi::OsString::new()),
7243            ),
7244            (
7245                SYSTEMD_CREDENTIALS_DIRECTORY_ENV,
7246                None,
7247                Some(std::ffi::OsString::from("relative/credentials")),
7248            ),
7249            (
7250                SYSTEMD_CREDENTIALS_DIRECTORY_ENV,
7251                None,
7252                Some(std::ffi::OsString::from("/secure/../other")),
7253            ),
7254        ] {
7255            let error = resolve_deployment_trust_paths_from(trust_dir, credentials_dir)
7256                .expect_err("invalid present override fell back");
7257            assert!(
7258                error.to_string().contains(name),
7259                "error did not identify {name}: {error}"
7260            );
7261        }
7262    }
7263
7264    #[cfg(unix)]
7265    #[test]
7266    fn trusted_artifact_accepts_effective_uid_owned_file_without_path_override_policy() {
7267        use std::os::unix::fs::PermissionsExt as _;
7268
7269        let fixture = tempfile::Builder::new()
7270            .prefix(".trust-owner-test-")
7271            .tempdir_in(env!("CARGO_MANIFEST_DIR"))
7272            .expect("secure fixture directory");
7273        let path = fixture.path().join("fixed-path-policy-artifact");
7274        std::fs::write(&path, b"service-owned trust").expect("trust artifact");
7275        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644))
7276            .expect("read-only trust artifact");
7277
7278        let bytes = read_trusted_artifact(
7279            &TrustedArtifactPath { path },
7280            "service-owned trust artifact",
7281        )
7282        .expect("effective-uid-owned 0644 artifact must be trusted");
7283        assert_eq!(bytes, b"service-owned trust");
7284    }
7285
7286    #[cfg(unix)]
7287    #[test]
7288    fn trusted_artifact_metadata_accepts_root_and_nonroot_service_but_rejects_other_uid() {
7289        const NONROOT_SERVICE_UID: u32 = 1_000;
7290        const OTHER_UID: u32 = 1_001;
7291
7292        validate_trusted_artifact_metadata(0, 0o100644, NONROOT_SERVICE_UID, "root artifact")
7293            .expect("root-owned 0644 artifact must remain trusted");
7294        validate_trusted_artifact_metadata(
7295            NONROOT_SERVICE_UID,
7296            0o100644,
7297            NONROOT_SERVICE_UID,
7298            "service artifact",
7299        )
7300        .expect("non-root effective-uid-owned 0644 artifact must be trusted");
7301
7302        let error = validate_trusted_artifact_metadata(
7303            OTHER_UID,
7304            0o100644,
7305            NONROOT_SERVICE_UID,
7306            "other-user artifact",
7307        )
7308        .expect_err("different non-service uid was trusted");
7309        assert!(error.to_string().contains("untrusted owner"), "{error}");
7310    }
7311
7312    #[cfg(unix)]
7313    fn user_service_trust_fixture() -> (
7314        tempfile::TempDir,
7315        DeploymentTrustPaths,
7316        TestDeploymentCa,
7317        SigningKey,
7318    ) {
7319        use base64::engine::general_purpose::STANDARD;
7320
7321        let fixture = tempfile::Builder::new()
7322            .prefix(".trust-loader-test-")
7323            .tempdir_in(env!("CARGO_MANIFEST_DIR"))
7324            .expect("secure fixture directory");
7325        let trust_dir = fixture.path().join("trust");
7326        let credentials_dir = fixture.path().join("credentials");
7327        std::fs::create_dir(&trust_dir).expect("trust directory");
7328        std::fs::create_dir(&credentials_dir).expect("credentials directory");
7329
7330        let ca = test_deployment_ca(0x71);
7331        let registry = SigningKey::from_bytes(&[0x72; 32]);
7332        let rotation_key = crate::did_op::HybridRotationKey::new(
7333            ca.ed25519.verifying_key().to_bytes(),
7334            hyprstream_crypto::pq::ml_dsa_sk_to_vk_bytes(&ca.ml_dsa_65),
7335        )
7336        .expect("hybrid rotation key");
7337        let genesis =
7338            crate::did_op::DidOp::signed_genesis(vec![rotation_key], &ca.ed25519, &ca.ml_dsa_65)
7339                .expect("signed authority genesis");
7340        let did = genesis.genesis_did().expect("authority-log DID");
7341        let active = crate::did_op::verify_did_op_log(&did, std::slice::from_ref(&genesis))
7342            .expect("verified authority log");
7343        let log = DeploymentAuthorityLog {
7344            schema: AUTHORITY_LOG_SCHEMA.to_owned(),
7345            deployment_domain: ca.verifying_keys.domain(),
7346            did: did.clone(),
7347            operations_b64: vec![STANDARD.encode(genesis.to_dag_cbor())],
7348        };
7349        let checkpoint = DeploymentAuthorityCheckpoint {
7350            schema: AUTHORITY_CHECKPOINT_SCHEMA.to_owned(),
7351            deployment_domain: ca.verifying_keys.domain(),
7352            did,
7353            sequence: active.sequence,
7354            head_cid: active.head_cid,
7355        };
7356        let mut public_ca = ca.verifying_keys.ed25519_bytes().to_vec();
7357        public_ca.extend_from_slice(&ca.verifying_keys.ml_dsa_65_bytes());
7358        std::fs::write(trust_dir.join(DEPLOYMENT_CA_ROOT_FILE), public_ca).expect("deployment CA");
7359        std::fs::write(
7360            trust_dir.join(DEPLOYMENT_AUTHORITY_LOG_FILE),
7361            serde_json::to_vec(&log).expect("authority-log JSON"),
7362        )
7363        .expect("authority log");
7364        std::fs::write(
7365            trust_dir.join(DEPLOYMENT_AUTHORITY_CHECKPOINT_FILE),
7366            serde_json::to_vec(&checkpoint).expect("checkpoint JSON"),
7367        )
7368        .expect("authority checkpoint");
7369        std::fs::write(
7370            credentials_dir.join(REGISTRY_DEPLOYMENT_CREDENTIAL_FILE),
7371            exact_registry_credential(&ca, &registry),
7372        )
7373        .expect("registry credential");
7374        let paths = resolve_deployment_trust_paths_from(
7375            Some(trust_dir.into_os_string()),
7376            Some(credentials_dir.into_os_string()),
7377        )
7378        .expect("fixture trust paths");
7379        (fixture, paths, ca, registry)
7380    }
7381
7382    #[cfg(unix)]
7383    #[test]
7384    fn user_service_paths_preserve_complete_enrolled_verification() {
7385        let (_fixture, paths, ca, registry) = user_service_trust_fixture();
7386        let credentials = load_trusted_registry_deployment_credentials_from(&paths)
7387            .expect("load explicit user-service artifacts");
7388        let witness = authenticate_registry_deployment_credentials(credentials)
7389            .expect("authenticate complete enrolled artifact set");
7390        assert!(witness.verifier.matches(&registry.verifying_key()));
7391        assert!(witness.verifier.matches_deployment_root(&ca.verifying_keys));
7392
7393        let (protected, mut claims) = exact_registry_credential_values(&ca, &registry);
7394        claims["aud"] = serde_json::Value::String("urn:hyprstream:service:other".to_owned());
7395        std::fs::write(
7396            &paths.registry_credential.path,
7397            sign_registry_credential_json(&ca, &protected.to_string(), &claims.to_string()),
7398        )
7399        .expect("invalid-audience credential");
7400        let credentials = load_trusted_registry_deployment_credentials_from(&paths)
7401            .expect("path loader must not pre-authenticate");
7402        assert!(
7403            authenticate_registry_deployment_credentials(credentials).is_err(),
7404            "explicit path bypassed the closed JWT audience/profile verifier"
7405        );
7406    }
7407
7408    #[cfg(unix)]
7409    #[test]
7410    fn user_service_paths_reject_writable_files_and_symlinks() {
7411        use std::os::unix::fs::{symlink, PermissionsExt as _};
7412
7413        let (_fixture, paths, _ca, _registry) = user_service_trust_fixture();
7414        for mode in [0o664, 0o666] {
7415            std::fs::set_permissions(
7416                &paths.registry_credential.path,
7417                std::fs::Permissions::from_mode(mode),
7418            )
7419            .expect("make credential writable");
7420            let error = match load_trusted_registry_deployment_credentials_from(&paths) {
7421                Ok(_) => panic!("group/world-writable credential was trusted"),
7422                Err(error) => error,
7423            };
7424            assert!(
7425                error.to_string().contains("group/world writable"),
7426                "mode {mode:o} failed for the wrong reason: {error}"
7427            );
7428        }
7429
7430        let real_credential = paths
7431            .registry_credential
7432            .path
7433            .with_file_name("registry-service.real.jwt");
7434        std::fs::rename(&paths.registry_credential.path, &real_credential)
7435            .expect("retain real credential");
7436        std::fs::set_permissions(&real_credential, std::fs::Permissions::from_mode(0o600))
7437            .expect("secure real credential");
7438        symlink(&real_credential, &paths.registry_credential.path).expect("credential symlink");
7439        let error = load_trusted_registry_deployment_credentials_from(&paths)
7440            .err()
7441            .expect("symlinked credential was trusted");
7442        assert!(
7443            error
7444                .to_string()
7445                .contains("Too many levels of symbolic links")
7446                || error
7447                    .to_string()
7448                    .contains("registry deployment credential is unavailable"),
7449            "unexpected symlink rejection: {error}"
7450        );
7451    }
7452
7453    #[cfg(unix)]
7454    #[test]
7455    fn user_service_paths_reject_writable_or_symlinked_ancestors() {
7456        use std::os::unix::fs::{symlink, PermissionsExt as _};
7457
7458        let (fixture, paths, _ca, _registry) = user_service_trust_fixture();
7459        std::fs::set_permissions(fixture.path(), std::fs::Permissions::from_mode(0o770))
7460            .expect("make fixture parent writable");
7461        let error = load_trusted_registry_deployment_credentials_from(&paths)
7462            .err()
7463            .expect("group-writable ancestor was trusted");
7464        assert!(error.to_string().contains("parent"));
7465        assert!(error.to_string().contains("group/world writable"));
7466
7467        std::fs::set_permissions(fixture.path(), std::fs::Permissions::from_mode(0o700))
7468            .expect("restore fixture parent");
7469        let trust_dir = fixture.path().join("trust");
7470        let real_trust_dir = fixture.path().join("real-trust");
7471        std::fs::rename(&trust_dir, &real_trust_dir).expect("retain real trust directory");
7472        symlink(&real_trust_dir, &trust_dir).expect("trust-directory symlink");
7473        let error = load_trusted_registry_deployment_credentials_from(&paths)
7474            .err()
7475            .expect("symlinked trust ancestor was trusted");
7476        assert!(
7477            error.to_string().contains("not a real directory"),
7478            "unexpected ancestor-symlink rejection: {error}"
7479        );
7480    }
7481
7482    /// H3: the OS-owned enrollment seam reads the chain through the trusted
7483    /// artifact policy, so a trust dir without the chain fails closed before
7484    /// any attestation bytes are even parsed.
7485    #[cfg(unix)]
7486    #[test]
7487    fn os_owned_enrollment_verification_fails_closed_without_chain() {
7488        static ENV_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
7489        let _serial = ENV_LOCK.lock();
7490        let fixture = tempfile::Builder::new()
7491            .prefix(".enrollment-trust-test-")
7492            .tempdir_in(env!("CARGO_MANIFEST_DIR"))
7493            .expect("secure fixture directory");
7494        let prev = std::env::var_os(DEPLOYMENT_TRUST_DIR_ENV);
7495        std::env::set_var(DEPLOYMENT_TRUST_DIR_ENV, fixture.path());
7496        let result = verify_os_owned_service_key_enrollment(b"{}");
7497        match &prev {
7498            Some(value) => std::env::set_var(DEPLOYMENT_TRUST_DIR_ENV, value),
7499            None => std::env::remove_var(DEPLOYMENT_TRUST_DIR_ENV),
7500        }
7501        let error = result.expect_err("attestation verified without a chain");
7502        assert!(
7503            error.to_string().contains("deployment CA root"),
7504            "unexpected failure mode: {error}"
7505        );
7506    }
7507
7508    #[test]
7509    fn verified_registry_identity_ignores_policy_and_post_start_environment_mutation() {
7510        const CHILD: &str = "HYPRSTREAM_TEST_POST_START_REGISTRY_ENV_CHILD";
7511        if std::env::var_os(CHILD).is_none() {
7512            let status = std::process::Command::new(
7513                std::env::current_exe().expect("discovery test executable"),
7514            )
7515            .arg("--exact")
7516            .arg("service::resolver_tests::verified_registry_identity_ignores_policy_and_post_start_environment_mutation")
7517            .arg("--nocapture")
7518            .env(CHILD, "1")
7519            .status()
7520            .expect("post-start environment subprocess");
7521            assert!(status.success(), "post-start environment subprocess failed");
7522            return;
7523        }
7524        let caller = SigningKey::from_bytes(&[0x61; 32]);
7525        hyprstream_service::global_trust_store().insert(
7526            caller.verifying_key(),
7527            hyprstream_service::Attestation {
7528                scopes: std::iter::once("registry".to_owned()).collect(),
7529                subject: None,
7530                jwt: None,
7531                expires_at: 0,
7532                attested_by: None,
7533            },
7534        );
7535        let ca = test_deployment_ca(0x62);
7536        let registry = SigningKey::from_bytes(&[0x63; 32]);
7537        let witness =
7538            authenticate_test_registry_credential(&ca, exact_registry_credential(&ca, &registry))
7539                .expect("explicit genesis-only trusted pair");
7540        std::env::set_var("CREDENTIALS_DIRECTORY", "post-start-attacker-directory");
7541        std::env::set_var("XDG_CONFIG_HOME", "post-start-attacker-config");
7542        assert!(witness.verifier.matches(&registry.verifying_key()));
7543        assert!(!witness.verifier.matches(&caller.verifying_key()));
7544    }
7545
7546    #[cfg(feature = "rocksdb")]
7547    #[test]
7548    fn process_bootstrap_precedes_first_generated_client() {
7549        const CHILD: &str = "HYPRSTREAM_TEST_RESOLVER_BOOTSTRAP_CHILD";
7550        if std::env::var_os(CHILD).is_some() {
7551            let store = hyprstream_service::deployment_data_dir()
7552                .expect("deployment-owned data root")
7553                .join("pds-store");
7554            std::fs::create_dir_all(&store).expect("checkpoint store directory");
7555            drop(rocksdb::DB::open_default(&store).expect("empty checkpoint store"));
7556            let signing = SigningKey::from_bytes(&[0x47; 32]);
7557            hyprstream_service::global_trust_store().insert(
7558                signing.verifying_key(),
7559                hyprstream_service::Attestation {
7560                    scopes: std::iter::once("registry".to_owned()).collect(),
7561                    subject: None,
7562                    jwt: None,
7563                    expires_at: 0,
7564                    attested_by: None,
7565                },
7566            );
7567            let authority =
7568                authenticate_discovery_bootstrap_identity(authenticated_registry_identity(0x51))
7569                    .expect("authenticate process resolver");
7570            DiscoveryService::bootstrap_authenticated_process(
7571                authority,
7572                hyprstream_rpc_std::discovery_client::DiscoveryClient::new(Arc::new(
7573                    NoopBootstrapClient,
7574                )),
7575                &hyprstream_pds::rds::RdsConfig::default(),
7576            )
7577            .expect("install process resolver");
7578            assert!(production_rpc_client("model", signing, None).is_ok());
7579            assert!(
7580                authenticate_discovery_bootstrap_identity(authenticated_registry_identity(0x52))
7581                    .is_err()
7582            );
7583            return;
7584        }
7585        let deployment = tempfile::tempdir().expect("deployment data root");
7586        let status =
7587            std::process::Command::new(std::env::current_exe().expect("discovery test executable"))
7588                .arg("--exact")
7589                .arg("service::resolver_tests::process_bootstrap_precedes_first_generated_client")
7590                .arg("--nocapture")
7591                .env(CHILD, "1")
7592                .env("XDG_DATA_HOME", deployment.path())
7593                .status()
7594                .expect("resolver bootstrap subprocess");
7595        assert!(status.success(), "resolver bootstrap subprocess failed");
7596    }
7597
7598    #[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
7599    fn seed_registry_for_bootstrap(tag: u8) {
7600        let registry = SigningKey::from_bytes(&[tag; 32]);
7601        hyprstream_service::global_trust_store().insert(
7602            registry.verifying_key(),
7603            hyprstream_service::Attestation {
7604                scopes: std::iter::once("registry".to_owned()).collect(),
7605                subject: None,
7606                jwt: None,
7607                expires_at: 0,
7608                attested_by: None,
7609            },
7610        );
7611    }
7612
7613    #[cfg(feature = "rocksdb")]
7614    #[test]
7615    fn authenticated_process_bootstrap_has_exactly_one_concurrent_consumer() {
7616        const CHILD: &str = "HYPRSTREAM_TEST_RESOLVER_CONCURRENT_CHILD";
7617        if std::env::var_os(CHILD).is_some() {
7618            let store = hyprstream_service::deployment_data_dir()
7619                .unwrap()
7620                .join("pds-store");
7621            std::fs::create_dir_all(&store).unwrap();
7622            drop(rocksdb::DB::open_default(&store).unwrap());
7623            seed_registry_for_bootstrap(0x48);
7624            let barrier = Arc::new(std::sync::Barrier::new(9));
7625            let attempts: Vec<_> = (0..8)
7626                .map(|_| {
7627                    let barrier = Arc::clone(&barrier);
7628                    std::thread::spawn(move || {
7629                        barrier.wait();
7630                        authenticate_discovery_bootstrap_identity(authenticated_registry_identity(
7631                            0x53,
7632                        ))
7633                        .and_then(|authority| {
7634                            DiscoveryService::bootstrap_authenticated_process(
7635                                authority,
7636                                hyprstream_rpc_std::discovery_client::DiscoveryClient::new(
7637                                    Arc::new(NoopBootstrapClient),
7638                                ),
7639                                &hyprstream_pds::rds::RdsConfig::default(),
7640                            )
7641                        })
7642                        .is_ok()
7643                    })
7644                })
7645                .collect();
7646            barrier.wait();
7647            assert_eq!(
7648                attempts
7649                    .into_iter()
7650                    .map(|attempt| attempt.join().unwrap())
7651                    .filter(|installed| *installed)
7652                    .count(),
7653                1
7654            );
7655            return;
7656        }
7657        let deployment = tempfile::tempdir().unwrap();
7658        let status = std::process::Command::new(std::env::current_exe().unwrap())
7659            .arg("--exact")
7660            .arg("service::resolver_tests::authenticated_process_bootstrap_has_exactly_one_concurrent_consumer")
7661            .arg("--nocapture")
7662            .env(CHILD, "1")
7663            .env("XDG_DATA_HOME", deployment.path())
7664            .status()
7665            .unwrap();
7666        assert!(status.success());
7667    }
7668
7669    #[cfg(feature = "rocksdb")]
7670    #[test]
7671    fn authenticated_process_bootstrap_failure_is_terminal() {
7672        const CHILD: &str = "HYPRSTREAM_TEST_RESOLVER_FAILURE_CHILD";
7673        if std::env::var_os(CHILD).is_some() {
7674            seed_registry_for_bootstrap(0x49);
7675            let authority =
7676                authenticate_discovery_bootstrap_identity(authenticated_registry_identity(0x54))
7677                    .unwrap();
7678            let first = DiscoveryService::bootstrap_authenticated_process(
7679                authority,
7680                hyprstream_rpc_std::discovery_client::DiscoveryClient::new(Arc::new(
7681                    NoopBootstrapClient,
7682                )),
7683                &hyprstream_pds::rds::RdsConfig::default(),
7684            )
7685            .expect_err("missing checkpoint store unexpectedly installed");
7686            assert!(first.to_string().contains("failed to open"));
7687            let store = hyprstream_service::deployment_data_dir()
7688                .unwrap()
7689                .join("pds-store");
7690            std::fs::create_dir_all(&store).unwrap();
7691            drop(rocksdb::DB::open_default(&store).unwrap());
7692            let second =
7693                authenticate_discovery_bootstrap_identity(authenticated_registry_identity(0x55))
7694                    .err()
7695                    .expect("failed bootstrap authority was replayed");
7696            assert!(second.to_string().contains("already sealed or consumed"));
7697            return;
7698        }
7699        let deployment = tempfile::tempdir().unwrap();
7700        let status = std::process::Command::new(std::env::current_exe().unwrap())
7701            .arg("--exact")
7702            .arg("service::resolver_tests::authenticated_process_bootstrap_failure_is_terminal")
7703            .arg("--nocapture")
7704            .env(CHILD, "1")
7705            .env("XDG_DATA_HOME", deployment.path())
7706            .status()
7707            .unwrap();
7708        assert!(status.success());
7709    }
7710
7711    #[tokio::test]
7712    async fn stale_or_expired_production_evidence_is_rejected() {
7713        let (resolver, _) = native_production_fixture(false);
7714        let expiry = resolver
7715            .state_store
7716            .announcements_for("model", unix_millis_now())
7717            .await
7718            .unwrap()[0]
7719            .live_until_unix_ms;
7720        {
7721            // Expired writes now correctly leave a prior valid value intact.
7722            // Advance this memory backend's clock to expire the real lease
7723            // instead of attempting to overwrite it with a rejected write.
7724            let _clock = crate::state_store::tests::ReplicaClock::at(expiry);
7725            assert!(resolver
7726                .resolve_service(ServiceQuery::network("model").expect("query"))
7727                .await
7728                .is_err());
7729        }
7730
7731        let (resolver, source) = native_production_fixture(false);
7732        source.0.lock().as_mut().expect("fixture state").expires_at =
7733            Some("2000-01-01T00:00:00Z".to_owned());
7734        assert!(resolver
7735            .resolve_service(ServiceQuery::network("model").expect("query"))
7736            .await
7737            .is_err());
7738    }
7739
7740    #[tokio::test]
7741    async fn malformed_candidate_does_not_poison_valid_alternative() {
7742        let (resolver, _) = native_production_fixture(false);
7743        let mut malformed = resolver
7744            .state_store
7745            .announcements_for("model", unix_millis_now())
7746            .await
7747            .expect("fixture service")
7748            .into_iter()
7749            .next()
7750            .expect("fixture endpoint");
7751        malformed.request_kem_recipient = vec![0xff];
7752        malformed.socket_kind = "quic".to_owned();
7753        malformed.endpoint = "quic://missing-port".to_owned();
7754        resolver
7755            .state_store
7756            .put_announcement("model", malformed)
7757            .await
7758            .expect("seed malformed alternative");
7759
7760        let resolved = resolver
7761            .resolve_service(ServiceQuery::network("model").expect("query"))
7762            .await
7763            .unwrap_or_else(|e| panic!("malformed alternative poisoned valid candidate: {e}"));
7764        assert_eq!(resolved.service_name(), "model");
7765    }
7766
7767    #[tokio::test]
7768    async fn rejected_or_forked_current_state_never_produces_candidate() {
7769        let (resolver, source) = production_fixture(false);
7770        *source.0.lock() = None;
7771        assert!(resolver
7772            .resolve_service(ServiceQuery::network("model").expect("query"))
7773            .await
7774            .is_err());
7775
7776        let (resolver, _) = production_fixture(false);
7777        mutate_endpoint(&resolver, "model", "quic", |endpoint| {
7778            endpoint.accepted_state_digest = vec![0x77; 64];
7779        })
7780        .await;
7781        assert!(resolver
7782            .resolve_service(ServiceQuery::network("model").expect("query"))
7783            .await
7784            .is_err());
7785    }
7786
7787    #[tokio::test]
7788    async fn resolver_uses_fresh_announced_quic_endpoint() {
7789        let svc = service();
7790        svc.state_store
7791            .put_announcement(
7792                "model",
7793                legacy_endpoint(
7794                    "quic",
7795                    "quic://model.hyprstream.svc.cluster.local:10.96.0.42:4433",
7796                    Instant::now(),
7797                ),
7798            )
7799            .await
7800            .expect("seed announcement");
7801
7802        let transport = match svc.resolve("model", SocketKind::Quic).await {
7803            Ok(transport) => transport,
7804            Err(err) => panic!("fresh announced QUIC endpoint must resolve: {err}"),
7805        };
7806        assert_eq!(transport.bind_mode(), BindMode::Connect);
7807        match transport.endpoint {
7808            EndpointType::Quic {
7809                addr, server_name, ..
7810            } => {
7811                assert_eq!(server_name, "model.hyprstream.svc.cluster.local");
7812                assert_eq!(addr, SocketAddr::from(([10, 96, 0, 42], 4433)));
7813            }
7814            other => panic!("expected announced QUIC endpoint, got {other:?}"),
7815        }
7816    }
7817
7818    #[tokio::test]
7819    async fn resolver_rejects_missing_announced_quic_endpoint() {
7820        let err = match service().resolve("model", SocketKind::Quic).await {
7821            Ok(transport) => panic!("missing announced QUIC endpoint resolved to {transport:?}"),
7822            Err(err) => err,
7823        };
7824        assert!(
7825            err.to_string().contains("no fresh announced QUIC endpoint"),
7826            "unexpected error: {err}"
7827        );
7828    }
7829
7830    #[tokio::test]
7831    async fn resolver_rejects_stale_announced_quic_endpoint() {
7832        let svc = service();
7833        svc.state_store
7834            .put_announcement(
7835                "model",
7836                legacy_endpoint(
7837                    "quic",
7838                    "quic://model.hyprstream.svc.cluster.local:10.96.0.42:4433",
7839                    Instant::now() - (ANNOUNCED_ENDPOINT_TTL + Duration::from_secs(1)),
7840                ),
7841            )
7842            .await
7843            .expect("seed stale announcement");
7844
7845        let err = match svc.resolve("model", SocketKind::Quic).await {
7846            Ok(transport) => panic!("stale announced QUIC endpoint resolved to {transport:?}"),
7847            Err(err) => err,
7848        };
7849        assert!(
7850            err.to_string().contains("no fresh announced QUIC endpoint"),
7851            "unexpected error: {err}"
7852        );
7853    }
7854
7855    #[test]
7856    fn announced_iroh_endpoint_parses_to_transport() {
7857        let node_hex = "0707070707070707070707070707070707070707070707070707070707070707";
7858        let transport = match parse_announced_iroh(&format!("iroh://{node_hex}")) {
7859            Ok(transport) => transport,
7860            Err(err) => panic!("valid iroh endpoint must parse: {err}"),
7861        };
7862        assert_eq!(transport.bind_mode(), BindMode::Connect);
7863        match transport.endpoint {
7864            EndpointType::Iroh {
7865                node_id,
7866                direct_addrs,
7867                relay_url,
7868            } => {
7869                assert_eq!(node_id, [7u8; 32]);
7870                assert!(direct_addrs.is_empty());
7871                assert!(relay_url.is_none());
7872            }
7873            other => panic!("expected iroh endpoint, got {other:?}"),
7874        }
7875    }
7876}
7877
7878/// Convert a `SocketKind` to a lowercase string for serialization.
7879fn socket_kind_to_string(kind: SocketKind) -> &'static str {
7880    match kind {
7881        SocketKind::Req => "req",
7882        SocketKind::Rep => "rep",
7883        SocketKind::Quic => "quic",
7884    }
7885}
7886
7887// ============================================================================
7888// DiscoveryHandler implementation (generated trait)
7889// ============================================================================
7890
7891#[async_trait(?Send)]
7892impl DiscoveryHandler for DiscoveryService {
7893    async fn authorize(
7894        &self,
7895        ctx: &EnvelopeContext,
7896        resource: &str,
7897        operation: &str,
7898    ) -> Result<()> {
7899        // Delegate to authorization provider if available
7900        if let Some(ref auth) = self.auth_provider {
7901            let subject = ctx.subject().to_string();
7902            let allowed = auth
7903                .check_context(ctx, resource, operation)
7904                .await
7905                .unwrap_or_else(|e| {
7906                    tracing::warn!("Discovery auth check failed for {}: {}", subject, e);
7907                    false
7908                });
7909            if allowed {
7910                Ok(())
7911            } else {
7912                anyhow::bail!(
7913                    "Unauthorized: {} cannot {} on {}",
7914                    subject,
7915                    operation,
7916                    resource
7917                )
7918            }
7919        } else {
7920            // No auth provider — allow (backward compat for local-only deployments)
7921            Ok(())
7922        }
7923    }
7924
7925    async fn handle_list_services(
7926        &self,
7927        _ctx: &EnvelopeContext,
7928        _request_id: u64,
7929    ) -> Result<DiscoveryResponseVariant> {
7930        trace!("Discovery: listing services");
7931
7932        // Process-local registry
7933        let reg = self.reg()?;
7934        let service_names = reg.list_services();
7935        let mut summaries: Vec<ServiceSummary> = service_names
7936            .iter()
7937            .filter_map(|name| {
7938                reg.service_entry(name).map(|entry| {
7939                    let socket_kinds: Vec<String> = entry
7940                        .endpoints
7941                        .keys()
7942                        .map(|k| socket_kind_to_string(*k).to_owned())
7943                        .collect();
7944                    ServiceSummary {
7945                        name: entry.name,
7946                        description: entry.description.unwrap_or_default(),
7947                        socket_kinds,
7948                        has_schema: entry.schema.is_some(),
7949                    }
7950                })
7951            })
7952            .collect();
7953        drop(reg);
7954
7955        // Merge live announcements from the configured state backend.
7956        let announced = self
7957            .state_store
7958            .all_announcements(unix_millis_now())
7959            .await?;
7960        let local_names: Vec<String> = summaries.iter().map(|s| s.name.clone()).collect();
7961        for (name, endpoints) in &announced {
7962            if local_names.iter().any(|n| n == name) {
7963                // Service exists locally — add announced socket kinds
7964                if let Some(summary) = summaries.iter_mut().find(|s| s.name == *name) {
7965                    for ep in endpoints {
7966                        if !summary.socket_kinds.contains(&ep.socket_kind) {
7967                            summary.socket_kinds.push(ep.socket_kind.clone());
7968                        }
7969                    }
7970                }
7971            } else {
7972                // Service only known from announcements
7973                summaries.push(ServiceSummary {
7974                    name: name.clone(),
7975                    description: String::new(),
7976                    socket_kinds: endpoints.iter().map(|e| e.socket_kind.clone()).collect(),
7977                    has_schema: false,
7978                });
7979            }
7980        }
7981
7982        Ok(DiscoveryResponseVariant::ListServicesResult(ServiceList {
7983            services: summaries,
7984        }))
7985    }
7986
7987    async fn handle_get_endpoints(
7988        &self,
7989        _ctx: &EnvelopeContext,
7990        _request_id: u64,
7991        data: &str,
7992    ) -> Result<DiscoveryResponseVariant> {
7993        let service_name = data;
7994        trace!("Discovery: getting endpoints for '{}'", service_name);
7995
7996        // Process-local registry
7997        let reg = self.reg()?;
7998        let endpoints_map = reg.service_endpoints(service_name);
7999        drop(reg);
8000
8001        // Build from local registry (no self-proof — discovery's own JWT not stored locally)
8002        let mut endpoints: Vec<EndpointInfo> = match endpoints_map {
8003            Some(map) => map
8004                .iter()
8005                .map(|(kind, transport)| EndpointInfo {
8006                    socket_kind: socket_kind_to_string(*kind).to_owned(),
8007                    endpoint: transport.endpoint_string(),
8008                    service_jwt: String::new(),
8009                    tls_endorsement: self.tls_endorsement.clone(),
8010                    tls_domain: self.tls_domain.clone(),
8011                    service_did: Did::default(),
8012                    capabilities: Vec::new(),
8013                    accepted_state_digest: Vec::new(),
8014                    accepted_state_epoch: 0,
8015                    response_key_id: String::new(),
8016                    request_kem_key_id: String::new(),
8017                    request_kem_recipient: Vec::new(),
8018                    expires_at_unix_ms: 0,
8019                    source_signer: Vec::new(),
8020                })
8021                .collect(),
8022            None => Vec::new(),
8023        };
8024
8025        // Merge announced endpoints from other processes. Their service JWTs
8026        // were verified at announcement time and are never persisted or
8027        // projected through discovery: bearer credentials must not become a
8028        // discovery-read capability.
8029        for ep in self
8030            .state_store
8031            .announcements_for(service_name, unix_millis_now())
8032            .await?
8033        {
8034            // An identity-bound announced endpoint must not be shadowed by an
8035            // identity-less process-local entry of the same socket kind: the
8036            // checkpoint-validating resolver can only use the announced one,
8037            // so a co-located service's unverifiable local entry would
8038            // otherwise win the dedup and make the service unresolvable.
8039            if !ep.service_did.as_str().is_empty() {
8040                endpoints.retain(|e| e.socket_kind != ep.socket_kind);
8041            }
8042            if !endpoints.iter().any(|e| e.socket_kind == ep.socket_kind) {
8043                endpoints.push(EndpointInfo {
8044                    socket_kind: ep.socket_kind,
8045                    endpoint: ep.endpoint,
8046                    service_jwt: String::new(),
8047                    tls_endorsement: self.tls_endorsement.clone(),
8048                    tls_domain: self.tls_domain.clone(),
8049                    service_did: ep.service_did,
8050                    capabilities: ep.capabilities.into_iter().collect(),
8051                    accepted_state_digest: ep.accepted_state_digest,
8052                    accepted_state_epoch: ep.accepted_state_epoch,
8053                    response_key_id: ep.response_key_id,
8054                    request_kem_key_id: ep.request_kem_key_id,
8055                    request_kem_recipient: ep.request_kem_recipient,
8056                    expires_at_unix_ms: ep.expires_at_unix_ms,
8057                    source_signer: ep.source_signer.to_vec(),
8058                });
8059            }
8060        }
8061
8062        if endpoints.is_empty() {
8063            Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8064                message: format!("Service '{}' not found", service_name),
8065                code: "NOT_FOUND".to_owned(),
8066                details: String::new(),
8067            }))
8068        } else {
8069            Ok(DiscoveryResponseVariant::GetEndpointsResult(
8070                ServiceEndpoints { endpoints },
8071            ))
8072        }
8073    }
8074
8075    async fn handle_get_schema(
8076        &self,
8077        _ctx: &EnvelopeContext,
8078        _request_id: u64,
8079        data: &str,
8080    ) -> Result<DiscoveryResponseVariant> {
8081        let service_name = data;
8082        trace!("Discovery: getting schema for '{}'", service_name);
8083
8084        let reg = self.reg()?;
8085        let schema = reg.service_schema(service_name);
8086        drop(reg);
8087
8088        match schema {
8089            Some(bytes) => Ok(DiscoveryResponseVariant::GetSchemaResult(bytes.to_vec())),
8090            None => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8091                message: format!("No schema registered for service '{}'", service_name),
8092                code: "NO_SCHEMA".to_owned(),
8093                details: String::new(),
8094            })),
8095        }
8096    }
8097
8098    async fn handle_ping(
8099        &self,
8100        _ctx: &EnvelopeContext,
8101        _request_id: u64,
8102    ) -> Result<DiscoveryResponseVariant> {
8103        trace!("Discovery: ping");
8104
8105        let reg = self.reg()?;
8106        let service_count = reg.list_services().len() as u32;
8107        drop(reg);
8108
8109        let uptime = self.started_at.elapsed().as_secs();
8110
8111        Ok(DiscoveryResponseVariant::PingResult(PingInfo {
8112            status: "ok".to_owned(),
8113            service_count,
8114            uptime,
8115        }))
8116    }
8117
8118    async fn handle_get_auth_metadata(
8119        &self,
8120        _ctx: &EnvelopeContext,
8121        _request_id: u64,
8122        data: &str,
8123    ) -> Result<DiscoveryResponseVariant> {
8124        let filter = data;
8125        trace!("Discovery: get auth metadata (filter='{}')", filter);
8126
8127        let issuer = match &self.oauth_issuer_url {
8128            Some(url) => url.clone(),
8129            None => {
8130                return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8131                    message: "OAuth issuer URL not configured".to_owned(),
8132                    code: "NOT_CONFIGURED".to_owned(),
8133                    details: String::new(),
8134                }));
8135            }
8136        };
8137
8138        let reg = self.reg()?;
8139        let service_names = reg.list_services();
8140
8141        let services: Vec<AuthMetadata> = service_names
8142            .iter()
8143            .filter(|name| filter.is_empty() || *name == filter)
8144            .filter_map(|name| {
8145                let quic_transport = match reg.try_endpoint(name, SocketKind::Quic) {
8146                    Ok(transport) => transport,
8147                    Err(err) => {
8148                        tracing::debug!(
8149                            service = %name,
8150                            error = %err,
8151                            "skipping auth metadata for service without registered QUIC reach"
8152                        );
8153                        return None;
8154                    }
8155                };
8156                let resource = quic_transport.quic_resource_url(name)?;
8157                Some(AuthMetadata {
8158                    service_name: name.clone(),
8159                    resource,
8160                    authorization_servers: vec![issuer.clone()],
8161                    scopes_supported: Vec::new(),
8162                    resource_name: format!("HyprStream {} Service", name),
8163                })
8164            })
8165            .collect();
8166        drop(reg);
8167
8168        Ok(DiscoveryResponseVariant::GetAuthMetadataResult(
8169            AuthMetadataList { services },
8170        ))
8171    }
8172
8173    async fn handle_prepare_stream(
8174        &self,
8175        _ctx: &EnvelopeContext,
8176        _request_id: u64,
8177    ) -> Result<DiscoveryResponseVariant> {
8178        Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8179            message: "prepareStream removed — use StreamChannel::prepare_identified_stream for authenticated streaming".to_owned(),
8180            code: "REMOVED".to_owned(),
8181            details: String::new(),
8182        }))
8183    }
8184
8185    async fn handle_get_stream(
8186        &self,
8187        _ctx: &EnvelopeContext,
8188        _request_id: u64,
8189    ) -> Result<DiscoveryResponseVariant> {
8190        Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8191            message:
8192                "getStream removed — use StreamChannel::prepare_identified_stream for authenticated streaming"
8193                    .to_owned(),
8194            code: "REMOVED".to_owned(),
8195            details: String::new(),
8196        }))
8197    }
8198
8199    async fn handle_list_streams(
8200        &self,
8201        _ctx: &EnvelopeContext,
8202        _request_id: u64,
8203    ) -> Result<DiscoveryResponseVariant> {
8204        Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8205            message: "listStreams removed — use StreamChannel::prepare_identified_stream for authenticated streaming".to_owned(),
8206            code: "REMOVED".to_owned(),
8207            details: String::new(),
8208        }))
8209    }
8210
8211    async fn handle_announce(
8212        &self,
8213        ctx: &EnvelopeContext,
8214        _request_id: u64,
8215        data: &ServiceAnnouncement,
8216    ) -> Result<DiscoveryResponseVariant> {
8217        self.store_announcement(ctx, data).await
8218    }
8219
8220    // ────────────────────────────────────────────────────────────────────
8221    // Phase 0.5 Stage D — federation directory
8222    // ────────────────────────────────────────────────────────────────────
8223
8224    async fn handle_register_entity_statement(
8225        &self,
8226        ctx: &EnvelopeContext,
8227        _request_id: u64,
8228        data: &RegisterEntityStatementRequest,
8229    ) -> Result<DiscoveryResponseVariant> {
8230        if data.issuer.is_empty() {
8231            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8232                message: "issuer is required".to_owned(),
8233                code: "INVALID_ARGUMENT".to_owned(),
8234                details: String::new(),
8235            }));
8236        }
8237        if data.jwt.is_empty() {
8238            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8239                message: "jwt is required".to_owned(),
8240                code: "INVALID_ARGUMENT".to_owned(),
8241                details: String::new(),
8242            }));
8243        }
8244
8245        // NOTE: we intentionally do NOT gate cache writes by the
8246        // `federation:register` policy here. The load-bearing trust
8247        // check happens at use time in FederationKeyResolver::get_key,
8248        // which queries policy before accepting any cached entity
8249        // statement for verification. Gating writes too would force
8250        // discovery to call PolicyService on every register, which is
8251        // a perf hit on a hot path that already requires a passing
8252        // `authorize()` check on the caller side. Cache may contain
8253        // statements for issuers no operator currently trusts —
8254        // they're inert until the resolver's policy check admits them.
8255        let cached = CachedEntityStatement {
8256            jwt: data.jwt.clone(),
8257            fetched_at: unix_seconds_now(),
8258        };
8259        self.state_store
8260            .put_entity_statement(&data.issuer, cached)
8261            .await?;
8262        let total = self.state_store.known_issuer_count().await?;
8263
8264        info!(
8265            issuer = %data.issuer,
8266            caller = %ctx.subject(),
8267            total_cached = total,
8268            "Discovery: entity statement registered"
8269        );
8270        Ok(DiscoveryResponseVariant::RegisterEntityStatementResult)
8271    }
8272
8273    async fn handle_get_entity_statement(
8274        &self,
8275        _ctx: &EnvelopeContext,
8276        _request_id: u64,
8277        data: &str,
8278    ) -> Result<DiscoveryResponseVariant> {
8279        let issuer = data;
8280        match self.state_store.entity_statement(issuer).await? {
8281            Some(cached) => {
8282                trace!(issuer = %issuer, "Discovery: entity statement cache hit");
8283                Ok(DiscoveryResponseVariant::GetEntityStatementResult(
8284                    EntityStatement {
8285                        issuer: issuer.to_owned(),
8286                        jwt: cached.jwt,
8287                        fetched_at: cached.fetched_at,
8288                    },
8289                ))
8290            }
8291            None => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8292                message: format!("no entity statement cached for issuer: {}", issuer),
8293                code: "NOT_FOUND".to_owned(),
8294                details: String::new(),
8295            })),
8296        }
8297    }
8298
8299    async fn handle_register_envelope_keyset(
8300        &self,
8301        ctx: &EnvelopeContext,
8302        _request_id: u64,
8303        data: &RegisterEnvelopeKeysetRequest,
8304    ) -> Result<DiscoveryResponseVariant> {
8305        if data.service_did.as_str().is_empty() {
8306            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8307                message: "serviceDid is required".to_owned(),
8308                code: "INVALID_ARGUMENT".to_owned(),
8309                details: String::new(),
8310            }));
8311        }
8312        if data.cose_keyset_cbor.is_empty() {
8313            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8314                message: "coseKeysetCbor is required".to_owned(),
8315                code: "INVALID_ARGUMENT".to_owned(),
8316                details: String::new(),
8317            }));
8318        }
8319
8320        let cached = CachedEnvelopeKeyset {
8321            cose_keyset_cbor: data.cose_keyset_cbor.clone(),
8322            fetched_at: unix_seconds_now(),
8323        };
8324        self.state_store
8325            .put_envelope_keyset(data.service_did.as_str(), cached)
8326            .await?;
8327
8328        info!(
8329            service_did = %data.service_did,
8330            caller = %ctx.subject(),
8331            "Discovery: envelope keyset registered"
8332        );
8333        Ok(DiscoveryResponseVariant::RegisterEnvelopeKeysetResult)
8334    }
8335
8336    async fn handle_get_envelope_keyset(
8337        &self,
8338        _ctx: &EnvelopeContext,
8339        _request_id: u64,
8340        data: &str,
8341    ) -> Result<DiscoveryResponseVariant> {
8342        let service_did = data;
8343        match self.state_store.envelope_keyset(service_did).await? {
8344            Some(cached) => {
8345                trace!(service_did = %service_did, "Discovery: envelope keyset cache hit");
8346                Ok(DiscoveryResponseVariant::GetEnvelopeKeysetResult(
8347                    EnvelopeKeyset {
8348                        service_did: hyprstream_rpc::identity::Did::new(service_did.to_owned()),
8349                        cose_keyset_cbor: cached.cose_keyset_cbor,
8350                        fetched_at: cached.fetched_at,
8351                    },
8352                ))
8353            }
8354            None => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8355                message: format!("no envelope keyset cached for service: {}", service_did),
8356                code: "NOT_FOUND".to_owned(),
8357                details: String::new(),
8358            })),
8359        }
8360    }
8361
8362    async fn handle_list_known_issuers(
8363        &self,
8364        ctx: &EnvelopeContext,
8365        _request_id: u64,
8366    ) -> Result<DiscoveryResponseVariant> {
8367        // Federation topology leak — require authentication. Per
8368        // Phase 0.5 plan Q10: getEntityStatement/getEnvelopeKeyset are
8369        // anonymous-readable (public artifacts), but listKnownIssuers
8370        // enumerates which partners we trust, which is operator-sensitive.
8371        if let Err(e) = self
8372            .authorize(ctx, "discovery:federation", "list-known-issuers")
8373            .await
8374        {
8375            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8376                message: format!("unauthorized: {}", e),
8377                code: "UNAUTHORIZED".to_owned(),
8378                details: String::new(),
8379            }));
8380        }
8381        let issuers = self.state_store.known_issuers().await?;
8382        Ok(DiscoveryResponseVariant::ListKnownIssuersResult(
8383            IssuerList { issuers },
8384        ))
8385    }
8386
8387    // ────────────────────────────────────────────────────────────────────
8388    // #431 — federated record lookup as a verifiable CAR proof
8389    // ────────────────────────────────────────────────────────────────────
8390
8391    async fn handle_get_record(
8392        &self,
8393        ctx: &EnvelopeContext,
8394        _request_id: u64,
8395        data: &GetRecordRequest,
8396    ) -> Result<DiscoveryResponseVariant> {
8397        // Resolve the three components from either the at:// URI or the structured
8398        // fields. When `uri` is set it wins (the fields are ignored, per schema).
8399        let (did, collection, rkey) = if !data.uri.is_empty() {
8400            match parse_at_uri(&data.uri) {
8401                Ok(parts) => parts,
8402                Err(e) => {
8403                    return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8404                        message: format!("invalid at-uri: {e}"),
8405                        code: "INVALID_ARGUMENT".to_owned(),
8406                        details: String::new(),
8407                    }));
8408                }
8409            }
8410        } else if !data.did.is_empty() && !data.collection.is_empty() && !data.rkey.is_empty() {
8411            (data.did.clone(), data.collection.clone(), data.rkey.clone())
8412        } else {
8413            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8414                message: "getRecord requires either `uri` or all of (did, collection, rkey)"
8415                    .to_owned(),
8416                code: "INVALID_ARGUMENT".to_owned(),
8417                details: String::new(),
8418            }));
8419        };
8420
8421        // ── Access control (CONFIDENTIALITY — the load-bearing check) ──
8422        // The schema's auto-generated discovery:query gate already ran in dispatch,
8423        // but that only authorizes "may call getRecord at all". An atproto record's
8424        // CID is content-derived and predictable, so presenting a valid at:// / CID
8425        // must NOT by itself grant a read. Independently require the caller to be
8426        // permitted to read THIS target DID's collection. Integrity (the signed CAR
8427        // proof, verified offline) does NOT substitute for this check.
8428        let resource = format!("discovery:record:{did}/{collection}");
8429        if let Err(e) = self.authorize(ctx, &resource, "query").await {
8430            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8431                message: format!("unauthorized to read {resource}: {e}"),
8432                code: "UNAUTHORIZED".to_owned(),
8433                details: String::new(),
8434            }));
8435        }
8436
8437        let resolver = match &self.record_resolver {
8438            Some(r) => r,
8439            None => {
8440                return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8441                    message: "no record resolver configured on this node".to_owned(),
8442                    code: "NOT_FOUND".to_owned(),
8443                    details: String::new(),
8444                }));
8445            }
8446        };
8447
8448        match resolver.resolve_record(&did, &collection, &rkey).await {
8449            Ok(Some(rec)) => Ok(DiscoveryResponseVariant::GetRecordResult(RecordCar {
8450                uri: rec.uri,
8451                car: rec.car,
8452            })),
8453            Ok(None) => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8454                message: format!("no record at at://{did}/{collection}/{rkey}"),
8455                code: "NOT_FOUND".to_owned(),
8456                details: String::new(),
8457            })),
8458            Err(e) => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8459                message: format!("record resolution failed: {e}"),
8460                code: "INTERNAL".to_owned(),
8461                details: String::new(),
8462            })),
8463        }
8464    }
8465
8466    async fn handle_get_repo(
8467        &self,
8468        ctx: &EnvelopeContext,
8469        _request_id: u64,
8470        value: &str,
8471    ) -> Result<DiscoveryResponseVariant> {
8472        let did = value;
8473        if did.is_empty() {
8474            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8475                message: "getRepo requires a non-empty DID".to_owned(),
8476                code: "INVALID_ARGUMENT".to_owned(),
8477                details: String::new(),
8478            }));
8479        }
8480
8481        // Access control: a full repo CAR exposes every record in the DID's
8482        // collections, so gate on the DID itself (broadest read of that repo).
8483        let resource = format!("discovery:record:{did}");
8484        if let Err(e) = self.authorize(ctx, &resource, "query").await {
8485            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8486                message: format!("unauthorized to read repo {did}: {e}"),
8487                code: "UNAUTHORIZED".to_owned(),
8488                details: String::new(),
8489            }));
8490        }
8491
8492        let resolver = match &self.record_resolver {
8493            Some(r) => r,
8494            None => {
8495                return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8496                    message: "no record resolver configured on this node".to_owned(),
8497                    code: "NOT_FOUND".to_owned(),
8498                    details: String::new(),
8499                }));
8500            }
8501        };
8502
8503        match resolver.resolve_repo(did).await {
8504            Ok(Some(rec)) => Ok(DiscoveryResponseVariant::GetRepoResult(RecordCar {
8505                uri: rec.uri,
8506                car: rec.car,
8507            })),
8508            Ok(None) => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8509                message: format!("no repo stored for DID {did}"),
8510                code: "NOT_FOUND".to_owned(),
8511                details: String::new(),
8512            })),
8513            Err(e) => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8514                message: format!("repo resolution failed: {e}"),
8515                code: "INTERNAL".to_owned(),
8516                details: String::new(),
8517            })),
8518        }
8519    }
8520
8521    /// #523 P0 / #524 — placement candidate query. The auto-generated dispatch
8522    /// gate already ran a `discovery:QueryCandidates`/`query` authz check
8523    /// before this handler was invoked; that only authorizes "may call
8524    /// queryCandidates at all". Each surviving candidate additionally gets its
8525    /// own fail-closed `placement:candidate:<did>` check below (a denied node
8526    /// is silently omitted, not surfaced as an error) — mirroring why
8527    /// `getRecord`/`getRepo` layer a per-target check on top of the generic gate.
8528    ///
8529    /// Pipeline: known node DIDs (placement directory) → hard-exclude any
8530    /// without a live `reportNodeLiveness` entry (decision: absence/expiry is
8531    /// exclusion, never a "stale" flag) → sync filter (label selectors AND,
8532    /// resource requests AND) via `scheduling::filter` → per-candidate async
8533    /// authz → rank (ascending load, DID tiebreak) via `scheduling::rank` →
8534    /// bound to `maxCandidates` (or [`DEFAULT_MAX_CANDIDATES`] when 0).
8535    /// `totalMatching` is the authorized-survivor count *before* the bound.
8536    async fn handle_query_candidates(
8537        &self,
8538        ctx: &EnvelopeContext,
8539        _request_id: u64,
8540        data: &QueryCandidatesRequest,
8541    ) -> Result<DiscoveryResponseVariant> {
8542        use futures::{stream, StreamExt, TryStreamExt};
8543        let query = async {
8544            let started = Instant::now();
8545            struct Candidate {
8546                did: String,
8547                record_uri: String,
8548                load_fraction: f32,
8549                allocatable: Vec<(String, String)>,
8550                last_seen: i64,
8551            }
8552
8553            let selectors: Vec<scheduling::LabelSelector> = data
8554                .selectors
8555                .iter()
8556                .map(|s| {
8557                    scheduling::LabelSelector::new(
8558                        s.key.clone(),
8559                        to_scheduling_op(s.op),
8560                        s.values.clone(),
8561                    )
8562                })
8563                .collect();
8564            let resources: Vec<scheduling::ResourceRequest> = data
8565                .resources
8566                .iter()
8567                .map(|r| scheduling::ResourceRequest::new(r.name.clone(), r.min_quantity.clone()))
8568                .collect();
8569
8570            // Exact totalMatching requires examining every eligible node, but
8571            // not one or two Policy RPCs and a point GET per node. Read bounded
8572            // shared snapshots and authorize in bounded vectors instead.
8573            let live_nodes = self.state_store.all_liveness(unix_millis_now()).await?;
8574            let eligible: Vec<_> = live_nodes
8575                .into_iter()
8576                .filter(|(node, live)| {
8577                    resources.iter().all(|req| {
8578                        live.allocatable
8579                            .iter()
8580                            .find(|(name, _)| name == &req.name)
8581                            .is_some_and(|(_, quantity)| req.satisfied_by(quantity))
8582                    }) && (self.placement_index.record_uri(node.as_str()).is_none()
8583                        || selectors.iter().all(|selector| {
8584                            selector.matches(&self.placement_index.effective_labels(node.as_str()))
8585                        }))
8586                })
8587                .map(|(node, _)| node)
8588                .collect();
8589            let authorized = stream::iter(eligible.chunks(256))
8590                .map(|nodes| async move {
8591                    anyhow::ensure!(
8592                        started.elapsed() < CANDIDATE_QUERY_TIMEOUT,
8593                        "candidate query deadline exceeded; retry"
8594                    );
8595                    let resources: Vec<_> = nodes
8596                        .iter()
8597                        .map(|node| format!("placement:candidate:{node}"))
8598                        .collect();
8599                    let decisions = match &self.auth_provider {
8600                        Some(auth) => auth.check_batch_context(ctx, &resources, "query").await?,
8601                        None => vec![true; resources.len()],
8602                    };
8603                    anyhow::ensure!(
8604                        decisions.len() == nodes.len(),
8605                        "invalid authorization decision count"
8606                    );
8607                    Ok::<_, anyhow::Error>(
8608                        nodes
8609                            .iter()
8610                            .zip(decisions)
8611                            .filter(|(_, allowed)| *allowed)
8612                            .map(|(node, _)| node.clone())
8613                            .collect::<Vec<_>>(),
8614                    )
8615                })
8616                .buffer_unordered(CANDIDATE_QUERY_CONCURRENCY)
8617                .try_collect::<Vec<_>>()
8618                .await?
8619                .into_iter()
8620                .flatten()
8621                .collect::<Vec<_>>();
8622
8623            // No denied node can trigger repository hydration. Completed
8624            // verified ingests survive a cold-query timeout, so retries progress.
8625            stream::iter(&authorized)
8626                .map(|node| async move {
8627                    anyhow::ensure!(
8628                        started.elapsed() < CANDIDATE_QUERY_TIMEOUT,
8629                        "candidate query deadline exceeded; retry"
8630                    );
8631                    anyhow::ensure!(
8632                        self.ensure_placement_ingested(node).await,
8633                        "candidate projection ingestion is pending; retry"
8634                    );
8635                    Ok::<_, anyhow::Error>(())
8636                })
8637                .buffer_unordered(CANDIDATE_QUERY_CONCURRENCY)
8638                .try_collect::<Vec<_>>()
8639                .await?;
8640
8641            // Recheck expiry in one shared-clock snapshot after possibly slow
8642            // hydration/authz, rather than issuing an awaited GET for each DID.
8643            let mut live: std::collections::HashMap<_, _> = self
8644                .state_store
8645                .all_liveness(unix_millis_now())
8646                .await?
8647                .into_iter()
8648                .collect();
8649            let mut candidates = Vec::new();
8650            for node in authorized {
8651                let did = node.as_str().to_owned();
8652                let Some(record_uri) = self.placement_index.record_uri(&did) else {
8653                    continue;
8654                };
8655                let Some(value) = live.remove(&node) else {
8656                    continue;
8657                };
8658                let labels = self.placement_index.effective_labels(&did);
8659                if !selectors.iter().all(|selector| selector.matches(&labels))
8660                    || !resources.iter().all(|req| {
8661                        value
8662                            .allocatable
8663                            .iter()
8664                            .find(|(name, _)| name == &req.name)
8665                            .is_some_and(|(_, quantity)| req.satisfied_by(quantity))
8666                    })
8667                {
8668                    continue;
8669                }
8670                candidates.push(Candidate {
8671                    did,
8672                    record_uri,
8673                    load_fraction: value.load_fraction,
8674                    allocatable: value.allocatable,
8675                    last_seen: value.last_seen,
8676                });
8677            }
8678            let authorized: Vec<_> = candidates.iter().collect();
8679            anyhow::ensure!(
8680                started.elapsed() < CANDIDATE_QUERY_TIMEOUT,
8681                "candidate query deadline exceeded; retry"
8682            );
8683
8684            // Post-filter, post-authz, pre-bound — so callers can tell truncation
8685            // apart from "that's really all of them".
8686            let total_matching = authorized.len() as u32;
8687
8688            let ranked = scheduling::rank(authorized, |a, b| {
8689                a.load_fraction
8690                    .partial_cmp(&b.load_fraction)
8691                    .unwrap_or(std::cmp::Ordering::Equal)
8692                    .then_with(|| a.did.cmp(&b.did))
8693            });
8694
8695            let max = if data.max_candidates == 0 {
8696                DEFAULT_MAX_CANDIDATES
8697            } else {
8698                data.max_candidates as usize
8699            };
8700            let candidates_out: Vec<PlacementCandidate> = ranked
8701                .into_iter()
8702                .take(max)
8703                .map(|c| PlacementCandidate {
8704                    node: c.did.clone(),
8705                    record_uri: c.record_uri.clone(),
8706                    load_fraction: c.load_fraction,
8707                    allocatable: c
8708                        .allocatable
8709                        .iter()
8710                        .map(|(name, quantity)| Resource {
8711                            name: name.clone(),
8712                            quantity: quantity.clone(),
8713                        })
8714                        .collect(),
8715                    last_seen: c.last_seen,
8716                })
8717                .collect();
8718
8719            Ok(DiscoveryResponseVariant::QueryCandidatesResult(
8720                PlacementCandidateSet {
8721                    candidates: candidates_out,
8722                    total_matching,
8723                },
8724            ))
8725        };
8726        #[cfg(not(target_arch = "wasm32"))]
8727        {
8728            tokio::time::timeout(CANDIDATE_QUERY_TIMEOUT, query)
8729                .await
8730                .map_err(|_| anyhow::anyhow!("candidate query deadline exceeded; retry"))?
8731        }
8732        #[cfg(target_arch = "wasm32")]
8733        query.await
8734    }
8735
8736    /// #524 P1 — node liveness heartbeat. The auto-generated dispatch gate
8737    /// already ran a `discovery:ReportNodeLiveness`/`write` authz check before
8738    /// this handler was invoked. Before accepting a DID-specific heartbeat,
8739    /// additionally require write admission for that candidate. This prevents
8740    /// an authorized caller from using arbitrary heartbeat DIDs to grow the
8741    /// durable placement directory or trigger unbounded resolver work.
8742    ///
8743    /// Re-inserting (heartbeating) an admitted node refreshes its TTL entry. The
8744    /// first heartbeat seen for a DID this directory hasn't ingested a
8745    /// `NodeRecord` for yet may lazily trigger a repo poll. The bounded retry
8746    /// gate records an attempt before resolver access, so absent/invalid/non-
8747    /// node repos are retried only after [`PLACEMENT_INGEST_RETRY_TTL`]; a
8748    /// failure there does not fail the admitted heartbeat itself.
8749    async fn handle_report_node_liveness(
8750        &self,
8751        ctx: &EnvelopeContext,
8752        _request_id: u64,
8753        data: &NodeLiveness,
8754    ) -> Result<DiscoveryResponseVariant> {
8755        let node_did = data.node.as_str().to_owned();
8756        if node_did.is_empty() {
8757            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8758                message: "node is required".to_owned(),
8759                code: "INVALID_ARGUMENT".to_owned(),
8760                details: String::new(),
8761            }));
8762        }
8763
8764        // The generic dispatch gate authorizes calling reportNodeLiveness; it
8765        // does not authorize claiming liveness for a particular DID. Gate the
8766        // DID before touching either the bounded volatile cache or the
8767        // unbounded-by-heartbeat placement snapshot index.
8768        let resource = format!("placement:candidate:{node_did}");
8769        if let Err(e) = self.authorize(ctx, &resource, "write").await {
8770            return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8771                message: format!("unauthorized to report liveness for {resource}: {e}"),
8772                code: "UNAUTHORIZED".to_owned(),
8773                details: String::new(),
8774            }));
8775        }
8776
8777        let received_at = unix_millis_now();
8778        let live = LiveAllocatable {
8779            allocatable: data
8780                .allocatable
8781                .iter()
8782                .map(|r| (r.name.clone(), r.quantity.clone()))
8783                .collect(),
8784            load_fraction: data.load_fraction,
8785            // Client clocks can move backwards or be arbitrarily future
8786            // skewed. Freshness and ordering describe this admitted receipt.
8787            last_seen: received_at,
8788            live_until_unix_ms: received_at.saturating_add(LIVENESS_TTL.as_millis() as i64),
8789        };
8790        self.state_store.put_liveness(&data.node, live).await?;
8791
8792        self.ensure_placement_ingested(&data.node).await;
8793
8794        Ok(DiscoveryResponseVariant::ReportNodeLivenessResult)
8795    }
8796}
8797
8798// ============================================================================
8799// RequestService implementation
8800// ============================================================================
8801
8802#[async_trait(?Send)]
8803impl RequestService for DiscoveryService {
8804    fn decode_request_body(
8805        &self,
8806        signed_body: &[u8],
8807    ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
8808        // The ONE bounded decode (v16 §5.2): the generated decoder derives
8809        // the full method leaf and returns the decoded message that policy,
8810        // MAC, and dispatch below all consume.
8811        crate::generated::discovery_client::decode_discovery_request_body(signed_body)
8812    }
8813
8814    async fn handle_request(
8815        &self,
8816        ctx: &EnvelopeContext,
8817        body: &hyprstream_rpc::service::DecodedRequestBody,
8818    ) -> Result<(Vec<u8>, Option<hyprstream_rpc::Continuation>)> {
8819        trace!(
8820            "Discovery request from {} (id={})",
8821            ctx.subject(),
8822            ctx.request_id
8823        );
8824        dispatch_discovery(self, ctx, body).await
8825    }
8826
8827    fn name(&self) -> &str {
8828        "discovery"
8829    }
8830
8831    fn transport(&self) -> &TransportConfig {
8832        &self.transport
8833    }
8834
8835    fn signing_key(&self) -> hyprstream_rpc::crypto::SigningKey {
8836        (*self.signing_key).clone()
8837    }
8838
8839    fn expected_audience(&self) -> Option<&str> {
8840        self.expected_audience.as_deref()
8841    }
8842
8843    fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
8844        self.jwt_key_source.clone()
8845    }
8846
8847    fn build_error_payload(&self, request_id: u64, error: &str) -> Vec<u8> {
8848        let variant = DiscoveryResponseVariant::Error(ErrorInfo {
8849            message: error.to_owned(),
8850            code: "INTERNAL".to_owned(),
8851            details: String::new(),
8852        });
8853        serialize_response(request_id, &variant).unwrap_or_default()
8854    }
8855}
8856
8857// ============================================================================
8858// #431 — getRecord / getRepo handler tests (access control + CAR proof)
8859// ============================================================================
8860
8861#[cfg(test)]
8862mod get_record_tests {
8863    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
8864    use super::*;
8865
8866    use std::collections::BTreeMap;
8867
8868    use hyprstream_pds::car::{build_record_proof_car, parse_car_v1, verify_record_proof};
8869    use hyprstream_pds::cid::Cid as PdsCid;
8870    use hyprstream_pds::commit::{Commit, UnsignedCommit};
8871    use hyprstream_pds::mst::{Node, Proof};
8872    use hyprstream_pds::record::{ModelRecord, COLLECTION_NSID};
8873    use hyprstream_pds::tid::Tid;
8874    use p256::ecdsa::{SigningKey as P256SigningKey, VerifyingKey as P256VerifyingKey};
8875
8876    const TEST_DID: &str = "did:web:alice.example.com";
8877
8878    /// Mock authorization provider: allows exactly the (resource, operation)
8879    /// pairs in its allow-list, denies everything else. Mirrors the Casbin gate
8880    /// the production PolicyAuthProvider wraps.
8881    struct MockAuth {
8882        allow: Vec<(String, String)>,
8883    }
8884    #[async_trait(?Send)]
8885    impl AuthorizationProvider for MockAuth {
8886        async fn check(
8887            &self,
8888            _subject: &str,
8889            _domain: &str,
8890            resource: &str,
8891            operation: &str,
8892            _bearer: Option<&str>,
8893        ) -> Result<bool> {
8894            Ok(self
8895                .allow
8896                .iter()
8897                .any(|(r, o)| r == resource && o == operation))
8898        }
8899    }
8900
8901    /// Mock record resolver backed by a real PDS-built CAR proof, so a returned
8902    /// RecordCar verifies under `verify_record_proof`. `None` records → NOT_FOUND.
8903    struct MockResolver {
8904        /// (collection, rkey) → (car bytes, uri). Built once in `build`.
8905        records: BTreeMap<(String, String), (Vec<u8>, String)>,
8906        /// The `#atproto` verifying key the test repo was signed with, so
8907        /// `ingest_did` can verify the commit (fail-closed #918).
8908        verifying_key: p256::ecdsa::VerifyingKey,
8909    }
8910
8911    /// Build a small repo with `n` records and return the resolver plus the
8912    /// published `#atproto` verifying key, target rkey, target record, and proof
8913    /// so tests can independently verify the CAR the handler returns.
8914    fn build_repo(
8915        n: u64,
8916    ) -> (
8917        MockResolver,
8918        P256VerifyingKey,
8919        String,      // target rkey
8920        ModelRecord, // target record
8921        PdsCid,      // target record CID
8922        Proof,       // target inclusion proof
8923    ) {
8924        let signing_key = P256SigningKey::random(&mut rand::rngs::OsRng);
8925        let verifying_key = P256VerifyingKey::from(&signing_key);
8926
8927        let mut records: BTreeMap<Tid, ModelRecord> = BTreeMap::new();
8928        let mut record_cids: BTreeMap<Tid, PdsCid> = BTreeMap::new();
8929        for i in 0..n {
8930            let tid = Tid::from_micros(1_700_000_000_000_000 + i * 1000, i as u16);
8931            let rec = ModelRecord::new(
8932                format!("at://{TEST_DID}"),
8933                format!("bafyreiexampleoid{i:020}"),
8934                "2026-06-24T00:00:00.000Z",
8935            )
8936            .unwrap();
8937            record_cids.insert(tid, rec.cid());
8938            records.insert(tid, rec);
8939        }
8940        let tree = Node::from_records(COLLECTION_NSID, &record_cids);
8941        let root = tree.root_cid();
8942        let unsigned = UnsignedCommit::new(TEST_DID.to_owned(), root, Tid::now(), None);
8943        let commit = Commit::sign(&unsigned, &signing_key);
8944        let (_root_data, node_blocks) = tree.to_node_data_with_blocks();
8945
8946        // Pick a middle record as the target.
8947        let target_tid = *records.keys().nth((n / 2) as usize).unwrap();
8948        let target_record = records.get(&target_tid).cloned().unwrap();
8949        let target_cid = record_cids[&target_tid];
8950        let target_rkey = target_tid.encode();
8951
8952        // Build a CAR proof for EVERY record so the resolver can answer any rkey.
8953        let mut out = BTreeMap::new();
8954        for (tid, rec) in &records {
8955            let proof = tree.proof(COLLECTION_NSID, tid).unwrap();
8956            let car = build_record_proof_car(&commit, &proof, &node_blocks, rec);
8957            let rkey = tid.encode();
8958            let uri = format!("at://{TEST_DID}/{COLLECTION_NSID}/{rkey}");
8959            out.insert((COLLECTION_NSID.to_owned(), rkey), (car, uri));
8960        }
8961        let target_proof = tree.proof(COLLECTION_NSID, &target_tid).unwrap();
8962
8963        (
8964            MockResolver {
8965                records: out,
8966                verifying_key,
8967            },
8968            verifying_key,
8969            target_rkey,
8970            target_record,
8971            target_cid,
8972            target_proof,
8973        )
8974    }
8975
8976    #[async_trait(?Send)]
8977    impl RecordResolver for MockResolver {
8978        async fn resolve_record(
8979            &self,
8980            _did: &str,
8981            collection: &str,
8982            rkey: &str,
8983        ) -> Result<Option<RecordCarData>> {
8984            Ok(self
8985                .records
8986                .get(&(collection.to_owned(), rkey.to_owned()))
8987                .map(|(car, uri)| RecordCarData {
8988                    uri: uri.clone(),
8989                    car: car.clone(),
8990                }))
8991        }
8992        async fn resolve_repo(&self, _did: &str) -> Result<Option<RecordCarData>> {
8993            Ok(self
8994                .records
8995                .values()
8996                .next()
8997                .map(|(car, uri)| RecordCarData {
8998                    uri: uri.clone(),
8999                    car: car.clone(),
9000                }))
9001        }
9002        async fn resolve_verifying_key(
9003            &self,
9004            _did: &str,
9005        ) -> Result<Option<p256::ecdsa::VerifyingKey>> {
9006            Ok(Some(self.verifying_key))
9007        }
9008    }
9009
9010    fn service_with(allow: Vec<(&str, &str)>, resolver: MockResolver) -> DiscoveryService {
9011        let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
9012        let allow = allow
9013            .into_iter()
9014            .map(|(r, o)| (r.to_owned(), o.to_owned()))
9015            .collect();
9016        DiscoveryService::new(
9017            Arc::new(sk),
9018            vk,
9019            hyprstream_rpc::transport::TransportConfig::inproc("discovery-test"),
9020        )
9021        .with_auth_provider(Box::new(MockAuth { allow }))
9022        .with_record_resolver(Arc::new(resolver))
9023    }
9024
9025    fn test_ctx() -> EnvelopeContext {
9026        // A genuine service-identity caller; subject() → "service:test-caller".
9027        EnvelopeContext::from_callback_service(1, "test-caller")
9028    }
9029
9030    /// Classical atproto peer interoperability is an explicit federation-edge
9031    /// surface. It remains P-256-only and does not construct, accept, or flow
9032    /// through the hybrid deployment-root capability.
9033    #[tokio::test]
9034    async fn classical_atproto_peer_key_remains_on_separate_interop_surface() {
9035        let (resolver, expected, _rkey, _record, _cid, _proof) = build_repo(1);
9036        let published = resolver
9037            .resolve_verifying_keys(TEST_DID)
9038            .await
9039            .expect("classical peer key resolution")
9040            .expect("published classical atproto key");
9041        let live = published
9042            .live_keys(chrono::Utc::now().timestamp())
9043            .collect::<Vec<_>>();
9044        assert_eq!(live, vec![&expected]);
9045    }
9046
9047    /// (a) An accessible record returns a valid CAR proof that verifies offline.
9048    #[tokio::test]
9049    async fn get_record_returns_verifiable_car_when_authorized() {
9050        let (resolver, vk, rkey, target_record, target_cid, proof) = build_repo(6);
9051        let resource = format!("discovery:record:{TEST_DID}/{COLLECTION_NSID}");
9052        let svc = service_with(vec![(&resource, "query")], resolver);
9053
9054        let req = GetRecordRequest {
9055            uri: format!("at://{TEST_DID}/{COLLECTION_NSID}/{rkey}"),
9056            did: String::new(),
9057            collection: String::new(),
9058            rkey: String::new(),
9059        };
9060        let resp = svc.handle_get_record(&test_ctx(), 1, &req).await.unwrap();
9061        let car = match resp {
9062            DiscoveryResponseVariant::GetRecordResult(rc) => rc,
9063            other => panic!("expected GetRecordResult, got {other:?}"),
9064        };
9065
9066        // The CAR parses and contains the target record block.
9067        let (roots, blocks) = parse_car_v1(&car.car).unwrap();
9068        assert!(!roots.is_empty(), "CAR must have a root commit CID");
9069        let block_cids: std::collections::BTreeSet<PdsCid> =
9070            blocks.iter().map(|(c, _)| *c).collect();
9071        assert!(
9072            block_cids.contains(&target_cid),
9073            "CAR must carry the target record block"
9074        );
9075
9076        // The returned proof verifies offline against the published #atproto key.
9077        let commit = blocks
9078            .iter()
9079            .find(|(c, _)| *c == roots[0])
9080            .map(|(_, b)| Commit::from_dag_cbor(b).unwrap())
9081            .expect("CAR must contain its commit block");
9082        verify_record_proof(&commit, &vk, &proof, &target_record)
9083            .expect("the returned CAR proof must verify offline (D5 untrusted-relay)");
9084    }
9085
9086    /// (b) THE LOAD-BEARING TEST: denied when the caller lacks read permission
9087    /// for the target DID/collection — even though the at:// / CID is valid.
9088    #[tokio::test]
9089    async fn get_record_denied_without_read_permission() {
9090        let (resolver, _vk, rkey, _rec, _cid, _proof) = build_repo(6);
9091        // Allow-list does NOT include discovery:record:<did>/<collection>.
9092        let svc = service_with(vec![("discovery:something-else", "query")], resolver);
9093
9094        let req = GetRecordRequest {
9095            uri: format!("at://{TEST_DID}/{COLLECTION_NSID}/{rkey}"),
9096            did: String::new(),
9097            collection: String::new(),
9098            rkey: String::new(),
9099        };
9100        let resp = svc.handle_get_record(&test_ctx(), 1, &req).await.unwrap();
9101        match resp {
9102            DiscoveryResponseVariant::Error(e) => {
9103                assert_eq!(e.code, "UNAUTHORIZED", "deny must surface as UNAUTHORIZED");
9104                // CRITICAL: the deny must NOT leak the record bytes.
9105                assert!(
9106                    !e.message.is_empty(),
9107                    "unauthorized error should explain the denied resource"
9108                );
9109            }
9110            DiscoveryResponseVariant::GetRecordResult(_) => {
9111                panic!("ACCESS-CONTROL FAILURE: a valid at:// returned a record without read permission");
9112            }
9113            other => panic!("expected UNAUTHORIZED error, got {other:?}"),
9114        }
9115    }
9116
9117    /// (c) A missing record errors cleanly with NOT_FOUND (authorized caller).
9118    #[tokio::test]
9119    async fn get_record_missing_returns_not_found() {
9120        let (resolver, _vk, _rkey, _rec, _cid, _proof) = build_repo(6);
9121        let resource = format!("discovery:record:{TEST_DID}/{COLLECTION_NSID}");
9122        let svc = service_with(vec![(&resource, "query")], resolver);
9123
9124        let req = GetRecordRequest {
9125            uri: format!("at://{TEST_DID}/{COLLECTION_NSID}/3zzzzzzzzzzzz"),
9126            did: String::new(),
9127            collection: String::new(),
9128            rkey: String::new(),
9129        };
9130        let resp = svc.handle_get_record(&test_ctx(), 1, &req).await.unwrap();
9131        match resp {
9132            DiscoveryResponseVariant::Error(e) => {
9133                assert_eq!(e.code, "NOT_FOUND", "absent record must be NOT_FOUND");
9134            }
9135            other => panic!("expected NOT_FOUND error, got {other:?}"),
9136        }
9137    }
9138
9139    /// at:// parser: rejects malformed URIs, accepts the canonical 3-part form.
9140    #[test]
9141    fn parse_at_uri_round_trip() {
9142        let (did, coll, rkey) =
9143            parse_at_uri("at://did:web:alice.example.com/ai.hyprstream.model/3zztslq4be52u")
9144                .unwrap();
9145        assert_eq!(did, "did:web:alice.example.com");
9146        assert_eq!(coll, "ai.hyprstream.model");
9147        assert_eq!(rkey, "3zztslq4be52u");
9148
9149        assert!(parse_at_uri("https://x").is_err());
9150        assert!(parse_at_uri("at://did:web:x").is_err()); // no collection/rkey
9151        assert!(parse_at_uri("at://did:web:x/coll").is_err()); // no rkey
9152    }
9153
9154    /// getRecord with structured (did, collection, rkey) fields (uri empty).
9155    #[tokio::test]
9156    async fn get_record_by_structured_fields() {
9157        let (resolver, _vk, rkey, _rec, target_cid, _proof) = build_repo(4);
9158        let resource = format!("discovery:record:{TEST_DID}/{COLLECTION_NSID}");
9159        let svc = service_with(vec![(&resource, "query")], resolver);
9160
9161        let req = GetRecordRequest {
9162            uri: String::new(),
9163            did: TEST_DID.to_owned(),
9164            collection: COLLECTION_NSID.to_owned(),
9165            rkey: rkey.clone(),
9166        };
9167        let resp = svc.handle_get_record(&test_ctx(), 1, &req).await.unwrap();
9168        let car = match resp {
9169            DiscoveryResponseVariant::GetRecordResult(rc) => rc,
9170            other => panic!("expected GetRecordResult, got {other:?}"),
9171        };
9172        let (_roots, blocks) = parse_car_v1(&car.car).unwrap();
9173        let block_cids: std::collections::BTreeSet<PdsCid> =
9174            blocks.iter().map(|(c, _)| *c).collect();
9175        assert!(block_cids.contains(&target_cid));
9176    }
9177}
9178
9179// ============================================================================
9180// #524 P1 — queryCandidates / reportNodeLiveness handler tests
9181// ============================================================================
9182
9183#[cfg(test)]
9184mod query_candidates_tests {
9185    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
9186    use super::*;
9187
9188    use std::collections::BTreeMap;
9189
9190    use hyprstream_pds::car::build_car_v1;
9191    use hyprstream_pds::cid::Cid;
9192    use hyprstream_pds::commit::{Commit, UnsignedCommit};
9193    use hyprstream_pds::mst::Node;
9194    use hyprstream_pds::placement::node::{self, NodeRecord};
9195    use hyprstream_pds::tid::Tid;
9196    use hyprstream_rpc_std::discovery_client::{LabelSelector, ResourceRequest, SelectorOp};
9197    use p256::ecdsa::{SigningKey as P256SigningKey, VerifyingKey as P256VerifyingKey};
9198
9199    /// Allows every (resource, operation) pair.
9200    struct AllowAll;
9201    #[async_trait(?Send)]
9202    impl AuthorizationProvider for AllowAll {
9203        async fn check(
9204            &self,
9205            _subject: &str,
9206            _domain: &str,
9207            _resource: &str,
9208            _operation: &str,
9209            _bearer: Option<&str>,
9210        ) -> Result<bool> {
9211            Ok(true)
9212        }
9213    }
9214
9215    /// Denies `query` access to exactly one `placement:candidate:<did>`
9216    /// resource; allows write admission so that test setup can heartbeat it.
9217    struct DenyNode(String);
9218    #[async_trait(?Send)]
9219    impl AuthorizationProvider for DenyNode {
9220        async fn check(
9221            &self,
9222            _subject: &str,
9223            _domain: &str,
9224            resource: &str,
9225            operation: &str,
9226            _bearer: Option<&str>,
9227        ) -> Result<bool> {
9228            Ok(resource != format!("placement:candidate:{}", self.0) || operation != "query")
9229        }
9230    }
9231
9232    /// Denies `write` admission for exactly one node DID.
9233    struct DenyNodeLiveness(String);
9234    #[async_trait(?Send)]
9235    impl AuthorizationProvider for DenyNodeLiveness {
9236        async fn check(
9237            &self,
9238            _subject: &str,
9239            _domain: &str,
9240            resource: &str,
9241            operation: &str,
9242            _bearer: Option<&str>,
9243        ) -> Result<bool> {
9244            Ok(resource != format!("placement:candidate:{}", self.0) || operation != "write")
9245        }
9246    }
9247
9248    /// Records repository lookups so admission tests can prove a denied DID
9249    /// never reaches `RecordResolver::resolve_repo`.
9250    struct TrackingRepoResolver {
9251        repos: HashMap<String, Vec<u8>>,
9252        resolved: parking_lot::Mutex<Vec<String>>,
9253    }
9254    #[async_trait(?Send)]
9255    impl RecordResolver for TrackingRepoResolver {
9256        async fn resolve_record(
9257            &self,
9258            _did: &str,
9259            _collection: &str,
9260            _rkey: &str,
9261        ) -> Result<Option<RecordCarData>> {
9262            Ok(None)
9263        }
9264        async fn resolve_repo(&self, did: &str) -> Result<Option<RecordCarData>> {
9265            self.resolved.lock().push(did.to_owned());
9266            Ok(self.repos.get(did).map(|car| RecordCarData {
9267                uri: format!("at://{did}"),
9268                car: car.clone(),
9269            }))
9270        }
9271    }
9272
9273    /// A resolver serving one fixed repo CAR per DID (built with a real
9274    /// `NodeRecord` encoding — not hand-hacked bytes).
9275    struct FixedRepoResolver {
9276        repos: HashMap<String, (Vec<u8>, P256VerifyingKey)>,
9277    }
9278    #[async_trait(?Send)]
9279    impl RecordResolver for FixedRepoResolver {
9280        async fn resolve_record(
9281            &self,
9282            _did: &str,
9283            _collection: &str,
9284            _rkey: &str,
9285        ) -> Result<Option<RecordCarData>> {
9286            Ok(None)
9287        }
9288        async fn resolve_repo(&self, did: &str) -> Result<Option<RecordCarData>> {
9289            Ok(self.repos.get(did).map(|(car, _vk)| RecordCarData {
9290                uri: format!("at://{did}"),
9291                car: car.clone(),
9292            }))
9293        }
9294        async fn resolve_verifying_key(&self, did: &str) -> Result<Option<P256VerifyingKey>> {
9295            Ok(self.repos.get(did).map(|(_, vk)| *vk))
9296        }
9297    }
9298
9299    /// Build a one-record repo CAR for `did` carrying `rec` as its
9300    /// `ai.hyprstream.placement.node` record.
9301    fn node_repo_car(did: &str, rec: &NodeRecord) -> (Vec<u8>, P256VerifyingKey) {
9302        let signing_key = P256SigningKey::random(&mut rand::rngs::OsRng);
9303        let verifying_key = P256VerifyingKey::from(&signing_key);
9304        let key = format!("{}/3a", node::COLLECTION_NSID);
9305        let record_cid = Cid::from_dag_cbor(&rec.to_dag_cbor());
9306        let mut keyed: BTreeMap<String, Cid> = BTreeMap::new();
9307        keyed.insert(key, record_cid);
9308        let tree = Node::from_keyed_records(&keyed);
9309        let root = tree.root_cid();
9310        let (_root_data, node_blocks) = tree.to_node_data_with_blocks();
9311        let unsigned = UnsignedCommit::new(did.to_owned(), root, Tid::now(), None);
9312        let commit = Commit::sign(&unsigned, &signing_key);
9313        let mut blocks: Vec<(Cid, Vec<u8>)> = vec![(commit.cid(), commit.to_dag_cbor())];
9314        for (cid, data) in node_blocks {
9315            blocks.push((cid, data.encode()));
9316        }
9317        blocks.push((record_cid, rec.to_dag_cbor()));
9318        (build_car_v1(&[commit.cid()], &blocks), verifying_key)
9319    }
9320
9321    fn sample_node_record(did: &str, labels: Vec<(&str, &str)>) -> NodeRecord {
9322        NodeRecord::new(
9323            format!("at://{did}"),
9324            labels
9325                .into_iter()
9326                .map(|(k, v)| node::Label {
9327                    key: k.to_owned(),
9328                    value: v.to_owned(),
9329                })
9330                .collect(),
9331            vec![],
9332            vec![],
9333            "2026-06-23T12:34:56.789Z",
9334        )
9335        .unwrap()
9336    }
9337
9338    fn service_with(
9339        auth: Box<dyn AuthorizationProvider>,
9340        repos: HashMap<String, (Vec<u8>, P256VerifyingKey)>,
9341    ) -> DiscoveryService {
9342        let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
9343        DiscoveryService::new(
9344            Arc::new(sk),
9345            vk,
9346            hyprstream_rpc::transport::TransportConfig::inproc("query-candidates-test"),
9347        )
9348        .with_auth_provider(auth)
9349        .with_record_resolver(Arc::new(FixedRepoResolver { repos }))
9350    }
9351
9352    struct ConcurrentRepoResolver {
9353        inner: FixedRepoResolver,
9354        barrier: Option<tokio::sync::Barrier>,
9355        stall: std::sync::atomic::AtomicBool,
9356        active: std::sync::atomic::AtomicUsize,
9357        peak: std::sync::atomic::AtomicUsize,
9358        resolved: parking_lot::Mutex<Vec<String>>,
9359    }
9360
9361    struct ActiveResolution<'a>(&'a std::sync::atomic::AtomicUsize);
9362    impl Drop for ActiveResolution<'_> {
9363        fn drop(&mut self) {
9364            self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
9365        }
9366    }
9367
9368    #[async_trait(?Send)]
9369    impl RecordResolver for ConcurrentRepoResolver {
9370        async fn resolve_record(
9371            &self,
9372            _did: &str,
9373            _collection: &str,
9374            _rkey: &str,
9375        ) -> Result<Option<RecordCarData>> {
9376            Ok(None)
9377        }
9378        async fn resolve_repo(&self, did: &str) -> Result<Option<RecordCarData>> {
9379            use std::sync::atomic::Ordering::SeqCst;
9380            self.resolved.lock().push(did.to_owned());
9381            let active = self.active.fetch_add(1, SeqCst) + 1;
9382            let _guard = ActiveResolution(&self.active);
9383            self.peak.fetch_max(active, SeqCst);
9384            if self.stall.load(SeqCst) {
9385                futures::future::pending::<()>().await;
9386            }
9387            if let Some(barrier) = &self.barrier {
9388                barrier.wait().await;
9389            }
9390            self.inner.resolve_repo(did).await
9391        }
9392        async fn resolve_verifying_key(&self, did: &str) -> Result<Option<P256VerifyingKey>> {
9393            self.inner.resolve_verifying_key(did).await
9394        }
9395    }
9396
9397    async fn assert_cold_query_concurrency(state: DiscoveryState) {
9398        use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst};
9399        let mut repos = HashMap::new();
9400        let now = unix_millis_now();
9401        for i in 0..33 {
9402            let did = format!("did:web:cold-{i}.example");
9403            repos.insert(
9404                did.clone(),
9405                node_repo_car(&did, &sample_node_record(&did, vec![])),
9406            );
9407            state
9408                .clone()
9409                .into_inner()
9410                .put_liveness(
9411                    &Did::new(did),
9412                    LiveAllocatable {
9413                        allocatable: vec![],
9414                        load_fraction: 0.1,
9415                        last_seen: now,
9416                        live_until_unix_ms: now + 45_000,
9417                    },
9418                )
9419                .await
9420                .unwrap();
9421        }
9422        let resolver = Arc::new(ConcurrentRepoResolver {
9423            inner: FixedRepoResolver { repos },
9424            // A sequential implementation cannot complete even one batch.
9425            barrier: Some(tokio::sync::Barrier::new(CANDIDATE_QUERY_CONCURRENCY)),
9426            stall: AtomicBool::new(false),
9427            active: AtomicUsize::new(0),
9428            peak: AtomicUsize::new(0),
9429            resolved: parking_lot::Mutex::new(vec![]),
9430        });
9431        let denied = "did:web:cold-32.example";
9432        let svc = service_with(Box::new(DenyNode(denied.to_owned())), HashMap::new())
9433            .with_record_resolver(resolver.clone())
9434            .with_state(state);
9435        let result = tokio::time::timeout(
9436            Duration::from_secs(4),
9437            svc.handle_query_candidates(&test_ctx(), 1, &empty_query(1)),
9438        )
9439        .await
9440        .unwrap()
9441        .unwrap();
9442        let result = as_set(result);
9443        assert_eq!(
9444            result.total_matching, 32,
9445            "maxCandidates must not hide incomplete hydration"
9446        );
9447        assert_eq!(result.candidates.len(), 1);
9448        assert_eq!(resolver.peak.load(SeqCst), CANDIDATE_QUERY_CONCURRENCY);
9449        assert_eq!(resolver.active.load(SeqCst), 0);
9450        assert_eq!(resolver.resolved.lock().len(), 32);
9451        assert!(!resolver.resolved.lock().iter().any(|did| did == denied));
9452    }
9453
9454    #[tokio::test]
9455    async fn cold_candidate_query_hydrates_concurrently_without_truncation() {
9456        let state = DiscoveryState::connect(&crate::DiscoveryStateConfig::default())
9457            .await
9458            .unwrap();
9459        assert_cold_query_concurrency(state).await;
9460    }
9461
9462    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
9463    #[tokio::test]
9464    async fn valkey_capacity_query_batches_exact_count_and_ranking() {
9465        use futures::{stream, StreamExt};
9466        use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
9467        struct BatchPolicy(Arc<AtomicUsize>);
9468        #[async_trait(?Send)]
9469        impl AuthorizationProvider for BatchPolicy {
9470            async fn check(
9471                &self,
9472                _: &str,
9473                _: &str,
9474                _: &str,
9475                _: &str,
9476                _: Option<&str>,
9477            ) -> Result<bool> {
9478                panic!("capacity query regressed to per-node Policy RPC");
9479            }
9480            async fn check_batch(
9481                &self,
9482                _: &str,
9483                _: &str,
9484                resources: &[String],
9485                _: &str,
9486                _: Option<&str>,
9487            ) -> Result<Vec<bool>> {
9488                assert!(resources.len() <= 256);
9489                self.0.fetch_add(1, SeqCst);
9490                tokio::time::sleep(Duration::from_millis(10)).await;
9491                Ok(resources
9492                    .iter()
9493                    .map(|r| !r.ends_with("capacity-00001.example"))
9494                    .collect())
9495            }
9496        }
9497        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
9498            return;
9499        };
9500        let config = crate::DiscoveryStateConfig {
9501            backend: crate::DiscoveryStateBackend::Tiered,
9502            active_active: true,
9503            valkey: crate::ValkeyStateConfig {
9504                url,
9505                key_prefix: format!(
9506                    "capacity-query-{}-{}",
9507                    std::process::id(),
9508                    unix_millis_now()
9509                ),
9510                ..crate::ValkeyStateConfig::default()
9511            },
9512            ..crate::DiscoveryStateConfig::default()
9513        };
9514        let capacity = config.valkey.liveness_capacity;
9515        assert_eq!(
9516            capacity, 65_536,
9517            "exercise advertised capacity without lowering it"
9518        );
9519        let state = DiscoveryState::connect(&config).await.unwrap();
9520        let calls = Arc::new(AtomicUsize::new(0));
9521        let svc =
9522            service_with(Box::new(BatchPolicy(calls.clone())), HashMap::new()).with_state(state);
9523        // Warm verified placement projection: signature/admission behavior has
9524        // separate real-CAR tests; this fixture isolates full-capacity querying.
9525        for i in 0..capacity {
9526            let did = format!("did:web:capacity-{i:05}.example");
9527            svc.placement_index.seed_warm_node_for_test(
9528                did.clone(),
9529                crate::placement_index::NodeFacts {
9530                    record_uri: format!("at://{did}/ai.hyprstream.placement.node/3a"),
9531                    labels: vec![(
9532                        "zone".to_owned(),
9533                        if i % 2 == 1 { "west" } else { "east" }.to_owned(),
9534                    )],
9535                    ..Default::default()
9536                },
9537            );
9538        }
9539        stream::iter(0..capacity)
9540            .map(|i| {
9541                let store = &svc.state_store;
9542                async move {
9543                    let now = unix_millis_now();
9544                    store
9545                        .put_liveness(
9546                            &Did::new(format!("did:web:capacity-{i:05}.example")),
9547                            LiveAllocatable {
9548                                allocatable: vec![("cpu".to_owned(), "8".to_owned())],
9549                                load_fraction: 1.0 - i as f32 / capacity as f32,
9550                                last_seen: now,
9551                                live_until_unix_ms: now + 45_000,
9552                            },
9553                        )
9554                        .await
9555                        .unwrap();
9556                }
9557            })
9558            .buffer_unordered(256)
9559            .collect::<Vec<_>>()
9560            .await;
9561        let mut request = empty_query(1);
9562        request.selectors = vec![LabelSelector {
9563            key: "zone".to_owned(),
9564            op: SelectorOp::In,
9565            values: vec!["west".to_owned()],
9566        }];
9567        request.resources = vec![ResourceRequest {
9568            name: "cpu".to_owned(),
9569            min_quantity: "4".to_owned(),
9570        }];
9571        let started = Instant::now();
9572        let result = as_set(
9573            svc.handle_query_candidates(&test_ctx(), 1, &request)
9574                .await
9575                .unwrap(),
9576        );
9577        assert!(started.elapsed() < CANDIDATE_QUERY_TIMEOUT);
9578        assert_eq!(result.total_matching, (capacity / 2 - 1) as u32);
9579        assert_eq!(result.candidates.len(), 1);
9580        assert_eq!(
9581            result.candidates[0].node,
9582            format!("did:web:capacity-{:05}.example", capacity - 1)
9583        );
9584        assert_eq!(
9585            calls.load(SeqCst),
9586            capacity / 2 / 256,
9587            "warm selector filtering precedes Policy RPC"
9588        );
9589    }
9590
9591    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
9592    #[tokio::test]
9593    async fn valkey_cold_candidate_query_hydrates_concurrently_without_truncation() {
9594        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
9595            return;
9596        };
9597        let config = crate::DiscoveryStateConfig {
9598            backend: crate::DiscoveryStateBackend::Tiered,
9599            active_active: true,
9600            valkey: crate::ValkeyStateConfig {
9601                url,
9602                key_prefix: format!("cold-query-{}-{}", std::process::id(), unix_millis_now()),
9603                ..crate::ValkeyStateConfig::default()
9604            },
9605            ..crate::DiscoveryStateConfig::default()
9606        };
9607        assert_cold_query_concurrency(DiscoveryState::connect(&config).await.unwrap()).await;
9608    }
9609
9610    #[tokio::test]
9611    async fn cold_candidate_deadline_cancels_ingest_and_retry_is_complete() {
9612        use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst};
9613        let did = "did:web:stalled.example";
9614        let resolver = Arc::new(ConcurrentRepoResolver {
9615            inner: FixedRepoResolver {
9616                repos: HashMap::from([(
9617                    did.to_owned(),
9618                    node_repo_car(did, &sample_node_record(did, vec![])),
9619                )]),
9620            },
9621            barrier: None,
9622            stall: AtomicBool::new(true),
9623            active: AtomicUsize::new(0),
9624            peak: AtomicUsize::new(0),
9625            resolved: parking_lot::Mutex::new(vec![]),
9626        });
9627        let svc =
9628            service_with(Box::new(AllowAll), HashMap::new()).with_record_resolver(resolver.clone());
9629        let now = unix_millis_now();
9630        svc.state_store
9631            .put_liveness(
9632                &Did::new(did.to_owned()),
9633                LiveAllocatable {
9634                    allocatable: vec![],
9635                    load_fraction: 0.1,
9636                    last_seen: now,
9637                    live_until_unix_ms: now + 45_000,
9638                },
9639            )
9640            .await
9641            .unwrap();
9642        let error = tokio::time::timeout(
9643            CANDIDATE_QUERY_TIMEOUT + Duration::from_secs(2),
9644            svc.handle_query_candidates(&test_ctx(), 1, &empty_query(1)),
9645        )
9646        .await
9647        .unwrap()
9648        .err()
9649        .unwrap();
9650        assert!(error.to_string().contains("deadline exceeded"));
9651        assert_eq!(
9652            resolver.active.load(SeqCst),
9653            0,
9654            "deadline must drop outstanding repository work"
9655        );
9656        resolver.stall.store(false, SeqCst);
9657        let result = as_set(
9658            svc.handle_query_candidates(&test_ctx(), 1, &empty_query(1))
9659                .await
9660                .unwrap(),
9661        );
9662        assert_eq!(
9663            result.total_matching, 1,
9664            "cancelled ingest must not be cached as absence"
9665        );
9666        assert_eq!(resolver.resolved.lock().len(), 2);
9667    }
9668
9669    #[tokio::test]
9670    async fn heartbeat_uses_receipt_time_after_client_clock_rollback() {
9671        let svc = service_with(Box::new(AllowAll), HashMap::new());
9672        let node = Did::new("did:web:rollback.example".to_owned());
9673        let start = unix_millis_now();
9674        for ts in [start + 86_400_000, start - 86_400_000] {
9675            let req = NodeLiveness {
9676                node: node.clone(),
9677                allocatable: vec![],
9678                load_fraction: 0.2,
9679                ts,
9680            };
9681            svc.handle_report_node_liveness(&test_ctx(), 1, &req)
9682                .await
9683                .unwrap();
9684            let value = svc
9685                .state_store
9686                .liveness(&node, unix_millis_now())
9687                .await
9688                .unwrap()
9689                .unwrap();
9690            assert!(value.last_seen >= start && value.last_seen <= unix_millis_now());
9691            assert_eq!(
9692                value.live_until_unix_ms,
9693                value.last_seen + LIVENESS_TTL.as_millis() as i64
9694            );
9695        }
9696    }
9697
9698    fn test_ctx() -> EnvelopeContext {
9699        EnvelopeContext::from_callback_service(1, "test-caller")
9700    }
9701
9702    async fn heartbeat(
9703        svc: &DiscoveryService,
9704        did: &str,
9705        allocatable: Vec<(&str, &str)>,
9706        load_fraction: f32,
9707    ) {
9708        let req = NodeLiveness {
9709            node: Did::new(did.to_owned()),
9710            allocatable: allocatable
9711                .into_iter()
9712                .map(|(n, q)| Resource {
9713                    name: n.to_owned(),
9714                    quantity: q.to_owned(),
9715                })
9716                .collect(),
9717            load_fraction,
9718            ts: 0,
9719        };
9720        let resp = svc
9721            .handle_report_node_liveness(&test_ctx(), 1, &req)
9722            .await
9723            .unwrap();
9724        assert!(matches!(
9725            resp,
9726            DiscoveryResponseVariant::ReportNodeLivenessResult
9727        ));
9728    }
9729
9730    async fn assert_cross_replica_candidates(a: DiscoveryState, b: DiscoveryState) {
9731        let did = "did:web:shared-node.example";
9732        let rec = sample_node_record(did, vec![("zone", "shared")]);
9733        let repos = HashMap::from([(did.to_owned(), node_repo_car(did, &rec))]);
9734        let replica_a = service_with(Box::new(AllowAll), repos.clone()).with_state(a);
9735        let replica_b = service_with(Box::new(AllowAll), repos.clone()).with_state(b.clone());
9736        heartbeat(&replica_a, did, vec![("cpu", "8")], 0.25).await;
9737        assert!(replica_b.placement_index.known_node_dids().is_empty());
9738        let req = QueryCandidatesRequest {
9739            selectors: vec![LabelSelector {
9740                key: "zone".to_owned(),
9741                op: SelectorOp::In,
9742                values: vec!["shared".to_owned()],
9743            }],
9744            resources: vec![ResourceRequest {
9745                name: "cpu".to_owned(),
9746                min_quantity: "4".to_owned(),
9747            }],
9748            max_candidates: 0,
9749        };
9750        let result = as_set(
9751            replica_b
9752                .handle_query_candidates(&test_ctx(), 1, &req)
9753                .await
9754                .unwrap(),
9755        );
9756        assert_eq!(result.total_matching, 1);
9757        assert_eq!(result.candidates[0].node, did);
9758        assert_eq!(
9759            result.candidates[0].record_uri,
9760            format!("at://{did}/{}/3a", node::COLLECTION_NSID)
9761        );
9762        // A restarted replica and a policy-denied replica start without local
9763        // placement state too. Only an authorized verified node is surfaced.
9764        let restarted = service_with(Box::new(AllowAll), repos.clone()).with_state(b.clone());
9765        assert_eq!(
9766            as_set(
9767                restarted
9768                    .handle_query_candidates(&test_ctx(), 1, &req)
9769                    .await
9770                    .unwrap()
9771            )
9772            .total_matching,
9773            1
9774        );
9775        let denied = service_with(Box::new(DenyNode(did.to_owned())), repos).with_state(b.clone());
9776        assert_eq!(
9777            as_set(
9778                denied
9779                    .handle_query_candidates(&test_ctx(), 1, &empty_query(0))
9780                    .await
9781                    .unwrap()
9782            )
9783            .total_matching,
9784            0
9785        );
9786        assert!(denied.placement_index.known_node_dids().is_empty());
9787        let missing_repo = service_with(Box::new(AllowAll), HashMap::new()).with_state(b);
9788        assert_eq!(
9789            as_set(
9790                missing_repo
9791                    .handle_query_candidates(&test_ctx(), 1, &empty_query(0))
9792                    .await
9793                    .unwrap()
9794            )
9795            .total_matching,
9796            0
9797        );
9798        // Expiring the shared record excludes it even from populated indexes.
9799        let now = unix_millis_now();
9800        replica_a
9801            .state_store
9802            .put_liveness(
9803                &Did::new(did.to_owned()),
9804                LiveAllocatable {
9805                    allocatable: vec![],
9806                    load_fraction: 0.1,
9807                    last_seen: now,
9808                    live_until_unix_ms: now + 20,
9809                },
9810            )
9811            .await
9812            .unwrap();
9813        // The shared clock cannot be advanced by passing a replica timestamp.
9814        tokio::time::sleep(Duration::from_millis(50)).await;
9815        assert_eq!(
9816            as_set(
9817                replica_b
9818                    .handle_query_candidates(&test_ctx(), 1, &req)
9819                    .await
9820                    .unwrap()
9821            )
9822            .total_matching,
9823            0
9824        );
9825    }
9826
9827    #[tokio::test]
9828    async fn shared_liveness_seeds_replica_without_local_placement() {
9829        let state = DiscoveryState::connect(&crate::DiscoveryStateConfig::default())
9830            .await
9831            .unwrap();
9832        assert_cross_replica_candidates(state.clone(), state).await;
9833    }
9834
9835    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
9836    #[tokio::test]
9837    async fn valkey_shared_liveness_seeds_other_replica_and_restart() {
9838        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
9839            return;
9840        };
9841        for backend in [
9842            crate::DiscoveryStateBackend::Valkey,
9843            crate::DiscoveryStateBackend::Tiered,
9844        ] {
9845            let config = crate::DiscoveryStateConfig {
9846                backend,
9847                active_active: true,
9848                valkey: crate::ValkeyStateConfig {
9849                    url: url.clone(),
9850                    key_prefix: format!(
9851                        "pr1560-candidates-{}-{}-{backend:?}",
9852                        std::process::id(),
9853                        unix_millis_now()
9854                    ),
9855                    pool_size: 2,
9856                    ..crate::ValkeyStateConfig::default()
9857                },
9858                ..crate::DiscoveryStateConfig::default()
9859            };
9860            let a = DiscoveryState::connect(&config).await.unwrap();
9861            let b = DiscoveryState::connect(&config).await.unwrap();
9862            assert_cross_replica_candidates(a, b).await;
9863        }
9864    }
9865
9866    #[tokio::test]
9867    async fn legacy_announcement_without_identity_expiry_uses_heartbeat_ttl() {
9868        let svc = service_with(Box::new(AllowAll), HashMap::new());
9869        let request = ServiceAnnouncement {
9870            service_name: "legacy".to_owned(),
9871            socket_kind: "rep".to_owned(),
9872            endpoint: "inproc://legacy".to_owned(),
9873            service_jwt: None,
9874            service_did: Did::new(String::new()),
9875            capabilities: vec![],
9876            accepted_state_digest: vec![],
9877            accepted_state_epoch: 0,
9878            response_key_id: String::new(),
9879            request_kem_key_id: String::new(),
9880            request_kem_recipient: vec![],
9881            expires_at_unix_ms: 0,
9882        };
9883        let now = unix_millis_now();
9884        assert!(matches!(
9885            svc.handle_announce(&test_ctx(), 1, &request).await.unwrap(),
9886            DiscoveryResponseVariant::AnnounceResult
9887        ));
9888        let entries = svc
9889            .state_store
9890            .announcements_for("legacy", now)
9891            .await
9892            .unwrap();
9893        assert_eq!(entries.len(), 1);
9894        assert!(entries[0].live_until_unix_ms >= now + ANNOUNCED_ENDPOINT_TTL.as_millis() as i64);
9895        assert_eq!(entries[0].expires_at_unix_ms, entries[0].live_until_unix_ms);
9896        assert!(svc
9897            .state_store
9898            .announcements_for("legacy", entries[0].live_until_unix_ms)
9899            .await
9900            .unwrap()
9901            .is_empty());
9902        let mut bound = request;
9903        bound.service_did = Did::new("did:at9p:identity-bound".to_owned());
9904        assert!(svc.handle_announce(&test_ctx(), 1, &bound).await.is_err());
9905    }
9906
9907    /// A denied heartbeat must not create a volatile liveness entry or trigger
9908    /// a first-seen repository poll, even if the caller passed the generic
9909    /// reportNodeLiveness dispatch gate.
9910    #[tokio::test]
9911    async fn liveness_admission_denial_skips_cache_and_repository_ingest() {
9912        let did = "did:web:unadmitted-node.example.com";
9913        let rec = sample_node_record(did, vec![]);
9914        let (car, _verifying_key) = node_repo_car(did, &rec);
9915        let resolver = Arc::new(TrackingRepoResolver {
9916            repos: HashMap::from([(did.to_owned(), car)]),
9917            resolved: parking_lot::Mutex::new(Vec::new()),
9918        });
9919        let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
9920        let svc = DiscoveryService::new(
9921            Arc::new(sk),
9922            vk,
9923            hyprstream_rpc::transport::TransportConfig::inproc("liveness-admission-test"),
9924        )
9925        .with_auth_provider(Box::new(DenyNodeLiveness(did.to_owned())))
9926        .with_record_resolver(resolver.clone());
9927        let req = NodeLiveness {
9928            node: Did::new(did.to_owned()),
9929            allocatable: vec![],
9930            load_fraction: 0.1,
9931            ts: 0,
9932        };
9933
9934        let response = svc
9935            .handle_report_node_liveness(&test_ctx(), 1, &req)
9936            .await
9937            .unwrap();
9938        assert!(matches!(
9939            response,
9940            DiscoveryResponseVariant::Error(ErrorInfo { ref code, .. }) if code == "UNAUTHORIZED"
9941        ));
9942        assert!(
9943            svc.state_store
9944                .liveness(&Did::new(did.to_owned()), unix_millis_now())
9945                .await
9946                .expect("read liveness")
9947                .is_none(),
9948            "denied DID must not receive a liveness entry"
9949        );
9950        assert!(
9951            svc.placement_index.record_uri(did).is_none(),
9952            "denied DID must not receive placement facts"
9953        );
9954        assert!(
9955            resolver.resolved.lock().is_empty(),
9956            "denied DID must not reach RecordResolver::resolve_repo"
9957        );
9958    }
9959
9960    /// An admitted DID with no repo is polled once per bounded retry window,
9961    /// not once per heartbeat. Its liveness report still refreshes normally.
9962    #[tokio::test]
9963    async fn absent_node_repo_does_not_poll_again_before_retry_window() {
9964        let did = "did:web:node-without-repo.example.com";
9965        let resolver = Arc::new(TrackingRepoResolver {
9966            repos: HashMap::new(),
9967            resolved: parking_lot::Mutex::new(Vec::new()),
9968        });
9969        let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
9970        let svc = DiscoveryService::new(
9971            Arc::new(sk),
9972            vk,
9973            hyprstream_rpc::transport::TransportConfig::inproc("placement-retry-test"),
9974        )
9975        .with_auth_provider(Box::new(AllowAll))
9976        .with_record_resolver(resolver.clone());
9977
9978        heartbeat(&svc, did, vec![], 0.9).await;
9979        heartbeat(&svc, did, vec![("cpu", "8")], 0.1).await;
9980
9981        assert_eq!(
9982            resolver.resolved.lock().as_slice(),
9983            [did],
9984            "two heartbeats before retry expiry must produce one repo poll"
9985        );
9986        let live = svc
9987            .state_store
9988            .liveness(&Did::new(did.to_owned()), unix_millis_now())
9989            .await
9990            .expect("read liveness")
9991            .expect("admitted heartbeat must still refresh liveness");
9992        assert!((live.load_fraction - 0.1).abs() < f32::EPSILON);
9993        assert_eq!(live.allocatable, vec![("cpu".to_owned(), "8".to_owned())]);
9994    }
9995
9996    fn empty_query(max_candidates: u32) -> QueryCandidatesRequest {
9997        QueryCandidatesRequest {
9998            selectors: vec![],
9999            resources: vec![],
10000            max_candidates,
10001        }
10002    }
10003
10004    fn as_set(resp: DiscoveryResponseVariant) -> PlacementCandidateSet {
10005        match resp {
10006            DiscoveryResponseVariant::QueryCandidatesResult(s) => s,
10007            other => panic!("expected QueryCandidatesResult, got {other:?}"),
10008        }
10009    }
10010
10011    /// A node with an ingested NodeRecord but NO liveness heartbeat must never
10012    /// appear — absence is hard exclusion, not a "stale" flag (decision #1).
10013    #[tokio::test]
10014    async fn liveness_hard_excludes_nodes_never_heartbeated() {
10015        let did = "did:web:node1.example.com";
10016        let rec = sample_node_record(did, vec![("zone", "us-east")]);
10017        let svc = service_with(
10018            Box::new(AllowAll),
10019            HashMap::from([(did.to_owned(), node_repo_car(did, &rec))]),
10020        );
10021
10022        let set = as_set(
10023            svc.handle_query_candidates(&test_ctx(), 1, &empty_query(0))
10024                .await
10025                .unwrap(),
10026        );
10027        assert!(set.candidates.is_empty());
10028        assert_eq!(set.total_matching, 0);
10029    }
10030
10031    #[tokio::test]
10032    async fn label_selector_matches_and_excludes_non_matching_nodes() {
10033        let did_a = "did:web:node-a.example.com";
10034        let did_b = "did:web:node-b.example.com";
10035        let repos = HashMap::from([
10036            (
10037                did_a.to_owned(),
10038                node_repo_car(did_a, &sample_node_record(did_a, vec![("zone", "us-east")])),
10039            ),
10040            (
10041                did_b.to_owned(),
10042                node_repo_car(did_b, &sample_node_record(did_b, vec![("zone", "us-west")])),
10043            ),
10044        ]);
10045        let svc = service_with(Box::new(AllowAll), repos);
10046        heartbeat(&svc, did_a, vec![], 0.1).await;
10047        heartbeat(&svc, did_b, vec![], 0.1).await;
10048
10049        let req = QueryCandidatesRequest {
10050            selectors: vec![LabelSelector {
10051                key: "zone".to_owned(),
10052                op: SelectorOp::In,
10053                values: vec!["us-east".to_owned()],
10054            }],
10055            resources: vec![],
10056            max_candidates: 0,
10057        };
10058        let set = as_set(
10059            svc.handle_query_candidates(&test_ctx(), 1, &req)
10060                .await
10061                .unwrap(),
10062        );
10063        assert_eq!(set.candidates.len(), 1);
10064        assert_eq!(set.candidates[0].node, did_a);
10065        assert_eq!(set.total_matching, 1);
10066    }
10067
10068    #[tokio::test]
10069    async fn resource_request_filters_insufficient_live_capacity() {
10070        let did = "did:web:node1.example.com";
10071        let rec = sample_node_record(did, vec![]);
10072        let svc = service_with(
10073            Box::new(AllowAll),
10074            HashMap::from([(did.to_owned(), node_repo_car(did, &rec))]),
10075        );
10076        heartbeat(&svc, did, vec![("nvidia.com/gpu", "2")], 0.1).await;
10077
10078        let req = QueryCandidatesRequest {
10079            selectors: vec![],
10080            resources: vec![ResourceRequest {
10081                name: "nvidia.com/gpu".to_owned(),
10082                min_quantity: "4".to_owned(),
10083            }],
10084            max_candidates: 0,
10085        };
10086        let set = as_set(
10087            svc.handle_query_candidates(&test_ctx(), 1, &req)
10088                .await
10089                .unwrap(),
10090        );
10091        assert!(
10092            set.candidates.is_empty(),
10093            "2 GPUs must not satisfy a request for >=4"
10094        );
10095
10096        // A satisfiable request against the same live report succeeds.
10097        let req_ok = QueryCandidatesRequest {
10098            selectors: vec![],
10099            resources: vec![ResourceRequest {
10100                name: "nvidia.com/gpu".to_owned(),
10101                min_quantity: "1".to_owned(),
10102            }],
10103            max_candidates: 0,
10104        };
10105        let set_ok = as_set(
10106            svc.handle_query_candidates(&test_ctx(), 1, &req_ok)
10107                .await
10108                .unwrap(),
10109        );
10110        assert_eq!(set_ok.candidates.len(), 1);
10111    }
10112
10113    /// THE LOAD-BEARING TEST: a per-candidate authz denial silently drops that
10114    /// node from the result — it is not surfaced as an RPC error.
10115    #[tokio::test]
10116    async fn per_candidate_authz_denial_is_silent_not_an_error() {
10117        let did_a = "did:web:node-a.example.com";
10118        let did_b = "did:web:node-b.example.com";
10119        let repos = HashMap::from([
10120            (
10121                did_a.to_owned(),
10122                node_repo_car(did_a, &sample_node_record(did_a, vec![])),
10123            ),
10124            (
10125                did_b.to_owned(),
10126                node_repo_car(did_b, &sample_node_record(did_b, vec![])),
10127            ),
10128        ]);
10129        let svc = service_with(Box::new(DenyNode(did_a.to_owned())), repos);
10130        heartbeat(&svc, did_a, vec![], 0.1).await;
10131        heartbeat(&svc, did_b, vec![], 0.1).await;
10132
10133        let resp = svc
10134            .handle_query_candidates(&test_ctx(), 1, &empty_query(0))
10135            .await
10136            .unwrap();
10137        let set = as_set(resp);
10138        assert_eq!(set.candidates.len(), 1);
10139        assert_eq!(set.candidates[0].node, did_b);
10140        assert_eq!(
10141            set.total_matching, 1,
10142            "denied candidate must not inflate totalMatching"
10143        );
10144    }
10145
10146    #[tokio::test]
10147    async fn bounding_respects_max_candidates_and_total_matching_is_pre_bound() {
10148        let dids: Vec<String> = (0..5)
10149            .map(|i| format!("did:web:node{i}.example.com"))
10150            .collect();
10151        let mut repos = HashMap::new();
10152        for d in &dids {
10153            repos.insert(d.clone(), node_repo_car(d, &sample_node_record(d, vec![])));
10154        }
10155        let svc = service_with(Box::new(AllowAll), repos);
10156        for (i, d) in dids.iter().enumerate() {
10157            heartbeat(&svc, d, vec![], i as f32 * 0.1).await;
10158        }
10159
10160        let set = as_set(
10161            svc.handle_query_candidates(&test_ctx(), 1, &empty_query(2))
10162                .await
10163                .unwrap(),
10164        );
10165        assert_eq!(set.candidates.len(), 2, "bounded to maxCandidates");
10166        assert_eq!(
10167            set.total_matching, 5,
10168            "totalMatching counts all authorized survivors, not just the bound"
10169        );
10170        // Ranked ascending by load_fraction: node0 (0.0) then node1 (0.1).
10171        assert_eq!(set.candidates[0].node, dids[0]);
10172        assert_eq!(set.candidates[1].node, dids[1]);
10173    }
10174
10175    #[tokio::test]
10176    async fn zero_max_candidates_uses_default_bound() {
10177        let dids: Vec<String> = (0..3)
10178            .map(|i| format!("did:web:node{i}.example.com"))
10179            .collect();
10180        let mut repos = HashMap::new();
10181        for d in &dids {
10182            repos.insert(d.clone(), node_repo_car(d, &sample_node_record(d, vec![])));
10183        }
10184        let svc = service_with(Box::new(AllowAll), repos);
10185        for d in &dids {
10186            heartbeat(&svc, d, vec![], 0.0).await;
10187        }
10188        let set = as_set(
10189            svc.handle_query_candidates(&test_ctx(), 1, &empty_query(0))
10190                .await
10191                .unwrap(),
10192        );
10193        assert_eq!(
10194            set.candidates.len(),
10195            3,
10196            "under the default bound, all 3 come back"
10197        );
10198    }
10199
10200    #[tokio::test]
10201    async fn heartbeat_refreshes_ttl_and_liveness_fields() {
10202        let did = "did:web:node1.example.com";
10203        let rec = sample_node_record(did, vec![]);
10204        let svc = service_with(
10205            Box::new(AllowAll),
10206            HashMap::from([(did.to_owned(), node_repo_car(did, &rec))]),
10207        );
10208        heartbeat(&svc, did, vec![("cpu", "1")], 0.9).await;
10209        heartbeat(&svc, did, vec![("cpu", "8")], 0.1).await;
10210
10211        let set = as_set(
10212            svc.handle_query_candidates(&test_ctx(), 1, &empty_query(0))
10213                .await
10214                .unwrap(),
10215        );
10216        assert_eq!(set.candidates.len(), 1);
10217        assert!(
10218            (set.candidates[0].load_fraction - 0.1).abs() < f32::EPSILON,
10219            "second heartbeat must win"
10220        );
10221        assert_eq!(set.candidates[0].allocatable[0].quantity, "8");
10222    }
10223}
10224
10225// ============================================================================
10226// Regression: eager-resolver bootstrap ordering (#fix-eager-resolver)
10227// ============================================================================
10228//
10229// Before the fix, `bootstrap_deployment_process`'s OsOwnedFiles arm called
10230// `DiscoveryClient::for_local_bootstrap`, which requires
10231// `registered_endpoint("discovery")` to be `Some`. At first-boot (before any
10232// service binds a socket) this is always `None`, so every `service start`
10233// aborted with "local bootstrap requires an explicitly registered service
10234// endpoint" — a startup-ordering inversion. The fix extracts the arm's
10235// resolver-install sequence into [`install_local_discovery_client`], which
10236// uses `resolve_local_discovery_transport` → `try_endpoint` (lazy default
10237// fallback) → `dial` (connects on first use) → `DiscoveryClient::new`.
10238
10239#[cfg(test)]
10240mod eager_resolver_regression {
10241    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
10242    use super::*;
10243
10244    /// Behavioral test: proves [`install_local_discovery_client`] succeeds at
10245    /// first boot with no registered endpoint — the lazy `try_endpoint`
10246    /// resolves a default UDS path and `dial` connects on first use (not now).
10247    ///
10248    /// This test exercises the extracted helper's mechanism (resolve → dial →
10249    /// DiscoveryClient::new) and asserts `registered_endpoint` is `None`. It
10250    /// does NOT by itself detect reversion of the OsOwnedFiles arm to
10251    /// `for_local_bootstrap` — that is the job of the structural test below.
10252    #[test]
10253    fn install_local_discovery_client_succeeds_with_no_registered_endpoint() {
10254        use hyprstream_rpc::registry::{EndpointMode, SocketKind};
10255
10256        // IPC mode with a temp runtime dir — the production same-node fabric.
10257        // Inproc mode makes `dial` error at the processor-lookup step, so it
10258        // cannot prove the lazy transport installs.
10259        let runtime_dir = std::env::temp_dir().join(format!(
10260            "hs-eager-resolver-regression-{}",
10261            std::process::id(),
10262        ));
10263        std::fs::create_dir_all(&runtime_dir).unwrap();
10264        hyprstream_rpc::registry::init(EndpointMode::Ipc, Some(runtime_dir.clone()));
10265
10266        let registry =
10267            hyprstream_rpc::registry::try_global().expect("EndpointRegistry must be initialized");
10268
10269        // First-boot posture: discovery has NOT registered an endpoint.
10270        let registered = registry.registered_endpoint("discovery", SocketKind::Rep);
10271        assert!(
10272            registered.is_none(),
10273            "test precondition: discovery must not be registered at first boot, \
10274             got {:?}",
10275            registered,
10276        );
10277
10278        // The resolver-install path: dial + DiscoveryClient::new.
10279        let signing_key = SigningKey::generate(&mut rand::rngs::OsRng);
10280        let discovery_vk = signing_key.verifying_key();
10281        let _client = install_local_discovery_client(signing_key, discovery_vk, None).expect(
10282            "install_local_discovery_client must succeed at first boot with \
10283                 no registered discovery endpoint",
10284        );
10285
10286        // Confirm the resolved transport is the default UDS path (IPC variant).
10287        let transport = resolve_local_discovery_transport()
10288            .expect("resolve_local_discovery_transport must resolve the lazy default");
10289        assert!(
10290            matches!(
10291                transport.endpoint,
10292                hyprstream_rpc::transport::EndpointType::Ipc { .. }
10293            ),
10294            "expected IPC transport (default discovery UDS), got {:?}",
10295            transport.endpoint,
10296        );
10297
10298        // Cleanup
10299        let _ = std::fs::remove_dir_all(&runtime_dir);
10300    }
10301
10302    #[test]
10303    fn production_bootstrap_discovery_dials_with_caller_credential() {
10304        let source = include_str!("service.rs");
10305        let production = source
10306            .split("// ============================================================================\n// Regression: PostgreSQL accepted-state authority selection")
10307            .next()
10308            .expect("production source");
10309        assert!(production.contains("None, signing_key, service_token, resolver,"));
10310        assert!(production
10311            .contains("install_local_discovery_client(signing_key, discovery_vk, service_token)"));
10312        assert!(production.contains("Some(discovery_vk), token_provider,"));
10313        assert!(production.contains("current_service_token(&self.signing_key, &self.token)"));
10314    }
10315
10316    #[test]
10317    fn process_client_observes_same_signer_jwt_renewal() {
10318        let signing_key = SigningKey::from_bytes(&[0x6d; 32]);
10319        let initial = Some("initial-service-jwt".to_owned());
10320        let provider = renewing_service_token_provider(&signing_key, initial.clone())
10321            .expect("credentialed client has provider");
10322        assert_eq!(provider(), initial);
10323
10324        hyprstream_service::global_trust_store().insert(
10325            signing_key.verifying_key(),
10326            hyprstream_service::Attestation {
10327                scopes: std::iter::once("model".to_owned()).collect(),
10328                subject: None,
10329                jwt: Some("renewed-service-jwt".to_owned()),
10330                expires_at: 0,
10331                attested_by: None,
10332            },
10333        );
10334        assert_eq!(provider().as_deref(), Some("renewed-service-jwt"));
10335        assert_eq!(
10336            current_service_token(&signing_key, &initial).as_deref(),
10337            Some("renewed-service-jwt")
10338        );
10339        assert!(renewing_service_token_provider(&signing_key, None).is_none());
10340    }
10341
10342    /// Structural guard: the OsOwnedFiles arm of
10343    /// [`bootstrap_deployment_process`] must call
10344    /// [`install_local_discovery_client`] (the lazy resolver), not
10345    /// `DiscoveryClient::for_local_bootstrap` (the eager resolver that fails at
10346    /// first boot). This test inspects the production source of the arm — if
10347    /// someone reverts the arm to `for_local_bootstrap`, this test fails.
10348    ///
10349    /// Together with the behavioral test above (which proves the helper
10350    /// succeeds with no registered endpoint), this guards the production
10351    /// install boundary: the arm calls the proven-correct helper, and the
10352    /// helper works.
10353    #[test]
10354    fn os_owned_arm_uses_lazy_resolver_not_for_local_bootstrap() {
10355        let source = include_str!("service.rs");
10356        // Production code only — exclude this regression test module.
10357        let production = source
10358            .split("// ============================================================================\n// Regression: eager-resolver")
10359            .next()
10360            .expect("production source before regression test module");
10361
10362        // Find the OsOwnedFiles arm of bootstrap_deployment_process.
10363        let arm = production
10364            .find("DeploymentTrustSource::OsOwnedFiles =>")
10365            .expect("OsOwnedFiles arm must exist in bootstrap_deployment_process");
10366        let arm_block = &production[arm..arm + 800];
10367
10368        assert!(
10369            arm_block.contains("install_local_discovery_client"),
10370            "OsOwnedFiles arm must call install_local_discovery_client (lazy resolver), \
10371             got:\n{}",
10372            arm_block,
10373        );
10374        assert!(
10375            !arm_block.contains("for_local_bootstrap"),
10376            "OsOwnedFiles arm must NOT call for_local_bootstrap (eager resolver), \
10377             got:\n{}",
10378            arm_block,
10379        );
10380    }
10381}
10382
10383// ============================================================================
10384// Regression: PostgreSQL accepted-state authority selection (#staging-resolver)
10385// ============================================================================
10386//
10387// The accepted-state authority must follow the deployment's resolved records
10388// binding: a configured `[rds]` PostgreSQL store serves the production
10389// resolver, roster, bootstrap endpoints, and browser currentness — with no
10390// local RocksDB fallback on error. These structural pins keep both
10391// construction callsites and the browser-currentness read path on the shared
10392// selection helper; the live behavior is covered by
10393// `checkpointed_pds::pg_tests` against a scratch database.
10394
10395#[cfg(test)]
10396mod pg_authority_wiring {
10397    #![allow(clippy::unwrap_used, clippy::expect_used)]
10398
10399    use super::SigningKey;
10400
10401    #[test]
10402    fn both_bootstrap_callsites_select_the_records_binding_backend() {
10403        let source = include_str!("service.rs");
10404        let production = source
10405            .split("// ============================================================================\n// Regression: PostgreSQL accepted-state authority selection")
10406            .next()
10407            .expect("production source before this regression module");
10408
10409        // The public bootstrap entry point takes the resolved records binding.
10410        assert!(
10411            production.contains(
10412                "pub async fn bootstrap_deployment_process(\n    signing_key: SigningKey,\n    service_token: Option<String>,\n    trust_source: crate::DeploymentTrustSource,\n    remote_node: bool,\n    network_required: bool,\n    records: &hyprstream_pds::rds::RdsConfig,\n) -> Result<()>"
10413            ),
10414            "bootstrap_deployment_process must take the resolved records binding"
10415        );
10416
10417        // Both concrete-source construction callsites go through the shared
10418        // selection helper — no direct RocksDB-only constructor remains.
10419        assert_eq!(
10420            production.matches("open_deployment_accepted_state_source(").count(),
10421            2,
10422            "exactly the network-required bootstrap arm and the install tail              must construct the accepted-state source"
10423        );
10424        assert!(
10425            !production.contains("CheckpointedPdsAcceptedStateSource::open("),
10426            "no callsite may bypass the records-binding selection helper"
10427        );
10428
10429        // The installed process resolver carries the selected source...
10430        assert!(
10431            production.contains("accepted_state_source: source,"),
10432            "the production resolver must carry the selected accepted-state source"
10433        );
10434        // ...and browser currentness reads through that installed resolver.
10435        assert!(
10436            production.contains(
10437                "pub fn production_browser_currentness_verifier() -> Result<Arc<dyn BrowserCurrentnessVerifier>> {\n    let resolver = PRODUCTION_RESOLVER"
10438            ),
10439            "browser currentness must read through the installed production resolver"
10440        );
10441    }
10442
10443    #[test]
10444    fn ephemeral_diagnostic_carrier_is_distinct_and_normal_identity_stays_stable() {
10445        let service_key = SigningKey::from_bytes(&[0x51; 32]);
10446        let stable = super::bootstrap_client_transport_key(&service_key, None);
10447        let expected_stable = hyprstream_rpc::node_identity::derive_purpose_key(
10448            &service_key,
10449            "hyprstream-bootstrap-client-transport-v1",
10450        );
10451        assert_eq!(stable.to_bytes(), expected_stable.to_bytes());
10452
10453        let ephemeral = SigningKey::generate(&mut rand::rngs::OsRng);
10454        let selected = super::bootstrap_client_transport_key(&service_key, Some(ephemeral.clone()));
10455        assert_eq!(selected.to_bytes(), ephemeral.to_bytes());
10456        assert_ne!(selected.verifying_key(), service_key.verifying_key());
10457        assert_ne!(selected.to_bytes(), stable.to_bytes());
10458
10459        let second_ephemeral = SigningKey::generate(&mut rand::rngs::OsRng);
10460        assert_ne!(selected.to_bytes(), second_ephemeral.to_bytes());
10461    }
10462
10463    #[test]
10464    fn ephemeral_bootstrap_preserves_authenticated_resolver_and_rpc_signer() {
10465        let source = include_str!("service.rs");
10466        let production = source
10467            .split("// ============================================================================\n// Regression: PostgreSQL accepted-state authority selection")
10468            .next()
10469            .expect("production source before this regression module");
10470        assert!(production.contains(
10471            "let ephemeral_transport_key = SigningKey::generate(&mut rand::rngs::OsRng);"
10472        ));
10473        assert!(production.contains(
10474            "bootstrap_deployment_process_with_carrier_identity(\n        signing_key,\n        service_token,"
10475        ));
10476        assert!(production.contains("authenticate_deployment_bootstrap()?"));
10477        assert!(production.contains(
10478            "ProductionRpcClient::new(\n                        \"discovery\",\n                        \"discovery\",\n                        None,\n                        signing_key,\n                        service_token,\n                        resolver,"
10479        ));
10480        assert!(production.contains(
10481            "let transport_key =\n                    bootstrap_client_transport_key(&signing_key, client_transport_key_override);"
10482        ));
10483        assert!(production.contains("open_deployment_accepted_state_source("));
10484        assert!(production.contains("source.bootstrap_endpoints(\"policy\")?;"));
10485    }
10486}