1#[cfg(all(test, feature = "rocksdb", not(target_arch = "wasm32")))]
7#[path = "network_bootstrap_tests.rs"]
8mod network_bootstrap_tests;
9
10#[cfg(all(test, feature = "rocksdb", not(target_arch = "wasm32")))]
11#[path = "event_network_bootstrap_tests.rs"]
12mod event_network_bootstrap_tests;
13
14#[cfg(all(test, feature = "rocksdb", not(target_arch = "wasm32")))]
15#[path = "moql_checkpoint_admission_tests.rs"]
16mod moql_checkpoint_admission_tests;
17
18use async_trait::async_trait;
19use hyprstream_rpc::browser_provisioning::{
20 BrowserCarrierProfile, BrowserCurrentnessVerifier, BrowserProvisioningDocument,
21 BrowserProvisioningMaterial, BrowserProvisioningRequest, BrowserRequestBinding,
22 BrowserRouteRole, BrowserTransportSecurity,
23};
24use hyprstream_rpc::registry::{self, EndpointRegistry, SocketKind};
25use hyprstream_rpc::resolver::{Resolver, ResolverProfile, ServiceQuery};
26use hyprstream_rpc::rpc_client::{CallOptions, RpcClient};
27use hyprstream_rpc::service::{EnvelopeContext, RequestService};
28use hyprstream_rpc::stream_consumer::StreamHandle;
29use hyprstream_rpc::transport::{EndpointType, TransportConfig};
30use hyprstream_rpc::{SigningKey, VerifyingKey};
31
32use crate::generated::discovery_client::{
33 dispatch_discovery, serialize_response, DiscoveryHandler,
34};
35use crate::placement_index::PlacementIndex;
36use crate::scheduling;
37use crate::state_store::{
38 unix_millis_now, AnnouncedEndpoint, CachedEntityStatement, CachedEnvelopeKeyset,
39 DiscoveryState, DiscoveryStateStore, LiveAllocatable, MemoryStateStore, ANNOUNCED_ENDPOINT_TTL,
40};
41use hyprstream_rpc_std::discovery_client::{
42 AuthMetadata, AuthMetadataList, DiscoveryResponseVariant, EndpointInfo, EntityStatement,
43 EnvelopeKeyset, ErrorInfo, GetRecordRequest, IssuerList, NodeLiveness, PingInfo,
44 PlacementCandidate, PlacementCandidateSet, QueryCandidatesRequest, RecordCar,
45 RegisterEntityStatementRequest, RegisterEnvelopeKeysetRequest, Resource, ServiceAnnouncement,
46 ServiceEndpoints, ServiceList, ServiceSummary,
47};
48
49use anyhow::{Context, Result};
50use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
51use hyprstream_rpc::identity::Did;
52use hyprstream_util::ttl_cache::TtlCache;
53use std::collections::BTreeSet;
54#[cfg(any(test, feature = "test-fixtures"))]
55use std::collections::HashMap;
56use std::net::SocketAddr;
57use std::str::FromStr;
58use std::sync::Arc;
59use std::time::{Duration, Instant};
60use tracing::{info, trace};
61
62const LIVENESS_TTL: Duration = Duration::from_secs(45);
68
69const LIVENESS_CACHE_MAX_ENTRIES: usize = 16_384;
73const LIVENESS_CACHE_REAP_BUDGET: usize = 32;
74const PLACEMENT_INGEST_RETRY_TTL: Duration = Duration::from_secs(300);
78const CANDIDATE_QUERY_CONCURRENCY: usize = 16;
79const CANDIDATE_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
80
81#[derive(Clone, Copy, PartialEq, Eq)]
82enum PlacementIngestStatus {
83 Pending,
84 Complete,
85 Cancelled,
86}
87
88struct PlacementIngestGuard(Arc<parking_lot::Mutex<PlacementIngestStatus>>);
89
90impl Drop for PlacementIngestGuard {
91 fn drop(&mut self) {
92 let mut status = self.0.lock();
93 if *status == PlacementIngestStatus::Pending {
94 *status = PlacementIngestStatus::Cancelled;
95 }
96 }
97}
98
99const DEFAULT_MAX_CANDIDATES: usize = 100;
103
104#[derive(Debug, Clone, PartialEq, Eq)]
107struct AcceptedStateEvidence {
108 service_did: Did,
109 digest: [u8; 64],
110 epoch: u64,
111 expires_at_unix_ms: i64,
112 response_ed25519: [u8; 32],
113 response_ml_dsa65: Vec<u8>,
114}
115
116#[derive(Debug, Clone, PartialEq, Eq)]
117struct AnchoredKemRecipient {
118 key_id: String,
119 recipient: hyprstream_rpc::crypto::hybrid_kem::RecipientPublic,
120 not_after_unix_ms: i64,
121}
122
123#[derive(Debug, Clone, PartialEq, Eq)]
124struct ServiceCandidate {
125 service_name: String,
126 service_did: Did,
127 response_verifying_key: [u8; 32],
128 response_ml_dsa65: Vec<u8>,
129 response_key_id: String,
130 request_kem_recipient: Option<AnchoredKemRecipient>,
131 transport: TransportConfig,
132 capabilities: BTreeSet<String>,
133 accepted_state: AcceptedStateEvidence,
134 source_signer: [u8; 32],
135 expires_at_unix_ms: i64,
136}
137
138#[derive(Debug, Clone, PartialEq, Eq)]
139struct CandidateDecision {
140 candidate_fingerprint: String,
141 accepted: bool,
142 reason: String,
143}
144
145#[derive(Debug, Clone, PartialEq, Eq)]
146struct ResolutionEvidence {
147 accepted_state_digest: [u8; 64],
148 accepted_state_epoch: u64,
149 selected_fingerprint: String,
150 ordered_decisions: Vec<CandidateDecision>,
151}
152
153#[derive(Debug, Clone)]
156struct SelectedService {
157 service_name: String,
158 service_did: Did,
159 response_verifying_key: VerifyingKey,
160 response_ml_dsa65: Vec<u8>,
161 response_key_id: String,
162 request_kem_recipient: AnchoredKemRecipient,
163 transport: TransportConfig,
164 expires_at_unix_ms: i64,
165 evidence: ResolutionEvidence,
166}
167
168impl SelectedService {
169 fn same_authority(&self, other: &Self) -> bool {
170 self.service_name == other.service_name
171 && self.service_did == other.service_did
172 && self.response_verifying_key == other.response_verifying_key
173 && self.response_ml_dsa65 == other.response_ml_dsa65
174 && self.response_key_id == other.response_key_id
175 && self.request_kem_recipient == other.request_kem_recipient
176 && self.evidence.accepted_state_digest == other.evidence.accepted_state_digest
177 && self.evidence.accepted_state_epoch == other.evidence.accepted_state_epoch
178 }
179 fn service_name(&self) -> &str {
180 &self.service_name
181 }
182
183 fn service_did(&self) -> &Did {
184 &self.service_did
185 }
186
187 fn response_verifying_key(&self) -> VerifyingKey {
188 self.response_verifying_key
189 }
190
191 fn response_ml_dsa65(&self) -> &[u8] {
192 &self.response_ml_dsa65
193 }
194
195 fn response_key_id(&self) -> &str {
196 &self.response_key_id
197 }
198 fn request_kem_recipient(&self) -> &AnchoredKemRecipient {
199 &self.request_kem_recipient
200 }
201
202 fn transport(&self) -> &TransportConfig {
203 &self.transport
204 }
205
206 fn evidence(&self) -> &ResolutionEvidence {
207 &self.evidence
208 }
209
210 fn expires_at_unix_ms(&self) -> i64 {
211 self.expires_at_unix_ms
212 }
213
214 fn ensure_fresh(&self, now_unix_ms: i64) -> Result<()> {
215 anyhow::ensure!(
216 now_unix_ms < self.expires_at_unix_ms,
217 "resolved service snapshot expired; re-resolution required"
218 );
219 anyhow::ensure!(
220 now_unix_ms < self.request_kem_recipient.not_after_unix_ms,
221 "resolved request KEM recipient expired; re-resolution required"
222 );
223 Ok(())
224 }
225
226 fn crypto_stores(
227 &self,
228 ) -> Result<(
229 Arc<dyn hyprstream_rpc::crypto::hybrid_kem::KemTrustStore>,
230 Arc<dyn hyprstream_rpc::envelope::PqTrustStore>,
231 )> {
232 let ed = self.response_verifying_key.to_bytes();
233 let mut kem = hyprstream_rpc::crypto::hybrid_kem::KeyedKemTrustStore::new();
234 kem.bind(ed, self.request_kem_recipient.recipient.clone());
235 let pq_key = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&self.response_ml_dsa65)
236 .map_err(|e| anyhow::anyhow!("invalid resolved ML-DSA-65 response key: {e}"))?;
237 let mut pq = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
238 pq.bind(ed, &pq_key);
239 Ok((Arc::new(kem), Arc::new(pq)))
240 }
241}
242
243fn canonical_service_name(name: &str) -> Result<String> {
244 let canonical = name.trim().to_ascii_lowercase();
245 anyhow::ensure!(!canonical.is_empty(), "service name is empty");
246 anyhow::ensure!(
247 canonical
248 .bytes()
249 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'),
250 "service name contains non-canonical characters"
251 );
252 anyhow::ensure!(canonical == name, "service name is not canonical");
253 Ok(canonical)
254}
255
256fn network_reach(transport: &TransportConfig) -> bool {
257 matches!(
258 transport.endpoint,
259 EndpointType::Quic { .. } | EndpointType::Iroh { .. }
260 )
261}
262
263fn native_iroh_reach(transport: &TransportConfig) -> bool {
264 matches!(transport.endpoint, EndpointType::Iroh { .. })
265}
266
267fn browser_webtransport_reach(transport: &TransportConfig) -> bool {
268 matches!(transport.endpoint, EndpointType::Quic { .. })
269}
270
271fn transport_fingerprint(transport: &TransportConfig) -> String {
272 blake3::hash(format!("{transport:?}").as_bytes())
273 .to_hex()
274 .to_string()
275}
276
277fn hash_framed(hasher: &mut blake3::Hasher, bytes: &[u8]) {
278 hasher.update(&(bytes.len() as u64).to_be_bytes());
279 hasher.update(bytes);
280}
281
282fn candidate_fingerprint(candidate: &ServiceCandidate) -> String {
283 let mut h = blake3::Hasher::new();
284 hash_framed(&mut h, candidate.service_name.as_bytes());
285 hash_framed(&mut h, candidate.service_did.as_str().as_bytes());
286 h.update(&candidate.response_verifying_key);
287 hash_framed(&mut h, &candidate.response_ml_dsa65);
288 hash_framed(&mut h, candidate.response_key_id.as_bytes());
289 if let Some(kem) = &candidate.request_kem_recipient {
290 hash_framed(&mut h, kem.key_id.as_bytes());
291 hash_framed(&mut h, &kem.recipient.encode());
292 }
293 hash_framed(
294 &mut h,
295 transport_fingerprint(&candidate.transport).as_bytes(),
296 );
297 for capability in &candidate.capabilities {
298 hash_framed(&mut h, capability.as_bytes());
299 }
300 h.update(&candidate.accepted_state.digest);
301 h.update(&candidate.accepted_state.epoch.to_be_bytes());
302 hash_framed(
303 &mut h,
304 candidate.accepted_state.service_did.as_str().as_bytes(),
305 );
306 h.update(&candidate.accepted_state.expires_at_unix_ms.to_be_bytes());
307 h.update(&candidate.accepted_state.response_ed25519);
308 hash_framed(&mut h, &candidate.accepted_state.response_ml_dsa65);
309 h.update(&candidate.source_signer);
310 h.update(&candidate.expires_at_unix_ms.to_be_bytes());
311 if let Some(kem) = &candidate.request_kem_recipient {
312 h.update(&kem.not_after_unix_ms.to_be_bytes());
313 }
314 h.finalize().to_hex().to_string()
315}
316
317fn rejection_reason(
318 query: &ServiceQuery,
319 candidate: &ServiceCandidate,
320 now_unix_ms: i64,
321) -> Option<&'static str> {
322 if canonical_service_name(&candidate.service_name)
323 .ok()
324 .as_deref()
325 != Some(query.service_name.as_str())
326 {
327 return Some("service-name-mismatch");
328 }
329 if !candidate.service_did.is_did_at9p()
330 || candidate.service_did != candidate.accepted_state.service_did
331 {
332 return Some("missing-or-mismatched-accepted-identity");
333 }
334 if candidate.source_signer != candidate.accepted_state.response_ed25519 {
335 return Some("announcement-signer-not-current");
336 }
337 if candidate.response_verifying_key != candidate.accepted_state.response_ed25519
338 || candidate.response_ml_dsa65 != candidate.accepted_state.response_ml_dsa65
339 {
340 return Some("response-key-not-current");
341 }
342 if !candidate
343 .response_key_id
344 .starts_with(&format!("{}#", candidate.service_did))
345 || hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&candidate.response_ml_dsa65).is_err()
346 {
347 return Some("missing-or-invalid-response-key-material");
348 }
349 if !query
350 .required_capabilities
351 .is_subset(&candidate.capabilities)
352 {
353 return Some("missing-capability");
354 }
355 if candidate.expires_at_unix_ms <= now_unix_ms
356 || candidate.accepted_state.expires_at_unix_ms <= now_unix_ms
357 {
358 return Some("expired-candidate-or-state");
359 }
360 match query.profile {
361 ResolverProfile::NetworkDiscovery if !network_reach(&candidate.transport) => {
362 return Some("local-reach-for-network-profile");
363 }
364 ResolverProfile::NativeIrohRequired if !native_iroh_reach(&candidate.transport) => {
365 return Some("non-iroh-reach-for-native-profile");
366 }
367 ResolverProfile::BrowserWebTransport
368 if !browser_webtransport_reach(&candidate.transport) =>
369 {
370 return Some("non-quic-reach-for-browser-profile");
371 }
372 ResolverProfile::LocalInproc
373 if !matches!(candidate.transport.endpoint, EndpointType::Inproc { .. }) =>
374 {
375 return Some("non-inproc-reach-for-local-profile");
376 }
377 ResolverProfile::Ipc
378 if !matches!(
379 candidate.transport.endpoint,
380 EndpointType::Ipc { .. } | EndpointType::SystemdFd { .. }
381 ) =>
382 {
383 return Some("non-ipc-reach-for-ipc-profile");
384 }
385 _ => {}
386 }
387 let Some(kem) = &candidate.request_kem_recipient else {
388 return Some("missing-request-kem-recipient");
389 };
390 if !kem
391 .key_id
392 .starts_with(&format!("{}#", candidate.service_did))
393 || kem.not_after_unix_ms <= now_unix_ms
394 || kem.recipient.suite_id
395 != hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768
396 || kem.recipient.eks.len() != kem.recipient.suite_id.components().len()
397 || hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::decode(&kem.recipient.encode())
398 .is_err()
399 {
400 return Some("invalid-expired-or-nonhybrid-kem-recipient");
401 }
402 None
403}
404
405fn select_service_candidate(
406 query: &ServiceQuery,
407 candidates: Vec<ServiceCandidate>,
408 now_unix_ms: i64,
409) -> Result<SelectedService> {
410 let mut decisions = Vec::with_capacity(candidates.len());
411 let mut valid = Vec::new();
412 for candidate in candidates {
413 let fingerprint = candidate_fingerprint(&candidate);
414 if let Some(reason) = rejection_reason(query, &candidate, now_unix_ms) {
415 decisions.push(CandidateDecision {
416 candidate_fingerprint: fingerprint,
417 accepted: false,
418 reason: reason.to_owned(),
419 });
420 } else {
421 decisions.push(CandidateDecision {
422 candidate_fingerprint: fingerprint.clone(),
423 accepted: true,
424 reason: "validated".to_owned(),
425 });
426 valid.push((fingerprint, candidate));
427 }
428 }
429 anyhow::ensure!(!valid.is_empty(), "no validated service candidates");
430 let authority = &valid[0].1;
431 anyhow::ensure!(
432 valid.iter().all(|(_, candidate)| {
433 candidate.service_did == authority.service_did
434 && candidate.response_verifying_key == authority.response_verifying_key
435 && candidate.response_ml_dsa65 == authority.response_ml_dsa65
436 && candidate.response_key_id == authority.response_key_id
437 && candidate.request_kem_recipient == authority.request_kem_recipient
438 && candidate.accepted_state == authority.accepted_state
439 }),
440 "ambiguous validated service authority"
441 );
442 valid.sort_by(|(af, a), (bf, b)| {
443 b.accepted_state
444 .epoch
445 .cmp(&a.accepted_state.epoch)
446 .then_with(|| b.expires_at_unix_ms.cmp(&a.expires_at_unix_ms))
447 .then_with(|| a.service_did.cmp(&b.service_did))
448 .then_with(|| {
449 transport_fingerprint(&a.transport).cmp(&transport_fingerprint(&b.transport))
450 })
451 .then_with(|| af.cmp(bf))
452 });
453 valid.dedup_by(|(af, _), (bf, _)| af == bf);
454 let (selected_fingerprint, selected) = valid.remove(0);
455 decisions.sort_by(|a, b| a.candidate_fingerprint.cmp(&b.candidate_fingerprint));
456 let response_verifying_key = VerifyingKey::from_bytes(&selected.response_verifying_key)
457 .map_err(|e| anyhow::anyhow!("invalid selected response verifying key: {e}"))?;
458 let request_kem_recipient = selected
459 .request_kem_recipient
460 .ok_or_else(|| anyhow::anyhow!("validated candidate lost request KEM recipient"))?;
461 let expires_at_unix_ms = selected
462 .expires_at_unix_ms
463 .min(selected.accepted_state.expires_at_unix_ms)
464 .min(request_kem_recipient.not_after_unix_ms);
465 Ok(SelectedService {
466 service_name: selected.service_name,
467 service_did: selected.service_did,
468 response_verifying_key,
469 response_ml_dsa65: selected.response_ml_dsa65,
470 response_key_id: selected.response_key_id,
471 request_kem_recipient,
472 transport: selected.transport,
473 expires_at_unix_ms,
474 evidence: ResolutionEvidence {
475 accepted_state_digest: selected.accepted_state.digest,
476 accepted_state_epoch: selected.accepted_state.epoch,
477 selected_fingerprint,
478 ordered_decisions: decisions,
479 },
480 })
481}
482
483fn select_service_candidates(
484 query: &ServiceQuery,
485 mut candidates: Vec<ServiceCandidate>,
486 now_unix_ms: i64,
487) -> Result<Vec<SelectedService>> {
488 let mut ordered = Vec::new();
489 loop {
490 let selected = match select_service_candidate(query, candidates.clone(), now_unix_ms) {
491 Ok(selected) => selected,
492 Err(error) if !ordered.is_empty() && error.to_string().contains("no validated") => {
493 break;
494 }
495 Err(error) => return Err(error),
496 };
497 let fingerprint = selected.evidence.selected_fingerprint.clone();
498 candidates.retain(|candidate| candidate_fingerprint(candidate) != fingerprint);
499 ordered.push(selected);
500 }
501 Ok(ordered)
502}
503
504fn to_scheduling_op(
509 op: hyprstream_rpc_std::discovery_client::SelectorOp,
510) -> scheduling::SelectorOp {
511 use hyprstream_rpc_std::discovery_client::SelectorOp as Wire;
512 match op {
513 Wire::In => scheduling::SelectorOp::In,
514 Wire::NotIn => scheduling::SelectorOp::NotIn,
515 Wire::Exists => scheduling::SelectorOp::Exists,
516 Wire::DoesNotExist => scheduling::SelectorOp::DoesNotExist,
517 Wire::Gt => scheduling::SelectorOp::Gt,
518 Wire::Lt => scheduling::SelectorOp::Lt,
519 }
520}
521
522#[async_trait(?Send)]
531pub trait AuthorizationProvider: Send + Sync {
532 async fn check_context(
535 &self,
536 ctx: &EnvelopeContext,
537 resource: &str,
538 operation: &str,
539 ) -> Result<bool> {
540 self.check(
541 &ctx.subject().to_string(),
542 "*",
543 resource,
544 operation,
545 ctx.jwt_token(),
546 )
547 .await
548 }
549
550 async fn check_batch_context(
551 &self,
552 ctx: &EnvelopeContext,
553 resources: &[String],
554 operation: &str,
555 ) -> Result<Vec<bool>> {
556 self.check_batch(
557 &ctx.subject().to_string(),
558 "*",
559 resources,
560 operation,
561 ctx.jwt_token(),
562 )
563 .await
564 }
565
566 async fn check_batch(
569 &self,
570 subject: &str,
571 domain: &str,
572 resources: &[String],
573 operation: &str,
574 bearer: Option<&str>,
575 ) -> Result<Vec<bool>> {
576 anyhow::ensure!(resources.len() <= 256, "authorization batch exceeds 256");
577 let mut allowed = Vec::with_capacity(resources.len());
578 for resource in resources {
579 allowed.push(
580 self.check(subject, domain, resource, operation, bearer)
581 .await
582 .unwrap_or(false),
583 );
584 }
585 Ok(allowed)
586 }
587
588 async fn check(
590 &self,
591 subject: &str,
592 domain: &str,
593 resource: &str,
594 operation: &str,
595 bearer: Option<&str>,
596 ) -> Result<bool>;
597}
598
599#[derive(Clone, Debug)]
611pub struct RecordCarData {
612 pub uri: String,
613 pub car: Vec<u8>,
614}
615
616#[async_trait(?Send)]
628pub trait RecordResolver: Send + Sync {
629 async fn resolve_record(
633 &self,
634 did: &str,
635 collection: &str,
636 rkey: &str,
637 ) -> Result<Option<RecordCarData>>;
638
639 async fn resolve_repo(&self, did: &str) -> Result<Option<RecordCarData>>;
642
643 async fn resolve_verifying_key(&self, _did: &str) -> Result<Option<p256::ecdsa::VerifyingKey>> {
655 Ok(None)
656 }
657
658 async fn resolve_verifying_keys(
662 &self,
663 did: &str,
664 ) -> Result<Option<hyprstream_pds::commit::PublishedAtprotoKeys>> {
665 Ok(self
666 .resolve_verifying_key(did)
667 .await?
668 .map(hyprstream_pds::commit::PublishedAtprotoKeys::single))
669 }
670}
671
672pub(super) trait AcceptedStateSource: Send + Sync {
679 fn accepted_state(
680 &self,
681 did: &str,
682 ) -> Result<Option<hyprstream_pds::at9p_duplicity::AcceptedAt9pState>>;
683
684 fn accepted_states(&self) -> Result<Vec<hyprstream_pds::at9p_duplicity::AcceptedAt9pState>>;
689
690 fn bootstrap_endpoints(&self, _service_name: &str) -> Result<Option<Vec<AnnouncedEndpoint>>> {
691 Ok(None)
692 }
693}
694
695#[derive(Clone)]
698struct ResolvedService {
699 selected: SelectedService,
700}
701
702impl ResolvedService {
703 fn mint(
704 selected: SelectedService,
705 state: &hyprstream_pds::at9p_duplicity::AcceptedAt9pState,
706 ) -> Result<Self> {
707 anyhow::ensure!(
708 selected.service_did().as_str() == state.did,
709 "selected service DID is not the checkpoint-verified subject"
710 );
711 anyhow::ensure!(
712 selected.evidence().accepted_state_epoch == state.epoch,
713 "selected service epoch is not checkpoint current"
714 );
715 anyhow::ensure!(
716 selected.evidence().accepted_state_digest == state.head_digest,
717 "selected service digest is not checkpoint current"
718 );
719 let current_key = state
720 .current
721 .subject_keys
722 .iter()
723 .find(|key| {
724 key.ed25519_pub.as_slice()
725 == selected.response_verifying_key().to_bytes().as_slice()
726 })
727 .ok_or_else(|| {
728 anyhow::anyhow!("selected response key is not one of the accepted current keys")
729 })?;
730 anyhow::ensure!(
731 selected.response_ml_dsa65() == current_key.mldsa65_pub.as_slice(),
732 "selected PQ response key is not the atomically-bound accepted current key"
733 );
734 anyhow::ensure!(
735 state
736 .current
737 .services
738 .iter()
739 .any(|service| service.id == format!("#{}", selected.service_name())),
740 "selected service is not present in accepted current state"
741 );
742 Ok(Self { selected })
743 }
744 fn same_authority(&self, other: &Self) -> bool {
745 self.selected.same_authority(&other.selected)
746 }
747}
748
749impl std::ops::Deref for ResolvedService {
750 type Target = SelectedService;
751 fn deref(&self) -> &Self::Target {
752 &self.selected
753 }
754}
755
756struct CurrentStreamHandle {
757 inner: Box<dyn StreamHandle>,
758 resolver: Arc<DiscoveryServiceResolver>,
759 snapshot: ResolvedService,
760 invalidated: bool,
761}
762
763#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)]
764#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))]
765impl StreamHandle for CurrentStreamHandle {
766 async fn next_payload(&mut self) -> Result<Option<hyprstream_rpc::streaming::StreamPayload>> {
767 anyhow::ensure!(!self.invalidated, "stream snapshot is invalidated");
768 if let Err(error) = self.resolver.ensure_current(&self.snapshot).await {
769 self.invalidated = true;
770 return Err(error);
771 }
772 self.inner.next_payload().await
773 }
774 async fn cancel(&mut self) -> Result<()> {
775 anyhow::ensure!(!self.invalidated, "stream snapshot is invalidated");
776 if let Err(error) = self.resolver.ensure_current(&self.snapshot).await {
777 self.invalidated = true;
778 return Err(error);
779 }
780 self.inner.cancel().await
781 }
782 fn stream_id(&self) -> &str {
783 self.inner.stream_id()
784 }
785 fn is_completed(&self) -> bool {
786 self.invalidated || self.inner.is_completed()
787 }
788}
789
790struct DiscoveryServiceResolver {
792 state_store: Arc<dyn DiscoveryStateStore>,
793 accepted_state_source: Arc<dyn AcceptedStateSource>,
794 discovery_client: Option<hyprstream_rpc_std::discovery_client::DiscoveryClient>,
795}
796
797fn parse_at_uri(uri: &str) -> Result<(String, String, String)> {
803 let rest = uri
804 .strip_prefix("at://")
805 .ok_or_else(|| anyhow::anyhow!("at-uri must start with \"at://\": {uri:?}"))?;
806 let mut parts = rest.splitn(3, '/');
807 let did = parts.next().filter(|s| !s.is_empty());
808 let collection = parts.next().filter(|s| !s.is_empty());
809 let rkey = parts.next().filter(|s| !s.is_empty());
810 match (did, collection, rkey) {
811 (Some(d), Some(c), Some(r)) => Ok((d.to_owned(), c.to_owned(), r.to_owned())),
812 _ => anyhow::bail!("at-uri must be at://<did>/<collection>/<rkey>: {uri:?}"),
813 }
814}
815
816fn unix_seconds_now() -> i64 {
817 use std::time::{SystemTime, UNIX_EPOCH};
818 SystemTime::now()
819 .duration_since(UNIX_EPOCH)
820 .map(|d| d.as_secs() as i64)
821 .unwrap_or(0)
822}
823
824pub struct DiscoveryService {
826 started_at: Instant,
828 signing_key: Arc<SigningKey>,
830 jwt_verifying_key: hyprstream_rpc::prelude::VerifyingKey,
832 jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
834 oauth_issuer_url: Option<String>,
836 expected_audience: Option<String>,
838 auth_provider: Option<Box<dyn AuthorizationProvider>>,
840 record_resolver: Option<Arc<dyn RecordResolver>>,
843 accepted_state_source: Option<Arc<dyn AcceptedStateSource>>,
844 state_store: Arc<dyn DiscoveryStateStore>,
847 tls_endorsement: Vec<u8>,
850 tls_domain: String,
852 placement_index: PlacementIndex,
856 placement_ingest_attempts: TtlCache<Did, Arc<parking_lot::Mutex<PlacementIngestStatus>>>,
860 transport: TransportConfig,
862}
863
864pub struct DiscoverySelfAnnouncer {
866 validator: DiscoveryService,
867}
868
869impl DiscoverySelfAnnouncer {
870 pub async fn publish(&self, announcement: &ServiceAnnouncement) -> Result<()> {
871 anyhow::ensure!(
872 announcement.service_name == "discovery",
873 "self publication is only for Discovery"
874 );
875 anyhow::ensure!(
876 announcement.service_did.is_did_at9p(),
877 "self publication requires an accepted identity"
878 );
879 let mut ctx = EnvelopeContext::from_callback_service(0, "discovery");
880 ctx.cnf = self.validator.signing_key.verifying_key().to_bytes();
881 self.validator
882 .store_announcement(&ctx, announcement)
883 .await?;
884 Ok(())
885 }
886}
887
888impl DiscoveryService {
889 pub fn self_announcer(&self) -> Result<DiscoverySelfAnnouncer> {
890 let source = self
891 .accepted_state_source
892 .clone()
893 .ok_or_else(|| anyhow::anyhow!("self publication requires accepted-state authority"))?;
894 let mut validator = Self::new(
895 self.signing_key.clone(),
896 self.jwt_verifying_key,
897 self.transport.clone(),
898 );
899 validator.state_store = self.state_store.clone();
900 validator.accepted_state_source = Some(source);
901 validator.jwt_key_source = self.jwt_key_source.clone();
902 validator.expected_audience = self.expected_audience.clone();
903 Ok(DiscoverySelfAnnouncer { validator })
904 }
905
906 async fn store_announcement(
907 &self,
908 ctx: &EnvelopeContext,
909 data: &ServiceAnnouncement,
910 ) -> Result<DiscoveryResponseVariant> {
911 info!(
912 "Discovery: service '{}' announced {} endpoint: {} (from {})",
913 data.service_name,
914 data.socket_kind,
915 data.endpoint,
916 ctx.subject()
917 );
918
919 let svc_name = data.service_name.clone();
920 let sock_kind = data.socket_kind.clone();
921 let endpoint = data.endpoint.clone();
922 let service_jwt = data.service_jwt.clone().unwrap_or_default();
923 let identity_bound = !data.service_did.as_str().is_empty()
924 || !data.accepted_state_digest.is_empty()
925 || !data.request_kem_recipient.is_empty();
926 if identity_bound {
927 anyhow::ensure!(
928 !service_jwt.is_empty(),
929 "identity-bound announcement requires a verified service JWT"
930 );
931 anyhow::ensure!(
932 data.service_did.is_did_at9p()
933 && data.accepted_state_digest.len() == 64
934 && !data.capabilities.is_empty()
935 && data
936 .response_key_id
937 .starts_with(&format!("{}#", data.service_did))
938 && data
939 .request_kem_key_id
940 .starts_with(&format!("{}#", data.service_did))
941 && data.expires_at_unix_ms > unix_millis_now(),
942 "identity-bound announcement metadata is incomplete or expired"
943 );
944 let recipient = hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::decode(
945 &data.request_kem_recipient,
946 )?;
947 anyhow::ensure!(
948 recipient.suite_id
949 == hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768
950 && recipient.eks.len() == recipient.suite_id.components().len(),
951 "identity-bound announcement requires suite-complete hybrid KEM material"
952 );
953 match data.socket_kind.as_str() {
954 "quic" => {
955 parse_announced_quic(&data.endpoint)?;
956 }
957 "iroh" => {
958 parse_announced_iroh(&data.endpoint)?;
959 }
960 _ => {
961 anyhow::bail!("identity-bound network announcement requires QUIC or Iroh reach")
962 }
963 }
964 }
965
966 if !service_jwt.is_empty() {
969 let is_composite = hyprstream_rpc::auth::jwt::header_alg(&service_jwt)
974 .ok()
975 .flatten()
976 .is_some_and(|alg| alg == "ML-DSA-65-Ed25519");
977 let verified = if is_composite {
978 let dispatch =
979 hyprstream_rpc::auth::jwt::parse_composite_dispatch(&service_jwt, &["wit+jwt"])
980 .map_err(|e| {
981 tracing::warn!("Service JWT dispatch failed in announce: {}", e);
982 anyhow::anyhow!("Invalid service JWT in announce: {}", e)
983 })?;
984 let pair = self
985 .jwt_key_source
986 .as_ref()
987 .and_then(|ks| ks.composite_pair(dispatch.kid()))
988 .ok_or_else(|| {
989 anyhow::anyhow!("Invalid service JWT in announce: unknown composite kid")
990 })?;
991 anyhow::ensure!(
998 pair.role() == hyprstream_rpc::auth::CompositePairRole::Policy,
999 "Invalid service JWT in announce: composite pair role is not authorized \
1000 for service certification"
1001 );
1002 hyprstream_rpc::auth::jwt::decode_composite(
1003 &service_jwt,
1004 pair.ml_dsa(),
1005 pair.ed25519(),
1006 self.expected_audience.as_deref(),
1007 &dispatch,
1008 )
1009 .map_err(|e| {
1010 tracing::warn!("Service JWT verification failed in announce: {}", e);
1011 anyhow::anyhow!("Invalid service JWT in announce: {}", e)
1012 })?
1013 } else {
1014 hyprstream_rpc::auth::jwt::decode_with_key(
1015 &service_jwt,
1016 &self.jwt_verifying_key,
1017 self.expected_audience.as_deref(),
1018 )
1019 .map_err(|e| {
1020 tracing::warn!("Service JWT verification failed in announce: {}", e);
1021 anyhow::anyhow!("Invalid service JWT in announce: {}", e)
1022 })?
1023 };
1024 let expected_sub = format!("service:{}", svc_name);
1026 if verified.sub != expected_sub {
1027 anyhow::bail!(
1028 "Service JWT subject mismatch: expected '{}', got '{}'",
1029 expected_sub,
1030 verified.sub
1031 );
1032 }
1033 anyhow::ensure!(
1034 verified.cnf_key_bytes() == Some(ctx.cnf),
1035 "service JWT confirmation key does not match verified announcement signer"
1036 );
1037 }
1038
1039 let now_unix_ms = unix_millis_now();
1040 let heartbeat_expiry =
1041 now_unix_ms.saturating_add(ANNOUNCED_ENDPOINT_TTL.as_millis() as i64);
1042 let expires_at_unix_ms = if identity_bound {
1045 data.expires_at_unix_ms
1046 } else {
1047 heartbeat_expiry
1048 };
1049 let mut live_until_unix_ms = expires_at_unix_ms.min(heartbeat_expiry);
1050 if identity_bound {
1051 if let Some(source) = &self.accepted_state_source {
1052 let state = source
1053 .accepted_state(data.service_did.as_str())?
1054 .ok_or_else(|| {
1055 anyhow::anyhow!("announcement DID has no accepted-current state")
1056 })?;
1057 anyhow::ensure!(
1058 state
1059 .current
1060 .subject_keys
1061 .iter()
1062 .any(|key| key.ed25519_pub == ctx.cnf),
1063 "announcement signer is not an accepted current subject key"
1064 );
1065 let service = state
1066 .current
1067 .services
1068 .iter()
1069 .find(|entry| entry.id == format!("#{svc_name}"))
1070 .ok_or_else(|| anyhow::anyhow!("announcement service is not accepted"))?;
1071 let kem = service.endpoint.request_kem.as_ref().ok_or_else(|| {
1078 anyhow::anyhow!(
1079 "accepted service entry lacks a signed request KEM; \
1080 reprovision service identity"
1081 )
1082 })?;
1083 anyhow::ensure!(
1084 kem == &data.request_kem_recipient,
1085 "announcement KEM differs from accepted service"
1086 );
1087 if data.socket_kind == "iroh" {
1088 anyhow::ensure!(
1089 service.endpoint.address == data.endpoint,
1090 "announcement reach differs from accepted service"
1091 );
1092 }
1093 anyhow::ensure!(
1094 state.epoch == data.accepted_state_epoch
1095 && state.head_digest.as_slice() == data.accepted_state_digest.as_slice(),
1096 "announcement does not match accepted-current state"
1097 );
1098 live_until_unix_ms = live_until_unix_ms.min(accepted_expiry_unix_ms(&state)?);
1099 }
1100 }
1101 anyhow::ensure!(
1102 live_until_unix_ms > now_unix_ms,
1103 "announcement effective lifetime is already expired"
1104 );
1105
1106 let replacement = AnnouncedEndpoint {
1107 socket_kind: sock_kind.clone(),
1108 endpoint: endpoint.clone(),
1109 service_did: data.service_did.clone(),
1110 capabilities: data.capabilities.iter().cloned().collect(),
1111 accepted_state_digest: data.accepted_state_digest.clone(),
1112 accepted_state_epoch: data.accepted_state_epoch,
1113 response_key_id: data.response_key_id.clone(),
1114 request_kem_key_id: data.request_kem_key_id.clone(),
1115 request_kem_recipient: data.request_kem_recipient.clone(),
1116 expires_at_unix_ms,
1117 source_signer: ctx.cnf,
1118 live_until_unix_ms,
1119 };
1120 self.state_store
1121 .put_announcement(&svc_name, replacement)
1122 .await?;
1123
1124 Ok(DiscoveryResponseVariant::AnnounceResult)
1125 }
1126
1127 async fn ensure_placement_ingested(&self, node: &Did) -> bool {
1130 if self.placement_index.record_uri(node.as_str()).is_some() {
1131 return true;
1132 }
1133 let attempt = Arc::new(parking_lot::Mutex::new(PlacementIngestStatus::Pending));
1134 let attempt = if self.placement_ingest_attempts.insert_if_absent(
1135 node.clone(),
1136 Arc::clone(&attempt),
1137 PLACEMENT_INGEST_RETRY_TTL,
1138 ) {
1139 attempt
1140 } else {
1141 let Some(existing) = self.placement_ingest_attempts.get(node) else {
1142 return false;
1143 };
1144 {
1145 let mut status = existing.lock();
1146 match *status {
1147 PlacementIngestStatus::Complete => return true,
1148 PlacementIngestStatus::Pending => return false,
1149 PlacementIngestStatus::Cancelled => *status = PlacementIngestStatus::Pending,
1150 }
1151 }
1152 existing
1153 };
1154 let _guard = PlacementIngestGuard(Arc::clone(&attempt));
1157 if let Some(resolver) = &self.record_resolver {
1158 if let Err(e) = self
1159 .placement_index
1160 .ingest_did(resolver.as_ref(), node.as_str())
1161 .await
1162 {
1163 tracing::warn!(
1164 node = %node,
1165 error = %e,
1166 "placement directory ingestion failed for live node (liveness still recorded)"
1167 );
1168 }
1169 }
1170 *attempt.lock() = PlacementIngestStatus::Complete;
1171 true
1172 }
1173
1174 pub fn new(
1180 signing_key: Arc<SigningKey>,
1181 jwt_verifying_key: hyprstream_rpc::prelude::VerifyingKey,
1182 transport: TransportConfig,
1183 ) -> Self {
1184 Self {
1185 started_at: Instant::now(),
1186 signing_key,
1187 jwt_verifying_key,
1188 jwt_key_source: None,
1189 oauth_issuer_url: None,
1190 expected_audience: None,
1191 auth_provider: None,
1192 record_resolver: None,
1193 accepted_state_source: None,
1194 state_store: MemoryStateStore::production_default(),
1195 tls_endorsement: Vec::new(),
1196 tls_domain: String::new(),
1197 placement_index: PlacementIndex::new(),
1198 placement_ingest_attempts: TtlCache::new(
1199 LIVENESS_CACHE_MAX_ENTRIES,
1200 LIVENESS_CACHE_REAP_BUDGET,
1201 ),
1202 transport,
1203 }
1204 }
1205
1206 pub fn with_state(mut self, state: DiscoveryState) -> Self {
1208 self.state_store = state.into_inner();
1209 self
1210 }
1211
1212 pub fn with_tls_endorsement(mut self, endorsement: Vec<u8>, domain: String) -> Self {
1218 self.tls_endorsement = endorsement;
1219 self.tls_domain = domain;
1220 self
1221 }
1222
1223 pub fn with_oauth_issuer(mut self, url: String) -> Self {
1225 self.oauth_issuer_url = Some(url);
1226 self
1227 }
1228
1229 pub fn with_expected_audience(mut self, audience: String) -> Self {
1231 self.expected_audience = Some(audience);
1232 self
1233 }
1234
1235 pub fn with_jwt_key_source(
1237 mut self,
1238 src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
1239 ) -> Self {
1240 self.jwt_key_source = Some(src);
1241 self
1242 }
1243
1244 pub fn with_auth_provider(mut self, provider: Box<dyn AuthorizationProvider>) -> Self {
1246 self.auth_provider = Some(provider);
1247 self
1248 }
1249
1250 pub fn with_record_resolver(mut self, resolver: Arc<dyn RecordResolver>) -> Self {
1252 self.record_resolver = Some(resolver);
1253 self
1254 }
1255
1256 #[cfg(test)]
1257 fn with_accepted_state_source(mut self, source: Arc<dyn AcceptedStateSource>) -> Self {
1258 self.accepted_state_source = Some(source);
1259 self
1260 }
1261
1262 #[cfg(not(target_arch = "wasm32"))]
1404 #[cfg_attr(
1407 not(any(feature = "rocksdb", feature = "postgres")),
1408 allow(unreachable_code, unused_variables)
1409 )]
1410 fn bootstrap_authenticated_process(
1411 authority: ProcessBootstrapAuthority,
1412 discovery_client: hyprstream_rpc_std::discovery_client::DiscoveryClient,
1413 records: &hyprstream_pds::rds::RdsConfig,
1414 ) -> Result<()> {
1415 let authority = {
1416 let mut state = PROCESS_BOOTSTRAP_AUTHORITY.lock();
1417 match std::mem::replace(&mut *state, ProcessBootstrapAuthorityState::Consumed) {
1418 ProcessBootstrapAuthorityState::Sealed => {}
1419 previous => {
1420 *state = previous;
1421 anyhow::bail!("authenticated Discovery bootstrap is unavailable or consumed");
1422 }
1423 }
1424 authority
1425 };
1426 let deployment_identity = match &authority.acceptance_identity {
1427 ProcessAcceptanceIdentity::Deployment(identity) => Some(identity.clone()),
1428 #[cfg(test)]
1429 ProcessAcceptanceIdentity::Test(_) => None,
1430 };
1431 let source: Arc<dyn AcceptedStateSource> = match authority.acceptance_identity {
1432 #[cfg(any(feature = "rocksdb", feature = "postgres"))]
1433 ProcessAcceptanceIdentity::Deployment(identity) => crate::checkpointed_pds::open_deployment_accepted_state_source(
1434 &authority.store_path,
1435 identity,
1436 records,
1437 authority.network_required,
1438 )?,
1439 #[cfg(not(any(feature = "rocksdb", feature = "postgres")))]
1442 ProcessAcceptanceIdentity::Deployment(_) => anyhow::bail!(
1443 "checkpointed PDS accepted-state authority requires the `rocksdb` or `postgres` feature"
1444 ),
1445 #[cfg(all(test, feature = "rocksdb"))]
1446 ProcessAcceptanceIdentity::Test(identity) => Arc::new(
1447 crate::checkpointed_pds::CheckpointedPdsAcceptedStateSource::open_test(
1448 &authority.store_path,
1449 identity,
1450 )?,
1451 ),
1452 #[cfg(all(test, not(feature = "rocksdb")))]
1453 ProcessAcceptanceIdentity::Test(_) => anyhow::bail!(
1454 "checkpointed PDS accepted-state authority requires the `rocksdb` feature"
1455 ),
1456 };
1457 PROCESS_ACCEPTED_STATE_SOURCE
1458 .set(Arc::clone(&source))
1459 .map_err(|_| anyhow::anyhow!("process Discovery authority is already consumed"))?;
1460 PROCESS_NATIVE_NETWORK_REQUIRED
1461 .set(authority.network_required)
1462 .map_err(|_| anyhow::anyhow!("process native profile is already installed"))?;
1463 if let Some(identity) = deployment_identity {
1464 PROCESS_REGISTRY_VERIFIER.set(identity).map_err(|_| {
1465 anyhow::anyhow!("deployment registry verifier is already installed")
1466 })?;
1467 }
1468 PRODUCTION_RESOLVER
1469 .set(Arc::new(DiscoveryServiceResolver {
1470 state_store: MemoryStateStore::production_default(),
1471 accepted_state_source: source,
1472 discovery_client: Some(discovery_client),
1473 }))
1474 .map_err(|_| anyhow::anyhow!("production service resolver is already installed"))?;
1475
1476 Ok(())
1477 }
1478
1479 pub fn attach_process_accepted_state_source(&mut self) -> Result<()> {
1482 self.accepted_state_source = Some(
1483 PROCESS_ACCEPTED_STATE_SOURCE
1484 .get()
1485 .cloned()
1486 .ok_or_else(|| anyhow::anyhow!("process Discovery authority is not installed"))?,
1487 );
1488 Ok(())
1489 }
1490
1491 #[cfg(test)]
1492 fn production_resolver(&self) -> Result<DiscoveryServiceResolver> {
1493 Ok(DiscoveryServiceResolver {
1494 state_store: Arc::clone(&self.state_store),
1495 accepted_state_source: self.accepted_state_source.clone().ok_or_else(|| {
1496 anyhow::anyhow!("Discovery accepted-state source is not installed")
1497 })?,
1498 discovery_client: None,
1499 })
1500 }
1501
1502 fn reg(&self) -> Result<impl std::ops::Deref<Target = EndpointRegistry> + '_> {
1504 registry::try_global().ok_or_else(|| anyhow::anyhow!("EndpointRegistry not initialized"))
1505 }
1506}
1507
1508#[async_trait]
1513impl Resolver for DiscoveryService {
1514 async fn resolve(&self, name: &str, kind: SocketKind) -> anyhow::Result<TransportConfig> {
1515 if let Some(transport) = self.resolve_announced_endpoint(name, kind).await? {
1516 return Ok(transport);
1517 }
1518
1519 if kind == SocketKind::Quic {
1522 anyhow::bail!("no fresh announced QUIC endpoint for service '{name}'");
1523 }
1524
1525 self.reg()?.try_endpoint(name, kind)
1526 }
1527}
1528
1529fn accepted_expiry_unix_ms(
1530 state: &hyprstream_pds::at9p_duplicity::AcceptedAt9pState,
1531) -> Result<i64> {
1532 let expiry = state.expires_at.as_deref().ok_or_else(|| {
1533 anyhow::anyhow!(
1534 "genesis-only accepted state has no bounded expiry; refusing production resolution"
1535 )
1536 })?;
1537 Ok(chrono::DateTime::parse_from_rfc3339(expiry)
1538 .map_err(|e| anyhow::anyhow!("invalid accepted-state expiry: {e}"))?
1539 .timestamp_millis())
1540}
1541
1542#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
1545pub(super) fn project_bootstrap_endpoint(
1546 states: &[hyprstream_pds::at9p_duplicity::AcceptedAt9pState],
1547 service_name: &str,
1548 key: &VerifyingKey,
1549) -> Result<AnnouncedEndpoint> {
1550 anyhow::ensure!(
1551 matches!(service_name, "discovery" | "policy"),
1552 "not a fixed bootstrap role"
1553 );
1554 let service_id = format!("#{service_name}");
1555 let mut matching = states.iter().filter(|state| {
1556 state
1557 .current
1558 .services
1559 .iter()
1560 .any(|entry| entry.id == service_id)
1561 && state
1562 .current
1563 .subject_keys
1564 .iter()
1565 .any(|subject| subject.ed25519_pub == key.as_bytes())
1566 });
1567 let state = matching
1568 .next()
1569 .ok_or_else(|| anyhow::anyhow!("no accepted bootstrap identity for {service_name}"))?;
1570 anyhow::ensure!(
1571 matching.next().is_none(),
1572 "ambiguous accepted bootstrap identity for {service_name}"
1573 );
1574 let expiry = accepted_expiry_unix_ms(state)?;
1575 anyhow::ensure!(
1576 state.epoch > 0 && expiry > unix_millis_now(),
1577 "bootstrap identity is unbounded or expired"
1578 );
1579 let entry = state
1580 .current
1581 .services
1582 .iter()
1583 .find(|entry| entry.id == service_id)
1584 .ok_or_else(|| anyhow::anyhow!("bootstrap service disappeared"))?;
1585 anyhow::ensure!(
1586 entry.endpoint.transport == hyprstream_pds::at9p::Transport::Iroh,
1587 "bootstrap service requires signed Iroh reach"
1588 );
1589 parse_announced_iroh(&entry.endpoint.address)?;
1590 let kem = entry.endpoint.request_kem.as_ref().ok_or_else(|| {
1591 anyhow::anyhow!("bootstrap service lacks signed request KEM; reprovision service identity")
1592 })?;
1593 let recipient = hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::decode(kem)?;
1594 recipient.validate()?;
1595 anyhow::ensure!(
1596 recipient.suite_id == hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
1597 "bootstrap requires hybrid KEM"
1598 );
1599 let subject_key = state
1600 .current
1601 .subject_keys
1602 .iter()
1603 .find(|subject| subject.ed25519_pub == key.as_bytes())
1604 .ok_or_else(|| anyhow::anyhow!("bootstrap response key disappeared"))?;
1605 hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(&subject_key.mldsa65_pub)?;
1606 Ok(AnnouncedEndpoint {
1607 socket_kind: "iroh".to_owned(),
1608 endpoint: entry.endpoint.address.clone(),
1609 service_did: Did::from(state.did.clone()),
1610 capabilities: ["hyprstream-rpc/1".to_owned()].into_iter().collect(),
1611 accepted_state_digest: state.head_digest.to_vec(),
1612 accepted_state_epoch: state.epoch,
1613 response_key_id: format!("{}#response", state.did),
1614 request_kem_key_id: format!("{}#mesh-kem", state.did),
1615 request_kem_recipient: kem.clone(),
1616 expires_at_unix_ms: expiry,
1617 source_signer: key.to_bytes(),
1618 live_until_unix_ms: expiry,
1619 })
1620}
1621
1622impl DiscoveryServiceResolver {
1623 #[cfg(test)]
1624 async fn resolve_service(&self, query: ServiceQuery) -> Result<ResolvedService> {
1625 let selected = select_service_candidate(
1626 &query,
1627 self.acquire_candidates(&query).await?,
1628 unix_millis_now(),
1629 )?;
1630 let state = self
1631 .accepted_state_source
1632 .accepted_state(selected.service_did().as_str())?
1633 .ok_or_else(|| anyhow::anyhow!("accepted state disappeared during resolution"))?;
1634 ResolvedService::mint(selected, &state)
1635 }
1636
1637 async fn acquire_candidates(&self, query: &ServiceQuery) -> Result<Vec<ServiceCandidate>> {
1638 let bootstrap = if query.profile == ResolverProfile::NativeIrohRequired {
1639 self.accepted_state_source
1640 .bootstrap_endpoints(&query.service_name)?
1641 } else {
1642 None
1643 };
1644 let entries = if let Some(entries) = bootstrap {
1645 entries
1646 } else if let Some(client) = &self.discovery_client {
1647 client
1648 .get_endpoints(&query.service_name)
1649 .await?
1650 .endpoints
1651 .into_iter()
1652 .filter_map(|endpoint| {
1653 let source_signer: [u8; 32] = endpoint.source_signer.try_into().ok()?;
1654 Some(AnnouncedEndpoint {
1655 socket_kind: endpoint.socket_kind,
1656 endpoint: endpoint.endpoint,
1657 service_did: endpoint.service_did,
1658 capabilities: endpoint.capabilities.into_iter().collect(),
1659 accepted_state_digest: endpoint.accepted_state_digest,
1660 accepted_state_epoch: endpoint.accepted_state_epoch,
1661 response_key_id: endpoint.response_key_id,
1662 request_kem_key_id: endpoint.request_kem_key_id,
1663 request_kem_recipient: endpoint.request_kem_recipient,
1664 expires_at_unix_ms: endpoint.expires_at_unix_ms,
1665 source_signer,
1666 live_until_unix_ms: endpoint
1667 .expires_at_unix_ms
1668 .min(unix_millis_now() + ANNOUNCED_ENDPOINT_TTL.as_millis() as i64),
1669 })
1670 })
1671 .collect()
1672 } else {
1673 self.state_store
1674 .announcements_for(&query.service_name, unix_millis_now())
1675 .await?
1676 };
1677
1678 let mut candidates = Vec::new();
1679 for entry in entries {
1680 if entry.service_did.as_str().is_empty() || entry.accepted_state_digest.len() != 64 {
1683 continue;
1684 }
1685 let Some(state) = self
1686 .accepted_state_source
1687 .accepted_state(entry.service_did.as_str())?
1688 else {
1689 continue;
1690 };
1691 if state.did != entry.service_did.as_str()
1692 || state.epoch != entry.accepted_state_epoch
1693 || state.head_digest.as_slice() != entry.accepted_state_digest.as_slice()
1694 {
1695 continue;
1696 }
1697 let Ok(state_expiry) = accepted_expiry_unix_ms(&state) else {
1698 continue;
1699 };
1700 let Some(current_key) = state
1701 .current
1702 .subject_keys
1703 .iter()
1704 .find(|key| key.ed25519_pub.as_slice() == entry.source_signer)
1705 else {
1706 continue;
1707 };
1708 let Ok(response_ed25519) = current_key.ed25519_pub.as_slice().try_into() else {
1709 continue;
1710 };
1711 let Ok(recipient) = hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::decode(
1712 &entry.request_kem_recipient,
1713 ) else {
1714 continue;
1715 };
1716 let Ok(transport) = announced_endpoint_to_transport(&entry) else {
1717 continue;
1718 };
1719 let mut digest = [0u8; 64];
1720 digest.copy_from_slice(&entry.accepted_state_digest);
1721 candidates.push(ServiceCandidate {
1722 service_name: query.service_name.clone(),
1723 service_did: entry.service_did,
1724 response_verifying_key: response_ed25519,
1725 response_ml_dsa65: current_key.mldsa65_pub.clone(),
1726 response_key_id: entry.response_key_id,
1727 request_kem_recipient: Some(AnchoredKemRecipient {
1728 key_id: entry.request_kem_key_id,
1729 recipient,
1730 not_after_unix_ms: entry.expires_at_unix_ms,
1731 }),
1732 transport,
1733 capabilities: entry.capabilities,
1734 accepted_state: AcceptedStateEvidence {
1735 service_did: Did::from(state.did),
1736 digest,
1737 epoch: state.epoch,
1738 expires_at_unix_ms: state_expiry,
1739 response_ed25519,
1740 response_ml_dsa65: current_key.mldsa65_pub.clone(),
1741 },
1742 source_signer: entry.source_signer,
1743 expires_at_unix_ms: entry.expires_at_unix_ms,
1744 });
1745 }
1746 Ok(candidates)
1747 }
1748}
1749
1750impl DiscoveryServiceResolver {
1751 async fn resolve_service_candidates(
1752 &self,
1753 query: ServiceQuery,
1754 ) -> Result<Vec<ResolvedService>> {
1755 let candidates = self.acquire_candidates(&query).await?;
1756 let selected = select_service_candidates(&query, candidates, unix_millis_now())?;
1757 let mut resolved = Vec::with_capacity(selected.len());
1758 for item in selected {
1759 let state = self
1760 .accepted_state_source
1761 .accepted_state(item.service_did().as_str())?
1762 .ok_or_else(|| anyhow::anyhow!("accepted state disappeared during resolution"))?;
1763 resolved.push(ResolvedService::mint(item, &state)?);
1764 }
1765 Ok(resolved)
1766 }
1767
1768 async fn ensure_current(&self, resolved: &ResolvedService) -> Result<()> {
1769 resolved.ensure_fresh(unix_millis_now())?;
1770 let state = self
1771 .accepted_state_source
1772 .accepted_state(resolved.service_did().as_str())?
1773 .ok_or_else(|| anyhow::anyhow!("accepted state disappeared; re-resolution required"))?;
1774 anyhow::ensure!(
1775 state.epoch == resolved.evidence().accepted_state_epoch
1776 && state.head_digest == resolved.evidence().accepted_state_digest,
1777 "accepted state advanced or forked; re-resolution required"
1778 );
1779 let state_expiry = accepted_expiry_unix_ms(&state)?;
1780 anyhow::ensure!(
1781 unix_millis_now() < state_expiry,
1782 "accepted state expired; re-resolution required"
1783 );
1784 Ok(())
1785 }
1786
1787 async fn browser_provisioning(
1788 &self,
1789 request: BrowserProvisioningRequest,
1790 ) -> Result<BrowserProvisioningDocument> {
1791 request.validate()?;
1792 match request.carrier_profile {
1793 BrowserCarrierProfile::OwnedHybridWebTransport => {
1794 anyhow::ensure!(
1795 request.capability == "hyprstream-rpc/1"
1796 && request.scope == request.service_name,
1797 "owned browser RPC provisioning requires hyprstream-rpc/1 and exact service scope"
1798 );
1799 }
1800 BrowserCarrierProfile::StandardPublicRelay => {
1801 anyhow::ensure!(
1802 request.capability == "hyprstream-moq/1",
1803 "standard-public-relay provisioning requires hyprstream-moq/1"
1804 );
1805 }
1806 }
1807
1808 let query = ServiceQuery::new(
1809 request.service_name.clone(),
1810 [request.capability.clone()],
1811 ResolverProfile::BrowserWebTransport,
1812 1,
1813 )?;
1814 let resolved = self
1815 .resolve_service_candidates(query)
1816 .await?
1817 .into_iter()
1818 .next()
1819 .ok_or_else(|| anyhow::anyhow!("resolver returned no validated browser reach"))?;
1820 self.ensure_current(&resolved).await?;
1821 let state = self
1822 .accepted_state_source
1823 .accepted_state(resolved.service_did().as_str())?
1824 .ok_or_else(|| {
1825 anyhow::anyhow!("accepted state disappeared during browser provisioning")
1826 })?;
1827 anyhow::ensure!(
1828 state.epoch == resolved.evidence().accepted_state_epoch
1829 && state.head_digest == resolved.evidence().accepted_state_digest,
1830 "accepted state advanced during browser provisioning; re-resolution required"
1831 );
1832 let accepted_service = state
1833 .current
1834 .services
1835 .iter()
1836 .find(|service| service.id == format!("#{}", request.service_name))
1837 .ok_or_else(|| {
1838 anyhow::anyhow!("accepted current state does not contain requested service")
1839 })?;
1840
1841 let (origin, route, certificate_hashes, route_role, transport_security, encrypted_objects) =
1842 match request.carrier_profile {
1843 BrowserCarrierProfile::OwnedHybridWebTransport => {
1844 let EndpointType::Quic {
1845 addr,
1846 server_name,
1847 auth,
1848 } = resolved.transport().endpoint.clone()
1849 else {
1850 anyhow::bail!(
1851 "browser provisioning requires selected QUIC/WebTransport network reach"
1852 );
1853 };
1854 let authority = if addr.port() == 443 {
1857 server_name.clone()
1858 } else {
1859 format!("{server_name}:{}", addr.port())
1860 };
1861 let origin = format!("https://{authority}/");
1862 let route = format!(
1863 "https://{authority}{}",
1864 hyprstream_rpc::browser_provisioning::BROWSER_RPC_PATH
1865 );
1866 (
1867 origin,
1868 route,
1869 auth.accept_cert_hashes().to_vec(),
1870 BrowserRouteRole::Origin,
1871 BrowserTransportSecurity::OwnedHybridRequired,
1872 false,
1873 )
1874 }
1875 BrowserCarrierProfile::StandardPublicRelay => {
1876 anyhow::ensure!(
1877 accepted_service.endpoint.transport == hyprstream_pds::at9p::Transport::Moq,
1878 "accepted service does not select the MoQT carrier"
1879 );
1880 let relay = accepted_service.endpoint.relay.clone().ok_or_else(|| {
1881 anyhow::anyhow!(
1882 "accepted service has no signed standard public relay route"
1883 )
1884 })?;
1885 (
1886 accepted_service.endpoint.address.clone(),
1887 relay,
1888 Vec::new(),
1889 BrowserRouteRole::Relay,
1890 BrowserTransportSecurity::ClassicalUntrusted,
1891 true,
1892 )
1893 }
1894 };
1895
1896 let now = unix_millis_now();
1897 let expires_at_unix_ms = resolved
1898 .expires_at_unix_ms()
1899 .min(now.saturating_add(30_000));
1900 anyhow::ensure!(
1901 expires_at_unix_ms > now,
1902 "resolved browser provisioning expired before projection"
1903 );
1904 Ok(BrowserProvisioningDocument::from_material(
1905 BrowserProvisioningMaterial {
1906 service_name: request.service_name,
1907 service_did: resolved.service_did().as_str().to_owned(),
1908 service_origin: origin,
1909 webtransport_url: route,
1910 capability: request.capability,
1911 scope: request.scope,
1912 carrier_profile: request.carrier_profile,
1913 route_role,
1914 transport_security,
1915 response_key_id: resolved.response_key_id().to_owned(),
1916 response_ed25519: resolved.response_verifying_key().to_bytes(),
1917 response_ml_dsa65: resolved.response_ml_dsa65().to_vec(),
1918 request_kem_key_id: resolved.request_kem_recipient().key_id.clone(),
1919 request_kem_recipient: resolved.request_kem_recipient().recipient.clone(),
1920 accepted_state_digest: resolved.evidence().accepted_state_digest,
1921 accepted_state_epoch: resolved.evidence().accepted_state_epoch,
1922 expires_at_unix_ms,
1923 certificate_hashes,
1924 encrypted_objects_required: encrypted_objects,
1925 },
1926 ))
1927 }
1928
1929 async fn verify_browser_binding(&self, binding: &BrowserRequestBinding) -> Result<()> {
1930 binding.validate_shape()?;
1931 let now = unix_millis_now();
1932 anyhow::ensure!(
1933 binding.expires_at_unix_ms > now,
1934 "sealed browser provisioning binding expired before dispatch"
1935 );
1936 let current = self
1937 .browser_provisioning(BrowserProvisioningRequest::new(
1938 binding.service_name.clone(),
1939 binding.capability.clone(),
1940 binding.scope.clone(),
1941 binding.carrier_profile,
1942 )?)
1943 .await?;
1944 anyhow::ensure!(
1945 binding.service_did == current.service_did
1946 && binding.service_origin == current.service_origin
1947 && binding.webtransport_url == current.webtransport_url
1948 && binding.certificate_hashes == current.certificate_hashes
1949 && binding.response_key_id == current.response_key_id
1950 && binding.request_kem_key_id == current.request_kem_key_id
1951 && binding.accepted_state_epoch == current.accepted_state_epoch
1952 && binding.accepted_state_digest == current.accepted_state_digest
1953 && binding.expires_at_unix_ms <= current.expires_at_unix_ms,
1954 "accepted-current browser authority changed before dispatch"
1955 );
1956 let response_key = URL_SAFE_NO_PAD
1957 .decode(¤t.response_ed25519)
1958 .context("invalid current response key projection")?;
1959 let kem = URL_SAFE_NO_PAD
1960 .decode(¤t.request_kem_recipient)
1961 .context("invalid current request KEM projection")?;
1962 anyhow::ensure!(
1963 binding.matches_response_key(&response_key)? && binding.matches_request_kem(&kem)?,
1964 "accepted-current browser key material changed before dispatch"
1965 );
1966 Ok(())
1967 }
1968}
1969
1970struct ProductionBrowserCurrentnessVerifier {
1971 resolver: Arc<DiscoveryServiceResolver>,
1972}
1973
1974#[async_trait]
1975impl BrowserCurrentnessVerifier for ProductionBrowserCurrentnessVerifier {
1976 async fn ensure_current(&self, binding: &BrowserRequestBinding) -> Result<()> {
1977 self.resolver.verify_browser_binding(binding).await
1978 }
1979}
1980
1981static PROCESS_ACCEPTED_STATE_SOURCE: std::sync::OnceLock<Arc<dyn AcceptedStateSource>> =
1982 std::sync::OnceLock::new();
1983static PRODUCTION_RESOLVER: std::sync::OnceLock<Arc<DiscoveryServiceResolver>> =
1984 std::sync::OnceLock::new();
1985static PROCESS_REGISTRY_VERIFIER: std::sync::OnceLock<RegistryDeploymentVerifier> =
1986 std::sync::OnceLock::new();
1987static PROCESS_NATIVE_NETWORK_REQUIRED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1988#[cfg(not(target_arch = "wasm32"))]
1989static PROCESS_BOOTSTRAP_CARRIER: std::sync::OnceLock<
1990 hyprstream_rpc::transport::iroh_substrate::IrohSubstrate,
1991> = std::sync::OnceLock::new();
1992
1993pub fn native_network_required() -> bool {
1995 PROCESS_NATIVE_NETWORK_REQUIRED
1996 .get()
1997 .copied()
1998 .unwrap_or(false)
1999}
2000
2001pub async fn production_moq_event_target() -> Result<(
2003 TransportConfig,
2004 hyprstream_rpc::stream_info::MoqlServerIdentity,
2005)> {
2006 let resolver = PRODUCTION_RESOLVER
2007 .get()
2008 .ok_or_else(|| anyhow::anyhow!("production resolver is not installed"))?;
2009 let resolved = resolver
2010 .resolve_service_candidates(ServiceQuery::network_moq("event")?)
2011 .await?
2012 .into_iter()
2013 .next()
2014 .ok_or_else(|| anyhow::anyhow!("no native Event reach"))?;
2015 resolver.ensure_current(&resolved).await?;
2016 Ok((
2017 resolved.transport().clone(),
2018 hyprstream_rpc::stream_info::MoqlServerIdentity {
2019 did: resolved.service_did().as_str().to_owned(),
2020 epoch: resolved.evidence().accepted_state_epoch,
2021 head_digest: resolved.evidence().accepted_state_digest.to_vec(),
2022 expires_at_unix_ms: resolved.expires_at_unix_ms(),
2023 ed25519: resolved.response_verifying_key().to_bytes(),
2024 ml_dsa65: resolved.response_ml_dsa65().to_vec(),
2025 },
2026 ))
2027}
2028
2029pub(crate) fn project_accepted_identity_state(
2034 state: &hyprstream_pds::at9p_duplicity::AcceptedAt9pState,
2035) -> Option<hyprstream_rpc::transport::moql_admission::AcceptedIdentityState> {
2036 use hyprstream_rpc::transport::moql_admission::{AcceptedIdentityState, AcceptedSubjectKey};
2037 let expires = chrono::DateTime::parse_from_rfc3339(state.expires_at.as_deref()?)
2038 .ok()?
2039 .timestamp_millis();
2040 let subject_keys = state
2041 .current
2042 .subject_keys
2043 .iter()
2044 .map(|key| {
2045 Some(AcceptedSubjectKey {
2046 ed25519: key.ed25519_pub.as_slice().try_into().ok()?,
2047 ml_dsa_65: (!key.mldsa65_pub.is_empty()).then(|| key.mldsa65_pub.clone())?,
2048 })
2049 })
2050 .collect::<Option<Vec<_>>>()?;
2051 let service_ids = state
2055 .current
2056 .services
2057 .iter()
2058 .map(|service| service.id.clone())
2059 .collect::<Vec<_>>();
2060 (!subject_keys.is_empty()).then_some(AcceptedIdentityState {
2061 epoch: state.epoch,
2062 head_digest: state.head_digest,
2063 subject_keys,
2064 service_ids,
2065 expires_at_unix_ms: Some(expires),
2066 })
2067}
2068
2069pub fn production_moql_accepted_state_authority(
2072) -> Result<Arc<dyn hyprstream_rpc::transport::moql_admission::AcceptedStateAuthority>> {
2073 let source = PROCESS_ACCEPTED_STATE_SOURCE
2074 .get()
2075 .cloned()
2076 .ok_or_else(|| anyhow::anyhow!("production accepted-state authority is not installed"))?;
2077 Ok(Arc::new(move |did: &str| {
2078 let state = source.accepted_state(did).ok().flatten()?;
2079 project_accepted_identity_state(&state)
2080 }))
2081}
2082
2083pub fn production_deployment_roster(
2091) -> Result<Arc<dyn crate::admission_roster::DeploymentRosterSource>> {
2092 let source = PROCESS_ACCEPTED_STATE_SOURCE
2093 .get()
2094 .cloned()
2095 .ok_or_else(|| anyhow::anyhow!("production accepted-state authority is not installed"))?;
2096 let load: std::sync::Arc<dyn Fn() -> Vec<crate::admission_roster::RosterEntry> + Send + Sync> =
2097 Arc::new(move || {
2098 source
2099 .accepted_states()
2100 .ok()
2101 .map(|states| {
2102 states
2103 .iter()
2104 .filter_map(|state| {
2105 project_accepted_identity_state(state)
2106 .map(|projected| (state.did.clone(), projected))
2107 })
2108 .collect::<Vec<_>>()
2109 })
2110 .unwrap_or_default()
2111 });
2112 Ok(Arc::new(crate::admission_roster::CachedRoster::new(
2113 load,
2114 crate::admission_roster::ROSTER_CACHE_TTL,
2115 )))
2116}
2117
2118const DEPLOYMENT_CA_ROOT_PATH: &str = "/etc/hyprstream/trust/deployment-ca.hybrid";
2119const DEPLOYMENT_AUTHORITY_LOG_PATH: &str = "/etc/hyprstream/trust/deployment-authority.log.json";
2120const DEPLOYMENT_AUTHORITY_CHECKPOINT_PATH: &str =
2121 "/etc/hyprstream/trust/deployment-authority.head.json";
2122const REGISTRY_DEPLOYMENT_CREDENTIAL_PATH: &str =
2123 "/run/hyprstream/credentials/registry-service.jwt";
2124const DEPLOYMENT_TRUST_DIR_ENV: &str = "HYPRSTREAM_DEPLOYMENT_TRUST_DIR";
2125const SYSTEMD_CREDENTIALS_DIRECTORY_ENV: &str = "CREDENTIALS_DIRECTORY";
2126const DEPLOYMENT_CA_ROOT_FILE: &str = "deployment-ca.hybrid";
2127const DEPLOYMENT_AUTHORITY_LOG_FILE: &str = "deployment-authority.log.json";
2128const DEPLOYMENT_AUTHORITY_CHECKPOINT_FILE: &str = "deployment-authority.head.json";
2129const REGISTRY_DEPLOYMENT_CREDENTIAL_FILE: &str = "registry-service.jwt";
2130const REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE: &str = "hyprstream.registry-deployment.v1";
2131const REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE: &str = "urn:hyprstream:service:registry";
2132const REGISTRY_DEPLOYMENT_CREDENTIAL_MAX_TTL_SECONDS: i64 = 3_600;
2133const REGISTRY_DEPLOYMENT_CREDENTIAL_CLOCK_SKEW_SECONDS: i64 = 60;
2134const ED25519_PUBLIC_KEY_BYTES: usize = 32;
2135const ML_DSA_65_PUBLIC_KEY_BYTES: usize = 1_952;
2136const DEPLOYMENT_CA_ROOT_BYTES: usize = ED25519_PUBLIC_KEY_BYTES + ML_DSA_65_PUBLIC_KEY_BYTES;
2137const HYBRID_JWT_SIGNATURE_BYTES: usize = 3_309 + 64;
2138const AUTHORITY_LOG_SCHEMA: &str = "hyprstream.deployment-authority-log.v1";
2139const AUTHORITY_CHECKPOINT_SCHEMA: &str = "hyprstream.deployment-authority-checkpoint.v1";
2140const REGISTRY_DELEGATION_SCHEMA: &str = "hyprstream.registry-delegation.v1";
2141const MAX_AUTHORITY_LOG_OPERATIONS: usize = 128;
2142const MAX_REGISTRY_DELEGATION_BYTES: usize = 256 * 1024;
2143const MAX_DEPLOYMENT_CLOUD_SECRET_BYTES: usize = 64 * 1024;
2144const REGISTRY_DELEGATION_ABILITY: &str = "mint-registry-jwt";
2145const SERVICE_KEY_ENROLLMENT_SCHEMA: &str = "hyprstream.service-key-enrollment.v1";
2146const SERVICE_KEY_ENROLLMENT_ABILITY: &str = "enroll-service-key";
2147const SERVICE_KEY_ENROLLMENT_KEY_TYPE: &str = "hybrid-ed25519-mldsa65";
2148const SERVICE_KEY_ENROLLMENT_MAX_TTL_SECONDS: i64 = 3_600;
2149const SERVICE_KEY_ENROLLMENT_SIGNATURE_CONTEXT: &[u8] = b"hyprstream.service-key-enrollment.v1";
2151pub const SERVICE_KEY_ENROLLMENT_ALLOWED_SERVICES: [&str; 2] = ["discovery", "policy"];
2155const MAX_SERVICE_KEY_ENROLLMENT_BYTES: usize = 256 * 1024;
2156
2157#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2159#[serde(deny_unknown_fields)]
2160pub struct DeploymentAuthorityLog {
2161 pub schema: String,
2162 pub deployment_domain: String,
2163 pub did: String,
2164 pub operations_b64: Vec<String>,
2165}
2166
2167#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2173#[serde(deny_unknown_fields)]
2174pub struct DeploymentAuthorityCheckpoint {
2175 pub schema: String,
2176 pub deployment_domain: String,
2177 pub did: String,
2178 pub sequence: u64,
2179 pub head_cid: String,
2180}
2181
2182#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2184#[serde(deny_unknown_fields)]
2185pub struct RegistryDelegationArtifact {
2186 pub schema: String,
2187 pub deployment_domain: String,
2188 pub authority_log_did: String,
2189 pub delegated_public_key_b64: String,
2190 pub ucan_b64: String,
2191}
2192
2193#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2198#[serde(deny_unknown_fields)]
2199pub struct ServiceKeyEnrollmentArtifact {
2200 pub schema: String,
2201 pub deployment_domain: String,
2202 pub service: String,
2203 pub hybrid_public_key_b64: String,
2206 pub not_before: i64,
2207 pub expires_at: i64,
2208 pub delegation: RegistryDelegationArtifact,
2210 pub signature_b64: String,
2213}
2214
2215#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
2219#[serde(deny_unknown_fields)]
2220pub struct ServiceKeyEnrollmentSigningBody {
2221 pub schema: String,
2222 pub deployment_domain: String,
2223 pub service: String,
2224 pub hybrid_public_key_b64: String,
2225 pub not_before: i64,
2226 pub expires_at: i64,
2227 pub delegation: RegistryDelegationArtifact,
2228}
2229
2230impl ServiceKeyEnrollmentArtifact {
2231 pub fn unsigned(body: ServiceKeyEnrollmentSigningBody) -> Self {
2233 Self {
2234 schema: body.schema,
2235 deployment_domain: body.deployment_domain,
2236 service: body.service,
2237 hybrid_public_key_b64: body.hybrid_public_key_b64,
2238 not_before: body.not_before,
2239 expires_at: body.expires_at,
2240 delegation: body.delegation,
2241 signature_b64: String::new(),
2242 }
2243 }
2244
2245 pub fn signing_bytes(&self) -> Result<Vec<u8>> {
2247 let body = ServiceKeyEnrollmentSigningBody {
2248 schema: self.schema.clone(),
2249 deployment_domain: self.deployment_domain.clone(),
2250 service: self.service.clone(),
2251 hybrid_public_key_b64: self.hybrid_public_key_b64.clone(),
2252 not_before: self.not_before,
2253 expires_at: self.expires_at,
2254 delegation: self.delegation.clone(),
2255 };
2256 serde_json::to_vec(&body)
2257 .map_err(|error| anyhow::anyhow!("encoding enrollment signing body: {error}"))
2258 }
2259}
2260
2261#[derive(Clone)]
2267pub(crate) struct HybridDeploymentCa {
2268 ed25519: VerifyingKey,
2269 ml_dsa_65: hyprstream_rpc::crypto::pq::MlDsaVerifyingKey,
2270}
2271
2272impl HybridDeploymentCa {
2273 pub(crate) fn from_public_key_bytes(ed25519: &[u8], ml_dsa_65: &[u8]) -> Result<Self> {
2274 let ed25519: [u8; ED25519_PUBLIC_KEY_BYTES] = ed25519
2275 .try_into()
2276 .map_err(|_| anyhow::anyhow!("deployment CA Ed25519 key must be 32 bytes"))?;
2277 let ed25519 = VerifyingKey::from_bytes(&ed25519)
2278 .map_err(|error| anyhow::anyhow!("deployment CA Ed25519 key is malformed: {error}"))?;
2279 let ml_dsa_65 =
2280 hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(ml_dsa_65).map_err(|error| {
2281 anyhow::anyhow!("deployment CA ML-DSA-65 key is malformed: {error}")
2282 })?;
2283 Ok(Self { ed25519, ml_dsa_65 })
2284 }
2285
2286 fn from_os_pin(bytes: &[u8]) -> Result<Self> {
2287 anyhow::ensure!(
2288 bytes.len() == DEPLOYMENT_CA_ROOT_BYTES,
2289 "deployment CA root must be exactly {DEPLOYMENT_CA_ROOT_BYTES} bytes \
2290 (32-byte Ed25519 followed by 1952-byte ML-DSA-65)"
2291 );
2292 let (ed25519, ml_dsa_65) = bytes.split_at(ED25519_PUBLIC_KEY_BYTES);
2293 Self::from_public_key_bytes(ed25519, ml_dsa_65)
2294 }
2295
2296 fn domain(&self) -> String {
2297 hyprstream_rpc::auth::composite_kid(&self.ml_dsa_65, &self.ed25519)
2298 }
2299
2300 pub(crate) fn ed25519_bytes(&self) -> [u8; ED25519_PUBLIC_KEY_BYTES] {
2301 self.ed25519.to_bytes()
2302 }
2303
2304 pub(crate) fn ml_dsa_65_bytes(&self) -> Vec<u8> {
2305 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&self.ml_dsa_65)
2306 }
2307}
2308
2309#[derive(Clone)]
2314pub struct RegistryDeploymentVerifier {
2315 verifying_key: VerifyingKey,
2316 _deployment_ca: Arc<HybridDeploymentCa>,
2319}
2320
2321impl RegistryDeploymentVerifier {
2322 pub fn verify_strict(
2323 &self,
2324 message: &[u8],
2325 signature: &ed25519_dalek::Signature,
2326 ) -> Result<()> {
2327 self.verifying_key
2328 .verify_strict(message, signature)
2329 .map_err(|error| anyhow::anyhow!("deployment registry signature rejected: {error}"))
2330 }
2331
2332 pub fn matches(&self, key: &VerifyingKey) -> bool {
2333 self.verifying_key == *key
2334 }
2335
2336 pub(crate) fn deployment_did_op_key(&self) -> Result<crate::did_op::HybridRotationKey> {
2343 crate::did_op::HybridRotationKey::new(
2344 self._deployment_ca.ed25519.to_bytes(),
2345 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&self._deployment_ca.ml_dsa_65),
2346 )
2347 }
2348
2349 #[cfg(test)]
2350 pub(crate) fn for_test_deployment_root(
2351 ed25519: &SigningKey,
2352 ml_dsa_65: &hyprstream_rpc::crypto::pq::MlDsaSigningKey,
2353 ) -> Result<Self> {
2354 let root = HybridDeploymentCa::from_public_key_bytes(
2355 ed25519.verifying_key().as_bytes(),
2356 &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(ml_dsa_65),
2357 )?;
2358 Ok(Self {
2359 verifying_key: ed25519.verifying_key(),
2360 _deployment_ca: Arc::new(root),
2361 })
2362 }
2363
2364 #[cfg(test)]
2365 fn matches_deployment_root(&self, root: &HybridDeploymentCa) -> bool {
2366 self._deployment_ca.domain() == root.domain()
2367 }
2368}
2369
2370pub fn deployment_registry_verifier() -> Result<RegistryDeploymentVerifier> {
2372 PROCESS_REGISTRY_VERIFIER
2373 .get()
2374 .cloned()
2375 .ok_or_else(|| anyhow::anyhow!("deployment registry verifier is not installed"))
2376}
2377
2378pub fn authenticate_local_deployment_registry() -> Result<RegistryDeploymentVerifier> {
2383 authenticate_registry_deployment_credentials(load_trusted_registry_deployment_credentials()?)
2384 .map(|identity| identity.verifier)
2385}
2386
2387struct AuthenticatedRegistryDeploymentIdentity {
2389 verifier: RegistryDeploymentVerifier,
2390}
2391
2392struct TrustedRegistryDeploymentCredentials {
2393 ca_verifying_key: HybridDeploymentCa,
2394 authority_log: DeploymentAuthorityLog,
2395 authority_checkpoint: DeploymentAuthorityCheckpoint,
2396 registry_credential: String,
2397}
2398
2399enum DeploymentAuthorityState<'a> {
2400 GenesisOnly,
2403 Enrolled {
2406 authority_log: &'a DeploymentAuthorityLog,
2407 authority_checkpoint: &'a DeploymentAuthorityCheckpoint,
2408 },
2409}
2410
2411struct UniqueJson(serde_json::Value);
2416
2417impl<'de> serde::Deserialize<'de> for UniqueJson {
2418 fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
2419 where
2420 D: serde::Deserializer<'de>,
2421 {
2422 struct Visitor;
2423
2424 impl<'de> serde::de::Visitor<'de> for Visitor {
2425 type Value = UniqueJson;
2426
2427 fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2428 formatter.write_str("JSON without duplicate object members")
2429 }
2430
2431 fn visit_unit<E>(self) -> std::result::Result<Self::Value, E> {
2432 Ok(UniqueJson(serde_json::Value::Null))
2433 }
2434
2435 fn visit_bool<E>(self, value: bool) -> std::result::Result<Self::Value, E> {
2436 Ok(UniqueJson(serde_json::Value::Bool(value)))
2437 }
2438
2439 fn visit_i64<E>(self, value: i64) -> std::result::Result<Self::Value, E> {
2440 Ok(UniqueJson(serde_json::Value::Number(value.into())))
2441 }
2442
2443 fn visit_u64<E>(self, value: u64) -> std::result::Result<Self::Value, E> {
2444 Ok(UniqueJson(serde_json::Value::Number(value.into())))
2445 }
2446
2447 fn visit_f64<E>(self, value: f64) -> std::result::Result<Self::Value, E>
2448 where
2449 E: serde::de::Error,
2450 {
2451 serde_json::Number::from_f64(value)
2452 .map(serde_json::Value::Number)
2453 .map(UniqueJson)
2454 .ok_or_else(|| E::custom("non-finite JSON number"))
2455 }
2456
2457 fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E> {
2458 Ok(UniqueJson(serde_json::Value::String(value.to_owned())))
2459 }
2460
2461 fn visit_string<E>(self, value: String) -> std::result::Result<Self::Value, E> {
2462 Ok(UniqueJson(serde_json::Value::String(value)))
2463 }
2464
2465 fn visit_none<E>(self) -> std::result::Result<Self::Value, E>
2466 where
2467 E: serde::de::Error,
2468 {
2469 self.visit_unit()
2470 }
2471
2472 fn visit_some<D>(self, deserializer: D) -> std::result::Result<Self::Value, D::Error>
2473 where
2474 D: serde::Deserializer<'de>,
2475 {
2476 <UniqueJson as serde::Deserialize>::deserialize(deserializer)
2477 }
2478
2479 fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
2480 where
2481 A: serde::de::SeqAccess<'de>,
2482 {
2483 let mut values = Vec::new();
2484 while let Some(value) = seq.next_element::<UniqueJson>()? {
2485 values.push(value.0);
2486 }
2487 Ok(UniqueJson(serde_json::Value::Array(values)))
2488 }
2489
2490 fn visit_map<A>(self, mut map: A) -> std::result::Result<Self::Value, A::Error>
2491 where
2492 A: serde::de::MapAccess<'de>,
2493 {
2494 let mut values = serde_json::Map::new();
2495 while let Some((key, value)) = map.next_entry::<String, UniqueJson>()? {
2496 if values.insert(key.clone(), value.0).is_some() {
2497 return Err(serde::de::Error::custom(format!(
2498 "duplicate JSON member {key:?}"
2499 )));
2500 }
2501 }
2502 Ok(UniqueJson(serde_json::Value::Object(values)))
2503 }
2504 }
2505
2506 deserializer.deserialize_any(Visitor)
2507 }
2508}
2509
2510fn exact_json_object<'a>(
2511 value: &'a serde_json::Value,
2512 expected: &[&str],
2513 description: &str,
2514) -> Result<&'a serde_json::Map<String, serde_json::Value>> {
2515 let object = value
2516 .as_object()
2517 .ok_or_else(|| anyhow::anyhow!("{description} must be a JSON object"))?;
2518 for member in object.keys() {
2519 anyhow::ensure!(
2520 expected.contains(&member.as_str()),
2521 "{description} contains unexpected member {member:?}"
2522 );
2523 }
2524 for member in expected {
2525 anyhow::ensure!(
2526 object.contains_key(*member),
2527 "{description} is missing member {member:?}"
2528 );
2529 }
2530 Ok(object)
2531}
2532
2533fn exact_string_member<'a>(
2534 object: &'a serde_json::Map<String, serde_json::Value>,
2535 member: &str,
2536 description: &str,
2537) -> Result<&'a str> {
2538 object[member]
2539 .as_str()
2540 .ok_or_else(|| anyhow::anyhow!("{description}.{member} must be a string"))
2541}
2542
2543fn exact_i64_member(
2544 object: &serde_json::Map<String, serde_json::Value>,
2545 member: &str,
2546 description: &str,
2547) -> Result<i64> {
2548 object[member]
2549 .as_i64()
2550 .ok_or_else(|| anyhow::anyhow!("{description}.{member} must be an integer"))
2551}
2552
2553fn decode_canonical_jwt_segment(segment: &str, description: &str, limit: usize) -> Result<Vec<u8>> {
2554 anyhow::ensure!(!segment.is_empty(), "{description} is empty");
2555 anyhow::ensure!(segment.len() <= limit, "{description} is too large");
2556 let decoded = URL_SAFE_NO_PAD
2557 .decode(segment)
2558 .map_err(|error| anyhow::anyhow!("{description} is not unpadded base64url: {error}"))?;
2559 anyhow::ensure!(
2560 URL_SAFE_NO_PAD.encode(&decoded) == segment,
2561 "{description} is not canonical unpadded base64url"
2562 );
2563 Ok(decoded)
2564}
2565
2566fn parse_unique_jwt_json(
2567 segment: &str,
2568 description: &str,
2569 limit: usize,
2570) -> Result<serde_json::Value> {
2571 let bytes = decode_canonical_jwt_segment(segment, description, limit)?;
2572 let mut deserializer = serde_json::Deserializer::from_slice(&bytes);
2573 let value = <UniqueJson as serde::Deserialize>::deserialize(&mut deserializer)
2574 .map_err(|error| anyhow::anyhow!("{description} is malformed: {error}"))?;
2575 deserializer
2576 .end()
2577 .map_err(|error| anyhow::anyhow!("{description} has trailing data: {error}"))?;
2578 Ok(value.0)
2579}
2580
2581fn deployment_domain(ca_verifying_key: &HybridDeploymentCa) -> String {
2582 ca_verifying_key.domain()
2583}
2584
2585fn validate_registry_credential_numeric_dates(
2586 exp: i64,
2587 nbf: i64,
2588 iat: i64,
2589 now: i64,
2590) -> Result<()> {
2591 for (name, value) in [("exp", exp), ("nbf", nbf), ("iat", iat)] {
2592 anyhow::ensure!(
2593 value >= 0,
2594 "credential {name} must not precede the Unix epoch"
2595 );
2596 }
2597
2598 anyhow::ensure!(exp > now, "credential has expired");
2599 let latest_future_time = now
2600 .checked_add(REGISTRY_DEPLOYMENT_CREDENTIAL_CLOCK_SKEW_SECONDS)
2601 .ok_or_else(|| anyhow::anyhow!("credential clock-skew boundary overflow"))?;
2602 anyhow::ensure!(nbf <= latest_future_time, "credential is not yet valid");
2603 anyhow::ensure!(
2604 iat <= latest_future_time,
2605 "credential was issued in the future"
2606 );
2607 anyhow::ensure!(nbf <= exp, "credential nbf is after exp");
2608 anyhow::ensure!(nbf <= iat, "credential nbf is after iat");
2609 anyhow::ensure!(iat < exp, "credential iat is not before exp");
2610 let lifetime = exp
2611 .checked_sub(iat)
2612 .ok_or_else(|| anyhow::anyhow!("credential lifetime arithmetic overflow"))?;
2613 anyhow::ensure!(
2614 lifetime <= REGISTRY_DEPLOYMENT_CREDENTIAL_MAX_TTL_SECONDS,
2615 "credential lifetime exceeds the inclusive one-hour profile limit"
2616 );
2617 Ok(())
2618}
2619
2620fn validate_registry_deployment_credential_profile(
2621 token: &str,
2622 ca_verifying_key: &HybridDeploymentCa,
2623 authority_state: DeploymentAuthorityState<'_>,
2624) -> Result<([u8; 32], i64)> {
2625 anyhow::ensure!(
2626 !token.is_empty() && !token.bytes().any(|byte| byte.is_ascii_whitespace()),
2627 "registry credential must be compact with no whitespace"
2628 );
2629 anyhow::ensure!(
2630 token.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
2631 "registry credential exceeds the 64 KiB cloud-secret contract"
2632 );
2633 let mut segments = token.split('.');
2634 let protected_segment = segments.next().unwrap_or_default();
2635 let claims_segment = segments.next().unwrap_or_default();
2636 let signature_segment = segments.next().unwrap_or_default();
2637 anyhow::ensure!(
2638 segments.next().is_none(),
2639 "credential must contain exactly three segments"
2640 );
2641 let signature = decode_canonical_jwt_segment(signature_segment, "credential signature", 8_192)?;
2642 anyhow::ensure!(
2643 signature.len() == HYBRID_JWT_SIGNATURE_BYTES,
2644 "credential signature must contain both ML-DSA-65 and Ed25519 components"
2645 );
2646
2647 let protected = parse_unique_jwt_json(protected_segment, "protected header", 4_096)?;
2648 let protected = exact_json_object(&protected, &["alg", "typ", "kid"], "protected header")?;
2649 anyhow::ensure!(
2650 exact_string_member(protected, "alg", "protected header")? == "ML-DSA-65-Ed25519",
2651 "protected header alg must be exactly ML-DSA-65-Ed25519"
2652 );
2653 anyhow::ensure!(
2654 exact_string_member(protected, "typ", "protected header")? == "wit+jwt",
2655 "protected header typ must be exactly wit+jwt"
2656 );
2657 let domain = deployment_domain(ca_verifying_key);
2658
2659 let claims = parse_unique_jwt_json(
2660 claims_segment,
2661 "credential claims",
2662 MAX_REGISTRY_DELEGATION_BYTES + 16_384,
2663 )?;
2664 let claims_object = claims
2665 .as_object()
2666 .ok_or_else(|| anyhow::anyhow!("credential claims must be a JSON object"))?;
2667 let delegated = claims_object.contains_key("delegation");
2668 let direct_members = [
2669 "iss",
2670 "sub",
2671 "aud",
2672 "exp",
2673 "nbf",
2674 "iat",
2675 "deployment_domain",
2676 "profile",
2677 "cnf",
2678 ];
2679 let delegated_members = [
2680 "iss",
2681 "sub",
2682 "aud",
2683 "exp",
2684 "nbf",
2685 "iat",
2686 "deployment_domain",
2687 "profile",
2688 "cnf",
2689 "delegation",
2690 ];
2691 let claims = exact_json_object(
2692 &claims,
2693 if delegated {
2694 &delegated_members
2695 } else {
2696 &direct_members
2697 },
2698 "credential claims",
2699 )?;
2700 anyhow::ensure!(
2701 exact_string_member(claims, "sub", "credential claims")? == "service:registry",
2702 "credential subject must be exactly service:registry"
2703 );
2704 let intended_issuer = format!("urn:hyprstream:deployment:{domain}");
2705 anyhow::ensure!(
2706 exact_string_member(claims, "iss", "credential claims")? == intended_issuer,
2707 "credential issuer does not bind the pinned deployment domain"
2708 );
2709 anyhow::ensure!(
2710 exact_string_member(claims, "aud", "credential claims")?
2711 == REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE,
2712 "credential audience must be exactly {REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE}"
2713 );
2714 anyhow::ensure!(
2715 exact_string_member(claims, "deployment_domain", "credential claims")? == domain,
2716 "credential deployment_domain does not bind the pinned deployment CA"
2717 );
2718 anyhow::ensure!(
2719 exact_string_member(claims, "profile", "credential claims")?
2720 == REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE,
2721 "credential profile must be exactly {REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE}"
2722 );
2723
2724 let now = chrono::Utc::now().timestamp();
2725 let exp = exact_i64_member(claims, "exp", "credential claims")?;
2726 let nbf = exact_i64_member(claims, "nbf", "credential claims")?;
2727 let iat = exact_i64_member(claims, "iat", "credential claims")?;
2728 validate_registry_credential_numeric_dates(exp, nbf, iat, now)?;
2729
2730 let cnf = exact_json_object(&claims["cnf"], &["jwk"], "credential claims.cnf")?;
2731 let jwk = exact_json_object(
2732 &cnf["jwk"],
2733 &["kty", "crv", "x"],
2734 "credential claims.cnf.jwk",
2735 )?;
2736 anyhow::ensure!(
2737 exact_string_member(jwk, "kty", "credential claims.cnf.jwk")? == "OKP",
2738 "credential cnf.jwk.kty must be exactly OKP"
2739 );
2740 anyhow::ensure!(
2741 exact_string_member(jwk, "crv", "credential claims.cnf.jwk")? == "Ed25519",
2742 "credential cnf.jwk.crv must be exactly Ed25519"
2743 );
2744 let x = exact_string_member(jwk, "x", "credential claims.cnf.jwk")?;
2745 let key = decode_canonical_jwt_segment(x, "credential cnf.jwk.x", 64)?;
2746 let key: [u8; 32] = key
2747 .try_into()
2748 .map_err(|_| anyhow::anyhow!("credential cnf.jwk.x must decode to exactly 32 bytes"))?;
2749
2750 let enrolled = match authority_state {
2751 DeploymentAuthorityState::GenesisOnly => None,
2752 DeploymentAuthorityState::Enrolled {
2753 authority_log,
2754 authority_checkpoint,
2755 } => Some((
2756 authority_log,
2757 validate_deployment_authority_log(
2758 ca_verifying_key,
2759 authority_log,
2760 authority_checkpoint,
2761 )?,
2762 )),
2763 };
2764 let delegated_ca;
2765 let signing_ca = if delegated {
2766 let (installed_authority_log, active) = enrolled.as_ref().ok_or_else(|| {
2767 anyhow::anyhow!(
2768 "delegated credential is forbidden during one-time genesis verification"
2769 )
2770 })?;
2771 let encoded = exact_string_member(claims, "delegation", "credential claims")?;
2772 let bytes = decode_canonical_jwt_segment(
2773 encoded,
2774 "credential delegation",
2775 MAX_REGISTRY_DELEGATION_BYTES,
2776 )?;
2777 let artifact: RegistryDelegationArtifact = serde_json::from_slice(&bytes)
2778 .map_err(|error| anyhow::anyhow!("credential delegation is malformed: {error}"))?;
2779 delegated_ca = validate_registry_delegation_artifact(
2780 ca_verifying_key,
2781 &artifact,
2782 installed_authority_log,
2783 active,
2784 u64::try_from(now).map_err(|_| anyhow::anyhow!("system clock precedes Unix epoch"))?,
2785 )?;
2786 &delegated_ca.delegated
2787 } else if let Some((_installed_authority_log, active)) = enrolled.as_ref() {
2788 anyhow::ensure!(
2789 active.rotation_keys.iter().any(|key| {
2790 key.ed25519_pub == ca_verifying_key.ed25519.to_bytes()
2791 && key.mldsa65_pub
2792 == hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&ca_verifying_key.ml_dsa_65)
2793 }),
2794 "direct root credential issuer is retired at the installed authority-log head"
2795 );
2796 ca_verifying_key
2797 } else {
2798 ca_verifying_key
2799 };
2800 anyhow::ensure!(
2801 exact_string_member(protected, "kid", "protected header")? == signing_ca.domain(),
2802 "protected header kid does not bind the authorized deployment signer"
2803 );
2804
2805 let dispatch = hyprstream_rpc::auth::jwt::parse_composite_dispatch(token, &["wit+jwt"])
2808 .map_err(|error| anyhow::anyhow!("credential composite dispatch rejected: {error}"))?;
2809 let verified = hyprstream_rpc::auth::jwt::decode_composite(
2810 token,
2811 &signing_ca.ml_dsa_65,
2812 &signing_ca.ed25519,
2813 Some(REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE),
2814 &dispatch,
2815 )
2816 .map_err(|error| anyhow::anyhow!("credential hybrid signature rejected: {error}"))?;
2817 anyhow::ensure!(
2818 verified.sub == "service:registry" && verified.cnf_key_bytes() == Some(key),
2819 "verified credential differs from the exact deployment profile"
2820 );
2821 Ok((key, exp))
2822}
2823
2824struct AuthoritySetUcanVerifier<'a> {
2825 keys: &'a [crate::did_op::HybridRotationKey],
2826}
2827
2828impl hyprstream_rpc::auth::ucan::UcanVerifier for AuthoritySetUcanVerifier<'_> {
2829 fn verify(
2830 &self,
2831 _issuer: &hyprstream_rpc::auth::ucan::Did,
2832 ed_key: &[u8; ED25519_PUBLIC_KEY_BYTES],
2833 payload: &[u8],
2834 signature: &[u8],
2835 ) -> std::result::Result<(), hyprstream_rpc::auth::ucan::UcanError> {
2836 use hyprstream_rpc::auth::ucan::UcanError;
2837 let authority = self
2838 .keys
2839 .iter()
2840 .find(|key| &key.ed25519_pub == ed_key)
2841 .ok_or_else(|| {
2842 UcanError::BadSignature("UCAN issuer is not in the active authority set".to_owned())
2843 })?;
2844 let (ed, pq) = authority
2845 .verifying_keys()
2846 .map_err(|error| UcanError::BadSignature(error.to_string()))?;
2847 hyprstream_rpc::crypto::cose_sign::verify_composite(
2848 signature,
2849 &ed,
2850 Some(&pq),
2851 payload,
2852 hyprstream_rpc::auth::ucan::token::UCAN_AAD,
2853 true,
2854 )
2855 .map(|_| ())
2856 .map_err(|error| UcanError::BadSignature(error.to_string()))
2857 }
2858}
2859
2860pub fn registry_mint_capability(
2866 deployment_domain: &str,
2867 delegated_public_key_b64: &str,
2868) -> hyprstream_rpc::auth::ucan::Capability {
2869 use hyprstream_rpc::auth::ucan::{Ability, Capability, CaveatValue, Caveats, Resource};
2870
2871 let mut caveats = std::collections::BTreeMap::new();
2872 caveats.insert(
2873 "audience".to_owned(),
2874 CaveatValue::Text(REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE.to_owned()),
2875 );
2876 caveats.insert(
2877 "deployment_domain".to_owned(),
2878 CaveatValue::Text(deployment_domain.to_owned()),
2879 );
2880 caveats.insert(
2881 "delegated_public_key_b64".to_owned(),
2882 CaveatValue::Text(delegated_public_key_b64.to_owned()),
2883 );
2884 caveats.insert(
2885 "max_ttl_seconds".to_owned(),
2886 CaveatValue::Int(REGISTRY_DEPLOYMENT_CREDENTIAL_MAX_TTL_SECONDS),
2887 );
2888 caveats.insert(
2889 "profile".to_owned(),
2890 CaveatValue::Text(REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE.to_owned()),
2891 );
2892 Capability::with_caveats(
2893 Resource::new(format!(
2894 "hyprstream://deployment/{deployment_domain}/service/registry"
2895 )),
2896 Ability::new(REGISTRY_DELEGATION_ABILITY),
2897 Caveats(caveats),
2898 )
2899}
2900
2901pub fn service_key_enrollment_capability(
2907 deployment_domain: &str,
2908 delegated_public_key_b64: &str,
2909) -> hyprstream_rpc::auth::ucan::Capability {
2910 use hyprstream_rpc::auth::ucan::{Ability, Capability, CaveatValue, Caveats, Resource};
2911
2912 let mut caveats = std::collections::BTreeMap::new();
2913 caveats.insert(
2914 "deployment_domain".to_owned(),
2915 CaveatValue::Text(deployment_domain.to_owned()),
2916 );
2917 caveats.insert(
2918 "delegated_public_key_b64".to_owned(),
2919 CaveatValue::Text(delegated_public_key_b64.to_owned()),
2920 );
2921 caveats.insert(
2922 "allowed_services".to_owned(),
2923 CaveatValue::List(
2924 SERVICE_KEY_ENROLLMENT_ALLOWED_SERVICES
2925 .iter()
2926 .map(|service| (*service).to_owned())
2927 .collect(),
2928 ),
2929 );
2930 caveats.insert(
2931 "key_type".to_owned(),
2932 CaveatValue::Text(SERVICE_KEY_ENROLLMENT_KEY_TYPE.to_owned()),
2933 );
2934 caveats.insert(
2935 "max_attestation_ttl_seconds".to_owned(),
2936 CaveatValue::Int(SERVICE_KEY_ENROLLMENT_MAX_TTL_SECONDS),
2937 );
2938 Capability::with_caveats(
2939 Resource::new(format!(
2940 "hyprstream://deployment/{deployment_domain}/service-key-enrollment"
2941 )),
2942 Ability::new(SERVICE_KEY_ENROLLMENT_ABILITY),
2943 Caveats(caveats),
2944 )
2945}
2946
2947pub fn delegation_capability_set_grants_enrollment(
2952 capabilities: &[hyprstream_rpc::auth::ucan::Capability],
2953 deployment_domain: &str,
2954 delegated_public_key_b64: &str,
2955) -> Result<bool> {
2956 let registry = registry_mint_capability(deployment_domain, delegated_public_key_b64);
2957 let enrollment = service_key_enrollment_capability(deployment_domain, delegated_public_key_b64);
2958 if capabilities.len() == 1 && capabilities[0] == registry {
2959 return Ok(false);
2960 }
2961 anyhow::ensure!(
2962 capabilities.len() == 2
2963 && capabilities.contains(®istry)
2964 && capabilities.contains(&enrollment),
2965 "delegation capability set is not the exact registry or registry+enrollment scope"
2966 );
2967 Ok(true)
2968}
2969
2970#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2973pub struct ValidatedDelegation {
2974 pub grants_service_key_enrollment: bool,
2975 pub expires_at: u64,
2976}
2977
2978pub fn validate_delegation_ucan(
2986 ucan: &hyprstream_rpc::auth::ucan::Ucan,
2987 deployment_domain: &str,
2988 delegated_public_key_b64: &str,
2989 active_keys: &[crate::did_op::HybridRotationKey],
2990 now: u64,
2991) -> Result<ValidatedDelegation> {
2992 use hyprstream_rpc::auth::ucan::validate as validate_ucan;
2993
2994 anyhow::ensure!(
2995 ucan.proofs.is_empty(),
2996 "registry delegation must be one authority-to-signer link"
2997 );
2998 validate_ucan(ucan, &AuthoritySetUcanVerifier { keys: active_keys }, now)
2999 .context("validating registry delegation UCAN")?;
3000 anyhow::ensure!(
3001 active_keys
3002 .iter()
3003 .any(|key| ucan.issuer().to_ed25519().ok() == Some(key.ed25519_pub)),
3004 "registry delegation issuer is not an active authority"
3005 );
3006 let delegated_public = base64::engine::general_purpose::STANDARD
3007 .decode(delegated_public_key_b64)
3008 .context("decoding delegated registry-signer public key")?;
3009 let delegated_ed: [u8; ED25519_PUBLIC_KEY_BYTES] = delegated_public
3010 .get(..ED25519_PUBLIC_KEY_BYTES)
3011 .ok_or_else(|| anyhow::anyhow!("delegated registry-signer public key is truncated"))?
3012 .try_into()
3013 .map_err(|_| anyhow::anyhow!("delegated Ed25519 key is malformed"))?;
3014 anyhow::ensure!(
3015 ucan.audience().to_ed25519()? == delegated_ed,
3016 "registry delegation audience does not match delegated signer"
3017 );
3018 let grants_service_key_enrollment = delegation_capability_set_grants_enrollment(
3019 ucan.capabilities(),
3020 deployment_domain,
3021 delegated_public_key_b64,
3022 )?;
3023 let expires_at = ucan
3024 .payload
3025 .expiration
3026 .ok_or_else(|| anyhow::anyhow!("registry delegation must expire"))?;
3027 Ok(ValidatedDelegation {
3028 grants_service_key_enrollment,
3029 expires_at,
3030 })
3031}
3032
3033struct ValidatedRegistryDelegation {
3037 delegated: HybridDeploymentCa,
3038 grants_service_key_enrollment: bool,
3039 expires_at: u64,
3040}
3041
3042fn validate_registry_delegation_artifact(
3043 root: &HybridDeploymentCa,
3044 artifact: &RegistryDelegationArtifact,
3045 installed_authority_log: &DeploymentAuthorityLog,
3046 active: &crate::did_op::VerifiedDidOpLog,
3047 now: u64,
3048) -> Result<ValidatedRegistryDelegation> {
3049 use hyprstream_rpc::auth::ucan::Ucan;
3050
3051 anyhow::ensure!(
3052 artifact.schema == REGISTRY_DELEGATION_SCHEMA,
3053 "unsupported registry delegation schema"
3054 );
3055 anyhow::ensure!(
3056 artifact.deployment_domain == root.domain(),
3057 "registry delegation deployment domain does not match pinned root"
3058 );
3059 anyhow::ensure!(
3060 artifact.authority_log_did == installed_authority_log.did,
3061 "registry delegation names a different authority log"
3062 );
3063 let active_keys = &active.rotation_keys;
3064
3065 let delegated_public = base64::engine::general_purpose::STANDARD
3066 .decode(&artifact.delegated_public_key_b64)
3067 .context("decoding delegated registry-signer public key")?;
3068 anyhow::ensure!(
3069 delegated_public.len() == DEPLOYMENT_CA_ROOT_BYTES,
3070 "delegated registry-signer public key has the wrong length"
3071 );
3072 let delegated = HybridDeploymentCa::from_public_key_bytes(
3073 &delegated_public[..ED25519_PUBLIC_KEY_BYTES],
3074 &delegated_public[ED25519_PUBLIC_KEY_BYTES..],
3075 )?;
3076
3077 let ucan_bytes = base64::engine::general_purpose::STANDARD
3078 .decode(&artifact.ucan_b64)
3079 .context("decoding registry delegation UCAN")?;
3080 anyhow::ensure!(
3081 ucan_bytes.len() <= MAX_REGISTRY_DELEGATION_BYTES,
3082 "registry delegation UCAN is too large"
3083 );
3084 let ucan = Ucan::from_cbor(&ucan_bytes).context("decoding registry delegation UCAN")?;
3085 let validated = validate_delegation_ucan(
3086 &ucan,
3087 &artifact.deployment_domain,
3088 &artifact.delegated_public_key_b64,
3089 active_keys,
3090 now,
3091 )?;
3092 Ok(ValidatedRegistryDelegation {
3093 delegated,
3094 grants_service_key_enrollment: validated.grants_service_key_enrollment,
3095 expires_at: validated.expires_at,
3096 })
3097}
3098
3099pub fn validate_registry_delegation(
3107 public_ca: &[u8],
3108 authority_log: &DeploymentAuthorityLog,
3109 authority_checkpoint: &DeploymentAuthorityCheckpoint,
3110 artifact: &RegistryDelegationArtifact,
3111 now: u64,
3112) -> Result<ValidatedDelegation> {
3113 let root = HybridDeploymentCa::from_os_pin(public_ca)?;
3114 let active = validate_deployment_authority_log(&root, authority_log, authority_checkpoint)?;
3115 let validated =
3116 validate_registry_delegation_artifact(&root, artifact, authority_log, &active, now)?;
3117 Ok(ValidatedDelegation {
3118 grants_service_key_enrollment: validated.grants_service_key_enrollment,
3119 expires_at: validated.expires_at,
3120 })
3121}
3122
3123fn validate_deployment_authority_log(
3124 root: &HybridDeploymentCa,
3125 log: &DeploymentAuthorityLog,
3126 checkpoint: &DeploymentAuthorityCheckpoint,
3127) -> Result<crate::did_op::VerifiedDidOpLog> {
3128 anyhow::ensure!(
3129 log.schema == AUTHORITY_LOG_SCHEMA,
3130 "unsupported deployment authority-log schema"
3131 );
3132 anyhow::ensure!(
3133 log.deployment_domain == root.domain(),
3134 "deployment authority-log domain does not match pinned root"
3135 );
3136 anyhow::ensure!(
3137 !log.operations_b64.is_empty() && log.operations_b64.len() <= MAX_AUTHORITY_LOG_OPERATIONS,
3138 "deployment authority-log operation count is invalid"
3139 );
3140 let operations = log
3141 .operations_b64
3142 .iter()
3143 .map(|encoded| {
3144 let bytes = base64::engine::general_purpose::STANDARD
3145 .decode(encoded)
3146 .context("decoding deployment authority operation")?;
3147 crate::did_op::DidOp::from_dag_cbor(&bytes)
3148 })
3149 .collect::<Result<Vec<_>>>()?;
3150 let active = crate::did_op::verify_did_op_log(&log.did, &operations)
3151 .context("verifying deployment authority log")?;
3152 anyhow::ensure!(
3153 checkpoint.schema == AUTHORITY_CHECKPOINT_SCHEMA,
3154 "unsupported deployment authority-checkpoint schema"
3155 );
3156 anyhow::ensure!(
3157 checkpoint.deployment_domain == root.domain(),
3158 "deployment authority checkpoint domain does not match pinned root"
3159 );
3160 anyhow::ensure!(
3161 checkpoint.did == log.did
3162 && checkpoint.did == active.did
3163 && checkpoint.sequence == active.sequence
3164 && checkpoint.head_cid == active.head_cid,
3165 "deployment authority-log head does not match the independently trusted checkpoint"
3166 );
3167 let genesis = operations
3168 .first()
3169 .ok_or_else(|| anyhow::anyhow!("deployment authority log is empty"))?;
3170 let genesis_composite = hyprstream_rpc::crypto::cose_sign::assemble_composite_nested(
3171 (
3172 root.ed25519.to_bytes().to_vec(),
3173 genesis.signature.ed25519.clone(),
3174 ),
3175 Some((
3176 hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&root.ml_dsa_65),
3177 genesis.signature.mldsa65.clone(),
3178 )),
3179 )
3180 .context("assembling pinned-root authority genesis signature")?;
3181 hyprstream_rpc::crypto::cose_sign::verify_composite(
3182 &genesis_composite,
3183 &root.ed25519,
3184 Some(&root.ml_dsa_65),
3185 &genesis.signable_bytes(),
3186 crate::did_op::DID_OP_SIGNATURE_CONTEXT,
3187 true,
3188 )
3189 .context("authority-log genesis was not signed by the pinned root")?;
3190 anyhow::ensure!(
3191 genesis.rotation_keys.iter().any(|key| {
3192 key.ed25519_pub == root.ed25519.to_bytes()
3193 && key.mldsa65_pub == hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&root.ml_dsa_65)
3194 }),
3195 "deployment authority-log genesis is not anchored by the pinned public root"
3196 );
3197 Ok(active)
3198}
3199
3200#[derive(Clone, Debug, Eq, PartialEq)]
3204pub struct VerifiedDeploymentArtifacts {
3205 pub deployment_domain: String,
3206 pub registry_public_key: [u8; ED25519_PUBLIC_KEY_BYTES],
3207 pub expires_at: i64,
3208}
3209
3210pub fn verify_deployment_public_ca(public_ca: &[u8]) -> Result<String> {
3214 HybridDeploymentCa::from_os_pin(public_ca).map(|ca| ca.domain())
3215}
3216
3217pub fn verify_genesis_deployment_artifacts(
3225 public_ca: &[u8],
3226 registry_credential: &str,
3227) -> Result<VerifiedDeploymentArtifacts> {
3228 let ca = HybridDeploymentCa::from_os_pin(public_ca)?;
3229 let (registry_public_key, expires_at) = validate_registry_deployment_credential_profile(
3230 registry_credential,
3231 &ca,
3232 DeploymentAuthorityState::GenesisOnly,
3233 )?;
3234 Ok(VerifiedDeploymentArtifacts {
3235 deployment_domain: ca.domain(),
3236 registry_public_key,
3237 expires_at,
3238 })
3239}
3240
3241pub fn verify_deployment_artifacts_with_authority_log(
3244 public_ca: &[u8],
3245 authority_log_json: &[u8],
3246 authority_checkpoint_json: &[u8],
3247 registry_credential: &str,
3248) -> Result<VerifiedDeploymentArtifacts> {
3249 anyhow::ensure!(
3250 authority_log_json.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3251 "installed deployment authority log is too large"
3252 );
3253 let ca = HybridDeploymentCa::from_os_pin(public_ca)?;
3254 let authority_log: DeploymentAuthorityLog = serde_json::from_slice(authority_log_json)
3255 .map_err(|error| {
3256 anyhow::anyhow!("installed deployment authority log is malformed: {error}")
3257 })?;
3258 anyhow::ensure!(
3259 authority_checkpoint_json.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3260 "installed deployment authority checkpoint is too large"
3261 );
3262 let authority_checkpoint: DeploymentAuthorityCheckpoint =
3263 serde_json::from_slice(authority_checkpoint_json).map_err(|error| {
3264 anyhow::anyhow!("installed deployment authority checkpoint is malformed: {error}")
3265 })?;
3266 let (registry_public_key, expires_at) = validate_registry_deployment_credential_profile(
3267 registry_credential,
3268 &ca,
3269 DeploymentAuthorityState::Enrolled {
3270 authority_log: &authority_log,
3271 authority_checkpoint: &authority_checkpoint,
3272 },
3273 )?;
3274 Ok(VerifiedDeploymentArtifacts {
3275 deployment_domain: ca.domain(),
3276 registry_public_key,
3277 expires_at,
3278 })
3279}
3280
3281#[derive(Clone, Debug, Eq, PartialEq)]
3285pub struct VerifiedServiceKeyEnrollment {
3286 pub deployment_domain: String,
3287 pub service: String,
3288 pub hybrid_public_key: Vec<u8>,
3291 pub expires_at: i64,
3292}
3293
3294pub fn verify_service_key_enrollment(
3304 public_ca: &[u8],
3305 authority_log_json: &[u8],
3306 authority_checkpoint_json: &[u8],
3307 attestation_json: &[u8],
3308) -> Result<VerifiedServiceKeyEnrollment> {
3309 anyhow::ensure!(
3310 authority_log_json.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3311 "installed deployment authority log is too large"
3312 );
3313 anyhow::ensure!(
3314 authority_checkpoint_json.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3315 "installed deployment authority checkpoint is too large"
3316 );
3317 anyhow::ensure!(
3318 attestation_json.len() <= MAX_SERVICE_KEY_ENROLLMENT_BYTES,
3319 "service-key enrollment attestation is too large"
3320 );
3321 let ca = HybridDeploymentCa::from_os_pin(public_ca)?;
3322 let authority_log: DeploymentAuthorityLog = serde_json::from_slice(authority_log_json)
3323 .map_err(|error| {
3324 anyhow::anyhow!("installed deployment authority log is malformed: {error}")
3325 })?;
3326 let authority_checkpoint: DeploymentAuthorityCheckpoint =
3327 serde_json::from_slice(authority_checkpoint_json).map_err(|error| {
3328 anyhow::anyhow!("installed deployment authority checkpoint is malformed: {error}")
3329 })?;
3330 let active = validate_deployment_authority_log(&ca, &authority_log, &authority_checkpoint)?;
3331 let attestation: ServiceKeyEnrollmentArtifact = serde_json::from_slice(attestation_json)
3332 .map_err(|error| {
3333 anyhow::anyhow!("service-key enrollment attestation is malformed: {error}")
3334 })?;
3335 anyhow::ensure!(
3336 attestation.schema == SERVICE_KEY_ENROLLMENT_SCHEMA,
3337 "unsupported service-key enrollment schema"
3338 );
3339 anyhow::ensure!(
3340 attestation.deployment_domain == ca.domain(),
3341 "enrollment deployment domain does not match pinned root"
3342 );
3343 anyhow::ensure!(
3344 SERVICE_KEY_ENROLLMENT_ALLOWED_SERVICES.contains(&attestation.service.as_str()),
3345 "enrollment service is outside the fixed discovery/policy allowlist"
3346 );
3347 let hybrid_public_key = base64::engine::general_purpose::STANDARD
3348 .decode(&attestation.hybrid_public_key_b64)
3349 .context("decoding enrolled hybrid public key")?;
3350 anyhow::ensure!(
3351 hybrid_public_key.len() == DEPLOYMENT_CA_ROOT_BYTES,
3352 "enrolled hybrid public key must be exactly {DEPLOYMENT_CA_ROOT_BYTES} bytes \
3353 (32-byte Ed25519 followed by 1952-byte ML-DSA-65)"
3354 );
3355 HybridDeploymentCa::from_public_key_bytes(
3359 &hybrid_public_key[..ED25519_PUBLIC_KEY_BYTES],
3360 &hybrid_public_key[ED25519_PUBLIC_KEY_BYTES..],
3361 )?;
3362
3363 let now = chrono::Utc::now().timestamp();
3364 let latest_future_time = now
3365 .checked_add(REGISTRY_DEPLOYMENT_CREDENTIAL_CLOCK_SKEW_SECONDS)
3366 .ok_or_else(|| anyhow::anyhow!("enrollment clock-skew arithmetic overflow"))?;
3367 anyhow::ensure!(
3368 attestation.not_before >= 0,
3369 "enrollment not_before is negative"
3370 );
3371 anyhow::ensure!(
3372 attestation.not_before <= latest_future_time,
3373 "enrollment is not yet valid"
3374 );
3375 anyhow::ensure!(
3376 attestation.not_before < attestation.expires_at,
3377 "enrollment not_before is not before expires_at"
3378 );
3379 let lifetime = attestation
3380 .expires_at
3381 .checked_sub(attestation.not_before)
3382 .ok_or_else(|| anyhow::anyhow!("enrollment lifetime arithmetic overflow"))?;
3383 anyhow::ensure!(
3384 lifetime <= SERVICE_KEY_ENROLLMENT_MAX_TTL_SECONDS,
3385 "enrollment lifetime exceeds the inclusive one-hour limit"
3386 );
3387 anyhow::ensure!(now < attestation.expires_at, "enrollment has expired");
3388
3389 let validated = validate_registry_delegation_artifact(
3390 &ca,
3391 &attestation.delegation,
3392 &authority_log,
3393 &active,
3394 u64::try_from(now).map_err(|_| anyhow::anyhow!("system clock precedes Unix epoch"))?,
3395 )?;
3396 anyhow::ensure!(
3397 validated.grants_service_key_enrollment,
3398 "delegation does not carry the service-key-enrollment capability"
3399 );
3400 anyhow::ensure!(
3401 attestation.deployment_domain == attestation.delegation.deployment_domain,
3402 "enrollment domain does not match its delegation"
3403 );
3404 anyhow::ensure!(
3405 attestation.expires_at >= 0
3406 && u64::try_from(attestation.expires_at)
3407 .map_err(|_| anyhow::anyhow!("enrollment expiry conversion failed"))?
3408 <= validated.expires_at,
3409 "enrollment expiry exceeds the delegation expiry"
3410 );
3411
3412 let signature = base64::engine::general_purpose::STANDARD
3413 .decode(&attestation.signature_b64)
3414 .context("decoding enrollment signature")?;
3415 hyprstream_rpc::crypto::cose_sign::verify_composite(
3419 &signature,
3420 &validated.delegated.ed25519,
3421 Some(&validated.delegated.ml_dsa_65),
3422 &attestation.signing_bytes()?,
3423 SERVICE_KEY_ENROLLMENT_SIGNATURE_CONTEXT,
3424 true,
3425 )
3426 .context("enrollment hybrid signature rejected")?;
3427 Ok(VerifiedServiceKeyEnrollment {
3428 deployment_domain: attestation.deployment_domain,
3429 service: attestation.service,
3430 hybrid_public_key,
3431 expires_at: attestation.expires_at,
3432 })
3433}
3434
3435pub fn verify_os_owned_service_key_enrollment(
3445 attestation_json: &[u8],
3446) -> Result<VerifiedServiceKeyEnrollment> {
3447 let paths = resolve_deployment_trust_paths()?;
3448 let public_ca = read_trusted_artifact(&paths.public_ca, "deployment CA root")?;
3449 let authority_log = read_trusted_artifact(&paths.authority_log, "deployment authority log")?;
3450 let authority_checkpoint = read_trusted_artifact(
3451 &paths.authority_checkpoint,
3452 "deployment authority checkpoint",
3453 )?;
3454 verify_service_key_enrollment(
3455 &public_ca,
3456 &authority_log,
3457 &authority_checkpoint,
3458 attestation_json,
3459 )
3460}
3461
3462#[derive(Clone, Debug, Eq, PartialEq)]
3463struct TrustedArtifactPath {
3464 path: std::path::PathBuf,
3465}
3466
3467#[derive(Clone, Debug, Eq, PartialEq)]
3468struct DeploymentTrustPaths {
3469 public_ca: TrustedArtifactPath,
3470 authority_log: TrustedArtifactPath,
3471 authority_checkpoint: TrustedArtifactPath,
3472 registry_credential: TrustedArtifactPath,
3473}
3474
3475fn explicit_absolute_directory(
3476 env_name: &str,
3477 value: std::ffi::OsString,
3478) -> Result<std::path::PathBuf> {
3479 use std::path::Component;
3480
3481 let path = std::path::PathBuf::from(value);
3482 anyhow::ensure!(
3483 !path.as_os_str().is_empty(),
3484 "{env_name} is present but empty"
3485 );
3486 anyhow::ensure!(path.is_absolute(), "{env_name} must be an absolute path");
3487 anyhow::ensure!(
3488 path.components()
3489 .all(|component| matches!(component, Component::RootDir | Component::Normal(_))),
3490 "{env_name} must not contain '..' components"
3491 );
3492 Ok(path)
3493}
3494
3495fn resolve_deployment_trust_paths_from(
3496 trust_directory: Option<std::ffi::OsString>,
3497 credentials_directory: Option<std::ffi::OsString>,
3498) -> Result<DeploymentTrustPaths> {
3499 let (public_ca, authority_log, authority_checkpoint) = match trust_directory {
3500 Some(value) => {
3501 let directory = explicit_absolute_directory(DEPLOYMENT_TRUST_DIR_ENV, value)?;
3502 (
3503 directory.join(DEPLOYMENT_CA_ROOT_FILE),
3504 directory.join(DEPLOYMENT_AUTHORITY_LOG_FILE),
3505 directory.join(DEPLOYMENT_AUTHORITY_CHECKPOINT_FILE),
3506 )
3507 }
3508 None => (
3509 std::path::PathBuf::from(DEPLOYMENT_CA_ROOT_PATH),
3510 std::path::PathBuf::from(DEPLOYMENT_AUTHORITY_LOG_PATH),
3511 std::path::PathBuf::from(DEPLOYMENT_AUTHORITY_CHECKPOINT_PATH),
3512 ),
3513 };
3514 let registry_credential = match credentials_directory {
3515 Some(value) => {
3516 let directory = explicit_absolute_directory(SYSTEMD_CREDENTIALS_DIRECTORY_ENV, value)?;
3517 directory.join(REGISTRY_DEPLOYMENT_CREDENTIAL_FILE)
3518 }
3519 None => std::path::PathBuf::from(REGISTRY_DEPLOYMENT_CREDENTIAL_PATH),
3520 };
3521 Ok(DeploymentTrustPaths {
3522 public_ca: TrustedArtifactPath { path: public_ca },
3523 authority_log: TrustedArtifactPath {
3524 path: authority_log,
3525 },
3526 authority_checkpoint: TrustedArtifactPath {
3527 path: authority_checkpoint,
3528 },
3529 registry_credential: TrustedArtifactPath {
3530 path: registry_credential,
3531 },
3532 })
3533}
3534
3535fn resolve_deployment_trust_paths() -> Result<DeploymentTrustPaths> {
3536 resolve_deployment_trust_paths_from(
3537 std::env::var_os(DEPLOYMENT_TRUST_DIR_ENV),
3538 std::env::var_os(SYSTEMD_CREDENTIALS_DIRECTORY_ENV),
3539 )
3540}
3541
3542#[cfg(unix)]
3543fn validate_trusted_artifact_metadata(
3544 owner_uid: u32,
3545 mode: u32,
3546 effective_uid: u32,
3547 subject: &str,
3548) -> Result<()> {
3549 anyhow::ensure!(
3550 owner_uid == 0 || owner_uid == effective_uid,
3551 "{subject} has an untrusted owner"
3552 );
3553 anyhow::ensure!(mode & 0o022 == 0, "{subject} is group/world writable");
3554 Ok(())
3555}
3556
3557#[cfg(unix)]
3558fn read_trusted_artifact(artifact: &TrustedArtifactPath, description: &str) -> Result<Vec<u8>> {
3559 use std::io::Read as _;
3560 use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _};
3561
3562 anyhow::ensure!(
3563 artifact.path.is_absolute(),
3564 "{description} path must be absolute"
3565 );
3566 let effective_uid = {
3567 unsafe { libc::geteuid() }
3569 };
3570 for parent in artifact.path.ancestors().skip(1) {
3571 let parent_metadata = std::fs::symlink_metadata(parent).map_err(|error| {
3572 anyhow::anyhow!("{description} parent {parent:?} is unavailable: {error}")
3573 })?;
3574 anyhow::ensure!(
3575 parent_metadata.file_type().is_dir(),
3576 "{description} parent {parent:?} is not a real directory"
3577 );
3578 validate_trusted_artifact_metadata(
3579 parent_metadata.uid(),
3580 parent_metadata.mode(),
3581 effective_uid,
3582 &format!("{description} parent {parent:?}"),
3583 )?;
3584 }
3585 let mut file = std::fs::OpenOptions::new()
3586 .read(true)
3587 .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW)
3588 .open(&artifact.path)
3589 .map_err(|error| {
3590 anyhow::anyhow!(
3591 "{description} is unavailable at {:?}: {error}",
3592 artifact.path
3593 )
3594 })?;
3595 let metadata = file.metadata().map_err(|error| {
3596 anyhow::anyhow!(
3597 "failed to inspect {description} at {:?}: {error}",
3598 artifact.path
3599 )
3600 })?;
3601 anyhow::ensure!(
3602 metadata.file_type().is_file(),
3603 "{description} is not a regular file"
3604 );
3605 validate_trusted_artifact_metadata(
3606 metadata.uid(),
3607 metadata.mode(),
3608 effective_uid,
3609 description,
3610 )?;
3611 let mut bytes = Vec::new();
3612 file.read_to_end(&mut bytes).map_err(|error| {
3613 anyhow::anyhow!(
3614 "failed to read {description} at {:?}: {error}",
3615 artifact.path
3616 )
3617 })?;
3618 Ok(bytes)
3619}
3620
3621#[cfg(not(unix))]
3622fn read_trusted_artifact(_artifact: &TrustedArtifactPath, description: &str) -> Result<Vec<u8>> {
3623 anyhow::bail!("{description} requires the trusted Unix deployment seam")
3624}
3625
3626fn load_registry_deployment_credential(paths: &DeploymentTrustPaths) -> Result<String> {
3627 String::from_utf8(read_trusted_artifact(
3628 &paths.registry_credential,
3629 "registry deployment credential",
3630 )?)
3631 .map_err(|error| anyhow::anyhow!("registry deployment credential is not UTF-8: {error}"))
3632}
3633
3634fn load_trusted_registry_deployment_credentials_from(
3635 paths: &DeploymentTrustPaths,
3636) -> Result<TrustedRegistryDeploymentCredentials> {
3637 let ca_bytes = read_trusted_artifact(&paths.public_ca, "deployment CA root")?;
3638 let ca_verifying_key = HybridDeploymentCa::from_os_pin(&ca_bytes)?;
3639 let authority_log_bytes =
3640 read_trusted_artifact(&paths.authority_log, "deployment authority log")?;
3641 anyhow::ensure!(
3642 authority_log_bytes.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3643 "deployment authority log exceeds the 64 KiB cloud-secret contract"
3644 );
3645 let authority_log = serde_json::from_slice(&authority_log_bytes)
3646 .map_err(|error| anyhow::anyhow!("deployment authority log is malformed: {error}"))?;
3647 let authority_checkpoint = load_deployment_authority_checkpoint_from(paths)?;
3648 let registry_credential = load_registry_deployment_credential(paths)?;
3649 Ok(TrustedRegistryDeploymentCredentials {
3650 ca_verifying_key,
3651 authority_log,
3652 authority_checkpoint,
3653 registry_credential,
3654 })
3655}
3656
3657fn load_trusted_registry_deployment_credentials() -> Result<TrustedRegistryDeploymentCredentials> {
3658 let paths = resolve_deployment_trust_paths()?;
3659 load_trusted_registry_deployment_credentials_from(&paths)
3660}
3661
3662fn load_deployment_authority_checkpoint_from(
3663 paths: &DeploymentTrustPaths,
3664) -> Result<DeploymentAuthorityCheckpoint> {
3665 let bytes = read_trusted_artifact(
3666 &paths.authority_checkpoint,
3667 "deployment authority checkpoint",
3668 )?;
3669 anyhow::ensure!(
3670 bytes.len() <= MAX_DEPLOYMENT_CLOUD_SECRET_BYTES,
3671 "deployment authority checkpoint exceeds the 64 KiB cloud-secret contract"
3672 );
3673 serde_json::from_slice(&bytes)
3674 .map_err(|error| anyhow::anyhow!("deployment authority checkpoint is malformed: {error}"))
3675}
3676
3677fn load_deployment_authority_checkpoint() -> Result<DeploymentAuthorityCheckpoint> {
3678 let paths = resolve_deployment_trust_paths()?;
3679 load_deployment_authority_checkpoint_from(&paths)
3680}
3681
3682fn authenticate_registry_deployment_credentials(
3683 credentials: TrustedRegistryDeploymentCredentials,
3684) -> Result<AuthenticatedRegistryDeploymentIdentity> {
3685 let (key_bytes, _expires_at) = validate_registry_deployment_credential_profile(
3686 &credentials.registry_credential,
3687 &credentials.ca_verifying_key,
3688 DeploymentAuthorityState::Enrolled {
3689 authority_log: &credentials.authority_log,
3690 authority_checkpoint: &credentials.authority_checkpoint,
3691 },
3692 )
3693 .map_err(|error| anyhow::anyhow!("registry deployment credential rejected: {error}"))?;
3694 let verifying_key = VerifyingKey::from_bytes(&key_bytes)
3695 .map_err(|error| anyhow::anyhow!("registry credential key is malformed: {error}"))?;
3696 Ok(AuthenticatedRegistryDeploymentIdentity {
3697 verifier: RegistryDeploymentVerifier {
3698 verifying_key,
3699 _deployment_ca: Arc::new(credentials.ca_verifying_key),
3700 },
3701 })
3702}
3703
3704#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
3707struct ProcessBootstrapAuthority {
3708 store_path: std::path::PathBuf,
3709 acceptance_identity: ProcessAcceptanceIdentity,
3710 network_required: bool,
3711}
3712
3713enum ProcessAcceptanceIdentity {
3714 Deployment(RegistryDeploymentVerifier),
3715 #[cfg(test)]
3716 #[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
3717 Test(VerifyingKey),
3718}
3719
3720enum ProcessBootstrapAuthorityState {
3721 Unsealed,
3722 Sealed,
3723 Consumed,
3724}
3725
3726static PROCESS_BOOTSTRAP_AUTHORITY: parking_lot::Mutex<ProcessBootstrapAuthorityState> =
3727 parking_lot::Mutex::new(ProcessBootstrapAuthorityState::Unsealed);
3728
3729fn resolve_local_discovery_transport() -> anyhow::Result<hyprstream_rpc::transport::TransportConfig>
3747{
3748 hyprstream_rpc::registry::try_global()
3749 .ok_or_else(|| {
3750 anyhow::anyhow!("EndpointRegistry not initialized — local discovery fabric unavailable")
3751 })?
3752 .try_endpoint("discovery", hyprstream_rpc::registry::SocketKind::Rep)
3753}
3754
3755fn install_local_discovery_client(
3767 signing_key: SigningKey,
3768 discovery_vk: VerifyingKey,
3769 token: Option<String>,
3770) -> Result<hyprstream_rpc_std::discovery_client::DiscoveryClient> {
3771 let transport = resolve_local_discovery_transport()?;
3772 let token_provider = renewing_service_token_provider(&signing_key, token);
3773 let signer = hyprstream_rpc::signer::LocalSigner::new(signing_key);
3774 let rpc = hyprstream_rpc::dial::dial_with_token_provider(
3775 &transport,
3776 signer,
3777 Some(discovery_vk),
3778 token_provider,
3779 )?;
3780 Ok(hyprstream_rpc_std::discovery_client::DiscoveryClient::new(
3781 rpc,
3782 ))
3783}
3784
3785fn current_service_token(signing_key: &SigningKey, initial: &Option<String>) -> Option<String> {
3789 initial.as_ref()?;
3790 hyprstream_service::global_trust_store()
3791 .get(&signing_key.verifying_key())
3792 .and_then(|att| att.jwt)
3793 .or_else(|| initial.clone())
3794}
3795
3796fn renewing_service_token_provider(
3797 signing_key: &SigningKey,
3798 initial: Option<String>,
3799) -> Option<Arc<dyn Fn() -> Option<String> + Send + Sync>> {
3800 initial.as_ref()?;
3801 let signer = signing_key.clone();
3802 Some(Arc::new(move || current_service_token(&signer, &initial)))
3803}
3804
3805fn bootstrap_client_transport_key(
3806 service_key: &SigningKey,
3807 ephemeral_override: Option<SigningKey>,
3808) -> SigningKey {
3809 ephemeral_override.unwrap_or_else(|| {
3810 hyprstream_rpc::node_identity::derive_purpose_key(
3811 service_key,
3812 "hyprstream-bootstrap-client-transport-v1",
3813 )
3814 })
3815}
3816
3817#[cfg(not(target_arch = "wasm32"))]
3836#[cfg_attr(
3839 not(any(feature = "rocksdb", feature = "postgres")),
3840 allow(unused_variables)
3841)]
3842pub async fn bootstrap_deployment_process(
3843 signing_key: SigningKey,
3844 service_token: Option<String>,
3845 trust_source: crate::DeploymentTrustSource,
3846 remote_node: bool,
3847 network_required: bool,
3848 records: &hyprstream_pds::rds::RdsConfig,
3849) -> Result<()> {
3850 bootstrap_deployment_process_with_carrier_identity(
3851 signing_key,
3852 service_token,
3853 trust_source,
3854 remote_node,
3855 network_required,
3856 records,
3857 None,
3858 )
3859 .await
3860}
3861
3862#[cfg(not(target_arch = "wasm32"))]
3867pub async fn bootstrap_deployment_process_with_ephemeral_client_transport(
3868 signing_key: SigningKey,
3869 service_token: Option<String>,
3870 trust_source: crate::DeploymentTrustSource,
3871 remote_node: bool,
3872 network_required: bool,
3873 records: &hyprstream_pds::rds::RdsConfig,
3874) -> Result<()> {
3875 let ephemeral_transport_key = SigningKey::generate(&mut rand::rngs::OsRng);
3876 bootstrap_deployment_process_with_carrier_identity(
3877 signing_key,
3878 service_token,
3879 trust_source,
3880 remote_node,
3881 network_required,
3882 records,
3883 Some(ephemeral_transport_key),
3884 )
3885 .await
3886}
3887
3888#[cfg(not(target_arch = "wasm32"))]
3889#[cfg_attr(
3890 not(any(feature = "rocksdb", feature = "postgres")),
3891 allow(unused_variables)
3892)]
3893async fn bootstrap_deployment_process_with_carrier_identity(
3894 signing_key: SigningKey,
3895 service_token: Option<String>,
3896 trust_source: crate::DeploymentTrustSource,
3897 remote_node: bool,
3898 network_required: bool,
3899 records: &hyprstream_pds::rds::RdsConfig,
3900 client_transport_key_override: Option<SigningKey>,
3901) -> Result<()> {
3902 let discovery_vk = hyprstream_service::global_trust_store()
3903 .resolve_one("discovery")
3904 .ok_or_else(|| anyhow::anyhow!("trust store has no authenticated discovery key"))?;
3905 let (authority, discovery_client) = match trust_source {
3906 crate::DeploymentTrustSource::OsOwnedFiles => {
3907 let mut authority = authenticate_deployment_bootstrap()?;
3908 authority.network_required = network_required;
3909 let discovery_client = if network_required {
3917 #[cfg(not(test))]
3918 let ProcessAcceptanceIdentity::Deployment(verifier) =
3919 &authority.acceptance_identity;
3920 #[cfg(test)]
3921 let verifier = match &authority.acceptance_identity {
3922 ProcessAcceptanceIdentity::Deployment(verifier) => verifier,
3923 #[cfg(test)]
3924 ProcessAcceptanceIdentity::Test(_) => {
3925 anyhow::bail!("native bootstrap requires deployment authority")
3926 }
3927 };
3928 #[cfg(any(feature = "rocksdb", feature = "postgres"))]
3929 let source = crate::checkpointed_pds::open_deployment_accepted_state_source(
3930 &authority.store_path,
3931 verifier.clone(),
3932 records,
3933 true,
3934 )?;
3935 #[cfg(not(any(feature = "rocksdb", feature = "postgres")))]
3938 let source: Arc<dyn AcceptedStateSource> =
3939 Err(anyhow::anyhow!(
3940 "checkpointed PDS accepted-state authority requires the `rocksdb` or `postgres` feature"
3941 ))?;
3942 source.bootstrap_endpoints("discovery")?;
3945 source.bootstrap_endpoints("policy")?;
3946 use hyprstream_rpc::transport::iroh_substrate::{IrohSubstrate, RefuseHandler};
3949 let transport_key =
3950 bootstrap_client_transport_key(&signing_key, client_transport_key_override);
3951 let carrier = IrohSubstrate::new(
3952 transport_key.to_bytes(),
3953 RefuseHandler::new("outbound bootstrap only"),
3954 RefuseHandler::new("outbound bootstrap only"),
3955 )
3956 .await?;
3957 let _ = hyprstream_rpc::transport::lazy_iroh::install_iroh_client_endpoint(
3958 carrier.owned_client_endpoint(),
3959 );
3960 PROCESS_BOOTSTRAP_CARRIER
3961 .set(carrier)
3962 .map_err(|_| anyhow::anyhow!("bootstrap carrier already installed"))?;
3963 let resolver = Arc::new(DiscoveryServiceResolver {
3964 state_store: MemoryStateStore::production_default(),
3965 accepted_state_source: source,
3966 discovery_client: None,
3967 });
3968 hyprstream_rpc_std::discovery_client::DiscoveryClient::new(Arc::new(
3969 ProductionRpcClient::new(
3970 "discovery",
3971 "discovery",
3972 None,
3973 signing_key,
3974 service_token,
3975 resolver,
3976 )?,
3977 ))
3978 } else {
3979 install_local_discovery_client(signing_key, discovery_vk, service_token)?
3980 };
3981 (authority, discovery_client)
3982 }
3983 crate::DeploymentTrustSource::DidAnchored(anchors) => {
3984 anyhow::ensure!(
3985 !network_required,
3986 "required native bootstrap needs provisioned OS-owned accepted service state"
3987 );
3988 let (authority, discovery_transport, mesh_kem_recipient, ml_dsa_65_keys) =
3989 authenticate_did_anchored_bootstrap(&anchors).await?;
3990 let token_provider = renewing_service_token_provider(&signing_key, service_token);
3991 let signer = hyprstream_rpc::signer::LocalSigner::new(signing_key);
3992 if remote_node {
3993 let recipient = mesh_kem_recipient.ok_or_else(|| {
3998 anyhow::anyhow!(
3999 "did:web deployment document publishes no #mesh-kem keyAgreement; \
4000 refusing cleartext downgrade for the remote-node bootstrap"
4001 )
4002 })?;
4003 let pq_key = match ml_dsa_65_keys.as_slice() {
4006 [key] => key.clone(),
4007 keys => anyhow::bail!(
4008 "did:web deployment document must publish exactly one ML-DSA-65 \
4009 verification method for the remote-node bootstrap (found {})",
4010 keys.len()
4011 ),
4012 };
4013 let mut kem_store = hyprstream_rpc::crypto::hybrid_kem::KeyedKemTrustStore::new();
4014 kem_store.bind(discovery_vk.to_bytes(), recipient);
4015 let mut pq_store = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
4016 pq_store.bind(discovery_vk.to_bytes(), &pq_key);
4017 let rpc = hyprstream_rpc::dial::dial_with_crypto_stores_and_token_provider(
4020 &discovery_transport,
4021 signer,
4022 Some(discovery_vk),
4023 token_provider,
4024 Some(Arc::new(kem_store)),
4025 Some(Arc::new(pq_store)),
4026 )?;
4027 let client = hyprstream_rpc_std::discovery_client::DiscoveryClient::new(rpc);
4028 let health = client
4029 .ping()
4030 .await
4031 .context("DID-anchored Discovery reach failed liveness check")?;
4032 anyhow::ensure!(
4033 health.status == "ok",
4034 "DID-anchored Discovery liveness returned status {:?}",
4035 health.status
4036 );
4037 (authority, client)
4038 } else {
4039 let transport = hyprstream_rpc::registry::try_global()
4046 .ok_or_else(|| {
4047 anyhow::anyhow!(
4048 "EndpointRegistry not initialized — same-node discovery fabric unavailable"
4049 )
4050 })?
4051 .try_endpoint("discovery", hyprstream_rpc::registry::SocketKind::Rep)?;
4052 let rpc = hyprstream_rpc::dial::dial_with_token_provider(
4053 &transport,
4054 signer,
4055 Some(discovery_vk),
4056 token_provider,
4057 )?;
4058 (
4059 authority,
4060 hyprstream_rpc_std::discovery_client::DiscoveryClient::new(rpc),
4061 )
4062 }
4063 }
4064 };
4065 DiscoveryService::bootstrap_authenticated_process(authority, discovery_client, records)
4066}
4067
4068#[cfg(not(target_arch = "wasm32"))]
4069fn seal_process_bootstrap_authority() -> Result<()> {
4070 let mut state = PROCESS_BOOTSTRAP_AUTHORITY.lock();
4071 anyhow::ensure!(
4072 matches!(*state, ProcessBootstrapAuthorityState::Unsealed),
4073 "Discovery bootstrap authority is already sealed or consumed"
4074 );
4075 *state = ProcessBootstrapAuthorityState::Sealed;
4076 Ok(())
4077}
4078
4079#[cfg(not(target_arch = "wasm32"))]
4080fn authenticate_deployment_bootstrap() -> Result<ProcessBootstrapAuthority> {
4081 seal_process_bootstrap_authority()?;
4082 let verifier = authenticate_local_deployment_registry()?;
4083 let store_path = hyprstream_service::deployment_data_dir()?.join("pds-store");
4084 Ok(ProcessBootstrapAuthority {
4085 store_path,
4086 acceptance_identity: ProcessAcceptanceIdentity::Deployment(verifier),
4087 network_required: false,
4088 })
4089}
4090
4091#[cfg(not(target_arch = "wasm32"))]
4092async fn authenticate_did_anchored_bootstrap(
4093 anchors: &crate::DidAnchors,
4094) -> Result<(
4095 ProcessBootstrapAuthority,
4096 TransportConfig,
4097 Option<hyprstream_rpc::crypto::hybrid_kem::RecipientPublic>,
4098 Vec<hyprstream_rpc::crypto::pq::MlDsaVerifyingKey>,
4099)> {
4100 seal_process_bootstrap_authority()?;
4101 let trust = crate::did_anchored::resolve_did_anchored_trust(anchors).await?;
4102 let authority_checkpoint = load_deployment_authority_checkpoint()?;
4103 let discovery_transport = trust.discovery_transport.clone();
4104 let mesh_kem_recipient = trust.mesh_kem_recipient.clone();
4105 let ml_dsa_65_keys = trust.ml_dsa_65_keys.clone();
4106 let verifier = authenticate_resolved_did_trust(trust, authority_checkpoint)?;
4107 let authority = ProcessBootstrapAuthority {
4108 store_path: hyprstream_service::deployment_data_dir()?.join("pds-store"),
4109 acceptance_identity: ProcessAcceptanceIdentity::Deployment(verifier),
4110 network_required: false,
4111 };
4112 Ok((
4113 authority,
4114 discovery_transport,
4115 mesh_kem_recipient,
4116 ml_dsa_65_keys,
4117 ))
4118}
4119
4120pub(crate) fn authenticate_resolved_did_trust(
4121 trust: crate::did_anchored::DidAnchoredTrust,
4122 authority_checkpoint: DeploymentAuthorityCheckpoint,
4123) -> Result<RegistryDeploymentVerifier> {
4124 authenticate_registry_deployment_credentials(TrustedRegistryDeploymentCredentials {
4125 ca_verifying_key: trust.ca_verifying_key,
4126 authority_log: trust.authority_log,
4127 authority_checkpoint,
4128 registry_credential: trust.registry_credential,
4129 })
4130 .map(|witness| witness.verifier)
4131}
4132
4133#[cfg(not(target_arch = "wasm32"))]
4143pub async fn resolve_and_authenticate_did_anchors(
4144 anchors: &crate::DidAnchors,
4145) -> Result<(RegistryDeploymentVerifier, TransportConfig)> {
4146 let trust = crate::did_anchored::resolve_did_anchored_trust(anchors).await?;
4147 let authority_checkpoint = load_deployment_authority_checkpoint()?;
4148 let discovery_transport = trust.discovery_transport.clone();
4149 let verifier = authenticate_resolved_did_trust(trust, authority_checkpoint)?;
4150 Ok((verifier, discovery_transport))
4151}
4152
4153#[cfg(test)]
4154#[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
4155fn authenticate_discovery_bootstrap_identity(
4156 acceptance_identity: ed25519_dalek::VerifyingKey,
4157) -> Result<ProcessBootstrapAuthority> {
4158 let mut state = PROCESS_BOOTSTRAP_AUTHORITY.lock();
4159 anyhow::ensure!(
4160 matches!(*state, ProcessBootstrapAuthorityState::Unsealed),
4161 "Discovery bootstrap authority is already sealed or consumed"
4162 );
4163 *state = ProcessBootstrapAuthorityState::Sealed;
4164 Ok(ProcessBootstrapAuthority {
4165 store_path: hyprstream_service::deployment_data_dir()?.join("pds-store"),
4166 acceptance_identity: ProcessAcceptanceIdentity::Test(acceptance_identity),
4167 network_required: false,
4168 })
4169}
4170
4171pub fn production_rpc_client(
4191 service_name: &str,
4192 signing_key: SigningKey,
4193 token: Option<String>,
4194) -> Result<Arc<dyn RpcClient>> {
4195 let resolver = PRODUCTION_RESOLVER
4196 .get()
4197 .cloned()
4198 .ok_or_else(|| anyhow::anyhow!("checkpoint-backed production resolver is not installed"))?;
4199 Ok(Arc::new(ProductionRpcClient::new(
4200 service_name,
4201 service_name,
4202 None,
4203 signing_key,
4204 token,
4205 resolver,
4206 )?))
4207}
4208
4209pub fn installed_bootstrap_discovery_client(
4225) -> Option<hyprstream_rpc_std::discovery_client::DiscoveryClient> {
4226 PRODUCTION_RESOLVER.get().cloned()?.discovery_client.clone()
4227}
4228
4229pub fn production_inference_rpc_client_at_transport(
4240 resolution_service_name: &str,
4241 selected_transport: &TransportConfig,
4242 signing_key: SigningKey,
4243 token: Option<String>,
4244) -> Result<Arc<dyn RpcClient>> {
4245 let resolver = PRODUCTION_RESOLVER
4246 .get()
4247 .cloned()
4248 .ok_or_else(|| anyhow::anyhow!("checkpoint-backed production resolver is not installed"))?;
4249 Ok(Arc::new(ProductionRpcClient::new(
4250 resolution_service_name,
4251 "inference",
4252 Some(selected_transport.clone()),
4253 signing_key,
4254 token,
4255 resolver,
4256 )?))
4257}
4258
4259pub async fn production_browser_provisioning(
4263 request: BrowserProvisioningRequest,
4264) -> Result<BrowserProvisioningDocument> {
4265 let resolver = PRODUCTION_RESOLVER
4266 .get()
4267 .cloned()
4268 .ok_or_else(|| anyhow::anyhow!("checkpoint-backed production resolver is not installed"))?;
4269 resolver.browser_provisioning(request).await
4270}
4271
4272pub fn production_browser_currentness_verifier() -> Result<Arc<dyn BrowserCurrentnessVerifier>> {
4276 let resolver = PRODUCTION_RESOLVER
4277 .get()
4278 .cloned()
4279 .ok_or_else(|| anyhow::anyhow!("checkpoint-backed production resolver is not installed"))?;
4280 Ok(Arc::new(ProductionBrowserCurrentnessVerifier { resolver }))
4281}
4282
4283struct ProductionRpcClient {
4284 resolution_service_name: String,
4285 service_domain: String,
4286 selected_transport: Option<TransportConfig>,
4287 signing_key: SigningKey,
4288 token: Option<String>,
4289 resolver: Arc<DiscoveryServiceResolver>,
4290 request_id: std::sync::atomic::AtomicU64,
4291}
4292
4293fn production_service_query(service_name: &str, required: bool) -> Result<ServiceQuery> {
4294 ServiceQuery::new(
4295 service_name,
4296 ["hyprstream-rpc/1".to_owned()],
4297 if required {
4298 ResolverProfile::NativeIrohRequired
4299 } else {
4300 ResolverProfile::NetworkDiscovery
4301 },
4302 3,
4303 )
4304}
4305
4306impl ProductionRpcClient {
4307 fn new(
4308 resolution_service_name: &str,
4309 service_domain: &str,
4310 selected_transport: Option<TransportConfig>,
4311 signing_key: SigningKey,
4312 token: Option<String>,
4313 resolver: Arc<DiscoveryServiceResolver>,
4314 ) -> Result<Self> {
4315 for (label, name) in [
4316 ("resolution service name", resolution_service_name),
4317 ("service domain", service_domain),
4318 ] {
4319 anyhow::ensure!(
4320 !name.is_empty()
4321 && name
4322 .bytes()
4323 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'),
4324 "{label} is not canonical"
4325 );
4326 }
4327 Ok(Self {
4328 resolution_service_name: resolution_service_name.to_owned(),
4329 service_domain: service_domain.to_owned(),
4330 selected_transport,
4331 signing_key,
4332 token,
4333 resolver,
4334 request_id: std::sync::atomic::AtomicU64::new(1),
4335 })
4336 }
4337 async fn snapshots(&self) -> Result<Vec<ResolvedService>> {
4338 let query =
4339 production_service_query(&self.resolution_service_name, native_network_required())?;
4340 let max_attempts = query.max_attempts;
4341 let mut snapshots = self.resolver.resolve_service_candidates(query).await?;
4342 let authority = snapshots
4343 .first()
4344 .ok_or_else(|| anyhow::anyhow!("resolver returned no validated alternatives"))?;
4345 anyhow::ensure!(
4346 snapshots.iter().all(|item| item.same_authority(authority)),
4347 "resolver retry set crosses service authority"
4348 );
4349 if let Some(selected) = &self.selected_transport {
4350 snapshots.retain(|snapshot| snapshot.transport() == selected);
4351 anyhow::ensure!(
4352 !snapshots.is_empty(),
4353 "router-selected reach is not a current authorized candidate for service '{}'",
4354 self.resolution_service_name
4355 );
4356 }
4357 Ok(snapshots.into_iter().take(max_attempts).collect())
4358 }
4359 fn client_for(&self, snapshot: &ResolvedService) -> Result<Arc<dyn RpcClient>> {
4360 let (kem, pq) = snapshot.crypto_stores()?;
4361 #[cfg(feature = "test-fixtures")]
4362 if let Some(client) = test_fixtures::dial_override(snapshot.transport())? {
4363 return Ok(client);
4364 }
4365 let signer = hyprstream_rpc::signer::LocalSigner::new(self.signing_key.clone());
4366 hyprstream_rpc::dial::dial_with_crypto_stores(
4367 snapshot.transport(),
4368 signer,
4369 Some(snapshot.response_verifying_key()),
4370 current_service_token(&self.signing_key, &self.token),
4371 Some(kem),
4372 Some(pq),
4373 )
4374 }
4375 async fn attempt<T, F, Fut>(&self, mut call: F) -> Result<(T, ResolvedService)>
4376 where
4377 F: FnMut(Arc<dyn RpcClient>) -> Fut,
4378 Fut: std::future::Future<Output = Result<T>>,
4379 {
4380 for refresh in 0..2 {
4381 let mut last_transport_error = None;
4382 let mut invalidated = None;
4383 for snapshot in self.snapshots().await? {
4384 if let Err(error) = self.resolver.ensure_current(&snapshot).await {
4385 invalidated = Some(error);
4386 break;
4387 }
4388 match call(self.client_for(&snapshot)?).await {
4389 Ok(value) => return Ok((value, snapshot)),
4390 Err(error)
4391 if hyprstream_rpc::transport_traits::is_pre_dispatch_transport_error(
4392 &error,
4393 ) =>
4394 {
4395 last_transport_error = Some(error);
4396 }
4397 Err(error) => return Err(error),
4398 }
4399 }
4400 if let Some(error) = invalidated {
4401 if refresh == 0 {
4402 continue;
4403 }
4404 return Err(anyhow::anyhow!(
4405 "resolved service remained invalid after re-resolution: {error}"
4406 ));
4407 }
4408 return Err(last_transport_error.unwrap_or_else(|| {
4409 anyhow::anyhow!("validated same-authority alternatives exhausted")
4410 }));
4411 }
4412 unreachable!("bounded re-resolution loop always returns")
4413 }
4414 fn checked_stream(
4415 &self,
4416 inner: Box<dyn StreamHandle>,
4417 snapshot: ResolvedService,
4418 ) -> Box<dyn StreamHandle> {
4419 Box::new(CurrentStreamHandle {
4420 inner,
4421 resolver: Arc::clone(&self.resolver),
4422 snapshot,
4423 invalidated: false,
4424 })
4425 }
4426}
4427
4428#[cfg(feature = "test-fixtures")]
4429#[doc(hidden)]
4430pub mod test_fixtures {
4431 use super::*;
4432 use crate::state_store::PutResult;
4433 use hyprstream_crypto::pq::ml_dsa_sk_to_vk_bytes;
4434 use hyprstream_pds::at9p::{
4435 CapsuleBody, HybridKeyPair, ServiceEndpoint, ServiceEntry, ServiceType,
4436 Transport as At9pTransport,
4437 };
4438 use hyprstream_pds::at9p_duplicity::AcceptedAt9pState;
4439 use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
4440
4441 type DialOverride = dyn Fn(&TransportConfig) -> Result<Arc<dyn RpcClient>> + Send + Sync;
4442
4443 static DIAL_OVERRIDE: std::sync::OnceLock<Arc<DialOverride>> = std::sync::OnceLock::new();
4444
4445 struct FixtureAcceptedStates(parking_lot::Mutex<HashMap<String, AcceptedAt9pState>>);
4446
4447 impl AcceptedStateSource for FixtureAcceptedStates {
4448 fn accepted_state(&self, did: &str) -> Result<Option<AcceptedAt9pState>> {
4449 Ok(self.0.lock().get(did).cloned())
4450 }
4451 fn accepted_states(&self) -> Result<Vec<AcceptedAt9pState>> {
4452 Ok(self.0.lock().values().cloned().collect())
4453 }
4454 }
4455
4456 #[derive(Clone)]
4457 struct FixtureAuthority {
4458 state: AcceptedAt9pState,
4459 signing: SigningKey,
4460 request_kem_recipient: hyprstream_rpc::crypto::hybrid_kem::RecipientPublic,
4461 }
4462
4463 #[derive(Default)]
4471 struct FixtureAnnouncementStore {
4472 services: parking_lot::Mutex<HashMap<String, Vec<AnnouncedEndpoint>>>,
4473 }
4474
4475 impl FixtureAnnouncementStore {
4476 fn put_announcement_sync(&self, service_name: &str, endpoint: AnnouncedEndpoint) {
4477 let mut services = self.services.lock();
4478 let endpoints = services.entry(service_name.to_owned()).or_default();
4479 endpoints.retain(|existing| existing.endpoint != endpoint.endpoint);
4480 endpoints.push(endpoint);
4481 }
4482
4483 fn announcements_for_sync(
4484 &self,
4485 service_name: &str,
4486 now_unix_ms: i64,
4487 ) -> Vec<AnnouncedEndpoint> {
4488 self.services
4489 .lock()
4490 .get(service_name)
4491 .into_iter()
4492 .flatten()
4493 .filter(|entry| entry.is_live_at(now_unix_ms))
4494 .cloned()
4495 .collect()
4496 }
4497
4498 fn clear_announcements_sync(&self, service_name: &str) {
4499 self.services.lock().remove(service_name);
4500 }
4501 }
4502
4503 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
4504 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
4505 impl DiscoveryStateStore for FixtureAnnouncementStore {
4506 async fn put_announcement(
4507 &self,
4508 service_name: &str,
4509 endpoint: AnnouncedEndpoint,
4510 ) -> Result<PutResult> {
4511 self.put_announcement_sync(service_name, endpoint);
4512 Ok(PutResult::Stored)
4513 }
4514
4515 async fn announcements_for(
4516 &self,
4517 service_name: &str,
4518 now_unix_ms: i64,
4519 ) -> Result<Vec<AnnouncedEndpoint>> {
4520 Ok(self.announcements_for_sync(service_name, now_unix_ms))
4521 }
4522
4523 async fn all_announcements(
4524 &self,
4525 now_unix_ms: i64,
4526 ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>> {
4527 Ok(self
4528 .services
4529 .lock()
4530 .iter()
4531 .map(|(service_name, endpoints)| {
4532 (
4533 service_name.clone(),
4534 endpoints
4535 .iter()
4536 .filter(|entry| entry.is_live_at(now_unix_ms))
4537 .cloned()
4538 .collect(),
4539 )
4540 })
4541 .collect())
4542 }
4543
4544 async fn put_liveness(&self, _node: &Did, _value: LiveAllocatable) -> Result<PutResult> {
4548 anyhow::bail!("fixture announcement store does not track liveness")
4549 }
4550
4551 #[cfg(test)]
4552 async fn liveness(
4553 &self,
4554 _node: &Did,
4555 _now_unix_ms: i64,
4556 ) -> Result<Option<LiveAllocatable>> {
4557 anyhow::bail!("fixture announcement store does not track liveness")
4558 }
4559
4560 async fn all_liveness(&self, _now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>> {
4561 anyhow::bail!("fixture announcement store does not track liveness")
4562 }
4563
4564 async fn put_entity_statement(
4565 &self,
4566 _issuer: &str,
4567 _value: CachedEntityStatement,
4568 ) -> Result<()> {
4569 anyhow::bail!("fixture announcement store does not track entity statements")
4570 }
4571
4572 async fn entity_statement(&self, _issuer: &str) -> Result<Option<CachedEntityStatement>> {
4573 anyhow::bail!("fixture announcement store does not track entity statements")
4574 }
4575
4576 async fn known_issuers(&self) -> Result<Vec<String>> {
4577 anyhow::bail!("fixture announcement store does not track entity statements")
4578 }
4579
4580 async fn known_issuer_count(&self) -> Result<usize> {
4581 anyhow::bail!("fixture announcement store does not track entity statements")
4582 }
4583
4584 async fn put_envelope_keyset(
4585 &self,
4586 _service_did: &str,
4587 _value: CachedEnvelopeKeyset,
4588 ) -> Result<()> {
4589 anyhow::bail!("fixture announcement store does not track envelope keysets")
4590 }
4591
4592 async fn envelope_keyset(
4593 &self,
4594 _service_did: &str,
4595 ) -> Result<Option<CachedEnvelopeKeyset>> {
4596 anyhow::bail!("fixture announcement store does not track envelope keysets")
4597 }
4598 }
4599
4600 #[derive(Clone)]
4605 pub struct ProductionInferenceFixture {
4606 service_name: String,
4607 announced: Arc<FixtureAnnouncementStore>,
4608 states: Arc<FixtureAcceptedStates>,
4609 primary: FixtureAuthority,
4610 foreign: FixtureAuthority,
4611 }
4612
4613 fn authority(service_name: &str, tag: u8) -> Result<FixtureAuthority> {
4614 let signing = SigningKey::from_bytes(&[tag; 32]);
4615 let pq_signing = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signing);
4616 let keys = HybridKeyPair::new(
4617 signing.verifying_key().to_bytes().to_vec(),
4618 ml_dsa_sk_to_vk_bytes(&pq_signing),
4619 )?;
4620 let endpoint = ServiceEndpoint::new(At9pTransport::Iroh, "iroh://fixture")?;
4621 let service = ServiceEntry::new(
4622 format!("#{service_name}"),
4623 ServiceType::NinePExport,
4624 endpoint,
4625 )?;
4626 let body = CapsuleBody::new(vec![keys], vec![service])?;
4627 let genesis = sign_capsule(body.clone(), &signing, &pq_signing)?;
4628 let subject = genesis.cid512()?;
4629 let update = sign_update_record(
4630 subject,
4631 1,
4632 [1; 64],
4633 body,
4634 "2099-01-01T00:00:00Z".to_owned(),
4635 &signing,
4636 &pq_signing,
4637 )?;
4638 let state = AcceptedAt9pState::from_persisted_update(&update.to_dag_cbor()?)?;
4639 let request_kem_recipient = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
4640 hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
4641 )?
4642 .public()
4643 .clone();
4644 Ok(FixtureAuthority {
4645 state,
4646 signing,
4647 request_kem_recipient,
4648 })
4649 }
4650
4651 fn announcement(
4652 authority: &FixtureAuthority,
4653 transport: &TransportConfig,
4654 last_heartbeat: Instant,
4655 ) -> Result<AnnouncedEndpoint> {
4656 let socket_kind = match &transport.endpoint {
4657 EndpointType::Quic { .. } => "quic",
4658 EndpointType::Iroh { .. } => "iroh",
4659 _ => anyhow::bail!("production inference fixture advertises only network candidates"),
4660 };
4661 Ok(AnnouncedEndpoint {
4662 socket_kind: socket_kind.to_owned(),
4663 endpoint: transport.endpoint_string(),
4664 service_did: Did::from(authority.state.did.clone()),
4665 capabilities: ["hyprstream-rpc/1".to_owned(), "hyprstream-moq/1".to_owned()]
4666 .into_iter()
4667 .collect(),
4668 accepted_state_digest: authority.state.head_digest.to_vec(),
4669 accepted_state_epoch: authority.state.epoch,
4670 response_key_id: format!("{}#response-current", authority.state.did),
4671 request_kem_key_id: format!("{}#kem-current", authority.state.did),
4672 request_kem_recipient: authority.request_kem_recipient.encode(),
4673 expires_at_unix_ms: 4_070_908_800_000,
4674 source_signer: authority.signing.verifying_key().to_bytes(),
4675 live_until_unix_ms: unix_millis_now()
4676 .saturating_add(ANNOUNCED_ENDPOINT_TTL.as_millis() as i64)
4677 .saturating_sub(
4678 Instant::now()
4679 .saturating_duration_since(last_heartbeat)
4680 .as_millis() as i64,
4681 ),
4682 })
4683 }
4684
4685 impl ProductionInferenceFixture {
4686 pub fn reset(&self, transports: &[TransportConfig]) -> Result<()> {
4688 {
4689 let mut states = self.states.0.lock();
4690 states.clear();
4691 states.insert(self.primary.state.did.clone(), self.primary.state.clone());
4692 }
4693 self.announced.clear_announcements_sync(&self.service_name);
4694 for transport in transports {
4695 self.announced.put_announcement_sync(
4696 &self.service_name,
4697 announcement(&self.primary, transport, Instant::now())?,
4698 );
4699 }
4700 Ok(())
4701 }
4702
4703 pub fn mark_stale(&self) {
4705 for mut endpoint in self
4706 .announced
4707 .announcements_for_sync(&self.service_name, unix_millis_now())
4708 {
4709 endpoint.live_until_unix_ms = unix_millis_now() - 1;
4710 self.announced
4711 .put_announcement_sync(&self.service_name, endpoint);
4712 }
4713 }
4714
4715 pub fn add_foreign_authority(&self, transport: &TransportConfig) -> Result<()> {
4717 self.states
4718 .0
4719 .lock()
4720 .insert(self.foreign.state.did.clone(), self.foreign.state.clone());
4721 self.announced.put_announcement_sync(
4722 &self.service_name,
4723 announcement(&self.foreign, transport, Instant::now())?,
4724 );
4725 Ok(())
4726 }
4727 }
4728
4729 pub fn install_production_inference_fixture(
4732 service_name: &str,
4733 transports: &[TransportConfig],
4734 dial_override: Arc<dyn Fn(&TransportConfig) -> Result<Arc<dyn RpcClient>> + Send + Sync>,
4735 ) -> Result<ProductionInferenceFixture> {
4736 anyhow::ensure!(
4737 PRODUCTION_RESOLVER.get().is_none() && DIAL_OVERRIDE.get().is_none(),
4738 "production inference fixture is already installed"
4739 );
4740 let primary = authority(service_name, 0x61)?;
4741 let foreign = authority(service_name, 0x62)?;
4742 let states = Arc::new(FixtureAcceptedStates(parking_lot::Mutex::new(
4743 HashMap::new(),
4744 )));
4745 let announced = Arc::new(FixtureAnnouncementStore::default());
4746 let fixture = ProductionInferenceFixture {
4747 service_name: service_name.to_owned(),
4748 announced: Arc::clone(&announced),
4749 states: Arc::clone(&states),
4750 primary,
4751 foreign,
4752 };
4753 fixture.reset(transports)?;
4754 DIAL_OVERRIDE.set(dial_override).map_err(|_| {
4755 anyhow::anyhow!("production inference fixture dial is already installed")
4756 })?;
4757 PRODUCTION_RESOLVER
4758 .set(Arc::new(DiscoveryServiceResolver {
4759 state_store: announced,
4760 accepted_state_source: states,
4761 discovery_client: None,
4762 }))
4763 .map_err(|_| {
4764 anyhow::anyhow!("production inference fixture resolver is already installed")
4765 })?;
4766 Ok(fixture)
4767 }
4768
4769 pub fn install_bootstrap_discovery_client_fixture(
4778 discovery_client: hyprstream_rpc_std::discovery_client::DiscoveryClient,
4779 ) -> Result<()> {
4780 anyhow::ensure!(
4781 PRODUCTION_RESOLVER.get().is_none(),
4782 "production resolver fixture is already installed"
4783 );
4784 PRODUCTION_RESOLVER
4785 .set(Arc::new(DiscoveryServiceResolver {
4786 state_store: MemoryStateStore::production_default(),
4787 accepted_state_source: Arc::new(FixtureAcceptedStates(parking_lot::Mutex::new(
4788 HashMap::new(),
4789 ))),
4790 discovery_client: Some(discovery_client),
4791 }))
4792 .map_err(|_| anyhow::anyhow!("production resolver is already installed"))
4793 }
4794
4795 pub(super) fn dial_override(transport: &TransportConfig) -> Result<Option<Arc<dyn RpcClient>>> {
4796 DIAL_OVERRIDE.get().map(|dial| dial(transport)).transpose()
4797 }
4798}
4799
4800#[async_trait]
4801impl RpcClient for ProductionRpcClient {
4802 async fn call(&self, payload: Vec<u8>) -> Result<Vec<u8>> {
4803 Ok(self
4804 .attempt(|c| {
4805 let p = payload.clone();
4806 async move { c.call(p).await }
4807 })
4808 .await?
4809 .0)
4810 }
4811 async fn call_for_service(&self, service: &str, payload: Vec<u8>) -> Result<Vec<u8>> {
4812 anyhow::ensure!(
4813 service == self.service_domain,
4814 "generated service authority mismatch"
4815 );
4816 let service = service.to_owned();
4817 Ok(self
4818 .attempt(|c| {
4819 let p = payload.clone();
4820 let s = service.clone();
4821 async move { c.call_for_service(&s, p).await }
4822 })
4823 .await?
4824 .0)
4825 }
4826 async fn call_for_service_with_method(
4827 &self,
4828 service: &str,
4829 method_discriminator: u16,
4830 payload: Vec<u8>,
4831 ) -> Result<Vec<u8>> {
4832 anyhow::ensure!(
4833 service == self.service_domain,
4834 "generated service authority mismatch"
4835 );
4836 let service = service.to_owned();
4837 Ok(self
4838 .attempt(|c| {
4839 let p = payload.clone();
4840 let s = service.clone();
4841 async move {
4842 c.call_for_service_with_method(&s, method_discriminator, p)
4843 .await
4844 }
4845 })
4846 .await?
4847 .0)
4848 }
4849
4850 async fn call_with_options(&self, payload: Vec<u8>, options: CallOptions) -> Result<Vec<u8>> {
4851 Ok(self
4852 .attempt(|c| {
4853 let p = payload.clone();
4854 let o = options.clone();
4855 async move { c.call_with_options(p, o).await }
4856 })
4857 .await?
4858 .0)
4859 }
4860 async fn call_with_options_for_service(
4861 &self,
4862 service: &str,
4863 payload: Vec<u8>,
4864 options: CallOptions,
4865 ) -> Result<Vec<u8>> {
4866 anyhow::ensure!(
4867 service == self.service_domain,
4868 "generated service authority mismatch"
4869 );
4870 let service = service.to_owned();
4871 Ok(self
4872 .attempt(|c| {
4873 let p = payload.clone();
4874 let o = options.clone();
4875 let s = service.clone();
4876 async move { c.call_with_options_for_service(&s, p, o).await }
4877 })
4878 .await?
4879 .0)
4880 }
4881 async fn call_with_options_for_service_with_method(
4882 &self,
4883 service: &str,
4884 method_discriminator: u16,
4885 payload: Vec<u8>,
4886 options: CallOptions,
4887 ) -> Result<Vec<u8>> {
4888 anyhow::ensure!(
4889 service == self.service_domain,
4890 "generated service authority mismatch"
4891 );
4892 let service = service.to_owned();
4893 Ok(self
4894 .attempt(|c| {
4895 let p = payload.clone();
4896 let o = options.clone();
4897 let s = service.clone();
4898 async move {
4899 c.call_with_options_for_service_with_method(&s, method_discriminator, p, o)
4900 .await
4901 }
4902 })
4903 .await?
4904 .0)
4905 }
4906 async fn call_streaming(&self, payload: Vec<u8>, ephemeral: [u8; 32]) -> Result<Vec<u8>> {
4907 Ok(self
4908 .attempt(|c| {
4909 let p = payload.clone();
4910 async move { c.call_streaming(p, ephemeral).await }
4911 })
4912 .await?
4913 .0)
4914 }
4915 async fn call_streaming_for_service(
4916 &self,
4917 service: &str,
4918 payload: Vec<u8>,
4919 ephemeral: [u8; 32],
4920 ) -> Result<Vec<u8>> {
4921 anyhow::ensure!(
4922 service == self.service_domain,
4923 "generated service authority mismatch"
4924 );
4925 let service = service.to_owned();
4926 Ok(self
4927 .attempt(|c| {
4928 let p = payload.clone();
4929 let s = service.clone();
4930 async move { c.call_streaming_for_service(&s, p, ephemeral).await }
4931 })
4932 .await?
4933 .0)
4934 }
4935 async fn call_streaming_with_options_for_service(
4936 &self,
4937 service: &str,
4938 payload: Vec<u8>,
4939 ephemeral: [u8; 32],
4940 options: CallOptions,
4941 ) -> Result<Vec<u8>> {
4942 anyhow::ensure!(
4943 service == self.service_domain,
4944 "generated service authority mismatch"
4945 );
4946 let service = service.to_owned();
4947 Ok(self
4948 .attempt(|c| {
4949 let p = payload.clone();
4950 let o = options.clone();
4951 let s = service.clone();
4952 async move {
4953 c.call_streaming_with_options_for_service(&s, p, ephemeral, o)
4954 .await
4955 }
4956 })
4957 .await?
4958 .0)
4959 }
4960 async fn call_streaming_for_service_with_method(
4961 &self,
4962 service: &str,
4963 method_discriminator: u16,
4964 payload: Vec<u8>,
4965 ephemeral: [u8; 32],
4966 ) -> Result<Vec<u8>> {
4967 anyhow::ensure!(
4968 service == self.service_domain,
4969 "generated service authority mismatch"
4970 );
4971 let service = service.to_owned();
4972 Ok(self
4973 .attempt(|c| {
4974 let p = payload.clone();
4975 let s = service.clone();
4976 async move {
4977 c.call_streaming_for_service_with_method(&s, method_discriminator, p, ephemeral)
4978 .await
4979 }
4980 })
4981 .await?
4982 .0)
4983 }
4984 async fn call_streaming_with_options_for_service_with_method(
4985 &self,
4986 service: &str,
4987 method_discriminator: u16,
4988 payload: Vec<u8>,
4989 ephemeral: [u8; 32],
4990 options: CallOptions,
4991 ) -> Result<Vec<u8>> {
4992 anyhow::ensure!(
4993 service == self.service_domain,
4994 "generated service authority mismatch"
4995 );
4996 let service = service.to_owned();
4997 Ok(self
4998 .attempt(|c| {
4999 let p = payload.clone();
5000 let o = options.clone();
5001 let s = service.clone();
5002 async move {
5003 c.call_streaming_with_options_for_service_with_method(
5004 &s,
5005 method_discriminator,
5006 p,
5007 ephemeral,
5008 o,
5009 )
5010 .await
5011 }
5012 })
5013 .await?
5014 .0)
5015 }
5016 async fn open_stream(&self, payload: Vec<u8>) -> Result<Box<dyn StreamHandle>> {
5017 let (inner, snapshot) = self
5018 .attempt(|c| {
5019 let p = payload.clone();
5020 async move { c.open_stream(p).await }
5021 })
5022 .await?;
5023 Ok(self.checked_stream(inner, snapshot))
5024 }
5025 async fn open_stream_from_info(
5026 &self,
5027 info: hyprstream_rpc::stream_info::StreamInfo,
5028 secret: [u8; 32],
5029 public: [u8; 32],
5030 ) -> Result<Box<dyn StreamHandle>> {
5031 let (inner, snapshot) = self
5032 .attempt(|c| {
5033 let i = info.clone();
5034 async move { c.open_stream_from_info(i, secret, public).await }
5035 })
5036 .await?;
5037 Ok(self.checked_stream(inner, snapshot))
5038 }
5039 fn next_id(&self) -> u64 {
5040 self.request_id
5041 .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
5042 }
5043}
5044
5045impl DiscoveryService {
5046 async fn resolve_announced_endpoint(
5047 &self,
5048 name: &str,
5049 kind: SocketKind,
5050 ) -> anyhow::Result<Option<TransportConfig>> {
5051 let wanted = socket_kind_to_string(kind);
5052 let endpoints = self
5053 .state_store
5054 .announcements_for(name, unix_millis_now())
5055 .await?;
5056 let Some(endpoint) = endpoints.iter().find(|ep| ep.socket_kind == wanted) else {
5057 return Ok(None);
5058 };
5059
5060 Ok(Some(announced_endpoint_to_transport(endpoint)?))
5061 }
5062}
5063
5064fn announced_endpoint_to_transport(
5065 endpoint: &AnnouncedEndpoint,
5066) -> anyhow::Result<TransportConfig> {
5067 match endpoint.socket_kind.as_str() {
5068 "quic" => parse_announced_quic(&endpoint.endpoint),
5069 "iroh" => parse_announced_iroh(&endpoint.endpoint),
5070 _ => Ok(TransportConfig::from_endpoint(&endpoint.endpoint)),
5071 }
5072}
5073
5074fn parse_announced_quic(endpoint: &str) -> anyhow::Result<TransportConfig> {
5075 let rest = endpoint
5076 .strip_prefix("quic://")
5077 .ok_or_else(|| anyhow::anyhow!("announced QUIC endpoint must start with quic://"))?;
5078 let (reach, encoded_hashes) = rest
5079 .split_once('#')
5080 .map_or((rest, None), |(reach, hashes)| (reach, Some(hashes)));
5081 let (server_name, addr) = reach.split_once(':').ok_or_else(|| {
5082 anyhow::anyhow!("announced QUIC endpoint must be quic://<server-name>:<socket-addr>")
5083 })?;
5084 anyhow::ensure!(
5085 !server_name.is_empty(),
5086 "announced QUIC endpoint is missing server name"
5087 );
5088 let addr = SocketAddr::from_str(addr)
5089 .map_err(|e| anyhow::anyhow!("invalid announced QUIC socket address '{addr}': {e}"))?;
5090 anyhow::ensure!(
5091 addr.port() != 0,
5092 "announced QUIC endpoint must not use port 0"
5093 );
5094 let transport = if let Some(encoded_hashes) = encoded_hashes {
5095 let hashes = encoded_hashes
5096 .split(',')
5097 .map(|value| {
5098 URL_SAFE_NO_PAD
5099 .decode(value)
5100 .context("invalid announced QUIC certificate hash")?
5101 .try_into()
5102 .map_err(|_| {
5103 anyhow::anyhow!("announced QUIC certificate hash must be 32 bytes")
5104 })
5105 })
5106 .collect::<Result<Vec<[u8; 32]>>>()?;
5107 anyhow::ensure!(!hashes.is_empty(), "announced QUIC pin set is empty");
5108 TransportConfig::quic_with_auth(
5109 addr,
5110 server_name,
5111 hyprstream_rpc::transport::QuicServerAuth::pinned(hashes)?,
5112 )
5113 } else {
5114 TransportConfig::quic(addr, server_name).with_connect_mode()
5115 };
5116 Ok(transport)
5117}
5118
5119fn parse_announced_iroh(endpoint: &str) -> anyhow::Result<TransportConfig> {
5120 let hex = endpoint
5121 .strip_prefix("iroh://")
5122 .ok_or_else(|| anyhow::anyhow!("announced iroh endpoint must start with iroh://"))?;
5123 anyhow::ensure!(
5124 hex.len() == 64,
5125 "announced iroh node id must be 32 bytes of hex"
5126 );
5127 let mut node_id = [0u8; 32];
5128 for (idx, byte) in node_id.iter_mut().enumerate() {
5129 let start = idx * 2;
5130 *byte = u8::from_str_radix(&hex[start..start + 2], 16)
5131 .map_err(|e| anyhow::anyhow!("invalid announced iroh node id hex: {e}"))?;
5132 }
5133 Ok(TransportConfig::iroh(node_id, Vec::new(), None))
5134}
5135
5136#[cfg(test)]
5137#[allow(clippy::expect_used, clippy::unwrap_used)]
5138mod resolver_tests {
5139 use super::*;
5140 use ed25519_dalek::Signer as _;
5141 use hyprstream_crypto::pq::ml_dsa_sk_to_vk_bytes;
5142 use hyprstream_pds::at9p::{
5143 CapsuleBody, HybridKeyPair, ServiceEndpoint, ServiceEntry, ServiceType,
5144 Transport as At9pTransport,
5145 };
5146 use hyprstream_pds::at9p_duplicity::AcceptedAt9pState;
5147 use hyprstream_pds::at9p_sign::{sign_capsule, sign_update_record};
5148 use hyprstream_rpc::transport::{BindMode, EndpointType};
5149
5150 fn checked_test_time(now: i64, offset: i64) -> i64 {
5151 now.checked_add(offset).expect("test NumericDate offset")
5152 }
5153
5154 struct TestDeploymentCa {
5155 ed25519: SigningKey,
5156 ml_dsa_65: hyprstream_rpc::crypto::pq::MlDsaSigningKey,
5157 verifying_keys: HybridDeploymentCa,
5158 }
5159
5160 fn test_deployment_ca(tag: u8) -> TestDeploymentCa {
5161 let ed25519 = SigningKey::from_bytes(&[tag; 32]);
5162 let (ml_dsa_65, ml_dsa_65_vk) = hyprstream_rpc::crypto::pq::ml_dsa_generate_keypair();
5163 let verifying_keys = HybridDeploymentCa::from_public_key_bytes(
5164 ed25519.verifying_key().as_bytes(),
5165 &hyprstream_rpc::crypto::pq::ml_dsa_vk_bytes(&ml_dsa_65_vk),
5166 )
5167 .expect("test hybrid deployment CA");
5168 TestDeploymentCa {
5169 ed25519,
5170 ml_dsa_65,
5171 verifying_keys,
5172 }
5173 }
5174
5175 fn sign_registry_credential_json(
5176 ca: &TestDeploymentCa,
5177 protected_json: &str,
5178 claims_json: &str,
5179 ) -> String {
5180 let protected = URL_SAFE_NO_PAD.encode(protected_json.as_bytes());
5181 let claims = URL_SAFE_NO_PAD.encode(claims_json.as_bytes());
5182 let signing_input = format!("{protected}.{claims}");
5183 let ml_dsa_signature =
5184 hyprstream_rpc::crypto::pq::ml_dsa_sign(&ca.ml_dsa_65, signing_input.as_bytes());
5185 let ed25519_signature = ca.ed25519.sign(signing_input.as_bytes());
5186 let mut signature = Vec::with_capacity(HYBRID_JWT_SIGNATURE_BYTES);
5187 signature.extend_from_slice(&ml_dsa_signature);
5188 signature.extend_from_slice(&ed25519_signature.to_bytes());
5189 format!("{signing_input}.{}", URL_SAFE_NO_PAD.encode(signature))
5190 }
5191
5192 fn exact_registry_credential_values(
5193 ca: &TestDeploymentCa,
5194 registry: &SigningKey,
5195 ) -> (serde_json::Value, serde_json::Value) {
5196 let domain = deployment_domain(&ca.verifying_keys);
5197 let now = chrono::Utc::now().timestamp();
5198 (
5199 serde_json::json!({
5200 "alg":"ML-DSA-65-Ed25519",
5201 "typ":"wit+jwt",
5202 "kid":domain
5203 }),
5204 serde_json::json!({
5205 "iss": format!("urn:hyprstream:deployment:{domain}"),
5206 "sub": "service:registry",
5207 "aud": REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE,
5208 "exp": checked_test_time(now, 3600),
5209 "nbf": now,
5210 "iat": now,
5211 "deployment_domain": domain,
5212 "profile": REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE,
5213 "cnf": {"jwk": {
5214 "kty": "OKP",
5215 "crv": "Ed25519",
5216 "x": URL_SAFE_NO_PAD.encode(registry.verifying_key().as_bytes()),
5217 }},
5218 }),
5219 )
5220 }
5221
5222 fn exact_registry_credential(ca: &TestDeploymentCa, registry: &SigningKey) -> String {
5223 let (protected, claims) = exact_registry_credential_values(ca, registry);
5224 sign_registry_credential_json(ca, &protected.to_string(), &claims.to_string())
5225 }
5226
5227 fn authenticate_test_registry_credential(
5228 ca: &TestDeploymentCa,
5229 credential: String,
5230 ) -> Result<AuthenticatedRegistryDeploymentIdentity> {
5231 let (key_bytes, _expires_at) = validate_registry_deployment_credential_profile(
5232 &credential,
5233 &ca.verifying_keys,
5234 DeploymentAuthorityState::GenesisOnly,
5235 )?;
5236 let verifying_key = VerifyingKey::from_bytes(&key_bytes)
5237 .map_err(|error| anyhow::anyhow!("registry credential key is malformed: {error}"))?;
5238 Ok(AuthenticatedRegistryDeploymentIdentity {
5239 verifier: RegistryDeploymentVerifier {
5240 verifying_key,
5241 _deployment_ca: Arc::new(ca.verifying_keys.clone()),
5242 },
5243 })
5244 }
5245
5246 fn assert_registry_credential_rejected(
5247 ca: &TestDeploymentCa,
5248 protected: &serde_json::Value,
5249 claims: &serde_json::Value,
5250 case: &str,
5251 ) {
5252 let credential =
5253 sign_registry_credential_json(ca, &protected.to_string(), &claims.to_string());
5254 assert!(
5255 authenticate_test_registry_credential(ca, credential).is_err(),
5256 "invalid deployment credential accepted: {case}"
5257 );
5258 }
5259
5260 #[test]
5261 fn exact_registry_deployment_credential_installs_the_exact_bound_jwk() {
5262 let ca = test_deployment_ca(0x31);
5263 let registry = SigningKey::from_bytes(&[0x32; 32]);
5264 let witness =
5265 authenticate_test_registry_credential(&ca, exact_registry_credential(&ca, ®istry))
5266 .expect("exact deployment credential profile");
5267 assert!(witness.verifier.matches(®istry.verifying_key()));
5268 assert!(
5269 witness.verifier.matches_deployment_root(&ca.verifying_keys),
5270 "registry verifier lost its hybrid deployment-root provenance"
5271 );
5272 }
5273
5274 #[test]
5275 fn classical_only_deployment_root_and_credential_fail_closed() {
5276 let classical_ca = SigningKey::from_bytes(&[0x41; 32]);
5277 assert!(
5278 HybridDeploymentCa::from_public_key_bytes(
5279 classical_ca.verifying_key().as_bytes(),
5280 &[],
5281 )
5282 .is_err(),
5283 "classical-only deployment CA was accepted"
5284 );
5285 assert!(
5286 HybridDeploymentCa::from_os_pin(classical_ca.verifying_key().as_bytes()).is_err(),
5287 "legacy 32-byte classical root pin was accepted"
5288 );
5289
5290 let ca = test_deployment_ca(0x42);
5291 let registry = SigningKey::from_bytes(&[0x43; 32]);
5292 let (protected, claims) = exact_registry_credential_values(&ca, ®istry);
5293 let protected = URL_SAFE_NO_PAD.encode(protected.to_string());
5294 let claims = URL_SAFE_NO_PAD.encode(claims.to_string());
5295 let signing_input = format!("{protected}.{claims}");
5296 let classical_signature = ca.ed25519.sign(signing_input.as_bytes());
5297 let classical_token = format!(
5298 "{signing_input}.{}",
5299 URL_SAFE_NO_PAD.encode(classical_signature.to_bytes())
5300 );
5301 assert!(
5302 authenticate_test_registry_credential(&ca, classical_token).is_err(),
5303 "classical-only credential was accepted beneath the hybrid root"
5304 );
5305 }
5306
5307 #[test]
5308 fn hybrid_deployment_credential_requires_each_signature_half() {
5309 let ca = test_deployment_ca(0x44);
5310 let registry = SigningKey::from_bytes(&[0x45; 32]);
5311 let credential = exact_registry_credential(&ca, ®istry);
5312 authenticate_test_registry_credential(&ca, credential.clone())
5313 .expect("intact hybrid deployment credential");
5314
5315 let (signing_input, signature_segment) =
5316 credential.rsplit_once('.').expect("credential signature");
5317 let signature = URL_SAFE_NO_PAD
5318 .decode(signature_segment)
5319 .expect("hybrid signature");
5320 for (case, index) in [("ML-DSA-65", 0usize), ("Ed25519", 3_309usize)] {
5321 let mut corrupted = signature.clone();
5322 corrupted[index] ^= 0x01;
5323 let token = format!("{signing_input}.{}", URL_SAFE_NO_PAD.encode(corrupted));
5324 assert!(
5325 authenticate_test_registry_credential(&ca, token).is_err(),
5326 "credential with invalid {case} half was accepted"
5327 );
5328 }
5329 }
5330
5331 #[test]
5332 fn deployment_credential_numeric_dates_enforce_all_boundaries_and_orderings() {
5333 let now = 1_000_000;
5334
5335 for (case, exp, nbf, iat) in [
5336 (
5337 "one-hour lifetime endpoint",
5338 checked_test_time(now, 3600),
5339 now,
5340 now,
5341 ),
5342 (
5343 "inside lifetime endpoint",
5344 checked_test_time(now, 3599),
5345 now,
5346 now,
5347 ),
5348 (
5349 "future-skew and lifetime endpoints",
5350 checked_test_time(now, 3660),
5351 checked_test_time(now, 60),
5352 checked_test_time(now, 60),
5353 ),
5354 ] {
5355 validate_registry_credential_numeric_dates(exp, nbf, iat, now)
5356 .unwrap_or_else(|error| panic!("valid {case} rejected: {error}"));
5357 }
5358
5359 for (case, exp, nbf, iat, validation_now) in [
5360 (
5361 "expired endpoint",
5362 now,
5363 now,
5364 checked_test_time(now, -1),
5365 now,
5366 ),
5367 (
5368 "outside lifetime endpoint",
5369 checked_test_time(now, 3601),
5370 now,
5371 now,
5372 now,
5373 ),
5374 (
5375 "future nbf outside skew",
5376 checked_test_time(now, 3600),
5377 checked_test_time(now, 61),
5378 now,
5379 now,
5380 ),
5381 (
5382 "future iat outside skew",
5383 checked_test_time(now, 3661),
5384 now,
5385 checked_test_time(now, 61),
5386 now,
5387 ),
5388 (
5389 "exp equal to iat",
5390 checked_test_time(now, 1),
5391 checked_test_time(now, 1),
5392 checked_test_time(now, 1),
5393 now,
5394 ),
5395 (
5396 "exp before iat",
5397 checked_test_time(now, 1),
5398 now,
5399 checked_test_time(now, 2),
5400 now,
5401 ),
5402 (
5403 "nbf after exp",
5404 checked_test_time(now, 2),
5405 checked_test_time(now, 3),
5406 now,
5407 now,
5408 ),
5409 (
5410 "nbf after iat",
5411 checked_test_time(now, 2),
5412 checked_test_time(now, 1),
5413 now,
5414 now,
5415 ),
5416 ("negative exp", -1, 0, 0, -2),
5417 ("negative nbf", checked_test_time(now, 1), -1, 0, now),
5418 ("negative iat", checked_test_time(now, 1), 0, -1, now),
5419 ("minimum and maximum extremes", i64::MAX, 0, i64::MIN, now),
5420 ("all minimum extremes", i64::MIN, i64::MIN, i64::MIN, now),
5421 ("all maximum extremes", i64::MAX, i64::MAX, i64::MAX, now),
5422 (
5423 "clock-skew boundary overflow",
5424 i64::MAX,
5425 i64::MAX - 1,
5426 i64::MAX - 1,
5427 i64::MAX - 1,
5428 ),
5429 ] {
5430 assert!(
5431 validate_registry_credential_numeric_dates(exp, nbf, iat, validation_now).is_err(),
5432 "invalid NumericDate profile accepted: {case}"
5433 );
5434 }
5435 }
5436
5437 #[test]
5438 fn deployment_credential_rejects_signed_wraparound_lifetime_in_all_profiles() {
5439 let ca = test_deployment_ca(0x6a);
5440 let registry = SigningKey::from_bytes(&[0x6b; 32]);
5441 let (protected, mut claims) = exact_registry_credential_values(&ca, ®istry);
5442 claims["exp"] = serde_json::json!(checked_test_time(
5443 chrono::Utc::now().timestamp(),
5444 REGISTRY_DEPLOYMENT_CREDENTIAL_MAX_TTL_SECONDS,
5445 ));
5446 claims["nbf"] = serde_json::json!(i64::MIN);
5447 claims["iat"] = serde_json::json!(i64::MIN);
5448
5449 assert_registry_credential_rejected(
5450 &ca,
5451 &protected,
5452 &claims,
5453 "signed exp-minus-iat wraparound",
5454 );
5455 }
5456
5457 #[test]
5458 fn deployment_credential_rejects_every_protected_and_trust_dimension() {
5459 let ca = test_deployment_ca(0x33);
5460 let other_ca = test_deployment_ca(0x34);
5461 let registry = SigningKey::from_bytes(&[0x35; 32]);
5462 let (protected, claims) = exact_registry_credential_values(&ca, ®istry);
5463
5464 for (case, member, value) in [
5465 ("access token type", "typ", serde_json::json!("at+jwt")),
5466 (
5467 "application access token type",
5468 "typ",
5469 serde_json::json!("application/at+jwt"),
5470 ),
5471 ("generic JWT type", "typ", serde_json::json!("JWT")),
5472 ("wrong-case type", "typ", serde_json::json!("WIT+JWT")),
5473 ("whitespace type", "typ", serde_json::json!("wit+jwt ")),
5474 ("alternate algorithm", "alg", serde_json::json!("ES256")),
5475 ("wrong key id", "kid", serde_json::json!("attacker")),
5476 ] {
5477 let mut changed = protected.clone();
5478 changed[member] = value;
5479 assert_registry_credential_rejected(&ca, &changed, &claims, case);
5480 }
5481 for missing in ["typ", "alg", "kid"] {
5482 let mut changed = protected.clone();
5483 changed
5484 .as_object_mut()
5485 .expect("header object")
5486 .remove(missing);
5487 assert_registry_credential_rejected(&ca, &changed, &claims, missing);
5488 }
5489 let mut unknown = protected.clone();
5490 unknown["crit"] = serde_json::json!([]);
5491 assert_registry_credential_rejected(&ca, &unknown, &claims, "unknown header member");
5492
5493 let credential = exact_registry_credential(&ca, ®istry);
5494 assert!(
5495 authenticate_test_registry_credential(&other_ca, credential).is_err(),
5496 "CA substitution accepted"
5497 );
5498 let signed_by_other = {
5499 let (other_header, _) = exact_registry_credential_values(&other_ca, ®istry);
5500 sign_registry_credential_json(&other_ca, &other_header.to_string(), &claims.to_string())
5501 };
5502 assert!(
5503 authenticate_test_registry_credential(&ca, signed_by_other).is_err(),
5504 "signature substitution accepted"
5505 );
5506 }
5507
5508 #[test]
5509 fn deployment_credential_rejects_every_claim_and_time_dimension() {
5510 let ca = test_deployment_ca(0x36);
5511 let registry = SigningKey::from_bytes(&[0x37; 32]);
5512 let (protected, claims) = exact_registry_credential_values(&ca, ®istry);
5513 let now = chrono::Utc::now().timestamp();
5514
5515 for (case, member, value) in [
5516 ("wrong subject", "sub", serde_json::json!("service:model")),
5517 (
5518 "wrong issuer",
5519 "iss",
5520 serde_json::json!("https://attacker.invalid"),
5521 ),
5522 (
5523 "wrong audience",
5524 "aud",
5525 serde_json::json!("service:registry"),
5526 ),
5527 (
5528 "audience array",
5529 "aud",
5530 serde_json::json!([REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE]),
5531 ),
5532 (
5533 "wrong deployment domain",
5534 "deployment_domain",
5535 serde_json::json!("attacker"),
5536 ),
5537 (
5538 "wrong profile",
5539 "profile",
5540 serde_json::json!("hyprstream.registry-deployment.v2"),
5541 ),
5542 (
5543 "expired",
5544 "exp",
5545 serde_json::json!(checked_test_time(now, -1)),
5546 ),
5547 (
5548 "future nbf",
5549 "nbf",
5550 serde_json::json!(checked_test_time(now, 3600)),
5551 ),
5552 (
5553 "future iat",
5554 "iat",
5555 serde_json::json!(checked_test_time(now, 3600)),
5556 ),
5557 (
5558 "nbf after iat",
5559 "nbf",
5560 serde_json::json!(checked_test_time(now, 1)),
5561 ),
5562 (
5563 "excessive lifetime",
5564 "exp",
5565 serde_json::json!(checked_test_time(now, 3601)),
5566 ),
5567 ] {
5568 let mut changed = claims.clone();
5569 changed[member] = value;
5570 assert_registry_credential_rejected(&ca, &protected, &changed, case);
5571 }
5572 for missing in [
5573 "iss",
5574 "sub",
5575 "aud",
5576 "exp",
5577 "nbf",
5578 "iat",
5579 "deployment_domain",
5580 "profile",
5581 "cnf",
5582 ] {
5583 let mut changed = claims.clone();
5584 changed
5585 .as_object_mut()
5586 .expect("claims object")
5587 .remove(missing);
5588 assert_registry_credential_rejected(&ca, &protected, &changed, missing);
5589 }
5590 let mut unknown = claims.clone();
5591 unknown["scope"] = serde_json::json!("registry");
5592 assert_registry_credential_rejected(&ca, &protected, &unknown, "unknown claim");
5593 }
5594
5595 #[test]
5596 fn deployment_credential_rejects_every_confirmation_and_jwk_dimension() {
5597 let ca = test_deployment_ca(0x38);
5598 let registry = SigningKey::from_bytes(&[0x39; 32]);
5599 let (protected, claims) = exact_registry_credential_values(&ca, ®istry);
5600
5601 for (case, member, value) in [
5602 ("RSA declaration", "kty", serde_json::json!("RSA")),
5603 ("EC declaration", "kty", serde_json::json!("EC")),
5604 ("P-256 curve", "crv", serde_json::json!("P-256")),
5605 ("alternate OKP curve", "crv", serde_json::json!("X25519")),
5606 ("malformed x", "x", serde_json::json!("%%%")),
5607 (
5608 "padded x",
5609 "x",
5610 serde_json::json!(format!(
5611 "{}=",
5612 URL_SAFE_NO_PAD.encode(registry.verifying_key().as_bytes())
5613 )),
5614 ),
5615 (
5616 "short x",
5617 "x",
5618 serde_json::json!(URL_SAFE_NO_PAD.encode([0u8; 31])),
5619 ),
5620 ] {
5621 let mut changed = claims.clone();
5622 changed["cnf"]["jwk"][member] = value;
5623 assert_registry_credential_rejected(&ca, &protected, &changed, case);
5624 }
5625 for missing in ["kty", "crv", "x"] {
5626 let mut changed = claims.clone();
5627 changed["cnf"]["jwk"]
5628 .as_object_mut()
5629 .expect("jwk object")
5630 .remove(missing);
5631 assert_registry_credential_rejected(&ca, &protected, &changed, missing);
5632 }
5633 for incompatible in ["alg", "use", "key_ops", "kid"] {
5634 let mut changed = claims.clone();
5635 changed["cnf"]["jwk"][incompatible] = serde_json::json!("unexpected");
5636 assert_registry_credential_rejected(&ca, &protected, &changed, incompatible);
5637 }
5638 let mut ambiguous = claims.clone();
5639 ambiguous["cnf"]["jkt"] = serde_json::json!("thumbprint");
5640 assert_registry_credential_rejected(&ca, &protected, &ambiguous, "jwk plus jkt");
5641 let mut alternate = claims.clone();
5642 alternate["cnf"] = serde_json::json!({"jkt":"thumbprint"});
5643 assert_registry_credential_rejected(&ca, &protected, &alternate, "jkt only");
5644 }
5645
5646 #[test]
5647 fn deployment_credential_duplicate_members_fail_closed_at_every_level() {
5648 let ca = test_deployment_ca(0x3a);
5649 let registry = SigningKey::from_bytes(&[0x3b; 32]);
5650 let domain = deployment_domain(&ca.verifying_keys);
5651 let (_, claims) = exact_registry_credential_values(&ca, ®istry);
5652 let duplicate_headers = [
5653 format!(
5654 r#"{{"alg":"ML-DSA-65-Ed25519","alg":"ES256","typ":"wit+jwt","kid":"{domain}"}}"#
5655 ),
5656 format!(
5657 r#"{{"alg":"ML-DSA-65-Ed25519","typ":"wit+jwt","typ":"at+jwt","kid":"{domain}"}}"#
5658 ),
5659 format!(
5660 r#"{{"alg":"ML-DSA-65-Ed25519","typ":"wit+jwt","kid":"{domain}","kid":"attacker"}}"#
5661 ),
5662 ];
5663 for header in duplicate_headers {
5664 let token = sign_registry_credential_json(&ca, &header, &claims.to_string());
5665 assert!(authenticate_test_registry_credential(&ca, token).is_err());
5666 }
5667
5668 let now = chrono::Utc::now().timestamp();
5669 let exp = checked_test_time(now, 3600);
5670 let x = URL_SAFE_NO_PAD.encode(registry.verifying_key().as_bytes());
5671 let protected = serde_json::json!({
5672 "alg":"ML-DSA-65-Ed25519",
5673 "typ":"wit+jwt",
5674 "kid":domain
5675 });
5676 for duplicate_claims in [
5677 format!(
5678 r#"{{"iss":"urn:hyprstream:deployment:{domain}","sub":"service:registry","sub":"service:model","aud":"{REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE}","exp":{},"nbf":{now},"iat":{now},"deployment_domain":"{domain}","profile":"{REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE}","cnf":{{"jwk":{{"kty":"OKP","crv":"Ed25519","x":"{x}"}}}}}}"#,
5679 exp
5680 ),
5681 format!(
5682 r#"{{"iss":"urn:hyprstream:deployment:{domain}","sub":"service:registry","aud":"{REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE}","exp":{},"nbf":{now},"iat":{now},"deployment_domain":"{domain}","profile":"{REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE}","cnf":{{"jwk":{{"kty":"OKP","crv":"Ed25519","x":"{x}"}},"jwk":{{"kty":"RSA","crv":"P-256","x":"{x}"}}}}}}"#,
5683 exp
5684 ),
5685 format!(
5686 r#"{{"iss":"urn:hyprstream:deployment:{domain}","sub":"service:registry","aud":"{REGISTRY_DEPLOYMENT_CREDENTIAL_AUDIENCE}","exp":{},"nbf":{now},"iat":{now},"deployment_domain":"{domain}","profile":"{REGISTRY_DEPLOYMENT_CREDENTIAL_PROFILE}","cnf":{{"jwk":{{"kty":"OKP","kty":"RSA","crv":"Ed25519","x":"{x}"}}}}}}"#,
5687 exp
5688 ),
5689 ] {
5690 let token =
5691 sign_registry_credential_json(&ca, &protected.to_string(), &duplicate_claims);
5692 assert!(authenticate_test_registry_credential(&ca, token).is_err());
5693 }
5694 }
5695
5696 #[test]
5697 fn production_authority_api_inventory_has_no_public_composition_chain() {
5698 let service_factory = include_str!(concat!(
5699 env!("CARGO_MANIFEST_DIR"),
5700 "/../hyprstream-service/src/service/factory.rs"
5701 ));
5702 for forbidden in [
5703 "pub fn seal_discovery_bootstrap_authority",
5704 "pub fn consume_discovery_bootstrap_authority",
5705 "struct DiscoveryBootstrapAuthority",
5706 "discovery_authority:",
5707 "pub fn authenticate_registry_deployment_credential",
5708 "pub fn take_authenticated_registry_identity",
5709 "pub fn authenticated_registry_identity",
5710 "pub struct AuthenticatedRegistryDeploymentIdentity",
5711 "into_verifying_key",
5712 ] {
5713 assert!(
5714 !service_factory.contains(forbidden),
5715 "authority seam returned: {forbidden}"
5716 );
5717 }
5718 let discovery = include_str!("service.rs");
5719 let production = discovery
5720 .split("mod resolver_tests {")
5721 .next()
5722 .expect("production Discovery source");
5723 assert!(!production.contains("\n pub fn install_process_bootstrap("));
5724 assert!(production.contains("struct ProcessBootstrapAuthority {"));
5725 assert!(production.contains("enum ProcessBootstrapAuthorityState {"));
5726 let authentication = production
5727 .split("fn authenticate_deployment_bootstrap()")
5728 .nth(1)
5729 .expect("private deployment authentication boundary")
5730 .split("#[cfg(test)]\nfn authenticate_discovery_bootstrap_identity")
5731 .next()
5732 .expect("authentication body");
5733 assert!(authentication.contains("hyprstream_service::deployment_data_dir()"));
5734 assert!(authentication.contains("authenticate_local_deployment_registry()"));
5735 assert!(authentication.contains("authenticate_registry_deployment_credentials("));
5736 assert!(!authentication.contains("global_trust_store()"));
5737 assert!(!authentication.contains("resolve_one("));
5738 assert!(!authentication.contains("FnOnce"));
5739 assert!(!production.contains("pub fn authenticate_discovery_bootstrap("));
5740 assert!(!production.contains("pub struct AuthenticatedDiscoveryBootstrap"));
5741 assert!(!production.contains("pub fn bootstrap_authenticated_process("));
5742 assert!(production.contains("/etc/hyprstream/trust/deployment-ca.hybrid"));
5743 assert!(production.contains("/run/hyprstream/credentials/registry-service.jwt"));
5744 assert!(production.contains("HYPRSTREAM_DEPLOYMENT_TRUST_DIR"));
5745 assert!(production.contains("CREDENTIALS_DIRECTORY"));
5746 let loader = production
5747 .split("fn load_trusted_registry_deployment_credentials()")
5748 .nth(1)
5749 .expect("deployment trust loader")
5750 .split("fn authenticate_registry_deployment_credentials(")
5751 .next()
5752 .expect("loader body");
5753 assert!(loader.contains("resolve_deployment_trust_paths()"));
5754 assert!(loader.contains("load_trusted_registry_deployment_credentials_from(&paths)"));
5755 assert!(!loader.contains("dirs::config_dir"));
5756 assert!(!loader.contains("HYPRSTREAM__SECRETS__PATH"));
5757 }
5758
5759 fn service() -> DiscoveryService {
5760 let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
5761 DiscoveryService::new(Arc::new(sk), vk, TransportConfig::inproc("resolver-test"))
5762 }
5763
5764 fn legacy_endpoint(
5765 socket_kind: &str,
5766 endpoint: &str,
5767 last_heartbeat: Instant,
5768 ) -> AnnouncedEndpoint {
5769 let age_ms = Instant::now()
5770 .saturating_duration_since(last_heartbeat)
5771 .as_millis() as i64;
5772 let live_until_unix_ms = unix_millis_now()
5773 .saturating_add(ANNOUNCED_ENDPOINT_TTL.as_millis() as i64)
5774 .saturating_sub(age_ms);
5775 AnnouncedEndpoint {
5776 socket_kind: socket_kind.to_owned(),
5777 endpoint: endpoint.to_owned(),
5778 service_did: Did::default(),
5779 capabilities: BTreeSet::new(),
5780 accepted_state_digest: Vec::new(),
5781 accepted_state_epoch: 0,
5782 response_key_id: String::new(),
5783 request_kem_key_id: String::new(),
5784 request_kem_recipient: Vec::new(),
5785 expires_at_unix_ms: i64::MAX,
5786 source_signer: [0; 32],
5787 live_until_unix_ms,
5788 }
5789 }
5790
5791 fn accepted_state(tag: u8) -> (AcceptedAt9pState, SigningKey) {
5792 accepted_state_with_expiry(tag, "2099-01-01T00:00:00Z")
5793 }
5794
5795 fn accepted_state_with_expiry(tag: u8, expiry: &str) -> (AcceptedAt9pState, SigningKey) {
5796 let signing = SigningKey::from_bytes(&[tag; 32]);
5797 let pq_signing = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signing);
5798 let keys = HybridKeyPair::new(
5799 signing.verifying_key().to_bytes().to_vec(),
5800 ml_dsa_sk_to_vk_bytes(&pq_signing),
5801 )
5802 .unwrap_or_else(|e| panic!("test hybrid keys invalid: {e}"));
5803 let endpoint = ServiceEndpoint::new(At9pTransport::Iroh, "iroh://reach")
5804 .unwrap_or_else(|e| panic!("test endpoint invalid: {e}"));
5805 let service = ServiceEntry::new("#model", ServiceType::NinePExport, endpoint)
5806 .unwrap_or_else(|e| panic!("test service invalid: {e}"));
5807 let body = CapsuleBody::new(vec![keys], vec![service])
5808 .unwrap_or_else(|e| panic!("test body invalid: {e}"));
5809 let genesis = sign_capsule(body.clone(), &signing, &pq_signing)
5810 .unwrap_or_else(|e| panic!("test genesis signing failed: {e}"));
5811 let subject = genesis
5812 .cid512()
5813 .unwrap_or_else(|e| panic!("test genesis CID failed: {e}"));
5814 let update = sign_update_record(
5815 subject,
5816 1,
5817 [1; 64],
5818 body,
5819 expiry.to_owned(),
5820 &signing,
5821 &pq_signing,
5822 )
5823 .unwrap_or_else(|e| panic!("test update signing failed: {e}"));
5824 let bytes = update
5825 .to_dag_cbor()
5826 .unwrap_or_else(|e| panic!("test update encoding failed: {e}"));
5827 let state = AcceptedAt9pState::from_persisted_update(&bytes)
5828 .unwrap_or_else(|e| panic!("test accepted state invalid: {e}"));
5829 (state, signing)
5830 }
5831
5832 fn accepted_service_state(
5835 tag: u8,
5836 address: &str,
5837 request_kem: Option<&hyprstream_rpc::crypto::hybrid_kem::RecipientPublic>,
5838 ) -> (AcceptedAt9pState, SigningKey) {
5839 let signing = SigningKey::from_bytes(&[tag; 32]);
5840 let pq_signing = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signing);
5841 let keys = HybridKeyPair::new(
5842 signing.verifying_key().to_bytes().to_vec(),
5843 ml_dsa_sk_to_vk_bytes(&pq_signing),
5844 )
5845 .unwrap_or_else(|e| panic!("test hybrid keys invalid: {e}"));
5846 let mut endpoint = ServiceEndpoint::new(At9pTransport::Iroh, address)
5847 .unwrap_or_else(|e| panic!("test endpoint invalid: {e}"));
5848 endpoint.request_kem =
5849 request_kem.map(hyprstream_rpc::crypto::hybrid_kem::RecipientPublic::encode);
5850 let service = ServiceEntry::new("#model", ServiceType::NinePExport, endpoint)
5851 .unwrap_or_else(|e| panic!("test service invalid: {e}"));
5852 let body = CapsuleBody::new(vec![keys], vec![service])
5853 .unwrap_or_else(|e| panic!("test body invalid: {e}"));
5854 let genesis = sign_capsule(body.clone(), &signing, &pq_signing)
5855 .unwrap_or_else(|e| panic!("test genesis signing failed: {e}"));
5856 let subject = genesis
5857 .cid512()
5858 .unwrap_or_else(|e| panic!("test genesis CID failed: {e}"));
5859 let update = sign_update_record(
5860 subject,
5861 1,
5862 [1; 64],
5863 body,
5864 "2099-01-01T00:00:00Z".to_owned(),
5865 &signing,
5866 &pq_signing,
5867 )
5868 .unwrap_or_else(|e| panic!("test update signing failed: {e}"));
5869 let bytes = update
5870 .to_dag_cbor()
5871 .unwrap_or_else(|e| panic!("test update encoding failed: {e}"));
5872 let state = AcceptedAt9pState::from_persisted_update(&bytes)
5873 .unwrap_or_else(|e| panic!("test accepted state invalid: {e}"));
5874 (state, signing)
5875 }
5876
5877 struct MutableAcceptedState(parking_lot::Mutex<Option<AcceptedAt9pState>>);
5878
5879 impl AcceptedStateSource for MutableAcceptedState {
5880 fn accepted_state(&self, _did: &str) -> Result<Option<AcceptedAt9pState>> {
5881 Ok(self.0.lock().clone())
5882 }
5883 fn accepted_states(&self) -> Result<Vec<AcceptedAt9pState>> {
5884 Ok(self.0.lock().clone().into_iter().collect())
5885 }
5886 }
5887
5888 fn production_fixture(
5889 local_reach: bool,
5890 ) -> (DiscoveryServiceResolver, Arc<MutableAcceptedState>) {
5891 production_fixture_with_transport(local_reach, false)
5892 }
5893
5894 fn native_production_fixture(
5895 local_reach: bool,
5896 ) -> (DiscoveryServiceResolver, Arc<MutableAcceptedState>) {
5897 production_fixture_with_transport(local_reach, true)
5898 }
5899
5900 fn production_fixture_with_transport(
5901 local_reach: bool,
5902 native_iroh: bool,
5903 ) -> (DiscoveryServiceResolver, Arc<MutableAcceptedState>) {
5904 let (state, signing) = accepted_state(11);
5905 let kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
5906 hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
5907 )
5908 .unwrap_or_else(|e| panic!("test KEM generation failed: {e}"));
5909 let endpoint = if local_reach {
5910 "inproc://hyprstream/model".to_owned()
5911 } else if native_iroh {
5912 format!("iroh://{}", hex::encode([0x51; 32]))
5913 } else {
5914 "quic://localhost:127.0.0.1:9".to_owned()
5915 };
5916 let state_store = Arc::new(MemoryStateStore::default());
5917 state_store
5918 .put_announcement_sync(
5919 "model",
5920 AnnouncedEndpoint {
5921 socket_kind: if local_reach {
5922 "rep"
5923 } else if native_iroh {
5924 "iroh"
5925 } else {
5926 "quic"
5927 }
5928 .to_owned(),
5929 endpoint,
5930 service_did: Did::from(state.did.clone()),
5931 capabilities: ["hyprstream-rpc/1".to_owned(), "hyprstream-moq/1".to_owned()]
5932 .into_iter()
5933 .collect(),
5934 accepted_state_digest: state.head_digest.to_vec(),
5935 accepted_state_epoch: state.epoch,
5936 response_key_id: format!("{}#response-current", state.did),
5937 request_kem_key_id: format!("{}#kem-current", state.did),
5938 request_kem_recipient: kem.public().encode(),
5939 expires_at_unix_ms: 4_070_908_800_000,
5940 source_signer: signing.verifying_key().to_bytes(),
5941 live_until_unix_ms: unix_millis_now()
5942 + ANNOUNCED_ENDPOINT_TTL.as_millis() as i64,
5943 },
5944 )
5945 .expect("seed announcement");
5946 let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(state))));
5947 (
5948 DiscoveryServiceResolver {
5949 state_store,
5950 accepted_state_source: Arc::clone(&source) as Arc<dyn AcceptedStateSource>,
5951 discovery_client: None,
5952 },
5953 source,
5954 )
5955 }
5956
5957 async fn mutate_endpoint(
5958 resolver: &DiscoveryServiceResolver,
5959 service_name: &str,
5960 socket_kind: &str,
5961 mutate: impl FnOnce(&mut AnnouncedEndpoint),
5962 ) {
5963 let mut endpoint = resolver
5964 .state_store
5965 .announcements_for(service_name, unix_millis_now())
5966 .await
5967 .expect("read fixture announcement")
5968 .into_iter()
5969 .find(|entry| entry.socket_kind == socket_kind)
5970 .expect("fixture announcement");
5971 mutate(&mut endpoint);
5972 resolver
5973 .state_store
5974 .put_announcement(service_name, endpoint)
5975 .await
5976 .expect("replace fixture announcement");
5977 }
5978
5979 #[tokio::test]
5980 async fn production_resolver_joins_announcement_to_current_pds_state() {
5981 let (resolver, _) = native_production_fixture(false);
5982 let resolved = resolver
5983 .resolve_service(ServiceQuery::network("model").expect("query"))
5984 .await
5985 .unwrap_or_else(|e| panic!("production candidate rejected: {e}"));
5986 assert_eq!(resolved.service_name(), "model");
5987 assert_eq!(resolved.evidence().accepted_state_epoch, 1);
5988 assert!(resolved.service_did().is_did_at9p());
5989 assert!(!resolved.request_kem_recipient().recipient.eks.is_empty());
5990 resolver
5991 .ensure_current(&resolved)
5992 .await
5993 .unwrap_or_else(|e| panic!("unchanged accepted state rejected: {e}"));
5994 }
5995
5996 #[tokio::test]
5997 async fn production_profile_preserves_compatibility_quic_without_local_fallback() {
5998 let (resolver, _) = production_fixture(false);
5999 assert!(resolver
6000 .resolve_service(production_service_query("model", false).expect("query"))
6001 .await
6002 .is_ok());
6003 assert!(resolver
6004 .resolve_service(production_service_query("model", true).expect("query"))
6005 .await
6006 .is_err());
6007 let (resolver, _) = native_production_fixture(false);
6008 for required in [false, true] {
6009 assert!(resolver
6010 .resolve_service(production_service_query("model", required).expect("query"))
6011 .await
6012 .is_ok());
6013 }
6014 let (resolver, _) = production_fixture(true);
6015 for required in [false, true] {
6016 assert!(resolver
6017 .resolve_service(production_service_query("model", required).expect("query"))
6018 .await
6019 .is_err());
6020 }
6021 }
6022
6023 fn owned_browser_request() -> BrowserProvisioningRequest {
6024 BrowserProvisioningRequest::new(
6025 "model",
6026 "hyprstream-rpc/1",
6027 "model",
6028 BrowserCarrierProfile::OwnedHybridWebTransport,
6029 )
6030 .expect("browser request")
6031 }
6032
6033 #[tokio::test]
6034 async fn production_browser_projection_is_current_and_resolution_bound() {
6035 let (resolver, _) = production_fixture(false);
6036 let document = resolver
6037 .browser_provisioning(owned_browser_request())
6038 .await
6039 .unwrap_or_else(|error| panic!("browser projection failed: {error}"))
6040 .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6041 .expect("sign accepted projection");
6042 let json = serde_json::to_vec(&document).expect("serialize provisioning");
6043 let validated = hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6044 &json,
6045 &owned_browser_request(),
6046 unix_millis_now(),
6047 )
6048 .unwrap_or_else(|error| panic!("browser projection did not validate: {error}"));
6049 assert_eq!(validated.service_name(), "model");
6050 assert_eq!(validated.accepted_state_epoch(), 1);
6051 assert!(validated.service_did().starts_with("did:at9p:"));
6052 }
6053
6054 #[tokio::test]
6055 async fn production_browser_projection_normalizes_default_https_port() {
6056 let (resolver, _) = production_fixture(false);
6057 mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6058 endpoint.endpoint = "quic://localhost:127.0.0.1:443".to_owned();
6059 })
6060 .await;
6061 let document = resolver
6062 .browser_provisioning(owned_browser_request())
6063 .await
6064 .expect("project default HTTPS port")
6065 .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6066 .expect("sign projection");
6067 assert_eq!(document.service_origin, "https://localhost/");
6068 assert_eq!(
6069 document.webtransport_url,
6070 "https://localhost/hyprstream/browser-rpc"
6071 );
6072 let json = serde_json::to_vec(&document).expect("serialize provisioning");
6073 hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6074 &json,
6075 &owned_browser_request(),
6076 unix_millis_now(),
6077 )
6078 .expect("default-port projection must validate in the browser");
6079 }
6080
6081 #[tokio::test]
6082 async fn network_iroh_profile_rejects_quic_and_browser_profile_rejects_iroh() {
6083 let (resolver, _) = production_fixture(false);
6084 let native = match resolver
6085 .resolve_service(ServiceQuery::network("model").expect("native query"))
6086 .await
6087 {
6088 Ok(_) => panic!("native Iroh profile accepted a valid QUIC candidate"),
6089 Err(error) => error,
6090 };
6091 assert!(native.to_string().contains("no validated"));
6092
6093 let (resolver, _) = native_production_fixture(false);
6094 let browser = resolver
6095 .browser_provisioning(owned_browser_request())
6096 .await
6097 .expect_err("browser provisioning must reject Iroh reach");
6098 assert!(browser.to_string().contains("no validated"));
6099 }
6100
6101 #[tokio::test]
6102 async fn browser_projection_rejects_state_advance_revocation_and_cross_service() {
6103 let (resolver, source) = production_fixture(false);
6104 source.0.lock().as_mut().expect("fixture state").epoch += 1;
6105 assert!(resolver
6106 .browser_provisioning(owned_browser_request())
6107 .await
6108 .is_err());
6109
6110 let (resolver, _) = production_fixture(false);
6111 mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6112 endpoint.request_kem_recipient = vec![0x01];
6113 })
6114 .await;
6115 assert!(resolver
6116 .browser_provisioning(owned_browser_request())
6117 .await
6118 .is_err());
6119
6120 let (resolver, source) = production_fixture(false);
6121 source
6122 .0
6123 .lock()
6124 .as_mut()
6125 .expect("fixture state")
6126 .current
6127 .services[0]
6128 .id = "#policy".to_owned();
6129 assert!(resolver
6130 .browser_provisioning(owned_browser_request())
6131 .await
6132 .is_err());
6133 }
6134
6135 #[tokio::test]
6136 async fn public_relay_requires_signed_moq_route_and_application_object_profile() {
6137 let (resolver, source) = production_fixture(false);
6138 let request = BrowserProvisioningRequest::new(
6139 "model",
6140 "hyprstream-moq/1",
6141 "tenant-a/track-a",
6142 BrowserCarrierProfile::StandardPublicRelay,
6143 )
6144 .expect("relay request");
6145 assert!(resolver
6146 .browser_provisioning(request.clone())
6147 .await
6148 .is_err());
6149
6150 {
6151 let mut state = source.0.lock();
6152 let endpoint = &mut state.as_mut().expect("fixture state").current.services[0].endpoint;
6153 endpoint.transport = At9pTransport::Moq;
6154 endpoint.address = "https://model.example/".to_owned();
6155 endpoint.relay = Some("https://relay.example/moq".to_owned());
6156 }
6157 let document = resolver
6158 .browser_provisioning(request.clone())
6159 .await
6160 .unwrap_or_else(|error| panic!("signed relay projection failed: {error}"))
6161 .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6162 .expect("sign accepted relay projection");
6163 assert_eq!(
6164 document.transport_security,
6165 BrowserTransportSecurity::ClassicalUntrusted
6166 );
6167 assert!(document.application_hybrid_required);
6168 assert!(document.encrypted_objects_required);
6169 let json = serde_json::to_vec(&document).expect("serialize relay provisioning");
6170 hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6171 &json,
6172 &request,
6173 unix_millis_now(),
6174 )
6175 .expect("validate relay provisioning");
6176 }
6177
6178 #[tokio::test]
6179 async fn post_refetch_pre_dispatch_state_advance_rejects_pinned_binding() {
6180 let (resolver, source) = production_fixture(false);
6181 let document = resolver
6182 .browser_provisioning(owned_browser_request())
6183 .await
6184 .expect("pre-seal refetch")
6185 .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6186 .expect("sign projection");
6187 let json = serde_json::to_vec(&document).expect("serialize projection");
6188 let binding = hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6189 &json,
6190 &owned_browser_request(),
6191 unix_millis_now(),
6192 )
6193 .expect("validate projection")
6194 .request_binding()
6195 .expect("bind accepted evidence");
6196
6197 source.0.lock().as_mut().expect("fixture state").epoch += 1;
6198 assert!(resolver.verify_browser_binding(&binding).await.is_err());
6199 }
6200
6201 #[tokio::test]
6202 async fn post_refetch_route_and_pin_rotation_rejects_dial_evidence() {
6203 async fn binding_for(
6204 resolver: &DiscoveryServiceResolver,
6205 ) -> hyprstream_rpc::browser_provisioning::BrowserRequestBinding {
6206 let document = resolver
6207 .browser_provisioning(owned_browser_request())
6208 .await
6209 .expect("pre-seal refetch")
6210 .sign_projection(&SigningKey::from_bytes(&[11; 32]))
6211 .expect("sign projection");
6212 let json = serde_json::to_vec(&document).expect("serialize projection");
6213 hyprstream_rpc::browser_provisioning::BrowserProvisioning::from_json(
6214 &json,
6215 &owned_browser_request(),
6216 unix_millis_now(),
6217 )
6218 .expect("validate projection")
6219 .request_binding()
6220 .expect("bind dial evidence")
6221 }
6222
6223 let (resolver, _) = production_fixture(false);
6224 let route_binding = binding_for(&resolver).await;
6225 mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6226 endpoint.endpoint = "quic://localhost:127.0.0.1:10".to_owned();
6227 })
6228 .await;
6229 assert!(resolver
6230 .verify_browser_binding(&route_binding)
6231 .await
6232 .is_err());
6233
6234 let (resolver, _) = production_fixture(false);
6235 mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6236 endpoint.endpoint = format!(
6237 "quic://localhost:127.0.0.1:9#{}",
6238 URL_SAFE_NO_PAD.encode([0x51; 32])
6239 );
6240 })
6241 .await;
6242 let pin_binding = binding_for(&resolver).await;
6243 mutate_endpoint(&resolver, "model", "quic", |endpoint| {
6244 endpoint.endpoint = format!(
6245 "quic://localhost:127.0.0.1:9#{}",
6246 URL_SAFE_NO_PAD.encode([0x52; 32])
6247 );
6248 })
6249 .await;
6250 assert!(resolver.verify_browser_binding(&pin_binding).await.is_err());
6251 }
6252
6253 #[tokio::test]
6254 async fn ordinary_iroh_announcement_handler_redacts_bearer_before_storage_and_discovery() {
6255 hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6256 let kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6257 hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6258 )
6259 .expect("test KEM");
6260 let reach = format!("iroh://{}", hex::encode([0x7a; 32]));
6261 let (state, service_signing) = accepted_service_state(12, &reach, Some(&kem.public()));
6262 let root = SigningKey::from_bytes(&[0x61; 32]);
6263 let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6264 state.clone(),
6265 ))));
6266 let service = DiscoveryService::new(
6267 Arc::new(root.clone()),
6268 root.verifying_key(),
6269 TransportConfig::inproc("announcement-rpc-test"),
6270 )
6271 .with_accepted_state_source(source);
6272 let claims = hyprstream_rpc::auth::Claims::new(
6273 "service:model".to_owned(),
6274 chrono::Utc::now().timestamp(),
6275 chrono::Utc::now().timestamp() + 3_600,
6276 )
6277 .with_cnf_jwk(service_signing.verifying_key().as_bytes());
6278 let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &root);
6279 let service_pq_signing =
6280 hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_signing);
6281 let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6282 hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6283 &service_signing,
6284 &service_pq_signing,
6285 );
6286 let ctx = EnvelopeContext::from_verified_as_system(&signed);
6287 let response = service
6288 .handle_announce(
6289 &ctx,
6290 1,
6291 &ServiceAnnouncement {
6292 service_name: "model".to_owned(),
6293 socket_kind: "iroh".to_owned(),
6294 endpoint: reach,
6295 service_jwt: Some(jwt.clone()),
6296 service_did: Did::from(state.did.clone()),
6297 capabilities: vec!["hyprstream-rpc/1".to_owned()],
6298 accepted_state_digest: state.head_digest.to_vec(),
6299 accepted_state_epoch: state.epoch,
6300 response_key_id: format!("{}#response-current", state.did),
6301 request_kem_key_id: format!("{}#kem-current", state.did),
6302 request_kem_recipient: kem.public().encode(),
6303 expires_at_unix_ms: 4_070_908_800_000,
6304 },
6305 )
6306 .await
6307 .expect("ordinary announcement handler");
6308 assert!(matches!(response, DiscoveryResponseVariant::AnnounceResult));
6309 let stored = service
6310 .state_store
6311 .announcements_for("model", unix_millis_now())
6312 .await
6313 .expect("read stored announcement");
6314 assert_eq!(stored.len(), 1);
6315 let stored_json = serde_json::to_string(&stored[0]).expect("serialize stored announcement");
6316 assert!(!stored_json.contains(&jwt), "bearer JWT must not persist");
6317
6318 let endpoints = service
6319 .handle_get_endpoints(&ctx, 2, "model")
6320 .await
6321 .expect("get endpoints after announcement");
6322 let DiscoveryResponseVariant::GetEndpointsResult(endpoints) = endpoints else {
6323 panic!("announcement must provide a discovery endpoint");
6324 };
6325 assert!(!endpoints.endpoints.is_empty());
6326 assert!(
6327 endpoints
6328 .endpoints
6329 .iter()
6330 .all(|endpoint| endpoint.service_jwt.is_empty()),
6331 "discovery must never project a stored bearer JWT"
6332 );
6333
6334 let resolver = service.production_resolver().expect("production resolver");
6335 let resolved = resolver
6336 .resolve_service(ServiceQuery::network("model").expect("query"))
6337 .await
6338 .expect("ordinary announcement must resolve");
6339 assert_eq!(resolved.evidence().accepted_state_digest, state.head_digest);
6340 assert!(matches!(
6341 &resolved.transport().endpoint,
6342 EndpointType::Iroh {
6343 node_id,
6344 direct_addrs,
6345 relay_url: None,
6346 } if *node_id == [0x7a; 32] && direct_addrs.is_empty()
6347 ));
6348 }
6349
6350 #[tokio::test]
6356 async fn identity_bound_announcement_refuses_legacy_identity_without_signed_kem() {
6357 let legacy_reach = format!("iroh://{}", hex::encode([0x7c; 32]));
6358 let (state, service_signing) = accepted_service_state(13, &legacy_reach, None);
6359 let root = SigningKey::from_bytes(&[0x62; 32]);
6360 let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6361 state.clone(),
6362 ))));
6363 let service = DiscoveryService::new(
6364 Arc::new(root.clone()),
6365 root.verifying_key(),
6366 TransportConfig::inproc("legacy-kem-announce-test"),
6367 )
6368 .with_accepted_state_source(source);
6369 let claims = hyprstream_rpc::auth::Claims::new(
6370 "service:model".to_owned(),
6371 chrono::Utc::now().timestamp(),
6372 chrono::Utc::now().timestamp() + 3_600,
6373 )
6374 .with_cnf_jwk(service_signing.verifying_key().as_bytes());
6375 let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &root);
6376 let service_pq_signing =
6377 hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_signing);
6378 let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6379 hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6380 &service_signing,
6381 &service_pq_signing,
6382 );
6383 let ctx = EnvelopeContext::from_verified_as_system(&signed);
6384 let kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6385 hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6386 )
6387 .expect("test KEM");
6388 let error = service
6389 .handle_announce(
6390 &ctx,
6391 1,
6392 &ServiceAnnouncement {
6393 service_name: "model".to_owned(),
6394 socket_kind: "iroh".to_owned(),
6395 endpoint: format!("iroh://{}", hex::encode([0x7b; 32])),
6396 service_jwt: Some(jwt),
6397 service_did: Did::from(state.did.clone()),
6398 capabilities: vec!["hyprstream-rpc/1".to_owned()],
6399 accepted_state_digest: state.head_digest.to_vec(),
6400 accepted_state_epoch: state.epoch,
6401 response_key_id: format!("{}#response-current", state.did),
6402 request_kem_key_id: format!("{}#kem-current", state.did),
6403 request_kem_recipient: kem.public().encode(),
6404 expires_at_unix_ms: 4_070_908_800_000,
6405 },
6406 )
6407 .await
6408 .expect_err(
6409 "identity-bound announcement over a KEM-less legacy identity must be refused",
6410 );
6411 assert!(error.to_string().contains("signed request KEM"));
6412 }
6413
6414 #[tokio::test]
6417 async fn identity_bound_announcement_refuses_kem_outside_accepted_service() {
6418 let accepted_kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6419 hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6420 )
6421 .expect("test accepted KEM");
6422 let reach = format!("iroh://{}", hex::encode([0x7d; 32]));
6423 let (state, service_signing) =
6424 accepted_service_state(14, &reach, Some(&accepted_kem.public()));
6425 let root = SigningKey::from_bytes(&[0x63; 32]);
6426 let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6427 state.clone(),
6428 ))));
6429 let service = DiscoveryService::new(
6430 Arc::new(root.clone()),
6431 root.verifying_key(),
6432 TransportConfig::inproc("foreign-kem-announce-test"),
6433 )
6434 .with_accepted_state_source(source);
6435 let claims = hyprstream_rpc::auth::Claims::new(
6436 "service:model".to_owned(),
6437 chrono::Utc::now().timestamp(),
6438 chrono::Utc::now().timestamp() + 3_600,
6439 )
6440 .with_cnf_jwk(service_signing.verifying_key().as_bytes());
6441 let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &root);
6442 let service_pq_signing =
6443 hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_signing);
6444 let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6445 hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6446 &service_signing,
6447 &service_pq_signing,
6448 );
6449 let ctx = EnvelopeContext::from_verified_as_system(&signed);
6450 let foreign_kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6451 hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6452 )
6453 .expect("test foreign KEM");
6454 let error = service
6455 .handle_announce(
6456 &ctx,
6457 1,
6458 &ServiceAnnouncement {
6459 service_name: "model".to_owned(),
6460 socket_kind: "iroh".to_owned(),
6461 endpoint: reach,
6462 service_jwt: Some(jwt),
6463 service_did: Did::from(state.did.clone()),
6464 capabilities: vec!["hyprstream-rpc/1".to_owned()],
6465 accepted_state_digest: state.head_digest.to_vec(),
6466 accepted_state_epoch: state.epoch,
6467 response_key_id: format!("{}#response-current", state.did),
6468 request_kem_key_id: format!("{}#kem-current", state.did),
6469 request_kem_recipient: foreign_kem.public().encode(),
6470 expires_at_unix_ms: 4_070_908_800_000,
6471 },
6472 )
6473 .await
6474 .expect_err("announcement KEM outside the accepted service must be refused");
6475 assert!(error.to_string().contains("announcement KEM differs"));
6476 }
6477
6478 #[tokio::test]
6482 async fn identity_bound_announcement_refuses_iroh_reach_outside_accepted_service() {
6483 let kem = hyprstream_rpc::crypto::hybrid_kem::generate_recipient(
6484 hyprstream_rpc::crypto::hybrid_kem::SuiteId::HyKemX25519MlKem768,
6485 )
6486 .expect("test KEM");
6487 let accepted_reach = format!("iroh://{}", hex::encode([0x7e; 32]));
6488 let (state, service_signing) =
6489 accepted_service_state(15, &accepted_reach, Some(&kem.public()));
6490 let root = SigningKey::from_bytes(&[0x64; 32]);
6491 let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6492 state.clone(),
6493 ))));
6494 let service = DiscoveryService::new(
6495 Arc::new(root.clone()),
6496 root.verifying_key(),
6497 TransportConfig::inproc("foreign-reach-announce-test"),
6498 )
6499 .with_accepted_state_source(source);
6500 let claims = hyprstream_rpc::auth::Claims::new(
6501 "service:model".to_owned(),
6502 chrono::Utc::now().timestamp(),
6503 chrono::Utc::now().timestamp() + 3_600,
6504 )
6505 .with_cnf_jwk(service_signing.verifying_key().as_bytes());
6506 let jwt = hyprstream_rpc::auth::jwt::encode_service_jwt(&claims, &root);
6507 let service_pq_signing =
6508 hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_signing);
6509 let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6510 hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6511 &service_signing,
6512 &service_pq_signing,
6513 );
6514 let ctx = EnvelopeContext::from_verified_as_system(&signed);
6515 let error = service
6516 .handle_announce(
6517 &ctx,
6518 1,
6519 &ServiceAnnouncement {
6520 service_name: "model".to_owned(),
6521 socket_kind: "iroh".to_owned(),
6522 endpoint: format!("iroh://{}", hex::encode([0x7f; 32])),
6523 service_jwt: Some(jwt),
6524 service_did: Did::from(state.did.clone()),
6525 capabilities: vec!["hyprstream-rpc/1".to_owned()],
6526 accepted_state_digest: state.head_digest.to_vec(),
6527 accepted_state_epoch: state.epoch,
6528 response_key_id: format!("{}#response-current", state.did),
6529 request_kem_key_id: format!("{}#kem-current", state.did),
6530 request_kem_recipient: kem.public().encode(),
6531 expires_at_unix_ms: 4_070_908_800_000,
6532 },
6533 )
6534 .await
6535 .expect_err("announcement reach outside the accepted service must be refused");
6536 assert!(error.to_string().contains("announcement reach differs"));
6537 }
6538
6539 #[tokio::test]
6540 async fn identity_bound_announcement_shadows_identity_less_local_entry() {
6541 hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6542 let service = service();
6543 let name = "shadowed-reach-fixture";
6544 hyprstream_rpc::registry::global().register(
6546 name,
6547 hyprstream_rpc::registry::SocketKind::Quic,
6548 hyprstream_rpc::transport::TransportConfig::inproc("local-shadowed-quic"),
6549 None,
6550 );
6551 let mut announced = legacy_endpoint(
6553 "quic",
6554 "quic://localhost:127.0.0.1:4433#announcement",
6555 Instant::now(),
6556 );
6557 announced.service_did = Did::from("did:at9p:shadow-fixture");
6558 service
6559 .state_store
6560 .put_announcement(name, announced)
6561 .await
6562 .expect("install identity-bound announcement fixture");
6563
6564 let signer = SigningKey::from_bytes(&[0x75; 32]);
6565 let pq_signer = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signer);
6566 let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6567 hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6568 &signer,
6569 &pq_signer,
6570 );
6571 let ctx = EnvelopeContext::from_verified_as_system(&signed);
6572 let response = service
6573 .handle_get_endpoints(&ctx, 1, name)
6574 .await
6575 .expect("get endpoints");
6576 let DiscoveryResponseVariant::GetEndpointsResult(endpoints) = response else {
6577 panic!("expected endpoints");
6578 };
6579 assert!(
6580 endpoints
6581 .endpoints
6582 .iter()
6583 .any(|e| e.endpoint == "quic://localhost:127.0.0.1:4433#announcement"
6584 && !e.service_did.as_str().is_empty()),
6585 "identity-bound announced endpoint must not be shadowed by the identity-less local entry: {:?}",
6586 endpoints.endpoints.iter().map(|e| e.endpoint.clone()).collect::<Vec<_>>(),
6587 );
6588 assert!(
6589 endpoints
6590 .endpoints
6591 .iter()
6592 .all(|e| !e.endpoint.contains("local-shadowed-quic")),
6593 "the identity-less local entry of the same socket kind must yield",
6594 );
6595 }
6596
6597 #[tokio::test]
6598 async fn identity_less_announcement_still_yields_to_local_entry() {
6599 hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6600 let service = service();
6601 let name = "legacy-yield-fixture";
6602 hyprstream_rpc::registry::global().register(
6603 name,
6604 hyprstream_rpc::registry::SocketKind::Quic,
6605 hyprstream_rpc::transport::TransportConfig::inproc("local-legacy-quic"),
6606 None,
6607 );
6608 service
6610 .state_store
6611 .put_announcement(
6612 name,
6613 legacy_endpoint(
6614 "quic",
6615 "quic://localhost:127.0.0.1:4499#legacy",
6616 Instant::now(),
6617 ),
6618 )
6619 .await
6620 .expect("install legacy announcement fixture");
6621 let signer = SigningKey::from_bytes(&[0x76; 32]);
6622 let pq_signer = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signer);
6623 let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6624 hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6625 &signer,
6626 &pq_signer,
6627 );
6628 let ctx = EnvelopeContext::from_verified_as_system(&signed);
6629 let response = service
6630 .handle_get_endpoints(&ctx, 1, name)
6631 .await
6632 .expect("get endpoints");
6633 let DiscoveryResponseVariant::GetEndpointsResult(endpoints) = response else {
6634 panic!("expected endpoints");
6635 };
6636 assert!(
6637 endpoints
6638 .endpoints
6639 .iter()
6640 .any(|e| e.endpoint.contains("local-legacy-quic")),
6641 "identity-less announced entry must still yield to the local entry",
6642 );
6643 assert!(
6644 endpoints
6645 .endpoints
6646 .iter()
6647 .all(|e| !e.endpoint.contains("4499")),
6648 "identity-less announced entry must not duplicate the local kind",
6649 );
6650 }
6651
6652 #[tokio::test]
6653 async fn remote_get_endpoints_omits_stale_and_fetch_cannot_refresh_it() {
6654 hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6655 let service = service();
6656 let name = "stale-remote-reach";
6657 service
6658 .state_store
6659 .put_announcement(
6660 name,
6661 legacy_endpoint(
6662 "iroh",
6663 &format!("iroh://{}", hex::encode([0x73; 32])),
6664 Instant::now() - ANNOUNCED_ENDPOINT_TTL - Duration::from_secs(1),
6665 ),
6666 )
6667 .await
6668 .expect("install stale endpoint fixture");
6669 let signer = SigningKey::from_bytes(&[0x74; 32]);
6670 let pq_signer = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signer);
6671 let signed = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6672 hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6673 &signer,
6674 &pq_signer,
6675 );
6676 let ctx = EnvelopeContext::from_verified_as_system(&signed);
6677
6678 let response = service
6679 .handle_get_endpoints(&ctx, 1, name)
6680 .await
6681 .expect("remote getEndpoints handler");
6682 assert!(matches!(response, DiscoveryResponseVariant::Error(_)));
6683 assert!(service
6684 .state_store
6685 .announcements_for(name, unix_millis_now())
6686 .await
6687 .expect("inspect state after fetch")
6688 .is_empty());
6689
6690 let listed = service
6691 .handle_list_services(&ctx, 2)
6692 .await
6693 .expect("remote listServices handler");
6694 let DiscoveryResponseVariant::ListServicesResult(services) = listed else {
6695 panic!("listServices must return service summaries");
6696 };
6697 assert!(services.services.iter().all(|summary| summary.name != name));
6698 }
6699
6700 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
6701 #[tokio::test]
6702 async fn valkey_announcement_handler_lease_and_result_follow_shared_time() {
6703 use crate::state_store::tests::ReplicaClock;
6704 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
6705 return;
6706 };
6707 for backend in [
6708 crate::DiscoveryStateBackend::Valkey,
6709 crate::DiscoveryStateBackend::Tiered,
6710 ] {
6711 let config = crate::DiscoveryStateConfig {
6712 backend,
6713 active_active: true,
6714 valkey: crate::ValkeyStateConfig {
6715 url: url.clone(),
6716 key_prefix: format!(
6717 "hs-handler-lease-{}-{}-{backend:?}",
6718 std::process::id(),
6719 unix_millis_now()
6720 ),
6721 ..Default::default()
6722 },
6723 ..Default::default()
6724 };
6725 let (state, signing) = accepted_state(12);
6726 let source = Arc::new(MutableAcceptedState(parking_lot::Mutex::new(Some(
6727 state.clone(),
6728 ))));
6729 let root = SigningKey::from_bytes(&[0x61; 32]);
6730 let service = DiscoveryService::new(
6731 Arc::new(root.clone()),
6732 root.verifying_key(),
6733 TransportConfig::inproc("lease-handler-test"),
6734 )
6735 .with_accepted_state_source(source.clone())
6736 .with_state(DiscoveryState::connect(&config).await.unwrap());
6737 let claims = hyprstream_rpc::auth::Claims::new(
6738 "service:model".to_owned(),
6739 chrono::Utc::now().timestamp(),
6740 chrono::Utc::now().timestamp() + 7_200,
6741 )
6742 .with_cnf_jwk(signing.verifying_key().as_bytes());
6743 let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&signing);
6744 let envelope = hyprstream_rpc::SignedEnvelope::new_signed_hybrid(
6745 hyprstream_rpc::RequestEnvelope::anonymous(Vec::new()),
6746 &signing,
6747 &pq,
6748 );
6749 let ctx = EnvelopeContext::from_verified_as_system(&envelope);
6750 let kem = hyprstream_rpc::node_identity::derive_mesh_kem_recipient(&signing).unwrap();
6751 let now = unix_millis_now();
6752 let mut request = ServiceAnnouncement {
6753 service_name: "model".to_owned(),
6754 socket_kind: "quic".to_owned(),
6755 endpoint: "quic://localhost:127.0.0.1:9".to_owned(),
6756 service_jwt: Some(hyprstream_rpc::auth::jwt::encode_service_jwt(
6757 &claims, &root,
6758 )),
6759 service_did: Did::from(state.did.clone()),
6760 capabilities: vec!["hyprstream-rpc/1".to_owned()],
6761 accepted_state_digest: state.head_digest.to_vec(),
6762 accepted_state_epoch: state.epoch,
6763 response_key_id: format!("{}#response-current", state.did),
6764 request_kem_key_id: format!("{}#kem-current", state.did),
6765 request_kem_recipient: kem.public().encode(),
6766 expires_at_unix_ms: now + 7_200_000,
6767 };
6768 for skew in [3_600_000, -3_600_000] {
6769 let _clock = ReplicaClock::at(now + skew);
6770 assert!(matches!(
6771 service.handle_announce(&ctx, 1, &request).await.unwrap(),
6772 DiscoveryResponseVariant::AnnounceResult
6773 ));
6774 let rows = service
6775 .state_store
6776 .announcements_for("model", now + skew)
6777 .await
6778 .unwrap();
6779 assert_eq!(
6780 rows.len(),
6781 1,
6782 "successful response requires a stored publication"
6783 );
6784 assert!((now + 89_000..=now + 91_000).contains(&rows[0].live_until_unix_ms));
6785 assert_eq!(rows[0].expires_at_unix_ms, request.expires_at_unix_ms);
6786 assert_eq!(rows[0].request_kem_recipient, request.request_kem_recipient);
6787 let stored_json = serde_json::to_string(&rows[0]).unwrap();
6788 assert!(!stored_json.contains(request.service_jwt.as_deref().unwrap()));
6789 assert!(service
6791 .resolve_announced_endpoint("model", SocketKind::Quic)
6792 .await
6793 .unwrap()
6794 .is_some());
6795 assert_eq!(
6796 service
6797 .state_store
6798 .all_announcements(now + skew)
6799 .await
6800 .unwrap()
6801 .len(),
6802 1
6803 );
6804 service
6805 .production_resolver()
6806 .unwrap()
6807 .resolve_service(ServiceQuery::network("model").unwrap())
6808 .await
6809 .unwrap();
6810 }
6811 let mut legacy = request.clone();
6814 legacy.service_name = "legacy".to_owned();
6815 legacy.service_jwt = None;
6816 legacy.service_did = Did::default();
6817 legacy.capabilities.clear();
6818 legacy.accepted_state_digest.clear();
6819 legacy.accepted_state_epoch = 0;
6820 legacy.response_key_id.clear();
6821 legacy.request_kem_key_id.clear();
6822 legacy.request_kem_recipient.clear();
6823 legacy.expires_at_unix_ms = 0;
6824 for skew in [3_600_000, -3_600_000] {
6825 let _clock = ReplicaClock::at(now + skew);
6826 assert!(matches!(
6827 service.handle_announce(&ctx, 1, &legacy).await.unwrap(),
6828 DiscoveryResponseVariant::AnnounceResult
6829 ));
6830 let rows = service
6831 .state_store
6832 .announcements_for("legacy", now + skew)
6833 .await
6834 .unwrap();
6835 assert_eq!(rows.len(), 1);
6836 assert!((now + 89_000..=now + 91_000).contains(&rows[0].live_until_unix_ms));
6837 assert_eq!(rows[0].expires_at_unix_ms, rows[0].live_until_unix_ms);
6838 }
6839
6840 request.expires_at_unix_ms = now + 20_000;
6844 assert!(service
6845 .handle_announce(&ctx, 2, &request)
6846 .await
6847 .unwrap_err()
6848 .to_string()
6849 .contains("not stored"));
6850
6851 let accepted_limit = (now / 1_000) * 1_000 + 20_000;
6854 let accepted_expiry = chrono::DateTime::from_timestamp_millis(accepted_limit)
6855 .unwrap()
6856 .to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
6857 let (bounded, _) = accepted_state_with_expiry(12, &accepted_expiry);
6858 *source.0.lock() = Some(bounded.clone());
6859 request.expires_at_unix_ms = now + 7_200_000;
6860 request.accepted_state_digest = bounded.head_digest.to_vec();
6861 assert!(matches!(
6862 service.handle_announce(&ctx, 3, &request).await.unwrap(),
6863 DiscoveryResponseVariant::AnnounceResult
6864 ));
6865 let values = service
6866 .state_store
6867 .announcements_for("model", now)
6868 .await
6869 .unwrap();
6870 assert_eq!(values[0].live_until_unix_ms, accepted_limit);
6871
6872 let _clock = ReplicaClock::at(now - 3_600_000);
6875 request.expires_at_unix_ms = now - 1;
6876 assert!(service
6877 .handle_announce(&ctx, 4, &request)
6878 .await
6879 .unwrap_err()
6880 .to_string()
6881 .contains("not stored"));
6882 let expired_text = chrono::DateTime::from_timestamp_millis(now - 1)
6883 .unwrap()
6884 .to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
6885 let (expired, _) = accepted_state_with_expiry(12, &expired_text);
6886 *source.0.lock() = Some(expired.clone());
6887 request.expires_at_unix_ms = now + 7_200_000;
6888 request.accepted_state_digest = expired.head_digest.to_vec();
6889 assert!(service
6890 .handle_announce(&ctx, 5, &request)
6891 .await
6892 .unwrap_err()
6893 .to_string()
6894 .contains("not stored"));
6895 }
6896 }
6897
6898 #[tokio::test]
6899 async fn accepted_state_advance_between_selection_and_dial_refuses() {
6900 let (resolver, source) = native_production_fixture(false);
6901 let resolved = resolver
6902 .resolve_service(ServiceQuery::network("model").expect("query"))
6903 .await
6904 .unwrap_or_else(|e| panic!("production candidate rejected: {e}"));
6905 {
6906 let mut guard = source.0.lock();
6907 let state = guard.as_mut().expect("fixture state");
6908 state.epoch += 1;
6909 state.head_digest = [0x55; 64];
6910 }
6911 let error = resolver
6912 .ensure_current(&resolved)
6913 .await
6914 .expect_err("advanced state accepted");
6915 assert!(error.to_string().contains("advanced or forked"));
6916 }
6917
6918 struct CountingStream(Arc<std::sync::atomic::AtomicUsize>);
6919
6920 #[async_trait]
6921 impl StreamHandle for CountingStream {
6922 async fn next_payload(
6923 &mut self,
6924 ) -> Result<Option<hyprstream_rpc::streaming::StreamPayload>> {
6925 self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6926 Ok(None)
6927 }
6928 async fn cancel(&mut self) -> Result<()> {
6929 self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6930 Ok(())
6931 }
6932 fn stream_id(&self) -> &str {
6933 "counting"
6934 }
6935 fn is_completed(&self) -> bool {
6936 false
6937 }
6938 }
6939
6940 #[tokio::test]
6941 async fn live_stream_continuation_fails_closed_after_snapshot_advance() {
6942 let (resolver, source) = native_production_fixture(false);
6943 let resolver = Arc::new(resolver);
6944 let snapshot = resolver
6945 .resolve_service_candidates(ServiceQuery::network("model").expect("query"))
6946 .await
6947 .expect("snapshot")
6948 .remove(0);
6949 let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
6950 let mut stream = CurrentStreamHandle {
6951 inner: Box::new(CountingStream(Arc::clone(&calls))),
6952 resolver,
6953 snapshot,
6954 invalidated: false,
6955 };
6956 {
6957 let mut guard = source.0.lock();
6958 let state = guard.as_mut().expect("fixture state");
6959 state.epoch += 1;
6960 state.head_digest = [0x77; 64];
6961 }
6962 assert!(stream.next_payload().await.is_err());
6963 assert!(stream.cancel().await.is_err());
6964 assert_eq!(
6965 calls.load(std::sync::atomic::Ordering::SeqCst),
6966 0,
6967 "invalid continuation dispatched an operation"
6968 );
6969 }
6970
6971 #[tokio::test]
6972 async fn production_network_resolver_never_falls_back_to_local_reach() {
6973 let (resolver, _) = production_fixture(true);
6974 assert!(resolver
6975 .resolve_service(ServiceQuery::network("model").expect("query"))
6976 .await
6977 .is_err());
6978 }
6979
6980 #[tokio::test]
6981 async fn router_selected_reach_must_match_current_authorized_candidate() {
6982 let (resolver, _) = native_production_fixture(false);
6983 let expected = resolver
6984 .resolve_service_candidates(ServiceQuery::network("model").expect("query"))
6985 .await
6986 .expect("candidate")
6987 .remove(0)
6988 .transport()
6989 .clone();
6990 let client = ProductionRpcClient::new(
6991 "model",
6992 "inference",
6993 Some(expected),
6994 SigningKey::from_bytes(&[0x71; 32]),
6995 None,
6996 Arc::new(resolver),
6997 )
6998 .expect("selected client");
6999 assert_eq!(
7000 client.snapshots().await.expect("selected snapshot").len(),
7001 1
7002 );
7003 }
7004
7005 #[tokio::test]
7006 async fn router_selected_reach_rejects_unadvertised_transport_and_wrong_domain() {
7007 let (resolver, _) = native_production_fixture(false);
7008 let client = ProductionRpcClient::new(
7009 "model",
7010 "inference",
7011 Some(TransportConfig::inproc("attacker-selected")),
7012 SigningKey::from_bytes(&[0x72; 32]),
7013 None,
7014 Arc::new(resolver),
7015 )
7016 .expect("selected client");
7017 let reach_error = match client.snapshots().await {
7018 Ok(_) => panic!("unadvertised transport was authorized"),
7019 Err(error) => error,
7020 };
7021 assert!(reach_error
7022 .to_string()
7023 .contains("not a current authorized candidate"));
7024 let domain_error = client
7025 .call_for_service("model", Vec::new())
7026 .await
7027 .expect_err("resolution name was accepted as generated domain");
7028 assert!(domain_error.to_string().contains("authority mismatch"));
7029 }
7030
7031 #[tokio::test]
7032 async fn generated_client_uses_ordinary_identity_bound_resolver_path() {
7033 let (resolver, _) = production_fixture(false);
7034 let entries = resolver
7035 .state_store
7036 .announcements_for("model", unix_millis_now())
7037 .await
7038 .expect("fixture announcement");
7039 for endpoint in entries {
7040 resolver
7041 .state_store
7042 .put_announcement("discovery", endpoint)
7043 .await
7044 .expect("seed discovery announcement");
7045 }
7046 let resolver = Arc::new(resolver);
7047 let _ = PRODUCTION_RESOLVER.set(resolver);
7048 let client_signing = SigningKey::from_bytes(&[0x44; 32]);
7049 let _client = hyprstream_rpc_std::discovery_client::DiscoveryClient::from_provider(
7050 &crate::ProductionRpcClientProvider,
7051 client_signing,
7052 None,
7053 )
7054 .unwrap_or_else(|e| panic!("generated resolver path failed: {e}"));
7055 }
7056
7057 #[test]
7058 fn isolated_process_without_bootstrap_fails_closed() {
7059 const CHILD: &str = "HYPRSTREAM_TEST_RESOLVER_ABSENT_CHILD";
7060 if std::env::var_os(CHILD).is_some() {
7061 let signing = SigningKey::from_bytes(&[0x46; 32]);
7062 let error = match production_rpc_client("model", signing, None) {
7063 Ok(_) => panic!("isolated process unexpectedly inherited another resolver"),
7064 Err(error) => error,
7065 };
7066 assert!(error.to_string().contains("not installed"));
7067 return;
7068 }
7069 let status =
7070 std::process::Command::new(std::env::current_exe().expect("discovery test executable"))
7071 .arg("--exact")
7072 .arg("service::resolver_tests::isolated_process_without_bootstrap_fails_closed")
7073 .arg("--nocapture")
7074 .env(CHILD, "1")
7075 .status()
7076 .expect("isolated resolver subprocess");
7077 assert!(status.success(), "isolated resolver subprocess failed");
7078 }
7079
7080 #[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
7081 struct NoopBootstrapClient;
7082
7083 #[async_trait::async_trait]
7084 impl RpcClient for NoopBootstrapClient {
7085 async fn call(&self, _: Vec<u8>) -> Result<Vec<u8>> {
7086 anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7087 }
7088 async fn call_for_service(&self, _: &str, _: Vec<u8>) -> Result<Vec<u8>> {
7089 anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7090 }
7091 async fn call_for_service_with_method(
7092 &self,
7093 _: &str,
7094 _: u16,
7095 _: Vec<u8>,
7096 ) -> Result<Vec<u8>> {
7097 anyhow::bail!("noop bootstrap client")
7098 }
7099 async fn call_with_options(&self, _: Vec<u8>, _: CallOptions) -> Result<Vec<u8>> {
7100 anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7101 }
7102 async fn call_with_options_for_service(
7103 &self,
7104 _: &str,
7105 _: Vec<u8>,
7106 _: CallOptions,
7107 ) -> Result<Vec<u8>> {
7108 anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7109 }
7110 async fn call_streaming(&self, _: Vec<u8>, _: [u8; 32]) -> Result<Vec<u8>> {
7111 anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7112 }
7113 async fn call_streaming_for_service(
7114 &self,
7115 _: &str,
7116 _: Vec<u8>,
7117 _: [u8; 32],
7118 ) -> Result<Vec<u8>> {
7119 anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7120 }
7121 async fn call_streaming_for_service_with_method(
7122 &self,
7123 _: &str,
7124 _: u16,
7125 _: Vec<u8>,
7126 _: [u8; 32],
7127 ) -> Result<Vec<u8>> {
7128 anyhow::bail!("noop bootstrap client")
7129 }
7130 async fn open_stream(&self, _: Vec<u8>) -> Result<Box<dyn StreamHandle>> {
7131 anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7132 }
7133 async fn open_stream_from_info(
7134 &self,
7135 _: hyprstream_rpc::stream_info::StreamInfo,
7136 _: [u8; 32],
7137 _: [u8; 32],
7138 ) -> Result<Box<dyn StreamHandle>> {
7139 anyhow::bail!("bootstrap transport was unexpectedly dispatched")
7140 }
7141 fn next_id(&self) -> u64 {
7142 1
7143 }
7144 }
7145
7146 #[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
7147 fn authenticated_registry_identity(tag: u8) -> ed25519_dalek::VerifyingKey {
7148 SigningKey::from_bytes(&[tag; 32]).verifying_key()
7149 }
7150
7151 #[test]
7152 fn deployment_trust_path_resolution_is_explicit_and_split() {
7153 let fixed = resolve_deployment_trust_paths_from(None, None).expect("fixed paths");
7154 assert_eq!(
7155 fixed.public_ca.path,
7156 std::path::Path::new(DEPLOYMENT_CA_ROOT_PATH)
7157 );
7158 assert_eq!(
7159 fixed.authority_log.path,
7160 std::path::Path::new(DEPLOYMENT_AUTHORITY_LOG_PATH)
7161 );
7162 assert_eq!(
7163 fixed.authority_checkpoint.path,
7164 std::path::Path::new(DEPLOYMENT_AUTHORITY_CHECKPOINT_PATH)
7165 );
7166 assert_eq!(
7167 fixed.registry_credential.path,
7168 std::path::Path::new(REGISTRY_DEPLOYMENT_CREDENTIAL_PATH)
7169 );
7170
7171 let trust_dir = std::path::Path::new("/run/user/1000/hyprstream/trust");
7172 let credentials_dir =
7173 std::path::Path::new("/run/user/1000/systemd/credentials/hyprstream.service");
7174 let overridden = resolve_deployment_trust_paths_from(
7175 Some(trust_dir.as_os_str().to_owned()),
7176 Some(credentials_dir.as_os_str().to_owned()),
7177 )
7178 .expect("explicit user-service paths");
7179 assert_eq!(
7180 overridden.public_ca.path,
7181 trust_dir.join(DEPLOYMENT_CA_ROOT_FILE)
7182 );
7183 assert_eq!(
7184 overridden.authority_log.path,
7185 trust_dir.join(DEPLOYMENT_AUTHORITY_LOG_FILE)
7186 );
7187 assert_eq!(
7188 overridden.authority_checkpoint.path,
7189 trust_dir.join(DEPLOYMENT_AUTHORITY_CHECKPOINT_FILE)
7190 );
7191 assert_eq!(
7192 overridden.registry_credential.path,
7193 credentials_dir.join(REGISTRY_DEPLOYMENT_CREDENTIAL_FILE)
7194 );
7195
7196 let trust_only =
7197 resolve_deployment_trust_paths_from(Some(trust_dir.as_os_str().to_owned()), None)
7198 .expect("trust-only override");
7199 assert_eq!(
7200 trust_only.public_ca.path,
7201 trust_dir.join(DEPLOYMENT_CA_ROOT_FILE)
7202 );
7203 assert_eq!(
7204 trust_only.registry_credential.path,
7205 std::path::Path::new(REGISTRY_DEPLOYMENT_CREDENTIAL_PATH)
7206 );
7207
7208 let credentials_only =
7209 resolve_deployment_trust_paths_from(None, Some(credentials_dir.as_os_str().to_owned()))
7210 .expect("credentials-only override");
7211 assert_eq!(
7212 credentials_only.public_ca.path,
7213 std::path::Path::new(DEPLOYMENT_CA_ROOT_PATH)
7214 );
7215 assert_eq!(
7216 credentials_only.registry_credential.path,
7217 credentials_dir.join(REGISTRY_DEPLOYMENT_CREDENTIAL_FILE)
7218 );
7219 }
7220
7221 #[test]
7222 fn invalid_present_trust_path_never_falls_back() {
7223 for (name, trust_dir, credentials_dir) in [
7224 (
7225 DEPLOYMENT_TRUST_DIR_ENV,
7226 Some(std::ffi::OsString::new()),
7227 None,
7228 ),
7229 (
7230 DEPLOYMENT_TRUST_DIR_ENV,
7231 Some(std::ffi::OsString::from("relative/trust")),
7232 None,
7233 ),
7234 (
7235 DEPLOYMENT_TRUST_DIR_ENV,
7236 Some(std::ffi::OsString::from("/secure/../other")),
7237 None,
7238 ),
7239 (
7240 SYSTEMD_CREDENTIALS_DIRECTORY_ENV,
7241 None,
7242 Some(std::ffi::OsString::new()),
7243 ),
7244 (
7245 SYSTEMD_CREDENTIALS_DIRECTORY_ENV,
7246 None,
7247 Some(std::ffi::OsString::from("relative/credentials")),
7248 ),
7249 (
7250 SYSTEMD_CREDENTIALS_DIRECTORY_ENV,
7251 None,
7252 Some(std::ffi::OsString::from("/secure/../other")),
7253 ),
7254 ] {
7255 let error = resolve_deployment_trust_paths_from(trust_dir, credentials_dir)
7256 .expect_err("invalid present override fell back");
7257 assert!(
7258 error.to_string().contains(name),
7259 "error did not identify {name}: {error}"
7260 );
7261 }
7262 }
7263
7264 #[cfg(unix)]
7265 #[test]
7266 fn trusted_artifact_accepts_effective_uid_owned_file_without_path_override_policy() {
7267 use std::os::unix::fs::PermissionsExt as _;
7268
7269 let fixture = tempfile::Builder::new()
7270 .prefix(".trust-owner-test-")
7271 .tempdir_in(env!("CARGO_MANIFEST_DIR"))
7272 .expect("secure fixture directory");
7273 let path = fixture.path().join("fixed-path-policy-artifact");
7274 std::fs::write(&path, b"service-owned trust").expect("trust artifact");
7275 std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644))
7276 .expect("read-only trust artifact");
7277
7278 let bytes = read_trusted_artifact(
7279 &TrustedArtifactPath { path },
7280 "service-owned trust artifact",
7281 )
7282 .expect("effective-uid-owned 0644 artifact must be trusted");
7283 assert_eq!(bytes, b"service-owned trust");
7284 }
7285
7286 #[cfg(unix)]
7287 #[test]
7288 fn trusted_artifact_metadata_accepts_root_and_nonroot_service_but_rejects_other_uid() {
7289 const NONROOT_SERVICE_UID: u32 = 1_000;
7290 const OTHER_UID: u32 = 1_001;
7291
7292 validate_trusted_artifact_metadata(0, 0o100644, NONROOT_SERVICE_UID, "root artifact")
7293 .expect("root-owned 0644 artifact must remain trusted");
7294 validate_trusted_artifact_metadata(
7295 NONROOT_SERVICE_UID,
7296 0o100644,
7297 NONROOT_SERVICE_UID,
7298 "service artifact",
7299 )
7300 .expect("non-root effective-uid-owned 0644 artifact must be trusted");
7301
7302 let error = validate_trusted_artifact_metadata(
7303 OTHER_UID,
7304 0o100644,
7305 NONROOT_SERVICE_UID,
7306 "other-user artifact",
7307 )
7308 .expect_err("different non-service uid was trusted");
7309 assert!(error.to_string().contains("untrusted owner"), "{error}");
7310 }
7311
7312 #[cfg(unix)]
7313 fn user_service_trust_fixture() -> (
7314 tempfile::TempDir,
7315 DeploymentTrustPaths,
7316 TestDeploymentCa,
7317 SigningKey,
7318 ) {
7319 use base64::engine::general_purpose::STANDARD;
7320
7321 let fixture = tempfile::Builder::new()
7322 .prefix(".trust-loader-test-")
7323 .tempdir_in(env!("CARGO_MANIFEST_DIR"))
7324 .expect("secure fixture directory");
7325 let trust_dir = fixture.path().join("trust");
7326 let credentials_dir = fixture.path().join("credentials");
7327 std::fs::create_dir(&trust_dir).expect("trust directory");
7328 std::fs::create_dir(&credentials_dir).expect("credentials directory");
7329
7330 let ca = test_deployment_ca(0x71);
7331 let registry = SigningKey::from_bytes(&[0x72; 32]);
7332 let rotation_key = crate::did_op::HybridRotationKey::new(
7333 ca.ed25519.verifying_key().to_bytes(),
7334 hyprstream_crypto::pq::ml_dsa_sk_to_vk_bytes(&ca.ml_dsa_65),
7335 )
7336 .expect("hybrid rotation key");
7337 let genesis =
7338 crate::did_op::DidOp::signed_genesis(vec![rotation_key], &ca.ed25519, &ca.ml_dsa_65)
7339 .expect("signed authority genesis");
7340 let did = genesis.genesis_did().expect("authority-log DID");
7341 let active = crate::did_op::verify_did_op_log(&did, std::slice::from_ref(&genesis))
7342 .expect("verified authority log");
7343 let log = DeploymentAuthorityLog {
7344 schema: AUTHORITY_LOG_SCHEMA.to_owned(),
7345 deployment_domain: ca.verifying_keys.domain(),
7346 did: did.clone(),
7347 operations_b64: vec![STANDARD.encode(genesis.to_dag_cbor())],
7348 };
7349 let checkpoint = DeploymentAuthorityCheckpoint {
7350 schema: AUTHORITY_CHECKPOINT_SCHEMA.to_owned(),
7351 deployment_domain: ca.verifying_keys.domain(),
7352 did,
7353 sequence: active.sequence,
7354 head_cid: active.head_cid,
7355 };
7356 let mut public_ca = ca.verifying_keys.ed25519_bytes().to_vec();
7357 public_ca.extend_from_slice(&ca.verifying_keys.ml_dsa_65_bytes());
7358 std::fs::write(trust_dir.join(DEPLOYMENT_CA_ROOT_FILE), public_ca).expect("deployment CA");
7359 std::fs::write(
7360 trust_dir.join(DEPLOYMENT_AUTHORITY_LOG_FILE),
7361 serde_json::to_vec(&log).expect("authority-log JSON"),
7362 )
7363 .expect("authority log");
7364 std::fs::write(
7365 trust_dir.join(DEPLOYMENT_AUTHORITY_CHECKPOINT_FILE),
7366 serde_json::to_vec(&checkpoint).expect("checkpoint JSON"),
7367 )
7368 .expect("authority checkpoint");
7369 std::fs::write(
7370 credentials_dir.join(REGISTRY_DEPLOYMENT_CREDENTIAL_FILE),
7371 exact_registry_credential(&ca, ®istry),
7372 )
7373 .expect("registry credential");
7374 let paths = resolve_deployment_trust_paths_from(
7375 Some(trust_dir.into_os_string()),
7376 Some(credentials_dir.into_os_string()),
7377 )
7378 .expect("fixture trust paths");
7379 (fixture, paths, ca, registry)
7380 }
7381
7382 #[cfg(unix)]
7383 #[test]
7384 fn user_service_paths_preserve_complete_enrolled_verification() {
7385 let (_fixture, paths, ca, registry) = user_service_trust_fixture();
7386 let credentials = load_trusted_registry_deployment_credentials_from(&paths)
7387 .expect("load explicit user-service artifacts");
7388 let witness = authenticate_registry_deployment_credentials(credentials)
7389 .expect("authenticate complete enrolled artifact set");
7390 assert!(witness.verifier.matches(®istry.verifying_key()));
7391 assert!(witness.verifier.matches_deployment_root(&ca.verifying_keys));
7392
7393 let (protected, mut claims) = exact_registry_credential_values(&ca, ®istry);
7394 claims["aud"] = serde_json::Value::String("urn:hyprstream:service:other".to_owned());
7395 std::fs::write(
7396 &paths.registry_credential.path,
7397 sign_registry_credential_json(&ca, &protected.to_string(), &claims.to_string()),
7398 )
7399 .expect("invalid-audience credential");
7400 let credentials = load_trusted_registry_deployment_credentials_from(&paths)
7401 .expect("path loader must not pre-authenticate");
7402 assert!(
7403 authenticate_registry_deployment_credentials(credentials).is_err(),
7404 "explicit path bypassed the closed JWT audience/profile verifier"
7405 );
7406 }
7407
7408 #[cfg(unix)]
7409 #[test]
7410 fn user_service_paths_reject_writable_files_and_symlinks() {
7411 use std::os::unix::fs::{symlink, PermissionsExt as _};
7412
7413 let (_fixture, paths, _ca, _registry) = user_service_trust_fixture();
7414 for mode in [0o664, 0o666] {
7415 std::fs::set_permissions(
7416 &paths.registry_credential.path,
7417 std::fs::Permissions::from_mode(mode),
7418 )
7419 .expect("make credential writable");
7420 let error = match load_trusted_registry_deployment_credentials_from(&paths) {
7421 Ok(_) => panic!("group/world-writable credential was trusted"),
7422 Err(error) => error,
7423 };
7424 assert!(
7425 error.to_string().contains("group/world writable"),
7426 "mode {mode:o} failed for the wrong reason: {error}"
7427 );
7428 }
7429
7430 let real_credential = paths
7431 .registry_credential
7432 .path
7433 .with_file_name("registry-service.real.jwt");
7434 std::fs::rename(&paths.registry_credential.path, &real_credential)
7435 .expect("retain real credential");
7436 std::fs::set_permissions(&real_credential, std::fs::Permissions::from_mode(0o600))
7437 .expect("secure real credential");
7438 symlink(&real_credential, &paths.registry_credential.path).expect("credential symlink");
7439 let error = load_trusted_registry_deployment_credentials_from(&paths)
7440 .err()
7441 .expect("symlinked credential was trusted");
7442 assert!(
7443 error
7444 .to_string()
7445 .contains("Too many levels of symbolic links")
7446 || error
7447 .to_string()
7448 .contains("registry deployment credential is unavailable"),
7449 "unexpected symlink rejection: {error}"
7450 );
7451 }
7452
7453 #[cfg(unix)]
7454 #[test]
7455 fn user_service_paths_reject_writable_or_symlinked_ancestors() {
7456 use std::os::unix::fs::{symlink, PermissionsExt as _};
7457
7458 let (fixture, paths, _ca, _registry) = user_service_trust_fixture();
7459 std::fs::set_permissions(fixture.path(), std::fs::Permissions::from_mode(0o770))
7460 .expect("make fixture parent writable");
7461 let error = load_trusted_registry_deployment_credentials_from(&paths)
7462 .err()
7463 .expect("group-writable ancestor was trusted");
7464 assert!(error.to_string().contains("parent"));
7465 assert!(error.to_string().contains("group/world writable"));
7466
7467 std::fs::set_permissions(fixture.path(), std::fs::Permissions::from_mode(0o700))
7468 .expect("restore fixture parent");
7469 let trust_dir = fixture.path().join("trust");
7470 let real_trust_dir = fixture.path().join("real-trust");
7471 std::fs::rename(&trust_dir, &real_trust_dir).expect("retain real trust directory");
7472 symlink(&real_trust_dir, &trust_dir).expect("trust-directory symlink");
7473 let error = load_trusted_registry_deployment_credentials_from(&paths)
7474 .err()
7475 .expect("symlinked trust ancestor was trusted");
7476 assert!(
7477 error.to_string().contains("not a real directory"),
7478 "unexpected ancestor-symlink rejection: {error}"
7479 );
7480 }
7481
7482 #[cfg(unix)]
7486 #[test]
7487 fn os_owned_enrollment_verification_fails_closed_without_chain() {
7488 static ENV_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
7489 let _serial = ENV_LOCK.lock();
7490 let fixture = tempfile::Builder::new()
7491 .prefix(".enrollment-trust-test-")
7492 .tempdir_in(env!("CARGO_MANIFEST_DIR"))
7493 .expect("secure fixture directory");
7494 let prev = std::env::var_os(DEPLOYMENT_TRUST_DIR_ENV);
7495 std::env::set_var(DEPLOYMENT_TRUST_DIR_ENV, fixture.path());
7496 let result = verify_os_owned_service_key_enrollment(b"{}");
7497 match &prev {
7498 Some(value) => std::env::set_var(DEPLOYMENT_TRUST_DIR_ENV, value),
7499 None => std::env::remove_var(DEPLOYMENT_TRUST_DIR_ENV),
7500 }
7501 let error = result.expect_err("attestation verified without a chain");
7502 assert!(
7503 error.to_string().contains("deployment CA root"),
7504 "unexpected failure mode: {error}"
7505 );
7506 }
7507
7508 #[test]
7509 fn verified_registry_identity_ignores_policy_and_post_start_environment_mutation() {
7510 const CHILD: &str = "HYPRSTREAM_TEST_POST_START_REGISTRY_ENV_CHILD";
7511 if std::env::var_os(CHILD).is_none() {
7512 let status = std::process::Command::new(
7513 std::env::current_exe().expect("discovery test executable"),
7514 )
7515 .arg("--exact")
7516 .arg("service::resolver_tests::verified_registry_identity_ignores_policy_and_post_start_environment_mutation")
7517 .arg("--nocapture")
7518 .env(CHILD, "1")
7519 .status()
7520 .expect("post-start environment subprocess");
7521 assert!(status.success(), "post-start environment subprocess failed");
7522 return;
7523 }
7524 let caller = SigningKey::from_bytes(&[0x61; 32]);
7525 hyprstream_service::global_trust_store().insert(
7526 caller.verifying_key(),
7527 hyprstream_service::Attestation {
7528 scopes: std::iter::once("registry".to_owned()).collect(),
7529 subject: None,
7530 jwt: None,
7531 expires_at: 0,
7532 attested_by: None,
7533 },
7534 );
7535 let ca = test_deployment_ca(0x62);
7536 let registry = SigningKey::from_bytes(&[0x63; 32]);
7537 let witness =
7538 authenticate_test_registry_credential(&ca, exact_registry_credential(&ca, ®istry))
7539 .expect("explicit genesis-only trusted pair");
7540 std::env::set_var("CREDENTIALS_DIRECTORY", "post-start-attacker-directory");
7541 std::env::set_var("XDG_CONFIG_HOME", "post-start-attacker-config");
7542 assert!(witness.verifier.matches(®istry.verifying_key()));
7543 assert!(!witness.verifier.matches(&caller.verifying_key()));
7544 }
7545
7546 #[cfg(feature = "rocksdb")]
7547 #[test]
7548 fn process_bootstrap_precedes_first_generated_client() {
7549 const CHILD: &str = "HYPRSTREAM_TEST_RESOLVER_BOOTSTRAP_CHILD";
7550 if std::env::var_os(CHILD).is_some() {
7551 let store = hyprstream_service::deployment_data_dir()
7552 .expect("deployment-owned data root")
7553 .join("pds-store");
7554 std::fs::create_dir_all(&store).expect("checkpoint store directory");
7555 drop(rocksdb::DB::open_default(&store).expect("empty checkpoint store"));
7556 let signing = SigningKey::from_bytes(&[0x47; 32]);
7557 hyprstream_service::global_trust_store().insert(
7558 signing.verifying_key(),
7559 hyprstream_service::Attestation {
7560 scopes: std::iter::once("registry".to_owned()).collect(),
7561 subject: None,
7562 jwt: None,
7563 expires_at: 0,
7564 attested_by: None,
7565 },
7566 );
7567 let authority =
7568 authenticate_discovery_bootstrap_identity(authenticated_registry_identity(0x51))
7569 .expect("authenticate process resolver");
7570 DiscoveryService::bootstrap_authenticated_process(
7571 authority,
7572 hyprstream_rpc_std::discovery_client::DiscoveryClient::new(Arc::new(
7573 NoopBootstrapClient,
7574 )),
7575 &hyprstream_pds::rds::RdsConfig::default(),
7576 )
7577 .expect("install process resolver");
7578 assert!(production_rpc_client("model", signing, None).is_ok());
7579 assert!(
7580 authenticate_discovery_bootstrap_identity(authenticated_registry_identity(0x52))
7581 .is_err()
7582 );
7583 return;
7584 }
7585 let deployment = tempfile::tempdir().expect("deployment data root");
7586 let status =
7587 std::process::Command::new(std::env::current_exe().expect("discovery test executable"))
7588 .arg("--exact")
7589 .arg("service::resolver_tests::process_bootstrap_precedes_first_generated_client")
7590 .arg("--nocapture")
7591 .env(CHILD, "1")
7592 .env("XDG_DATA_HOME", deployment.path())
7593 .status()
7594 .expect("resolver bootstrap subprocess");
7595 assert!(status.success(), "resolver bootstrap subprocess failed");
7596 }
7597
7598 #[cfg_attr(not(feature = "rocksdb"), allow(dead_code))]
7599 fn seed_registry_for_bootstrap(tag: u8) {
7600 let registry = SigningKey::from_bytes(&[tag; 32]);
7601 hyprstream_service::global_trust_store().insert(
7602 registry.verifying_key(),
7603 hyprstream_service::Attestation {
7604 scopes: std::iter::once("registry".to_owned()).collect(),
7605 subject: None,
7606 jwt: None,
7607 expires_at: 0,
7608 attested_by: None,
7609 },
7610 );
7611 }
7612
7613 #[cfg(feature = "rocksdb")]
7614 #[test]
7615 fn authenticated_process_bootstrap_has_exactly_one_concurrent_consumer() {
7616 const CHILD: &str = "HYPRSTREAM_TEST_RESOLVER_CONCURRENT_CHILD";
7617 if std::env::var_os(CHILD).is_some() {
7618 let store = hyprstream_service::deployment_data_dir()
7619 .unwrap()
7620 .join("pds-store");
7621 std::fs::create_dir_all(&store).unwrap();
7622 drop(rocksdb::DB::open_default(&store).unwrap());
7623 seed_registry_for_bootstrap(0x48);
7624 let barrier = Arc::new(std::sync::Barrier::new(9));
7625 let attempts: Vec<_> = (0..8)
7626 .map(|_| {
7627 let barrier = Arc::clone(&barrier);
7628 std::thread::spawn(move || {
7629 barrier.wait();
7630 authenticate_discovery_bootstrap_identity(authenticated_registry_identity(
7631 0x53,
7632 ))
7633 .and_then(|authority| {
7634 DiscoveryService::bootstrap_authenticated_process(
7635 authority,
7636 hyprstream_rpc_std::discovery_client::DiscoveryClient::new(
7637 Arc::new(NoopBootstrapClient),
7638 ),
7639 &hyprstream_pds::rds::RdsConfig::default(),
7640 )
7641 })
7642 .is_ok()
7643 })
7644 })
7645 .collect();
7646 barrier.wait();
7647 assert_eq!(
7648 attempts
7649 .into_iter()
7650 .map(|attempt| attempt.join().unwrap())
7651 .filter(|installed| *installed)
7652 .count(),
7653 1
7654 );
7655 return;
7656 }
7657 let deployment = tempfile::tempdir().unwrap();
7658 let status = std::process::Command::new(std::env::current_exe().unwrap())
7659 .arg("--exact")
7660 .arg("service::resolver_tests::authenticated_process_bootstrap_has_exactly_one_concurrent_consumer")
7661 .arg("--nocapture")
7662 .env(CHILD, "1")
7663 .env("XDG_DATA_HOME", deployment.path())
7664 .status()
7665 .unwrap();
7666 assert!(status.success());
7667 }
7668
7669 #[cfg(feature = "rocksdb")]
7670 #[test]
7671 fn authenticated_process_bootstrap_failure_is_terminal() {
7672 const CHILD: &str = "HYPRSTREAM_TEST_RESOLVER_FAILURE_CHILD";
7673 if std::env::var_os(CHILD).is_some() {
7674 seed_registry_for_bootstrap(0x49);
7675 let authority =
7676 authenticate_discovery_bootstrap_identity(authenticated_registry_identity(0x54))
7677 .unwrap();
7678 let first = DiscoveryService::bootstrap_authenticated_process(
7679 authority,
7680 hyprstream_rpc_std::discovery_client::DiscoveryClient::new(Arc::new(
7681 NoopBootstrapClient,
7682 )),
7683 &hyprstream_pds::rds::RdsConfig::default(),
7684 )
7685 .expect_err("missing checkpoint store unexpectedly installed");
7686 assert!(first.to_string().contains("failed to open"));
7687 let store = hyprstream_service::deployment_data_dir()
7688 .unwrap()
7689 .join("pds-store");
7690 std::fs::create_dir_all(&store).unwrap();
7691 drop(rocksdb::DB::open_default(&store).unwrap());
7692 let second =
7693 authenticate_discovery_bootstrap_identity(authenticated_registry_identity(0x55))
7694 .err()
7695 .expect("failed bootstrap authority was replayed");
7696 assert!(second.to_string().contains("already sealed or consumed"));
7697 return;
7698 }
7699 let deployment = tempfile::tempdir().unwrap();
7700 let status = std::process::Command::new(std::env::current_exe().unwrap())
7701 .arg("--exact")
7702 .arg("service::resolver_tests::authenticated_process_bootstrap_failure_is_terminal")
7703 .arg("--nocapture")
7704 .env(CHILD, "1")
7705 .env("XDG_DATA_HOME", deployment.path())
7706 .status()
7707 .unwrap();
7708 assert!(status.success());
7709 }
7710
7711 #[tokio::test]
7712 async fn stale_or_expired_production_evidence_is_rejected() {
7713 let (resolver, _) = native_production_fixture(false);
7714 let expiry = resolver
7715 .state_store
7716 .announcements_for("model", unix_millis_now())
7717 .await
7718 .unwrap()[0]
7719 .live_until_unix_ms;
7720 {
7721 let _clock = crate::state_store::tests::ReplicaClock::at(expiry);
7725 assert!(resolver
7726 .resolve_service(ServiceQuery::network("model").expect("query"))
7727 .await
7728 .is_err());
7729 }
7730
7731 let (resolver, source) = native_production_fixture(false);
7732 source.0.lock().as_mut().expect("fixture state").expires_at =
7733 Some("2000-01-01T00:00:00Z".to_owned());
7734 assert!(resolver
7735 .resolve_service(ServiceQuery::network("model").expect("query"))
7736 .await
7737 .is_err());
7738 }
7739
7740 #[tokio::test]
7741 async fn malformed_candidate_does_not_poison_valid_alternative() {
7742 let (resolver, _) = native_production_fixture(false);
7743 let mut malformed = resolver
7744 .state_store
7745 .announcements_for("model", unix_millis_now())
7746 .await
7747 .expect("fixture service")
7748 .into_iter()
7749 .next()
7750 .expect("fixture endpoint");
7751 malformed.request_kem_recipient = vec![0xff];
7752 malformed.socket_kind = "quic".to_owned();
7753 malformed.endpoint = "quic://missing-port".to_owned();
7754 resolver
7755 .state_store
7756 .put_announcement("model", malformed)
7757 .await
7758 .expect("seed malformed alternative");
7759
7760 let resolved = resolver
7761 .resolve_service(ServiceQuery::network("model").expect("query"))
7762 .await
7763 .unwrap_or_else(|e| panic!("malformed alternative poisoned valid candidate: {e}"));
7764 assert_eq!(resolved.service_name(), "model");
7765 }
7766
7767 #[tokio::test]
7768 async fn rejected_or_forked_current_state_never_produces_candidate() {
7769 let (resolver, source) = production_fixture(false);
7770 *source.0.lock() = None;
7771 assert!(resolver
7772 .resolve_service(ServiceQuery::network("model").expect("query"))
7773 .await
7774 .is_err());
7775
7776 let (resolver, _) = production_fixture(false);
7777 mutate_endpoint(&resolver, "model", "quic", |endpoint| {
7778 endpoint.accepted_state_digest = vec![0x77; 64];
7779 })
7780 .await;
7781 assert!(resolver
7782 .resolve_service(ServiceQuery::network("model").expect("query"))
7783 .await
7784 .is_err());
7785 }
7786
7787 #[tokio::test]
7788 async fn resolver_uses_fresh_announced_quic_endpoint() {
7789 let svc = service();
7790 svc.state_store
7791 .put_announcement(
7792 "model",
7793 legacy_endpoint(
7794 "quic",
7795 "quic://model.hyprstream.svc.cluster.local:10.96.0.42:4433",
7796 Instant::now(),
7797 ),
7798 )
7799 .await
7800 .expect("seed announcement");
7801
7802 let transport = match svc.resolve("model", SocketKind::Quic).await {
7803 Ok(transport) => transport,
7804 Err(err) => panic!("fresh announced QUIC endpoint must resolve: {err}"),
7805 };
7806 assert_eq!(transport.bind_mode(), BindMode::Connect);
7807 match transport.endpoint {
7808 EndpointType::Quic {
7809 addr, server_name, ..
7810 } => {
7811 assert_eq!(server_name, "model.hyprstream.svc.cluster.local");
7812 assert_eq!(addr, SocketAddr::from(([10, 96, 0, 42], 4433)));
7813 }
7814 other => panic!("expected announced QUIC endpoint, got {other:?}"),
7815 }
7816 }
7817
7818 #[tokio::test]
7819 async fn resolver_rejects_missing_announced_quic_endpoint() {
7820 let err = match service().resolve("model", SocketKind::Quic).await {
7821 Ok(transport) => panic!("missing announced QUIC endpoint resolved to {transport:?}"),
7822 Err(err) => err,
7823 };
7824 assert!(
7825 err.to_string().contains("no fresh announced QUIC endpoint"),
7826 "unexpected error: {err}"
7827 );
7828 }
7829
7830 #[tokio::test]
7831 async fn resolver_rejects_stale_announced_quic_endpoint() {
7832 let svc = service();
7833 svc.state_store
7834 .put_announcement(
7835 "model",
7836 legacy_endpoint(
7837 "quic",
7838 "quic://model.hyprstream.svc.cluster.local:10.96.0.42:4433",
7839 Instant::now() - (ANNOUNCED_ENDPOINT_TTL + Duration::from_secs(1)),
7840 ),
7841 )
7842 .await
7843 .expect("seed stale announcement");
7844
7845 let err = match svc.resolve("model", SocketKind::Quic).await {
7846 Ok(transport) => panic!("stale announced QUIC endpoint resolved to {transport:?}"),
7847 Err(err) => err,
7848 };
7849 assert!(
7850 err.to_string().contains("no fresh announced QUIC endpoint"),
7851 "unexpected error: {err}"
7852 );
7853 }
7854
7855 #[test]
7856 fn announced_iroh_endpoint_parses_to_transport() {
7857 let node_hex = "0707070707070707070707070707070707070707070707070707070707070707";
7858 let transport = match parse_announced_iroh(&format!("iroh://{node_hex}")) {
7859 Ok(transport) => transport,
7860 Err(err) => panic!("valid iroh endpoint must parse: {err}"),
7861 };
7862 assert_eq!(transport.bind_mode(), BindMode::Connect);
7863 match transport.endpoint {
7864 EndpointType::Iroh {
7865 node_id,
7866 direct_addrs,
7867 relay_url,
7868 } => {
7869 assert_eq!(node_id, [7u8; 32]);
7870 assert!(direct_addrs.is_empty());
7871 assert!(relay_url.is_none());
7872 }
7873 other => panic!("expected iroh endpoint, got {other:?}"),
7874 }
7875 }
7876}
7877
7878fn socket_kind_to_string(kind: SocketKind) -> &'static str {
7880 match kind {
7881 SocketKind::Req => "req",
7882 SocketKind::Rep => "rep",
7883 SocketKind::Quic => "quic",
7884 }
7885}
7886
7887#[async_trait(?Send)]
7892impl DiscoveryHandler for DiscoveryService {
7893 async fn authorize(
7894 &self,
7895 ctx: &EnvelopeContext,
7896 resource: &str,
7897 operation: &str,
7898 ) -> Result<()> {
7899 if let Some(ref auth) = self.auth_provider {
7901 let subject = ctx.subject().to_string();
7902 let allowed = auth
7903 .check_context(ctx, resource, operation)
7904 .await
7905 .unwrap_or_else(|e| {
7906 tracing::warn!("Discovery auth check failed for {}: {}", subject, e);
7907 false
7908 });
7909 if allowed {
7910 Ok(())
7911 } else {
7912 anyhow::bail!(
7913 "Unauthorized: {} cannot {} on {}",
7914 subject,
7915 operation,
7916 resource
7917 )
7918 }
7919 } else {
7920 Ok(())
7922 }
7923 }
7924
7925 async fn handle_list_services(
7926 &self,
7927 _ctx: &EnvelopeContext,
7928 _request_id: u64,
7929 ) -> Result<DiscoveryResponseVariant> {
7930 trace!("Discovery: listing services");
7931
7932 let reg = self.reg()?;
7934 let service_names = reg.list_services();
7935 let mut summaries: Vec<ServiceSummary> = service_names
7936 .iter()
7937 .filter_map(|name| {
7938 reg.service_entry(name).map(|entry| {
7939 let socket_kinds: Vec<String> = entry
7940 .endpoints
7941 .keys()
7942 .map(|k| socket_kind_to_string(*k).to_owned())
7943 .collect();
7944 ServiceSummary {
7945 name: entry.name,
7946 description: entry.description.unwrap_or_default(),
7947 socket_kinds,
7948 has_schema: entry.schema.is_some(),
7949 }
7950 })
7951 })
7952 .collect();
7953 drop(reg);
7954
7955 let announced = self
7957 .state_store
7958 .all_announcements(unix_millis_now())
7959 .await?;
7960 let local_names: Vec<String> = summaries.iter().map(|s| s.name.clone()).collect();
7961 for (name, endpoints) in &announced {
7962 if local_names.iter().any(|n| n == name) {
7963 if let Some(summary) = summaries.iter_mut().find(|s| s.name == *name) {
7965 for ep in endpoints {
7966 if !summary.socket_kinds.contains(&ep.socket_kind) {
7967 summary.socket_kinds.push(ep.socket_kind.clone());
7968 }
7969 }
7970 }
7971 } else {
7972 summaries.push(ServiceSummary {
7974 name: name.clone(),
7975 description: String::new(),
7976 socket_kinds: endpoints.iter().map(|e| e.socket_kind.clone()).collect(),
7977 has_schema: false,
7978 });
7979 }
7980 }
7981
7982 Ok(DiscoveryResponseVariant::ListServicesResult(ServiceList {
7983 services: summaries,
7984 }))
7985 }
7986
7987 async fn handle_get_endpoints(
7988 &self,
7989 _ctx: &EnvelopeContext,
7990 _request_id: u64,
7991 data: &str,
7992 ) -> Result<DiscoveryResponseVariant> {
7993 let service_name = data;
7994 trace!("Discovery: getting endpoints for '{}'", service_name);
7995
7996 let reg = self.reg()?;
7998 let endpoints_map = reg.service_endpoints(service_name);
7999 drop(reg);
8000
8001 let mut endpoints: Vec<EndpointInfo> = match endpoints_map {
8003 Some(map) => map
8004 .iter()
8005 .map(|(kind, transport)| EndpointInfo {
8006 socket_kind: socket_kind_to_string(*kind).to_owned(),
8007 endpoint: transport.endpoint_string(),
8008 service_jwt: String::new(),
8009 tls_endorsement: self.tls_endorsement.clone(),
8010 tls_domain: self.tls_domain.clone(),
8011 service_did: Did::default(),
8012 capabilities: Vec::new(),
8013 accepted_state_digest: Vec::new(),
8014 accepted_state_epoch: 0,
8015 response_key_id: String::new(),
8016 request_kem_key_id: String::new(),
8017 request_kem_recipient: Vec::new(),
8018 expires_at_unix_ms: 0,
8019 source_signer: Vec::new(),
8020 })
8021 .collect(),
8022 None => Vec::new(),
8023 };
8024
8025 for ep in self
8030 .state_store
8031 .announcements_for(service_name, unix_millis_now())
8032 .await?
8033 {
8034 if !ep.service_did.as_str().is_empty() {
8040 endpoints.retain(|e| e.socket_kind != ep.socket_kind);
8041 }
8042 if !endpoints.iter().any(|e| e.socket_kind == ep.socket_kind) {
8043 endpoints.push(EndpointInfo {
8044 socket_kind: ep.socket_kind,
8045 endpoint: ep.endpoint,
8046 service_jwt: String::new(),
8047 tls_endorsement: self.tls_endorsement.clone(),
8048 tls_domain: self.tls_domain.clone(),
8049 service_did: ep.service_did,
8050 capabilities: ep.capabilities.into_iter().collect(),
8051 accepted_state_digest: ep.accepted_state_digest,
8052 accepted_state_epoch: ep.accepted_state_epoch,
8053 response_key_id: ep.response_key_id,
8054 request_kem_key_id: ep.request_kem_key_id,
8055 request_kem_recipient: ep.request_kem_recipient,
8056 expires_at_unix_ms: ep.expires_at_unix_ms,
8057 source_signer: ep.source_signer.to_vec(),
8058 });
8059 }
8060 }
8061
8062 if endpoints.is_empty() {
8063 Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8064 message: format!("Service '{}' not found", service_name),
8065 code: "NOT_FOUND".to_owned(),
8066 details: String::new(),
8067 }))
8068 } else {
8069 Ok(DiscoveryResponseVariant::GetEndpointsResult(
8070 ServiceEndpoints { endpoints },
8071 ))
8072 }
8073 }
8074
8075 async fn handle_get_schema(
8076 &self,
8077 _ctx: &EnvelopeContext,
8078 _request_id: u64,
8079 data: &str,
8080 ) -> Result<DiscoveryResponseVariant> {
8081 let service_name = data;
8082 trace!("Discovery: getting schema for '{}'", service_name);
8083
8084 let reg = self.reg()?;
8085 let schema = reg.service_schema(service_name);
8086 drop(reg);
8087
8088 match schema {
8089 Some(bytes) => Ok(DiscoveryResponseVariant::GetSchemaResult(bytes.to_vec())),
8090 None => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8091 message: format!("No schema registered for service '{}'", service_name),
8092 code: "NO_SCHEMA".to_owned(),
8093 details: String::new(),
8094 })),
8095 }
8096 }
8097
8098 async fn handle_ping(
8099 &self,
8100 _ctx: &EnvelopeContext,
8101 _request_id: u64,
8102 ) -> Result<DiscoveryResponseVariant> {
8103 trace!("Discovery: ping");
8104
8105 let reg = self.reg()?;
8106 let service_count = reg.list_services().len() as u32;
8107 drop(reg);
8108
8109 let uptime = self.started_at.elapsed().as_secs();
8110
8111 Ok(DiscoveryResponseVariant::PingResult(PingInfo {
8112 status: "ok".to_owned(),
8113 service_count,
8114 uptime,
8115 }))
8116 }
8117
8118 async fn handle_get_auth_metadata(
8119 &self,
8120 _ctx: &EnvelopeContext,
8121 _request_id: u64,
8122 data: &str,
8123 ) -> Result<DiscoveryResponseVariant> {
8124 let filter = data;
8125 trace!("Discovery: get auth metadata (filter='{}')", filter);
8126
8127 let issuer = match &self.oauth_issuer_url {
8128 Some(url) => url.clone(),
8129 None => {
8130 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8131 message: "OAuth issuer URL not configured".to_owned(),
8132 code: "NOT_CONFIGURED".to_owned(),
8133 details: String::new(),
8134 }));
8135 }
8136 };
8137
8138 let reg = self.reg()?;
8139 let service_names = reg.list_services();
8140
8141 let services: Vec<AuthMetadata> = service_names
8142 .iter()
8143 .filter(|name| filter.is_empty() || *name == filter)
8144 .filter_map(|name| {
8145 let quic_transport = match reg.try_endpoint(name, SocketKind::Quic) {
8146 Ok(transport) => transport,
8147 Err(err) => {
8148 tracing::debug!(
8149 service = %name,
8150 error = %err,
8151 "skipping auth metadata for service without registered QUIC reach"
8152 );
8153 return None;
8154 }
8155 };
8156 let resource = quic_transport.quic_resource_url(name)?;
8157 Some(AuthMetadata {
8158 service_name: name.clone(),
8159 resource,
8160 authorization_servers: vec![issuer.clone()],
8161 scopes_supported: Vec::new(),
8162 resource_name: format!("HyprStream {} Service", name),
8163 })
8164 })
8165 .collect();
8166 drop(reg);
8167
8168 Ok(DiscoveryResponseVariant::GetAuthMetadataResult(
8169 AuthMetadataList { services },
8170 ))
8171 }
8172
8173 async fn handle_prepare_stream(
8174 &self,
8175 _ctx: &EnvelopeContext,
8176 _request_id: u64,
8177 ) -> Result<DiscoveryResponseVariant> {
8178 Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8179 message: "prepareStream removed — use StreamChannel::prepare_identified_stream for authenticated streaming".to_owned(),
8180 code: "REMOVED".to_owned(),
8181 details: String::new(),
8182 }))
8183 }
8184
8185 async fn handle_get_stream(
8186 &self,
8187 _ctx: &EnvelopeContext,
8188 _request_id: u64,
8189 ) -> Result<DiscoveryResponseVariant> {
8190 Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8191 message:
8192 "getStream removed — use StreamChannel::prepare_identified_stream for authenticated streaming"
8193 .to_owned(),
8194 code: "REMOVED".to_owned(),
8195 details: String::new(),
8196 }))
8197 }
8198
8199 async fn handle_list_streams(
8200 &self,
8201 _ctx: &EnvelopeContext,
8202 _request_id: u64,
8203 ) -> Result<DiscoveryResponseVariant> {
8204 Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8205 message: "listStreams removed — use StreamChannel::prepare_identified_stream for authenticated streaming".to_owned(),
8206 code: "REMOVED".to_owned(),
8207 details: String::new(),
8208 }))
8209 }
8210
8211 async fn handle_announce(
8212 &self,
8213 ctx: &EnvelopeContext,
8214 _request_id: u64,
8215 data: &ServiceAnnouncement,
8216 ) -> Result<DiscoveryResponseVariant> {
8217 self.store_announcement(ctx, data).await
8218 }
8219
8220 async fn handle_register_entity_statement(
8225 &self,
8226 ctx: &EnvelopeContext,
8227 _request_id: u64,
8228 data: &RegisterEntityStatementRequest,
8229 ) -> Result<DiscoveryResponseVariant> {
8230 if data.issuer.is_empty() {
8231 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8232 message: "issuer is required".to_owned(),
8233 code: "INVALID_ARGUMENT".to_owned(),
8234 details: String::new(),
8235 }));
8236 }
8237 if data.jwt.is_empty() {
8238 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8239 message: "jwt is required".to_owned(),
8240 code: "INVALID_ARGUMENT".to_owned(),
8241 details: String::new(),
8242 }));
8243 }
8244
8245 let cached = CachedEntityStatement {
8256 jwt: data.jwt.clone(),
8257 fetched_at: unix_seconds_now(),
8258 };
8259 self.state_store
8260 .put_entity_statement(&data.issuer, cached)
8261 .await?;
8262 let total = self.state_store.known_issuer_count().await?;
8263
8264 info!(
8265 issuer = %data.issuer,
8266 caller = %ctx.subject(),
8267 total_cached = total,
8268 "Discovery: entity statement registered"
8269 );
8270 Ok(DiscoveryResponseVariant::RegisterEntityStatementResult)
8271 }
8272
8273 async fn handle_get_entity_statement(
8274 &self,
8275 _ctx: &EnvelopeContext,
8276 _request_id: u64,
8277 data: &str,
8278 ) -> Result<DiscoveryResponseVariant> {
8279 let issuer = data;
8280 match self.state_store.entity_statement(issuer).await? {
8281 Some(cached) => {
8282 trace!(issuer = %issuer, "Discovery: entity statement cache hit");
8283 Ok(DiscoveryResponseVariant::GetEntityStatementResult(
8284 EntityStatement {
8285 issuer: issuer.to_owned(),
8286 jwt: cached.jwt,
8287 fetched_at: cached.fetched_at,
8288 },
8289 ))
8290 }
8291 None => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8292 message: format!("no entity statement cached for issuer: {}", issuer),
8293 code: "NOT_FOUND".to_owned(),
8294 details: String::new(),
8295 })),
8296 }
8297 }
8298
8299 async fn handle_register_envelope_keyset(
8300 &self,
8301 ctx: &EnvelopeContext,
8302 _request_id: u64,
8303 data: &RegisterEnvelopeKeysetRequest,
8304 ) -> Result<DiscoveryResponseVariant> {
8305 if data.service_did.as_str().is_empty() {
8306 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8307 message: "serviceDid is required".to_owned(),
8308 code: "INVALID_ARGUMENT".to_owned(),
8309 details: String::new(),
8310 }));
8311 }
8312 if data.cose_keyset_cbor.is_empty() {
8313 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8314 message: "coseKeysetCbor is required".to_owned(),
8315 code: "INVALID_ARGUMENT".to_owned(),
8316 details: String::new(),
8317 }));
8318 }
8319
8320 let cached = CachedEnvelopeKeyset {
8321 cose_keyset_cbor: data.cose_keyset_cbor.clone(),
8322 fetched_at: unix_seconds_now(),
8323 };
8324 self.state_store
8325 .put_envelope_keyset(data.service_did.as_str(), cached)
8326 .await?;
8327
8328 info!(
8329 service_did = %data.service_did,
8330 caller = %ctx.subject(),
8331 "Discovery: envelope keyset registered"
8332 );
8333 Ok(DiscoveryResponseVariant::RegisterEnvelopeKeysetResult)
8334 }
8335
8336 async fn handle_get_envelope_keyset(
8337 &self,
8338 _ctx: &EnvelopeContext,
8339 _request_id: u64,
8340 data: &str,
8341 ) -> Result<DiscoveryResponseVariant> {
8342 let service_did = data;
8343 match self.state_store.envelope_keyset(service_did).await? {
8344 Some(cached) => {
8345 trace!(service_did = %service_did, "Discovery: envelope keyset cache hit");
8346 Ok(DiscoveryResponseVariant::GetEnvelopeKeysetResult(
8347 EnvelopeKeyset {
8348 service_did: hyprstream_rpc::identity::Did::new(service_did.to_owned()),
8349 cose_keyset_cbor: cached.cose_keyset_cbor,
8350 fetched_at: cached.fetched_at,
8351 },
8352 ))
8353 }
8354 None => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8355 message: format!("no envelope keyset cached for service: {}", service_did),
8356 code: "NOT_FOUND".to_owned(),
8357 details: String::new(),
8358 })),
8359 }
8360 }
8361
8362 async fn handle_list_known_issuers(
8363 &self,
8364 ctx: &EnvelopeContext,
8365 _request_id: u64,
8366 ) -> Result<DiscoveryResponseVariant> {
8367 if let Err(e) = self
8372 .authorize(ctx, "discovery:federation", "list-known-issuers")
8373 .await
8374 {
8375 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8376 message: format!("unauthorized: {}", e),
8377 code: "UNAUTHORIZED".to_owned(),
8378 details: String::new(),
8379 }));
8380 }
8381 let issuers = self.state_store.known_issuers().await?;
8382 Ok(DiscoveryResponseVariant::ListKnownIssuersResult(
8383 IssuerList { issuers },
8384 ))
8385 }
8386
8387 async fn handle_get_record(
8392 &self,
8393 ctx: &EnvelopeContext,
8394 _request_id: u64,
8395 data: &GetRecordRequest,
8396 ) -> Result<DiscoveryResponseVariant> {
8397 let (did, collection, rkey) = if !data.uri.is_empty() {
8400 match parse_at_uri(&data.uri) {
8401 Ok(parts) => parts,
8402 Err(e) => {
8403 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8404 message: format!("invalid at-uri: {e}"),
8405 code: "INVALID_ARGUMENT".to_owned(),
8406 details: String::new(),
8407 }));
8408 }
8409 }
8410 } else if !data.did.is_empty() && !data.collection.is_empty() && !data.rkey.is_empty() {
8411 (data.did.clone(), data.collection.clone(), data.rkey.clone())
8412 } else {
8413 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8414 message: "getRecord requires either `uri` or all of (did, collection, rkey)"
8415 .to_owned(),
8416 code: "INVALID_ARGUMENT".to_owned(),
8417 details: String::new(),
8418 }));
8419 };
8420
8421 let resource = format!("discovery:record:{did}/{collection}");
8429 if let Err(e) = self.authorize(ctx, &resource, "query").await {
8430 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8431 message: format!("unauthorized to read {resource}: {e}"),
8432 code: "UNAUTHORIZED".to_owned(),
8433 details: String::new(),
8434 }));
8435 }
8436
8437 let resolver = match &self.record_resolver {
8438 Some(r) => r,
8439 None => {
8440 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8441 message: "no record resolver configured on this node".to_owned(),
8442 code: "NOT_FOUND".to_owned(),
8443 details: String::new(),
8444 }));
8445 }
8446 };
8447
8448 match resolver.resolve_record(&did, &collection, &rkey).await {
8449 Ok(Some(rec)) => Ok(DiscoveryResponseVariant::GetRecordResult(RecordCar {
8450 uri: rec.uri,
8451 car: rec.car,
8452 })),
8453 Ok(None) => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8454 message: format!("no record at at://{did}/{collection}/{rkey}"),
8455 code: "NOT_FOUND".to_owned(),
8456 details: String::new(),
8457 })),
8458 Err(e) => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8459 message: format!("record resolution failed: {e}"),
8460 code: "INTERNAL".to_owned(),
8461 details: String::new(),
8462 })),
8463 }
8464 }
8465
8466 async fn handle_get_repo(
8467 &self,
8468 ctx: &EnvelopeContext,
8469 _request_id: u64,
8470 value: &str,
8471 ) -> Result<DiscoveryResponseVariant> {
8472 let did = value;
8473 if did.is_empty() {
8474 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8475 message: "getRepo requires a non-empty DID".to_owned(),
8476 code: "INVALID_ARGUMENT".to_owned(),
8477 details: String::new(),
8478 }));
8479 }
8480
8481 let resource = format!("discovery:record:{did}");
8484 if let Err(e) = self.authorize(ctx, &resource, "query").await {
8485 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8486 message: format!("unauthorized to read repo {did}: {e}"),
8487 code: "UNAUTHORIZED".to_owned(),
8488 details: String::new(),
8489 }));
8490 }
8491
8492 let resolver = match &self.record_resolver {
8493 Some(r) => r,
8494 None => {
8495 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8496 message: "no record resolver configured on this node".to_owned(),
8497 code: "NOT_FOUND".to_owned(),
8498 details: String::new(),
8499 }));
8500 }
8501 };
8502
8503 match resolver.resolve_repo(did).await {
8504 Ok(Some(rec)) => Ok(DiscoveryResponseVariant::GetRepoResult(RecordCar {
8505 uri: rec.uri,
8506 car: rec.car,
8507 })),
8508 Ok(None) => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8509 message: format!("no repo stored for DID {did}"),
8510 code: "NOT_FOUND".to_owned(),
8511 details: String::new(),
8512 })),
8513 Err(e) => Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8514 message: format!("repo resolution failed: {e}"),
8515 code: "INTERNAL".to_owned(),
8516 details: String::new(),
8517 })),
8518 }
8519 }
8520
8521 async fn handle_query_candidates(
8537 &self,
8538 ctx: &EnvelopeContext,
8539 _request_id: u64,
8540 data: &QueryCandidatesRequest,
8541 ) -> Result<DiscoveryResponseVariant> {
8542 use futures::{stream, StreamExt, TryStreamExt};
8543 let query = async {
8544 let started = Instant::now();
8545 struct Candidate {
8546 did: String,
8547 record_uri: String,
8548 load_fraction: f32,
8549 allocatable: Vec<(String, String)>,
8550 last_seen: i64,
8551 }
8552
8553 let selectors: Vec<scheduling::LabelSelector> = data
8554 .selectors
8555 .iter()
8556 .map(|s| {
8557 scheduling::LabelSelector::new(
8558 s.key.clone(),
8559 to_scheduling_op(s.op),
8560 s.values.clone(),
8561 )
8562 })
8563 .collect();
8564 let resources: Vec<scheduling::ResourceRequest> = data
8565 .resources
8566 .iter()
8567 .map(|r| scheduling::ResourceRequest::new(r.name.clone(), r.min_quantity.clone()))
8568 .collect();
8569
8570 let live_nodes = self.state_store.all_liveness(unix_millis_now()).await?;
8574 let eligible: Vec<_> = live_nodes
8575 .into_iter()
8576 .filter(|(node, live)| {
8577 resources.iter().all(|req| {
8578 live.allocatable
8579 .iter()
8580 .find(|(name, _)| name == &req.name)
8581 .is_some_and(|(_, quantity)| req.satisfied_by(quantity))
8582 }) && (self.placement_index.record_uri(node.as_str()).is_none()
8583 || selectors.iter().all(|selector| {
8584 selector.matches(&self.placement_index.effective_labels(node.as_str()))
8585 }))
8586 })
8587 .map(|(node, _)| node)
8588 .collect();
8589 let authorized = stream::iter(eligible.chunks(256))
8590 .map(|nodes| async move {
8591 anyhow::ensure!(
8592 started.elapsed() < CANDIDATE_QUERY_TIMEOUT,
8593 "candidate query deadline exceeded; retry"
8594 );
8595 let resources: Vec<_> = nodes
8596 .iter()
8597 .map(|node| format!("placement:candidate:{node}"))
8598 .collect();
8599 let decisions = match &self.auth_provider {
8600 Some(auth) => auth.check_batch_context(ctx, &resources, "query").await?,
8601 None => vec![true; resources.len()],
8602 };
8603 anyhow::ensure!(
8604 decisions.len() == nodes.len(),
8605 "invalid authorization decision count"
8606 );
8607 Ok::<_, anyhow::Error>(
8608 nodes
8609 .iter()
8610 .zip(decisions)
8611 .filter(|(_, allowed)| *allowed)
8612 .map(|(node, _)| node.clone())
8613 .collect::<Vec<_>>(),
8614 )
8615 })
8616 .buffer_unordered(CANDIDATE_QUERY_CONCURRENCY)
8617 .try_collect::<Vec<_>>()
8618 .await?
8619 .into_iter()
8620 .flatten()
8621 .collect::<Vec<_>>();
8622
8623 stream::iter(&authorized)
8626 .map(|node| async move {
8627 anyhow::ensure!(
8628 started.elapsed() < CANDIDATE_QUERY_TIMEOUT,
8629 "candidate query deadline exceeded; retry"
8630 );
8631 anyhow::ensure!(
8632 self.ensure_placement_ingested(node).await,
8633 "candidate projection ingestion is pending; retry"
8634 );
8635 Ok::<_, anyhow::Error>(())
8636 })
8637 .buffer_unordered(CANDIDATE_QUERY_CONCURRENCY)
8638 .try_collect::<Vec<_>>()
8639 .await?;
8640
8641 let mut live: std::collections::HashMap<_, _> = self
8644 .state_store
8645 .all_liveness(unix_millis_now())
8646 .await?
8647 .into_iter()
8648 .collect();
8649 let mut candidates = Vec::new();
8650 for node in authorized {
8651 let did = node.as_str().to_owned();
8652 let Some(record_uri) = self.placement_index.record_uri(&did) else {
8653 continue;
8654 };
8655 let Some(value) = live.remove(&node) else {
8656 continue;
8657 };
8658 let labels = self.placement_index.effective_labels(&did);
8659 if !selectors.iter().all(|selector| selector.matches(&labels))
8660 || !resources.iter().all(|req| {
8661 value
8662 .allocatable
8663 .iter()
8664 .find(|(name, _)| name == &req.name)
8665 .is_some_and(|(_, quantity)| req.satisfied_by(quantity))
8666 })
8667 {
8668 continue;
8669 }
8670 candidates.push(Candidate {
8671 did,
8672 record_uri,
8673 load_fraction: value.load_fraction,
8674 allocatable: value.allocatable,
8675 last_seen: value.last_seen,
8676 });
8677 }
8678 let authorized: Vec<_> = candidates.iter().collect();
8679 anyhow::ensure!(
8680 started.elapsed() < CANDIDATE_QUERY_TIMEOUT,
8681 "candidate query deadline exceeded; retry"
8682 );
8683
8684 let total_matching = authorized.len() as u32;
8687
8688 let ranked = scheduling::rank(authorized, |a, b| {
8689 a.load_fraction
8690 .partial_cmp(&b.load_fraction)
8691 .unwrap_or(std::cmp::Ordering::Equal)
8692 .then_with(|| a.did.cmp(&b.did))
8693 });
8694
8695 let max = if data.max_candidates == 0 {
8696 DEFAULT_MAX_CANDIDATES
8697 } else {
8698 data.max_candidates as usize
8699 };
8700 let candidates_out: Vec<PlacementCandidate> = ranked
8701 .into_iter()
8702 .take(max)
8703 .map(|c| PlacementCandidate {
8704 node: c.did.clone(),
8705 record_uri: c.record_uri.clone(),
8706 load_fraction: c.load_fraction,
8707 allocatable: c
8708 .allocatable
8709 .iter()
8710 .map(|(name, quantity)| Resource {
8711 name: name.clone(),
8712 quantity: quantity.clone(),
8713 })
8714 .collect(),
8715 last_seen: c.last_seen,
8716 })
8717 .collect();
8718
8719 Ok(DiscoveryResponseVariant::QueryCandidatesResult(
8720 PlacementCandidateSet {
8721 candidates: candidates_out,
8722 total_matching,
8723 },
8724 ))
8725 };
8726 #[cfg(not(target_arch = "wasm32"))]
8727 {
8728 tokio::time::timeout(CANDIDATE_QUERY_TIMEOUT, query)
8729 .await
8730 .map_err(|_| anyhow::anyhow!("candidate query deadline exceeded; retry"))?
8731 }
8732 #[cfg(target_arch = "wasm32")]
8733 query.await
8734 }
8735
8736 async fn handle_report_node_liveness(
8750 &self,
8751 ctx: &EnvelopeContext,
8752 _request_id: u64,
8753 data: &NodeLiveness,
8754 ) -> Result<DiscoveryResponseVariant> {
8755 let node_did = data.node.as_str().to_owned();
8756 if node_did.is_empty() {
8757 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8758 message: "node is required".to_owned(),
8759 code: "INVALID_ARGUMENT".to_owned(),
8760 details: String::new(),
8761 }));
8762 }
8763
8764 let resource = format!("placement:candidate:{node_did}");
8769 if let Err(e) = self.authorize(ctx, &resource, "write").await {
8770 return Ok(DiscoveryResponseVariant::Error(ErrorInfo {
8771 message: format!("unauthorized to report liveness for {resource}: {e}"),
8772 code: "UNAUTHORIZED".to_owned(),
8773 details: String::new(),
8774 }));
8775 }
8776
8777 let received_at = unix_millis_now();
8778 let live = LiveAllocatable {
8779 allocatable: data
8780 .allocatable
8781 .iter()
8782 .map(|r| (r.name.clone(), r.quantity.clone()))
8783 .collect(),
8784 load_fraction: data.load_fraction,
8785 last_seen: received_at,
8788 live_until_unix_ms: received_at.saturating_add(LIVENESS_TTL.as_millis() as i64),
8789 };
8790 self.state_store.put_liveness(&data.node, live).await?;
8791
8792 self.ensure_placement_ingested(&data.node).await;
8793
8794 Ok(DiscoveryResponseVariant::ReportNodeLivenessResult)
8795 }
8796}
8797
8798#[async_trait(?Send)]
8803impl RequestService for DiscoveryService {
8804 fn decode_request_body(
8805 &self,
8806 signed_body: &[u8],
8807 ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
8808 crate::generated::discovery_client::decode_discovery_request_body(signed_body)
8812 }
8813
8814 async fn handle_request(
8815 &self,
8816 ctx: &EnvelopeContext,
8817 body: &hyprstream_rpc::service::DecodedRequestBody,
8818 ) -> Result<(Vec<u8>, Option<hyprstream_rpc::Continuation>)> {
8819 trace!(
8820 "Discovery request from {} (id={})",
8821 ctx.subject(),
8822 ctx.request_id
8823 );
8824 dispatch_discovery(self, ctx, body).await
8825 }
8826
8827 fn name(&self) -> &str {
8828 "discovery"
8829 }
8830
8831 fn transport(&self) -> &TransportConfig {
8832 &self.transport
8833 }
8834
8835 fn signing_key(&self) -> hyprstream_rpc::crypto::SigningKey {
8836 (*self.signing_key).clone()
8837 }
8838
8839 fn expected_audience(&self) -> Option<&str> {
8840 self.expected_audience.as_deref()
8841 }
8842
8843 fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
8844 self.jwt_key_source.clone()
8845 }
8846
8847 fn build_error_payload(&self, request_id: u64, error: &str) -> Vec<u8> {
8848 let variant = DiscoveryResponseVariant::Error(ErrorInfo {
8849 message: error.to_owned(),
8850 code: "INTERNAL".to_owned(),
8851 details: String::new(),
8852 });
8853 serialize_response(request_id, &variant).unwrap_or_default()
8854 }
8855}
8856
8857#[cfg(test)]
8862mod get_record_tests {
8863 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
8864 use super::*;
8865
8866 use std::collections::BTreeMap;
8867
8868 use hyprstream_pds::car::{build_record_proof_car, parse_car_v1, verify_record_proof};
8869 use hyprstream_pds::cid::Cid as PdsCid;
8870 use hyprstream_pds::commit::{Commit, UnsignedCommit};
8871 use hyprstream_pds::mst::{Node, Proof};
8872 use hyprstream_pds::record::{ModelRecord, COLLECTION_NSID};
8873 use hyprstream_pds::tid::Tid;
8874 use p256::ecdsa::{SigningKey as P256SigningKey, VerifyingKey as P256VerifyingKey};
8875
8876 const TEST_DID: &str = "did:web:alice.example.com";
8877
8878 struct MockAuth {
8882 allow: Vec<(String, String)>,
8883 }
8884 #[async_trait(?Send)]
8885 impl AuthorizationProvider for MockAuth {
8886 async fn check(
8887 &self,
8888 _subject: &str,
8889 _domain: &str,
8890 resource: &str,
8891 operation: &str,
8892 _bearer: Option<&str>,
8893 ) -> Result<bool> {
8894 Ok(self
8895 .allow
8896 .iter()
8897 .any(|(r, o)| r == resource && o == operation))
8898 }
8899 }
8900
8901 struct MockResolver {
8904 records: BTreeMap<(String, String), (Vec<u8>, String)>,
8906 verifying_key: p256::ecdsa::VerifyingKey,
8909 }
8910
8911 fn build_repo(
8915 n: u64,
8916 ) -> (
8917 MockResolver,
8918 P256VerifyingKey,
8919 String, ModelRecord, PdsCid, Proof, ) {
8924 let signing_key = P256SigningKey::random(&mut rand::rngs::OsRng);
8925 let verifying_key = P256VerifyingKey::from(&signing_key);
8926
8927 let mut records: BTreeMap<Tid, ModelRecord> = BTreeMap::new();
8928 let mut record_cids: BTreeMap<Tid, PdsCid> = BTreeMap::new();
8929 for i in 0..n {
8930 let tid = Tid::from_micros(1_700_000_000_000_000 + i * 1000, i as u16);
8931 let rec = ModelRecord::new(
8932 format!("at://{TEST_DID}"),
8933 format!("bafyreiexampleoid{i:020}"),
8934 "2026-06-24T00:00:00.000Z",
8935 )
8936 .unwrap();
8937 record_cids.insert(tid, rec.cid());
8938 records.insert(tid, rec);
8939 }
8940 let tree = Node::from_records(COLLECTION_NSID, &record_cids);
8941 let root = tree.root_cid();
8942 let unsigned = UnsignedCommit::new(TEST_DID.to_owned(), root, Tid::now(), None);
8943 let commit = Commit::sign(&unsigned, &signing_key);
8944 let (_root_data, node_blocks) = tree.to_node_data_with_blocks();
8945
8946 let target_tid = *records.keys().nth((n / 2) as usize).unwrap();
8948 let target_record = records.get(&target_tid).cloned().unwrap();
8949 let target_cid = record_cids[&target_tid];
8950 let target_rkey = target_tid.encode();
8951
8952 let mut out = BTreeMap::new();
8954 for (tid, rec) in &records {
8955 let proof = tree.proof(COLLECTION_NSID, tid).unwrap();
8956 let car = build_record_proof_car(&commit, &proof, &node_blocks, rec);
8957 let rkey = tid.encode();
8958 let uri = format!("at://{TEST_DID}/{COLLECTION_NSID}/{rkey}");
8959 out.insert((COLLECTION_NSID.to_owned(), rkey), (car, uri));
8960 }
8961 let target_proof = tree.proof(COLLECTION_NSID, &target_tid).unwrap();
8962
8963 (
8964 MockResolver {
8965 records: out,
8966 verifying_key,
8967 },
8968 verifying_key,
8969 target_rkey,
8970 target_record,
8971 target_cid,
8972 target_proof,
8973 )
8974 }
8975
8976 #[async_trait(?Send)]
8977 impl RecordResolver for MockResolver {
8978 async fn resolve_record(
8979 &self,
8980 _did: &str,
8981 collection: &str,
8982 rkey: &str,
8983 ) -> Result<Option<RecordCarData>> {
8984 Ok(self
8985 .records
8986 .get(&(collection.to_owned(), rkey.to_owned()))
8987 .map(|(car, uri)| RecordCarData {
8988 uri: uri.clone(),
8989 car: car.clone(),
8990 }))
8991 }
8992 async fn resolve_repo(&self, _did: &str) -> Result<Option<RecordCarData>> {
8993 Ok(self
8994 .records
8995 .values()
8996 .next()
8997 .map(|(car, uri)| RecordCarData {
8998 uri: uri.clone(),
8999 car: car.clone(),
9000 }))
9001 }
9002 async fn resolve_verifying_key(
9003 &self,
9004 _did: &str,
9005 ) -> Result<Option<p256::ecdsa::VerifyingKey>> {
9006 Ok(Some(self.verifying_key))
9007 }
9008 }
9009
9010 fn service_with(allow: Vec<(&str, &str)>, resolver: MockResolver) -> DiscoveryService {
9011 let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
9012 let allow = allow
9013 .into_iter()
9014 .map(|(r, o)| (r.to_owned(), o.to_owned()))
9015 .collect();
9016 DiscoveryService::new(
9017 Arc::new(sk),
9018 vk,
9019 hyprstream_rpc::transport::TransportConfig::inproc("discovery-test"),
9020 )
9021 .with_auth_provider(Box::new(MockAuth { allow }))
9022 .with_record_resolver(Arc::new(resolver))
9023 }
9024
9025 fn test_ctx() -> EnvelopeContext {
9026 EnvelopeContext::from_callback_service(1, "test-caller")
9028 }
9029
9030 #[tokio::test]
9034 async fn classical_atproto_peer_key_remains_on_separate_interop_surface() {
9035 let (resolver, expected, _rkey, _record, _cid, _proof) = build_repo(1);
9036 let published = resolver
9037 .resolve_verifying_keys(TEST_DID)
9038 .await
9039 .expect("classical peer key resolution")
9040 .expect("published classical atproto key");
9041 let live = published
9042 .live_keys(chrono::Utc::now().timestamp())
9043 .collect::<Vec<_>>();
9044 assert_eq!(live, vec![&expected]);
9045 }
9046
9047 #[tokio::test]
9049 async fn get_record_returns_verifiable_car_when_authorized() {
9050 let (resolver, vk, rkey, target_record, target_cid, proof) = build_repo(6);
9051 let resource = format!("discovery:record:{TEST_DID}/{COLLECTION_NSID}");
9052 let svc = service_with(vec![(&resource, "query")], resolver);
9053
9054 let req = GetRecordRequest {
9055 uri: format!("at://{TEST_DID}/{COLLECTION_NSID}/{rkey}"),
9056 did: String::new(),
9057 collection: String::new(),
9058 rkey: String::new(),
9059 };
9060 let resp = svc.handle_get_record(&test_ctx(), 1, &req).await.unwrap();
9061 let car = match resp {
9062 DiscoveryResponseVariant::GetRecordResult(rc) => rc,
9063 other => panic!("expected GetRecordResult, got {other:?}"),
9064 };
9065
9066 let (roots, blocks) = parse_car_v1(&car.car).unwrap();
9068 assert!(!roots.is_empty(), "CAR must have a root commit CID");
9069 let block_cids: std::collections::BTreeSet<PdsCid> =
9070 blocks.iter().map(|(c, _)| *c).collect();
9071 assert!(
9072 block_cids.contains(&target_cid),
9073 "CAR must carry the target record block"
9074 );
9075
9076 let commit = blocks
9078 .iter()
9079 .find(|(c, _)| *c == roots[0])
9080 .map(|(_, b)| Commit::from_dag_cbor(b).unwrap())
9081 .expect("CAR must contain its commit block");
9082 verify_record_proof(&commit, &vk, &proof, &target_record)
9083 .expect("the returned CAR proof must verify offline (D5 untrusted-relay)");
9084 }
9085
9086 #[tokio::test]
9089 async fn get_record_denied_without_read_permission() {
9090 let (resolver, _vk, rkey, _rec, _cid, _proof) = build_repo(6);
9091 let svc = service_with(vec![("discovery:something-else", "query")], resolver);
9093
9094 let req = GetRecordRequest {
9095 uri: format!("at://{TEST_DID}/{COLLECTION_NSID}/{rkey}"),
9096 did: String::new(),
9097 collection: String::new(),
9098 rkey: String::new(),
9099 };
9100 let resp = svc.handle_get_record(&test_ctx(), 1, &req).await.unwrap();
9101 match resp {
9102 DiscoveryResponseVariant::Error(e) => {
9103 assert_eq!(e.code, "UNAUTHORIZED", "deny must surface as UNAUTHORIZED");
9104 assert!(
9106 !e.message.is_empty(),
9107 "unauthorized error should explain the denied resource"
9108 );
9109 }
9110 DiscoveryResponseVariant::GetRecordResult(_) => {
9111 panic!("ACCESS-CONTROL FAILURE: a valid at:// returned a record without read permission");
9112 }
9113 other => panic!("expected UNAUTHORIZED error, got {other:?}"),
9114 }
9115 }
9116
9117 #[tokio::test]
9119 async fn get_record_missing_returns_not_found() {
9120 let (resolver, _vk, _rkey, _rec, _cid, _proof) = build_repo(6);
9121 let resource = format!("discovery:record:{TEST_DID}/{COLLECTION_NSID}");
9122 let svc = service_with(vec![(&resource, "query")], resolver);
9123
9124 let req = GetRecordRequest {
9125 uri: format!("at://{TEST_DID}/{COLLECTION_NSID}/3zzzzzzzzzzzz"),
9126 did: String::new(),
9127 collection: String::new(),
9128 rkey: String::new(),
9129 };
9130 let resp = svc.handle_get_record(&test_ctx(), 1, &req).await.unwrap();
9131 match resp {
9132 DiscoveryResponseVariant::Error(e) => {
9133 assert_eq!(e.code, "NOT_FOUND", "absent record must be NOT_FOUND");
9134 }
9135 other => panic!("expected NOT_FOUND error, got {other:?}"),
9136 }
9137 }
9138
9139 #[test]
9141 fn parse_at_uri_round_trip() {
9142 let (did, coll, rkey) =
9143 parse_at_uri("at://did:web:alice.example.com/ai.hyprstream.model/3zztslq4be52u")
9144 .unwrap();
9145 assert_eq!(did, "did:web:alice.example.com");
9146 assert_eq!(coll, "ai.hyprstream.model");
9147 assert_eq!(rkey, "3zztslq4be52u");
9148
9149 assert!(parse_at_uri("https://x").is_err());
9150 assert!(parse_at_uri("at://did:web:x").is_err()); assert!(parse_at_uri("at://did:web:x/coll").is_err()); }
9153
9154 #[tokio::test]
9156 async fn get_record_by_structured_fields() {
9157 let (resolver, _vk, rkey, _rec, target_cid, _proof) = build_repo(4);
9158 let resource = format!("discovery:record:{TEST_DID}/{COLLECTION_NSID}");
9159 let svc = service_with(vec![(&resource, "query")], resolver);
9160
9161 let req = GetRecordRequest {
9162 uri: String::new(),
9163 did: TEST_DID.to_owned(),
9164 collection: COLLECTION_NSID.to_owned(),
9165 rkey: rkey.clone(),
9166 };
9167 let resp = svc.handle_get_record(&test_ctx(), 1, &req).await.unwrap();
9168 let car = match resp {
9169 DiscoveryResponseVariant::GetRecordResult(rc) => rc,
9170 other => panic!("expected GetRecordResult, got {other:?}"),
9171 };
9172 let (_roots, blocks) = parse_car_v1(&car.car).unwrap();
9173 let block_cids: std::collections::BTreeSet<PdsCid> =
9174 blocks.iter().map(|(c, _)| *c).collect();
9175 assert!(block_cids.contains(&target_cid));
9176 }
9177}
9178
9179#[cfg(test)]
9184mod query_candidates_tests {
9185 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
9186 use super::*;
9187
9188 use std::collections::BTreeMap;
9189
9190 use hyprstream_pds::car::build_car_v1;
9191 use hyprstream_pds::cid::Cid;
9192 use hyprstream_pds::commit::{Commit, UnsignedCommit};
9193 use hyprstream_pds::mst::Node;
9194 use hyprstream_pds::placement::node::{self, NodeRecord};
9195 use hyprstream_pds::tid::Tid;
9196 use hyprstream_rpc_std::discovery_client::{LabelSelector, ResourceRequest, SelectorOp};
9197 use p256::ecdsa::{SigningKey as P256SigningKey, VerifyingKey as P256VerifyingKey};
9198
9199 struct AllowAll;
9201 #[async_trait(?Send)]
9202 impl AuthorizationProvider for AllowAll {
9203 async fn check(
9204 &self,
9205 _subject: &str,
9206 _domain: &str,
9207 _resource: &str,
9208 _operation: &str,
9209 _bearer: Option<&str>,
9210 ) -> Result<bool> {
9211 Ok(true)
9212 }
9213 }
9214
9215 struct DenyNode(String);
9218 #[async_trait(?Send)]
9219 impl AuthorizationProvider for DenyNode {
9220 async fn check(
9221 &self,
9222 _subject: &str,
9223 _domain: &str,
9224 resource: &str,
9225 operation: &str,
9226 _bearer: Option<&str>,
9227 ) -> Result<bool> {
9228 Ok(resource != format!("placement:candidate:{}", self.0) || operation != "query")
9229 }
9230 }
9231
9232 struct DenyNodeLiveness(String);
9234 #[async_trait(?Send)]
9235 impl AuthorizationProvider for DenyNodeLiveness {
9236 async fn check(
9237 &self,
9238 _subject: &str,
9239 _domain: &str,
9240 resource: &str,
9241 operation: &str,
9242 _bearer: Option<&str>,
9243 ) -> Result<bool> {
9244 Ok(resource != format!("placement:candidate:{}", self.0) || operation != "write")
9245 }
9246 }
9247
9248 struct TrackingRepoResolver {
9251 repos: HashMap<String, Vec<u8>>,
9252 resolved: parking_lot::Mutex<Vec<String>>,
9253 }
9254 #[async_trait(?Send)]
9255 impl RecordResolver for TrackingRepoResolver {
9256 async fn resolve_record(
9257 &self,
9258 _did: &str,
9259 _collection: &str,
9260 _rkey: &str,
9261 ) -> Result<Option<RecordCarData>> {
9262 Ok(None)
9263 }
9264 async fn resolve_repo(&self, did: &str) -> Result<Option<RecordCarData>> {
9265 self.resolved.lock().push(did.to_owned());
9266 Ok(self.repos.get(did).map(|car| RecordCarData {
9267 uri: format!("at://{did}"),
9268 car: car.clone(),
9269 }))
9270 }
9271 }
9272
9273 struct FixedRepoResolver {
9276 repos: HashMap<String, (Vec<u8>, P256VerifyingKey)>,
9277 }
9278 #[async_trait(?Send)]
9279 impl RecordResolver for FixedRepoResolver {
9280 async fn resolve_record(
9281 &self,
9282 _did: &str,
9283 _collection: &str,
9284 _rkey: &str,
9285 ) -> Result<Option<RecordCarData>> {
9286 Ok(None)
9287 }
9288 async fn resolve_repo(&self, did: &str) -> Result<Option<RecordCarData>> {
9289 Ok(self.repos.get(did).map(|(car, _vk)| RecordCarData {
9290 uri: format!("at://{did}"),
9291 car: car.clone(),
9292 }))
9293 }
9294 async fn resolve_verifying_key(&self, did: &str) -> Result<Option<P256VerifyingKey>> {
9295 Ok(self.repos.get(did).map(|(_, vk)| *vk))
9296 }
9297 }
9298
9299 fn node_repo_car(did: &str, rec: &NodeRecord) -> (Vec<u8>, P256VerifyingKey) {
9302 let signing_key = P256SigningKey::random(&mut rand::rngs::OsRng);
9303 let verifying_key = P256VerifyingKey::from(&signing_key);
9304 let key = format!("{}/3a", node::COLLECTION_NSID);
9305 let record_cid = Cid::from_dag_cbor(&rec.to_dag_cbor());
9306 let mut keyed: BTreeMap<String, Cid> = BTreeMap::new();
9307 keyed.insert(key, record_cid);
9308 let tree = Node::from_keyed_records(&keyed);
9309 let root = tree.root_cid();
9310 let (_root_data, node_blocks) = tree.to_node_data_with_blocks();
9311 let unsigned = UnsignedCommit::new(did.to_owned(), root, Tid::now(), None);
9312 let commit = Commit::sign(&unsigned, &signing_key);
9313 let mut blocks: Vec<(Cid, Vec<u8>)> = vec![(commit.cid(), commit.to_dag_cbor())];
9314 for (cid, data) in node_blocks {
9315 blocks.push((cid, data.encode()));
9316 }
9317 blocks.push((record_cid, rec.to_dag_cbor()));
9318 (build_car_v1(&[commit.cid()], &blocks), verifying_key)
9319 }
9320
9321 fn sample_node_record(did: &str, labels: Vec<(&str, &str)>) -> NodeRecord {
9322 NodeRecord::new(
9323 format!("at://{did}"),
9324 labels
9325 .into_iter()
9326 .map(|(k, v)| node::Label {
9327 key: k.to_owned(),
9328 value: v.to_owned(),
9329 })
9330 .collect(),
9331 vec![],
9332 vec![],
9333 "2026-06-23T12:34:56.789Z",
9334 )
9335 .unwrap()
9336 }
9337
9338 fn service_with(
9339 auth: Box<dyn AuthorizationProvider>,
9340 repos: HashMap<String, (Vec<u8>, P256VerifyingKey)>,
9341 ) -> DiscoveryService {
9342 let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
9343 DiscoveryService::new(
9344 Arc::new(sk),
9345 vk,
9346 hyprstream_rpc::transport::TransportConfig::inproc("query-candidates-test"),
9347 )
9348 .with_auth_provider(auth)
9349 .with_record_resolver(Arc::new(FixedRepoResolver { repos }))
9350 }
9351
9352 struct ConcurrentRepoResolver {
9353 inner: FixedRepoResolver,
9354 barrier: Option<tokio::sync::Barrier>,
9355 stall: std::sync::atomic::AtomicBool,
9356 active: std::sync::atomic::AtomicUsize,
9357 peak: std::sync::atomic::AtomicUsize,
9358 resolved: parking_lot::Mutex<Vec<String>>,
9359 }
9360
9361 struct ActiveResolution<'a>(&'a std::sync::atomic::AtomicUsize);
9362 impl Drop for ActiveResolution<'_> {
9363 fn drop(&mut self) {
9364 self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
9365 }
9366 }
9367
9368 #[async_trait(?Send)]
9369 impl RecordResolver for ConcurrentRepoResolver {
9370 async fn resolve_record(
9371 &self,
9372 _did: &str,
9373 _collection: &str,
9374 _rkey: &str,
9375 ) -> Result<Option<RecordCarData>> {
9376 Ok(None)
9377 }
9378 async fn resolve_repo(&self, did: &str) -> Result<Option<RecordCarData>> {
9379 use std::sync::atomic::Ordering::SeqCst;
9380 self.resolved.lock().push(did.to_owned());
9381 let active = self.active.fetch_add(1, SeqCst) + 1;
9382 let _guard = ActiveResolution(&self.active);
9383 self.peak.fetch_max(active, SeqCst);
9384 if self.stall.load(SeqCst) {
9385 futures::future::pending::<()>().await;
9386 }
9387 if let Some(barrier) = &self.barrier {
9388 barrier.wait().await;
9389 }
9390 self.inner.resolve_repo(did).await
9391 }
9392 async fn resolve_verifying_key(&self, did: &str) -> Result<Option<P256VerifyingKey>> {
9393 self.inner.resolve_verifying_key(did).await
9394 }
9395 }
9396
9397 async fn assert_cold_query_concurrency(state: DiscoveryState) {
9398 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst};
9399 let mut repos = HashMap::new();
9400 let now = unix_millis_now();
9401 for i in 0..33 {
9402 let did = format!("did:web:cold-{i}.example");
9403 repos.insert(
9404 did.clone(),
9405 node_repo_car(&did, &sample_node_record(&did, vec![])),
9406 );
9407 state
9408 .clone()
9409 .into_inner()
9410 .put_liveness(
9411 &Did::new(did),
9412 LiveAllocatable {
9413 allocatable: vec![],
9414 load_fraction: 0.1,
9415 last_seen: now,
9416 live_until_unix_ms: now + 45_000,
9417 },
9418 )
9419 .await
9420 .unwrap();
9421 }
9422 let resolver = Arc::new(ConcurrentRepoResolver {
9423 inner: FixedRepoResolver { repos },
9424 barrier: Some(tokio::sync::Barrier::new(CANDIDATE_QUERY_CONCURRENCY)),
9426 stall: AtomicBool::new(false),
9427 active: AtomicUsize::new(0),
9428 peak: AtomicUsize::new(0),
9429 resolved: parking_lot::Mutex::new(vec![]),
9430 });
9431 let denied = "did:web:cold-32.example";
9432 let svc = service_with(Box::new(DenyNode(denied.to_owned())), HashMap::new())
9433 .with_record_resolver(resolver.clone())
9434 .with_state(state);
9435 let result = tokio::time::timeout(
9436 Duration::from_secs(4),
9437 svc.handle_query_candidates(&test_ctx(), 1, &empty_query(1)),
9438 )
9439 .await
9440 .unwrap()
9441 .unwrap();
9442 let result = as_set(result);
9443 assert_eq!(
9444 result.total_matching, 32,
9445 "maxCandidates must not hide incomplete hydration"
9446 );
9447 assert_eq!(result.candidates.len(), 1);
9448 assert_eq!(resolver.peak.load(SeqCst), CANDIDATE_QUERY_CONCURRENCY);
9449 assert_eq!(resolver.active.load(SeqCst), 0);
9450 assert_eq!(resolver.resolved.lock().len(), 32);
9451 assert!(!resolver.resolved.lock().iter().any(|did| did == denied));
9452 }
9453
9454 #[tokio::test]
9455 async fn cold_candidate_query_hydrates_concurrently_without_truncation() {
9456 let state = DiscoveryState::connect(&crate::DiscoveryStateConfig::default())
9457 .await
9458 .unwrap();
9459 assert_cold_query_concurrency(state).await;
9460 }
9461
9462 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
9463 #[tokio::test]
9464 async fn valkey_capacity_query_batches_exact_count_and_ranking() {
9465 use futures::{stream, StreamExt};
9466 use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
9467 struct BatchPolicy(Arc<AtomicUsize>);
9468 #[async_trait(?Send)]
9469 impl AuthorizationProvider for BatchPolicy {
9470 async fn check(
9471 &self,
9472 _: &str,
9473 _: &str,
9474 _: &str,
9475 _: &str,
9476 _: Option<&str>,
9477 ) -> Result<bool> {
9478 panic!("capacity query regressed to per-node Policy RPC");
9479 }
9480 async fn check_batch(
9481 &self,
9482 _: &str,
9483 _: &str,
9484 resources: &[String],
9485 _: &str,
9486 _: Option<&str>,
9487 ) -> Result<Vec<bool>> {
9488 assert!(resources.len() <= 256);
9489 self.0.fetch_add(1, SeqCst);
9490 tokio::time::sleep(Duration::from_millis(10)).await;
9491 Ok(resources
9492 .iter()
9493 .map(|r| !r.ends_with("capacity-00001.example"))
9494 .collect())
9495 }
9496 }
9497 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
9498 return;
9499 };
9500 let config = crate::DiscoveryStateConfig {
9501 backend: crate::DiscoveryStateBackend::Tiered,
9502 active_active: true,
9503 valkey: crate::ValkeyStateConfig {
9504 url,
9505 key_prefix: format!(
9506 "capacity-query-{}-{}",
9507 std::process::id(),
9508 unix_millis_now()
9509 ),
9510 ..crate::ValkeyStateConfig::default()
9511 },
9512 ..crate::DiscoveryStateConfig::default()
9513 };
9514 let capacity = config.valkey.liveness_capacity;
9515 assert_eq!(
9516 capacity, 65_536,
9517 "exercise advertised capacity without lowering it"
9518 );
9519 let state = DiscoveryState::connect(&config).await.unwrap();
9520 let calls = Arc::new(AtomicUsize::new(0));
9521 let svc =
9522 service_with(Box::new(BatchPolicy(calls.clone())), HashMap::new()).with_state(state);
9523 for i in 0..capacity {
9526 let did = format!("did:web:capacity-{i:05}.example");
9527 svc.placement_index.seed_warm_node_for_test(
9528 did.clone(),
9529 crate::placement_index::NodeFacts {
9530 record_uri: format!("at://{did}/ai.hyprstream.placement.node/3a"),
9531 labels: vec![(
9532 "zone".to_owned(),
9533 if i % 2 == 1 { "west" } else { "east" }.to_owned(),
9534 )],
9535 ..Default::default()
9536 },
9537 );
9538 }
9539 stream::iter(0..capacity)
9540 .map(|i| {
9541 let store = &svc.state_store;
9542 async move {
9543 let now = unix_millis_now();
9544 store
9545 .put_liveness(
9546 &Did::new(format!("did:web:capacity-{i:05}.example")),
9547 LiveAllocatable {
9548 allocatable: vec![("cpu".to_owned(), "8".to_owned())],
9549 load_fraction: 1.0 - i as f32 / capacity as f32,
9550 last_seen: now,
9551 live_until_unix_ms: now + 45_000,
9552 },
9553 )
9554 .await
9555 .unwrap();
9556 }
9557 })
9558 .buffer_unordered(256)
9559 .collect::<Vec<_>>()
9560 .await;
9561 let mut request = empty_query(1);
9562 request.selectors = vec![LabelSelector {
9563 key: "zone".to_owned(),
9564 op: SelectorOp::In,
9565 values: vec!["west".to_owned()],
9566 }];
9567 request.resources = vec![ResourceRequest {
9568 name: "cpu".to_owned(),
9569 min_quantity: "4".to_owned(),
9570 }];
9571 let started = Instant::now();
9572 let result = as_set(
9573 svc.handle_query_candidates(&test_ctx(), 1, &request)
9574 .await
9575 .unwrap(),
9576 );
9577 assert!(started.elapsed() < CANDIDATE_QUERY_TIMEOUT);
9578 assert_eq!(result.total_matching, (capacity / 2 - 1) as u32);
9579 assert_eq!(result.candidates.len(), 1);
9580 assert_eq!(
9581 result.candidates[0].node,
9582 format!("did:web:capacity-{:05}.example", capacity - 1)
9583 );
9584 assert_eq!(
9585 calls.load(SeqCst),
9586 capacity / 2 / 256,
9587 "warm selector filtering precedes Policy RPC"
9588 );
9589 }
9590
9591 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
9592 #[tokio::test]
9593 async fn valkey_cold_candidate_query_hydrates_concurrently_without_truncation() {
9594 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
9595 return;
9596 };
9597 let config = crate::DiscoveryStateConfig {
9598 backend: crate::DiscoveryStateBackend::Tiered,
9599 active_active: true,
9600 valkey: crate::ValkeyStateConfig {
9601 url,
9602 key_prefix: format!("cold-query-{}-{}", std::process::id(), unix_millis_now()),
9603 ..crate::ValkeyStateConfig::default()
9604 },
9605 ..crate::DiscoveryStateConfig::default()
9606 };
9607 assert_cold_query_concurrency(DiscoveryState::connect(&config).await.unwrap()).await;
9608 }
9609
9610 #[tokio::test]
9611 async fn cold_candidate_deadline_cancels_ingest_and_retry_is_complete() {
9612 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst};
9613 let did = "did:web:stalled.example";
9614 let resolver = Arc::new(ConcurrentRepoResolver {
9615 inner: FixedRepoResolver {
9616 repos: HashMap::from([(
9617 did.to_owned(),
9618 node_repo_car(did, &sample_node_record(did, vec![])),
9619 )]),
9620 },
9621 barrier: None,
9622 stall: AtomicBool::new(true),
9623 active: AtomicUsize::new(0),
9624 peak: AtomicUsize::new(0),
9625 resolved: parking_lot::Mutex::new(vec![]),
9626 });
9627 let svc =
9628 service_with(Box::new(AllowAll), HashMap::new()).with_record_resolver(resolver.clone());
9629 let now = unix_millis_now();
9630 svc.state_store
9631 .put_liveness(
9632 &Did::new(did.to_owned()),
9633 LiveAllocatable {
9634 allocatable: vec![],
9635 load_fraction: 0.1,
9636 last_seen: now,
9637 live_until_unix_ms: now + 45_000,
9638 },
9639 )
9640 .await
9641 .unwrap();
9642 let error = tokio::time::timeout(
9643 CANDIDATE_QUERY_TIMEOUT + Duration::from_secs(2),
9644 svc.handle_query_candidates(&test_ctx(), 1, &empty_query(1)),
9645 )
9646 .await
9647 .unwrap()
9648 .err()
9649 .unwrap();
9650 assert!(error.to_string().contains("deadline exceeded"));
9651 assert_eq!(
9652 resolver.active.load(SeqCst),
9653 0,
9654 "deadline must drop outstanding repository work"
9655 );
9656 resolver.stall.store(false, SeqCst);
9657 let result = as_set(
9658 svc.handle_query_candidates(&test_ctx(), 1, &empty_query(1))
9659 .await
9660 .unwrap(),
9661 );
9662 assert_eq!(
9663 result.total_matching, 1,
9664 "cancelled ingest must not be cached as absence"
9665 );
9666 assert_eq!(resolver.resolved.lock().len(), 2);
9667 }
9668
9669 #[tokio::test]
9670 async fn heartbeat_uses_receipt_time_after_client_clock_rollback() {
9671 let svc = service_with(Box::new(AllowAll), HashMap::new());
9672 let node = Did::new("did:web:rollback.example".to_owned());
9673 let start = unix_millis_now();
9674 for ts in [start + 86_400_000, start - 86_400_000] {
9675 let req = NodeLiveness {
9676 node: node.clone(),
9677 allocatable: vec![],
9678 load_fraction: 0.2,
9679 ts,
9680 };
9681 svc.handle_report_node_liveness(&test_ctx(), 1, &req)
9682 .await
9683 .unwrap();
9684 let value = svc
9685 .state_store
9686 .liveness(&node, unix_millis_now())
9687 .await
9688 .unwrap()
9689 .unwrap();
9690 assert!(value.last_seen >= start && value.last_seen <= unix_millis_now());
9691 assert_eq!(
9692 value.live_until_unix_ms,
9693 value.last_seen + LIVENESS_TTL.as_millis() as i64
9694 );
9695 }
9696 }
9697
9698 fn test_ctx() -> EnvelopeContext {
9699 EnvelopeContext::from_callback_service(1, "test-caller")
9700 }
9701
9702 async fn heartbeat(
9703 svc: &DiscoveryService,
9704 did: &str,
9705 allocatable: Vec<(&str, &str)>,
9706 load_fraction: f32,
9707 ) {
9708 let req = NodeLiveness {
9709 node: Did::new(did.to_owned()),
9710 allocatable: allocatable
9711 .into_iter()
9712 .map(|(n, q)| Resource {
9713 name: n.to_owned(),
9714 quantity: q.to_owned(),
9715 })
9716 .collect(),
9717 load_fraction,
9718 ts: 0,
9719 };
9720 let resp = svc
9721 .handle_report_node_liveness(&test_ctx(), 1, &req)
9722 .await
9723 .unwrap();
9724 assert!(matches!(
9725 resp,
9726 DiscoveryResponseVariant::ReportNodeLivenessResult
9727 ));
9728 }
9729
9730 async fn assert_cross_replica_candidates(a: DiscoveryState, b: DiscoveryState) {
9731 let did = "did:web:shared-node.example";
9732 let rec = sample_node_record(did, vec![("zone", "shared")]);
9733 let repos = HashMap::from([(did.to_owned(), node_repo_car(did, &rec))]);
9734 let replica_a = service_with(Box::new(AllowAll), repos.clone()).with_state(a);
9735 let replica_b = service_with(Box::new(AllowAll), repos.clone()).with_state(b.clone());
9736 heartbeat(&replica_a, did, vec![("cpu", "8")], 0.25).await;
9737 assert!(replica_b.placement_index.known_node_dids().is_empty());
9738 let req = QueryCandidatesRequest {
9739 selectors: vec![LabelSelector {
9740 key: "zone".to_owned(),
9741 op: SelectorOp::In,
9742 values: vec!["shared".to_owned()],
9743 }],
9744 resources: vec![ResourceRequest {
9745 name: "cpu".to_owned(),
9746 min_quantity: "4".to_owned(),
9747 }],
9748 max_candidates: 0,
9749 };
9750 let result = as_set(
9751 replica_b
9752 .handle_query_candidates(&test_ctx(), 1, &req)
9753 .await
9754 .unwrap(),
9755 );
9756 assert_eq!(result.total_matching, 1);
9757 assert_eq!(result.candidates[0].node, did);
9758 assert_eq!(
9759 result.candidates[0].record_uri,
9760 format!("at://{did}/{}/3a", node::COLLECTION_NSID)
9761 );
9762 let restarted = service_with(Box::new(AllowAll), repos.clone()).with_state(b.clone());
9765 assert_eq!(
9766 as_set(
9767 restarted
9768 .handle_query_candidates(&test_ctx(), 1, &req)
9769 .await
9770 .unwrap()
9771 )
9772 .total_matching,
9773 1
9774 );
9775 let denied = service_with(Box::new(DenyNode(did.to_owned())), repos).with_state(b.clone());
9776 assert_eq!(
9777 as_set(
9778 denied
9779 .handle_query_candidates(&test_ctx(), 1, &empty_query(0))
9780 .await
9781 .unwrap()
9782 )
9783 .total_matching,
9784 0
9785 );
9786 assert!(denied.placement_index.known_node_dids().is_empty());
9787 let missing_repo = service_with(Box::new(AllowAll), HashMap::new()).with_state(b);
9788 assert_eq!(
9789 as_set(
9790 missing_repo
9791 .handle_query_candidates(&test_ctx(), 1, &empty_query(0))
9792 .await
9793 .unwrap()
9794 )
9795 .total_matching,
9796 0
9797 );
9798 let now = unix_millis_now();
9800 replica_a
9801 .state_store
9802 .put_liveness(
9803 &Did::new(did.to_owned()),
9804 LiveAllocatable {
9805 allocatable: vec![],
9806 load_fraction: 0.1,
9807 last_seen: now,
9808 live_until_unix_ms: now + 20,
9809 },
9810 )
9811 .await
9812 .unwrap();
9813 tokio::time::sleep(Duration::from_millis(50)).await;
9815 assert_eq!(
9816 as_set(
9817 replica_b
9818 .handle_query_candidates(&test_ctx(), 1, &req)
9819 .await
9820 .unwrap()
9821 )
9822 .total_matching,
9823 0
9824 );
9825 }
9826
9827 #[tokio::test]
9828 async fn shared_liveness_seeds_replica_without_local_placement() {
9829 let state = DiscoveryState::connect(&crate::DiscoveryStateConfig::default())
9830 .await
9831 .unwrap();
9832 assert_cross_replica_candidates(state.clone(), state).await;
9833 }
9834
9835 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
9836 #[tokio::test]
9837 async fn valkey_shared_liveness_seeds_other_replica_and_restart() {
9838 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
9839 return;
9840 };
9841 for backend in [
9842 crate::DiscoveryStateBackend::Valkey,
9843 crate::DiscoveryStateBackend::Tiered,
9844 ] {
9845 let config = crate::DiscoveryStateConfig {
9846 backend,
9847 active_active: true,
9848 valkey: crate::ValkeyStateConfig {
9849 url: url.clone(),
9850 key_prefix: format!(
9851 "pr1560-candidates-{}-{}-{backend:?}",
9852 std::process::id(),
9853 unix_millis_now()
9854 ),
9855 pool_size: 2,
9856 ..crate::ValkeyStateConfig::default()
9857 },
9858 ..crate::DiscoveryStateConfig::default()
9859 };
9860 let a = DiscoveryState::connect(&config).await.unwrap();
9861 let b = DiscoveryState::connect(&config).await.unwrap();
9862 assert_cross_replica_candidates(a, b).await;
9863 }
9864 }
9865
9866 #[tokio::test]
9867 async fn legacy_announcement_without_identity_expiry_uses_heartbeat_ttl() {
9868 let svc = service_with(Box::new(AllowAll), HashMap::new());
9869 let request = ServiceAnnouncement {
9870 service_name: "legacy".to_owned(),
9871 socket_kind: "rep".to_owned(),
9872 endpoint: "inproc://legacy".to_owned(),
9873 service_jwt: None,
9874 service_did: Did::new(String::new()),
9875 capabilities: vec![],
9876 accepted_state_digest: vec![],
9877 accepted_state_epoch: 0,
9878 response_key_id: String::new(),
9879 request_kem_key_id: String::new(),
9880 request_kem_recipient: vec![],
9881 expires_at_unix_ms: 0,
9882 };
9883 let now = unix_millis_now();
9884 assert!(matches!(
9885 svc.handle_announce(&test_ctx(), 1, &request).await.unwrap(),
9886 DiscoveryResponseVariant::AnnounceResult
9887 ));
9888 let entries = svc
9889 .state_store
9890 .announcements_for("legacy", now)
9891 .await
9892 .unwrap();
9893 assert_eq!(entries.len(), 1);
9894 assert!(entries[0].live_until_unix_ms >= now + ANNOUNCED_ENDPOINT_TTL.as_millis() as i64);
9895 assert_eq!(entries[0].expires_at_unix_ms, entries[0].live_until_unix_ms);
9896 assert!(svc
9897 .state_store
9898 .announcements_for("legacy", entries[0].live_until_unix_ms)
9899 .await
9900 .unwrap()
9901 .is_empty());
9902 let mut bound = request;
9903 bound.service_did = Did::new("did:at9p:identity-bound".to_owned());
9904 assert!(svc.handle_announce(&test_ctx(), 1, &bound).await.is_err());
9905 }
9906
9907 #[tokio::test]
9911 async fn liveness_admission_denial_skips_cache_and_repository_ingest() {
9912 let did = "did:web:unadmitted-node.example.com";
9913 let rec = sample_node_record(did, vec![]);
9914 let (car, _verifying_key) = node_repo_car(did, &rec);
9915 let resolver = Arc::new(TrackingRepoResolver {
9916 repos: HashMap::from([(did.to_owned(), car)]),
9917 resolved: parking_lot::Mutex::new(Vec::new()),
9918 });
9919 let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
9920 let svc = DiscoveryService::new(
9921 Arc::new(sk),
9922 vk,
9923 hyprstream_rpc::transport::TransportConfig::inproc("liveness-admission-test"),
9924 )
9925 .with_auth_provider(Box::new(DenyNodeLiveness(did.to_owned())))
9926 .with_record_resolver(resolver.clone());
9927 let req = NodeLiveness {
9928 node: Did::new(did.to_owned()),
9929 allocatable: vec![],
9930 load_fraction: 0.1,
9931 ts: 0,
9932 };
9933
9934 let response = svc
9935 .handle_report_node_liveness(&test_ctx(), 1, &req)
9936 .await
9937 .unwrap();
9938 assert!(matches!(
9939 response,
9940 DiscoveryResponseVariant::Error(ErrorInfo { ref code, .. }) if code == "UNAUTHORIZED"
9941 ));
9942 assert!(
9943 svc.state_store
9944 .liveness(&Did::new(did.to_owned()), unix_millis_now())
9945 .await
9946 .expect("read liveness")
9947 .is_none(),
9948 "denied DID must not receive a liveness entry"
9949 );
9950 assert!(
9951 svc.placement_index.record_uri(did).is_none(),
9952 "denied DID must not receive placement facts"
9953 );
9954 assert!(
9955 resolver.resolved.lock().is_empty(),
9956 "denied DID must not reach RecordResolver::resolve_repo"
9957 );
9958 }
9959
9960 #[tokio::test]
9963 async fn absent_node_repo_does_not_poll_again_before_retry_window() {
9964 let did = "did:web:node-without-repo.example.com";
9965 let resolver = Arc::new(TrackingRepoResolver {
9966 repos: HashMap::new(),
9967 resolved: parking_lot::Mutex::new(Vec::new()),
9968 });
9969 let (sk, vk) = hyprstream_rpc::crypto::generate_signing_keypair();
9970 let svc = DiscoveryService::new(
9971 Arc::new(sk),
9972 vk,
9973 hyprstream_rpc::transport::TransportConfig::inproc("placement-retry-test"),
9974 )
9975 .with_auth_provider(Box::new(AllowAll))
9976 .with_record_resolver(resolver.clone());
9977
9978 heartbeat(&svc, did, vec![], 0.9).await;
9979 heartbeat(&svc, did, vec![("cpu", "8")], 0.1).await;
9980
9981 assert_eq!(
9982 resolver.resolved.lock().as_slice(),
9983 [did],
9984 "two heartbeats before retry expiry must produce one repo poll"
9985 );
9986 let live = svc
9987 .state_store
9988 .liveness(&Did::new(did.to_owned()), unix_millis_now())
9989 .await
9990 .expect("read liveness")
9991 .expect("admitted heartbeat must still refresh liveness");
9992 assert!((live.load_fraction - 0.1).abs() < f32::EPSILON);
9993 assert_eq!(live.allocatable, vec![("cpu".to_owned(), "8".to_owned())]);
9994 }
9995
9996 fn empty_query(max_candidates: u32) -> QueryCandidatesRequest {
9997 QueryCandidatesRequest {
9998 selectors: vec![],
9999 resources: vec![],
10000 max_candidates,
10001 }
10002 }
10003
10004 fn as_set(resp: DiscoveryResponseVariant) -> PlacementCandidateSet {
10005 match resp {
10006 DiscoveryResponseVariant::QueryCandidatesResult(s) => s,
10007 other => panic!("expected QueryCandidatesResult, got {other:?}"),
10008 }
10009 }
10010
10011 #[tokio::test]
10014 async fn liveness_hard_excludes_nodes_never_heartbeated() {
10015 let did = "did:web:node1.example.com";
10016 let rec = sample_node_record(did, vec![("zone", "us-east")]);
10017 let svc = service_with(
10018 Box::new(AllowAll),
10019 HashMap::from([(did.to_owned(), node_repo_car(did, &rec))]),
10020 );
10021
10022 let set = as_set(
10023 svc.handle_query_candidates(&test_ctx(), 1, &empty_query(0))
10024 .await
10025 .unwrap(),
10026 );
10027 assert!(set.candidates.is_empty());
10028 assert_eq!(set.total_matching, 0);
10029 }
10030
10031 #[tokio::test]
10032 async fn label_selector_matches_and_excludes_non_matching_nodes() {
10033 let did_a = "did:web:node-a.example.com";
10034 let did_b = "did:web:node-b.example.com";
10035 let repos = HashMap::from([
10036 (
10037 did_a.to_owned(),
10038 node_repo_car(did_a, &sample_node_record(did_a, vec![("zone", "us-east")])),
10039 ),
10040 (
10041 did_b.to_owned(),
10042 node_repo_car(did_b, &sample_node_record(did_b, vec![("zone", "us-west")])),
10043 ),
10044 ]);
10045 let svc = service_with(Box::new(AllowAll), repos);
10046 heartbeat(&svc, did_a, vec![], 0.1).await;
10047 heartbeat(&svc, did_b, vec![], 0.1).await;
10048
10049 let req = QueryCandidatesRequest {
10050 selectors: vec![LabelSelector {
10051 key: "zone".to_owned(),
10052 op: SelectorOp::In,
10053 values: vec!["us-east".to_owned()],
10054 }],
10055 resources: vec![],
10056 max_candidates: 0,
10057 };
10058 let set = as_set(
10059 svc.handle_query_candidates(&test_ctx(), 1, &req)
10060 .await
10061 .unwrap(),
10062 );
10063 assert_eq!(set.candidates.len(), 1);
10064 assert_eq!(set.candidates[0].node, did_a);
10065 assert_eq!(set.total_matching, 1);
10066 }
10067
10068 #[tokio::test]
10069 async fn resource_request_filters_insufficient_live_capacity() {
10070 let did = "did:web:node1.example.com";
10071 let rec = sample_node_record(did, vec![]);
10072 let svc = service_with(
10073 Box::new(AllowAll),
10074 HashMap::from([(did.to_owned(), node_repo_car(did, &rec))]),
10075 );
10076 heartbeat(&svc, did, vec![("nvidia.com/gpu", "2")], 0.1).await;
10077
10078 let req = QueryCandidatesRequest {
10079 selectors: vec![],
10080 resources: vec![ResourceRequest {
10081 name: "nvidia.com/gpu".to_owned(),
10082 min_quantity: "4".to_owned(),
10083 }],
10084 max_candidates: 0,
10085 };
10086 let set = as_set(
10087 svc.handle_query_candidates(&test_ctx(), 1, &req)
10088 .await
10089 .unwrap(),
10090 );
10091 assert!(
10092 set.candidates.is_empty(),
10093 "2 GPUs must not satisfy a request for >=4"
10094 );
10095
10096 let req_ok = QueryCandidatesRequest {
10098 selectors: vec![],
10099 resources: vec![ResourceRequest {
10100 name: "nvidia.com/gpu".to_owned(),
10101 min_quantity: "1".to_owned(),
10102 }],
10103 max_candidates: 0,
10104 };
10105 let set_ok = as_set(
10106 svc.handle_query_candidates(&test_ctx(), 1, &req_ok)
10107 .await
10108 .unwrap(),
10109 );
10110 assert_eq!(set_ok.candidates.len(), 1);
10111 }
10112
10113 #[tokio::test]
10116 async fn per_candidate_authz_denial_is_silent_not_an_error() {
10117 let did_a = "did:web:node-a.example.com";
10118 let did_b = "did:web:node-b.example.com";
10119 let repos = HashMap::from([
10120 (
10121 did_a.to_owned(),
10122 node_repo_car(did_a, &sample_node_record(did_a, vec![])),
10123 ),
10124 (
10125 did_b.to_owned(),
10126 node_repo_car(did_b, &sample_node_record(did_b, vec![])),
10127 ),
10128 ]);
10129 let svc = service_with(Box::new(DenyNode(did_a.to_owned())), repos);
10130 heartbeat(&svc, did_a, vec![], 0.1).await;
10131 heartbeat(&svc, did_b, vec![], 0.1).await;
10132
10133 let resp = svc
10134 .handle_query_candidates(&test_ctx(), 1, &empty_query(0))
10135 .await
10136 .unwrap();
10137 let set = as_set(resp);
10138 assert_eq!(set.candidates.len(), 1);
10139 assert_eq!(set.candidates[0].node, did_b);
10140 assert_eq!(
10141 set.total_matching, 1,
10142 "denied candidate must not inflate totalMatching"
10143 );
10144 }
10145
10146 #[tokio::test]
10147 async fn bounding_respects_max_candidates_and_total_matching_is_pre_bound() {
10148 let dids: Vec<String> = (0..5)
10149 .map(|i| format!("did:web:node{i}.example.com"))
10150 .collect();
10151 let mut repos = HashMap::new();
10152 for d in &dids {
10153 repos.insert(d.clone(), node_repo_car(d, &sample_node_record(d, vec![])));
10154 }
10155 let svc = service_with(Box::new(AllowAll), repos);
10156 for (i, d) in dids.iter().enumerate() {
10157 heartbeat(&svc, d, vec![], i as f32 * 0.1).await;
10158 }
10159
10160 let set = as_set(
10161 svc.handle_query_candidates(&test_ctx(), 1, &empty_query(2))
10162 .await
10163 .unwrap(),
10164 );
10165 assert_eq!(set.candidates.len(), 2, "bounded to maxCandidates");
10166 assert_eq!(
10167 set.total_matching, 5,
10168 "totalMatching counts all authorized survivors, not just the bound"
10169 );
10170 assert_eq!(set.candidates[0].node, dids[0]);
10172 assert_eq!(set.candidates[1].node, dids[1]);
10173 }
10174
10175 #[tokio::test]
10176 async fn zero_max_candidates_uses_default_bound() {
10177 let dids: Vec<String> = (0..3)
10178 .map(|i| format!("did:web:node{i}.example.com"))
10179 .collect();
10180 let mut repos = HashMap::new();
10181 for d in &dids {
10182 repos.insert(d.clone(), node_repo_car(d, &sample_node_record(d, vec![])));
10183 }
10184 let svc = service_with(Box::new(AllowAll), repos);
10185 for d in &dids {
10186 heartbeat(&svc, d, vec![], 0.0).await;
10187 }
10188 let set = as_set(
10189 svc.handle_query_candidates(&test_ctx(), 1, &empty_query(0))
10190 .await
10191 .unwrap(),
10192 );
10193 assert_eq!(
10194 set.candidates.len(),
10195 3,
10196 "under the default bound, all 3 come back"
10197 );
10198 }
10199
10200 #[tokio::test]
10201 async fn heartbeat_refreshes_ttl_and_liveness_fields() {
10202 let did = "did:web:node1.example.com";
10203 let rec = sample_node_record(did, vec![]);
10204 let svc = service_with(
10205 Box::new(AllowAll),
10206 HashMap::from([(did.to_owned(), node_repo_car(did, &rec))]),
10207 );
10208 heartbeat(&svc, did, vec![("cpu", "1")], 0.9).await;
10209 heartbeat(&svc, did, vec![("cpu", "8")], 0.1).await;
10210
10211 let set = as_set(
10212 svc.handle_query_candidates(&test_ctx(), 1, &empty_query(0))
10213 .await
10214 .unwrap(),
10215 );
10216 assert_eq!(set.candidates.len(), 1);
10217 assert!(
10218 (set.candidates[0].load_fraction - 0.1).abs() < f32::EPSILON,
10219 "second heartbeat must win"
10220 );
10221 assert_eq!(set.candidates[0].allocatable[0].quantity, "8");
10222 }
10223}
10224
10225#[cfg(test)]
10240mod eager_resolver_regression {
10241 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
10242 use super::*;
10243
10244 #[test]
10253 fn install_local_discovery_client_succeeds_with_no_registered_endpoint() {
10254 use hyprstream_rpc::registry::{EndpointMode, SocketKind};
10255
10256 let runtime_dir = std::env::temp_dir().join(format!(
10260 "hs-eager-resolver-regression-{}",
10261 std::process::id(),
10262 ));
10263 std::fs::create_dir_all(&runtime_dir).unwrap();
10264 hyprstream_rpc::registry::init(EndpointMode::Ipc, Some(runtime_dir.clone()));
10265
10266 let registry =
10267 hyprstream_rpc::registry::try_global().expect("EndpointRegistry must be initialized");
10268
10269 let registered = registry.registered_endpoint("discovery", SocketKind::Rep);
10271 assert!(
10272 registered.is_none(),
10273 "test precondition: discovery must not be registered at first boot, \
10274 got {:?}",
10275 registered,
10276 );
10277
10278 let signing_key = SigningKey::generate(&mut rand::rngs::OsRng);
10280 let discovery_vk = signing_key.verifying_key();
10281 let _client = install_local_discovery_client(signing_key, discovery_vk, None).expect(
10282 "install_local_discovery_client must succeed at first boot with \
10283 no registered discovery endpoint",
10284 );
10285
10286 let transport = resolve_local_discovery_transport()
10288 .expect("resolve_local_discovery_transport must resolve the lazy default");
10289 assert!(
10290 matches!(
10291 transport.endpoint,
10292 hyprstream_rpc::transport::EndpointType::Ipc { .. }
10293 ),
10294 "expected IPC transport (default discovery UDS), got {:?}",
10295 transport.endpoint,
10296 );
10297
10298 let _ = std::fs::remove_dir_all(&runtime_dir);
10300 }
10301
10302 #[test]
10303 fn production_bootstrap_discovery_dials_with_caller_credential() {
10304 let source = include_str!("service.rs");
10305 let production = source
10306 .split("// ============================================================================\n// Regression: PostgreSQL accepted-state authority selection")
10307 .next()
10308 .expect("production source");
10309 assert!(production.contains("None, signing_key, service_token, resolver,"));
10310 assert!(production
10311 .contains("install_local_discovery_client(signing_key, discovery_vk, service_token)"));
10312 assert!(production.contains("Some(discovery_vk), token_provider,"));
10313 assert!(production.contains("current_service_token(&self.signing_key, &self.token)"));
10314 }
10315
10316 #[test]
10317 fn process_client_observes_same_signer_jwt_renewal() {
10318 let signing_key = SigningKey::from_bytes(&[0x6d; 32]);
10319 let initial = Some("initial-service-jwt".to_owned());
10320 let provider = renewing_service_token_provider(&signing_key, initial.clone())
10321 .expect("credentialed client has provider");
10322 assert_eq!(provider(), initial);
10323
10324 hyprstream_service::global_trust_store().insert(
10325 signing_key.verifying_key(),
10326 hyprstream_service::Attestation {
10327 scopes: std::iter::once("model".to_owned()).collect(),
10328 subject: None,
10329 jwt: Some("renewed-service-jwt".to_owned()),
10330 expires_at: 0,
10331 attested_by: None,
10332 },
10333 );
10334 assert_eq!(provider().as_deref(), Some("renewed-service-jwt"));
10335 assert_eq!(
10336 current_service_token(&signing_key, &initial).as_deref(),
10337 Some("renewed-service-jwt")
10338 );
10339 assert!(renewing_service_token_provider(&signing_key, None).is_none());
10340 }
10341
10342 #[test]
10354 fn os_owned_arm_uses_lazy_resolver_not_for_local_bootstrap() {
10355 let source = include_str!("service.rs");
10356 let production = source
10358 .split("// ============================================================================\n// Regression: eager-resolver")
10359 .next()
10360 .expect("production source before regression test module");
10361
10362 let arm = production
10364 .find("DeploymentTrustSource::OsOwnedFiles =>")
10365 .expect("OsOwnedFiles arm must exist in bootstrap_deployment_process");
10366 let arm_block = &production[arm..arm + 800];
10367
10368 assert!(
10369 arm_block.contains("install_local_discovery_client"),
10370 "OsOwnedFiles arm must call install_local_discovery_client (lazy resolver), \
10371 got:\n{}",
10372 arm_block,
10373 );
10374 assert!(
10375 !arm_block.contains("for_local_bootstrap"),
10376 "OsOwnedFiles arm must NOT call for_local_bootstrap (eager resolver), \
10377 got:\n{}",
10378 arm_block,
10379 );
10380 }
10381}
10382
10383#[cfg(test)]
10396mod pg_authority_wiring {
10397 #![allow(clippy::unwrap_used, clippy::expect_used)]
10398
10399 use super::SigningKey;
10400
10401 #[test]
10402 fn both_bootstrap_callsites_select_the_records_binding_backend() {
10403 let source = include_str!("service.rs");
10404 let production = source
10405 .split("// ============================================================================\n// Regression: PostgreSQL accepted-state authority selection")
10406 .next()
10407 .expect("production source before this regression module");
10408
10409 assert!(
10411 production.contains(
10412 "pub async fn bootstrap_deployment_process(\n signing_key: SigningKey,\n service_token: Option<String>,\n trust_source: crate::DeploymentTrustSource,\n remote_node: bool,\n network_required: bool,\n records: &hyprstream_pds::rds::RdsConfig,\n) -> Result<()>"
10413 ),
10414 "bootstrap_deployment_process must take the resolved records binding"
10415 );
10416
10417 assert_eq!(
10420 production.matches("open_deployment_accepted_state_source(").count(),
10421 2,
10422 "exactly the network-required bootstrap arm and the install tail must construct the accepted-state source"
10423 );
10424 assert!(
10425 !production.contains("CheckpointedPdsAcceptedStateSource::open("),
10426 "no callsite may bypass the records-binding selection helper"
10427 );
10428
10429 assert!(
10431 production.contains("accepted_state_source: source,"),
10432 "the production resolver must carry the selected accepted-state source"
10433 );
10434 assert!(
10436 production.contains(
10437 "pub fn production_browser_currentness_verifier() -> Result<Arc<dyn BrowserCurrentnessVerifier>> {\n let resolver = PRODUCTION_RESOLVER"
10438 ),
10439 "browser currentness must read through the installed production resolver"
10440 );
10441 }
10442
10443 #[test]
10444 fn ephemeral_diagnostic_carrier_is_distinct_and_normal_identity_stays_stable() {
10445 let service_key = SigningKey::from_bytes(&[0x51; 32]);
10446 let stable = super::bootstrap_client_transport_key(&service_key, None);
10447 let expected_stable = hyprstream_rpc::node_identity::derive_purpose_key(
10448 &service_key,
10449 "hyprstream-bootstrap-client-transport-v1",
10450 );
10451 assert_eq!(stable.to_bytes(), expected_stable.to_bytes());
10452
10453 let ephemeral = SigningKey::generate(&mut rand::rngs::OsRng);
10454 let selected = super::bootstrap_client_transport_key(&service_key, Some(ephemeral.clone()));
10455 assert_eq!(selected.to_bytes(), ephemeral.to_bytes());
10456 assert_ne!(selected.verifying_key(), service_key.verifying_key());
10457 assert_ne!(selected.to_bytes(), stable.to_bytes());
10458
10459 let second_ephemeral = SigningKey::generate(&mut rand::rngs::OsRng);
10460 assert_ne!(selected.to_bytes(), second_ephemeral.to_bytes());
10461 }
10462
10463 #[test]
10464 fn ephemeral_bootstrap_preserves_authenticated_resolver_and_rpc_signer() {
10465 let source = include_str!("service.rs");
10466 let production = source
10467 .split("// ============================================================================\n// Regression: PostgreSQL accepted-state authority selection")
10468 .next()
10469 .expect("production source before this regression module");
10470 assert!(production.contains(
10471 "let ephemeral_transport_key = SigningKey::generate(&mut rand::rngs::OsRng);"
10472 ));
10473 assert!(production.contains(
10474 "bootstrap_deployment_process_with_carrier_identity(\n signing_key,\n service_token,"
10475 ));
10476 assert!(production.contains("authenticate_deployment_bootstrap()?"));
10477 assert!(production.contains(
10478 "ProductionRpcClient::new(\n \"discovery\",\n \"discovery\",\n None,\n signing_key,\n service_token,\n resolver,"
10479 ));
10480 assert!(production.contains(
10481 "let transport_key =\n bootstrap_client_transport_key(&signing_key, client_transport_key_override);"
10482 ));
10483 assert!(production.contains("open_deployment_accepted_state_source("));
10484 assert!(production.contains("source.bootstrap_endpoints(\"policy\")?;"));
10485 }
10486}