1use anyhow::{bail, Result};
9use async_trait::async_trait;
10use hyprstream_rpc::identity::Did;
11use parking_lot::Mutex;
12use serde::{Deserialize, Serialize};
13use std::cmp::Reverse;
14use std::collections::{BTreeSet, BinaryHeap, HashMap};
15use std::sync::Arc;
16
17pub(crate) const ANNOUNCED_ENDPOINT_TTL: std::time::Duration = std::time::Duration::from_secs(90);
18
19#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
20#[serde(rename_all = "kebab-case")]
21pub enum DiscoveryStateBackend {
22 #[default]
23 Memory,
24 Valkey,
25 Tiered,
26}
27
28#[derive(Clone, Debug, Serialize, Deserialize)]
29#[serde(default)]
30pub struct MemoryStateConfig {
31 pub announcement_capacity: usize,
32 pub liveness_capacity: usize,
33 pub artifact_capacity: usize,
34}
35
36impl Default for MemoryStateConfig {
37 fn default() -> Self {
38 Self {
39 announcement_capacity: 16_384,
40 liveness_capacity: 16_384,
41 artifact_capacity: 4_096,
42 }
43 }
44}
45
46#[derive(Clone, Debug, Serialize, Deserialize)]
47#[serde(default)]
48pub struct ValkeyStateConfig {
49 pub url: String,
50 pub key_prefix: String,
51 pub pool_size: usize,
52 pub announcement_capacity: usize,
53 pub liveness_capacity: usize,
54 pub artifact_capacity: usize,
55 pub command_timeout_ms: u64,
58}
59
60impl Default for ValkeyStateConfig {
61 fn default() -> Self {
62 Self {
63 url: String::new(),
64 key_prefix: "hs".to_owned(),
65 pool_size: 8,
66 announcement_capacity: 65_536,
67 liveness_capacity: 65_536,
68 artifact_capacity: 16_384,
69 command_timeout_ms: 2_000,
70 }
71 }
72}
73
74#[derive(Clone, Debug, Serialize, Deserialize)]
75#[serde(default)]
76pub struct TieredStateConfig {
77 pub l1_max_ttl_ms: u64,
80}
81
82impl Default for TieredStateConfig {
83 fn default() -> Self {
84 Self {
85 l1_max_ttl_ms: 1_000,
86 }
87 }
88}
89
90#[derive(Clone, Debug, Default, Serialize, Deserialize)]
91#[serde(default)]
92pub struct DiscoveryStateConfig {
93 pub backend: DiscoveryStateBackend,
94 pub active_active: bool,
97 pub memory: MemoryStateConfig,
98 pub valkey: ValkeyStateConfig,
99 pub tiered: TieredStateConfig,
100}
101
102#[derive(Clone)]
104pub struct DiscoveryState(Arc<dyn DiscoveryStateStore>);
105
106impl DiscoveryState {
107 pub async fn connect(config: &DiscoveryStateConfig) -> Result<Self> {
108 anyhow::ensure!(
109 config.memory.announcement_capacity > 0
110 && config.memory.liveness_capacity > 0
111 && config.memory.artifact_capacity > 0,
112 "Discovery memory capacities must be positive"
113 );
114 if config.active_active && config.backend == DiscoveryStateBackend::Memory {
115 bail!("discovery.state.active_active requires valkey or tiered backend");
116 }
117 if config.backend == DiscoveryStateBackend::Tiered {
118 anyhow::ensure!(
119 config.tiered.l1_max_ttl_ms > 0,
120 "Discovery tiered l1_max_ttl_ms must be positive"
121 );
122 }
123 let memory = || {
124 Arc::new(MemoryStateStore::new(
125 config.memory.announcement_capacity,
126 config.memory.liveness_capacity,
127 config.memory.artifact_capacity,
128 ))
129 };
130
131 match config.backend {
132 DiscoveryStateBackend::Memory => Ok(Self(memory())),
133 DiscoveryStateBackend::Valkey | DiscoveryStateBackend::Tiered => {
134 #[cfg(target_arch = "wasm32")]
135 {
136 bail!("Valkey-backed Discovery state is unavailable on wasm32")
137 }
138 #[cfg(all(not(target_arch = "wasm32"), not(feature = "valkey")))]
139 {
140 bail!("Valkey-backed Discovery state requires the valkey feature")
141 }
142 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
143 {
144 let valkey = Arc::new(ValkeyStateStore::connect(&config.valkey).await?);
145 if config.backend == DiscoveryStateBackend::Valkey {
146 Ok(Self(valkey))
147 } else {
148 Ok(Self(Arc::new(TieredStateStore::new(
149 memory(),
150 valkey,
151 config.tiered.l1_max_ttl_ms,
152 ))))
153 }
154 }
155 }
156 }
157 }
158
159 pub(crate) fn into_inner(self) -> Arc<dyn DiscoveryStateStore> {
160 self.0
161 }
162}
163
164#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
171pub(crate) struct AnnouncedEndpoint {
172 pub(crate) socket_kind: String,
173 pub(crate) endpoint: String,
174 pub(crate) service_did: Did,
175 pub(crate) capabilities: BTreeSet<String>,
176 pub(crate) accepted_state_digest: Vec<u8>,
177 pub(crate) accepted_state_epoch: u64,
178 pub(crate) response_key_id: String,
179 pub(crate) request_kem_key_id: String,
180 pub(crate) request_kem_recipient: Vec<u8>,
181 pub(crate) expires_at_unix_ms: i64,
184 pub(crate) source_signer: [u8; 32],
185 pub(crate) live_until_unix_ms: i64,
188}
189
190impl AnnouncedEndpoint {
191 fn receipt_lease(mut self, now: i64) -> Option<Self> {
192 let heartbeat_expiry = now.saturating_add(ANNOUNCED_ENDPOINT_TTL.as_millis() as i64);
193 if self.expires_at_unix_ms == 0 {
194 self.expires_at_unix_ms = heartbeat_expiry;
195 }
196 self.live_until_unix_ms = heartbeat_expiry
197 .min(self.expires_at_unix_ms)
198 .min(self.live_until_unix_ms);
199 self.is_live_at(now).then_some(self)
200 }
201
202 pub(crate) fn is_live_at(&self, now_unix_ms: i64) -> bool {
203 now_unix_ms < self.live_until_unix_ms && now_unix_ms < self.expires_at_unix_ms
204 }
205
206 fn order(&self) -> (u64, i64) {
207 (self.accepted_state_epoch, self.expires_at_unix_ms)
208 }
209}
210
211#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
212pub(crate) struct LiveAllocatable {
213 pub(crate) allocatable: Vec<(String, String)>,
214 pub(crate) load_fraction: f32,
215 pub(crate) last_seen: i64,
217 pub(crate) live_until_unix_ms: i64,
218}
219
220impl LiveAllocatable {
221 fn is_live_at(&self, now_unix_ms: i64) -> bool {
222 now_unix_ms < self.live_until_unix_ms
223 }
224}
225
226#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
227pub(crate) struct CachedEntityStatement {
228 pub(crate) jwt: String,
229 pub(crate) fetched_at: i64,
230}
231
232#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
233pub(crate) struct CachedEnvelopeKeyset {
234 pub(crate) cose_keyset_cbor: Vec<u8>,
235 pub(crate) fetched_at: i64,
236}
237
238#[derive(Clone, Copy, Debug, PartialEq, Eq)]
239pub(crate) enum PutResult {
240 Stored,
241 IgnoredOlder,
242}
243
244#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
250#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
251pub(crate) trait DiscoveryStateStore: Send + Sync {
252 async fn put_announcement(
253 &self,
254 service_name: &str,
255 endpoint: AnnouncedEndpoint,
256 ) -> Result<PutResult>;
257 async fn announcements_for(
258 &self,
259 service_name: &str,
260 now_unix_ms: i64,
261 ) -> Result<Vec<AnnouncedEndpoint>>;
262 async fn all_announcements(
263 &self,
264 now_unix_ms: i64,
265 ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>>;
266
267 async fn put_liveness(&self, node: &Did, value: LiveAllocatable) -> Result<PutResult>;
268 #[cfg(test)]
269 async fn liveness(&self, node: &Did, now_unix_ms: i64) -> Result<Option<LiveAllocatable>>;
270 async fn all_liveness(&self, now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>>;
272
273 async fn put_entity_statement(&self, issuer: &str, value: CachedEntityStatement) -> Result<()>;
274 async fn entity_statement(&self, issuer: &str) -> Result<Option<CachedEntityStatement>>;
275 async fn known_issuers(&self) -> Result<Vec<String>>;
276 async fn known_issuer_count(&self) -> Result<usize>;
277
278 async fn put_envelope_keyset(
279 &self,
280 service_did: &str,
281 value: CachedEnvelopeKeyset,
282 ) -> Result<()>;
283 async fn envelope_keyset(&self, service_did: &str) -> Result<Option<CachedEnvelopeKeyset>>;
284}
285
286#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
287struct AnnouncementKey {
288 service_name: String,
289 socket_kind: String,
290}
291
292#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
293enum ExpiringKey {
294 Announcement(AnnouncementKey),
295 Liveness(String),
296}
297
298#[derive(Clone, Debug, PartialEq, Eq)]
299struct ExpiryEntry {
300 expires_at_unix_ms: i64,
301 version: u64,
302 key: ExpiringKey,
303}
304
305impl Ord for ExpiryEntry {
306 fn cmp(&self, other: &Self) -> std::cmp::Ordering {
307 self.expires_at_unix_ms
308 .cmp(&other.expires_at_unix_ms)
309 .then_with(|| self.version.cmp(&other.version))
310 .then_with(|| self.key.cmp(&other.key))
311 }
312}
313
314impl PartialOrd for ExpiryEntry {
315 fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
316 Some(self.cmp(other))
317 }
318}
319
320#[derive(Clone)]
321struct Versioned<T> {
322 value: T,
323 version: u64,
324}
325
326struct MemoryInner {
327 announcements: HashMap<AnnouncementKey, Versioned<AnnouncedEndpoint>>,
328 service_index: HashMap<String, BTreeSet<String>>,
329 liveness: HashMap<String, Versioned<LiveAllocatable>>,
330 expiry: BinaryHeap<Reverse<ExpiryEntry>>,
331 entity_statements: HashMap<String, CachedEntityStatement>,
332 envelope_keysets: HashMap<String, CachedEnvelopeKeyset>,
333 next_version: u64,
334}
335
336pub(crate) struct MemoryStateStore {
338 inner: Mutex<MemoryInner>,
339 announcement_capacity: usize,
340 liveness_capacity: usize,
341 artifact_capacity: usize,
342}
343
344impl MemoryStateStore {
345 pub(crate) fn new(
346 announcement_capacity: usize,
347 liveness_capacity: usize,
348 artifact_capacity: usize,
349 ) -> Self {
350 Self {
351 inner: Mutex::new(MemoryInner {
352 announcements: HashMap::new(),
353 service_index: HashMap::new(),
354 liveness: HashMap::new(),
355 expiry: BinaryHeap::new(),
356 entity_statements: HashMap::new(),
357 envelope_keysets: HashMap::new(),
358 next_version: 1,
359 }),
360 announcement_capacity,
361 liveness_capacity,
362 artifact_capacity,
363 }
364 }
365
366 pub(crate) fn production_default() -> Arc<dyn DiscoveryStateStore> {
367 Arc::new(Self::new(16_384, 16_384, 4_096))
368 }
369
370 fn next_version(inner: &mut MemoryInner) -> u64 {
371 let version = inner.next_version;
372 inner.next_version = inner.next_version.saturating_add(1);
373 version
374 }
375
376 fn compact_expiry(inner: &mut MemoryInner) {
379 let limit = 2 * (inner.announcements.len() + inner.liveness.len());
380 if inner.expiry.len() > limit {
381 let MemoryInner {
382 announcements,
383 liveness,
384 expiry,
385 ..
386 } = inner;
387 expiry.retain(|Reverse(entry)| match &entry.key {
388 ExpiringKey::Announcement(key) => announcements
389 .get(key)
390 .is_some_and(|value| value.version == entry.version),
391 ExpiringKey::Liveness(node) => liveness
392 .get(node)
393 .is_some_and(|value| value.version == entry.version),
394 });
395 }
396 }
397
398 fn reap(inner: &mut MemoryInner, now_unix_ms: i64) {
399 while let Some(Reverse(head)) = inner.expiry.peek() {
400 if head.expires_at_unix_ms > now_unix_ms {
401 break;
402 }
403 let Some(Reverse(expired)) = inner.expiry.pop() else {
404 break;
405 };
406 match &expired.key {
407 ExpiringKey::Announcement(key) => {
408 let remove = inner
409 .announcements
410 .get(key)
411 .is_some_and(|entry| entry.version == expired.version);
412 if remove {
413 inner.announcements.remove(key);
414 if let Some(kinds) = inner.service_index.get_mut(&key.service_name) {
415 kinds.remove(&key.socket_kind);
416 if kinds.is_empty() {
417 inner.service_index.remove(&key.service_name);
418 }
419 }
420 }
421 }
422 ExpiringKey::Liveness(node) => {
423 let remove = inner
424 .liveness
425 .get(node)
426 .is_some_and(|entry| entry.version == expired.version);
427 if remove {
428 inner.liveness.remove(node);
429 }
430 }
431 }
432 }
433 }
434
435 #[cfg(test)]
436 pub(crate) fn put_announcement_sync(
437 &self,
438 service_name: &str,
439 endpoint: AnnouncedEndpoint,
440 ) -> Result<PutResult> {
441 self.put_announcement_at(service_name, endpoint, unix_millis_now())
442 }
443
444 fn put_announcement_at(
445 &self,
446 service_name: &str,
447 endpoint: AnnouncedEndpoint,
448 now_unix_ms: i64,
449 ) -> Result<PutResult> {
450 let mut inner = self.inner.lock();
451 Self::reap(&mut inner, now_unix_ms);
452 let key = AnnouncementKey {
453 service_name: service_name.to_owned(),
454 socket_kind: endpoint.socket_kind.clone(),
455 };
456 if let Some(existing) = inner.announcements.get(&key) {
457 if existing.value.order() > endpoint.order() {
458 return Ok(PutResult::IgnoredOlder);
459 }
460 } else if inner.announcements.len() >= self.announcement_capacity {
461 bail!("Discovery memory announcement capacity exhausted");
462 }
463 let version = Self::next_version(&mut inner);
464 inner
465 .service_index
466 .entry(service_name.to_owned())
467 .or_default()
468 .insert(endpoint.socket_kind.clone());
469 inner.expiry.push(Reverse(ExpiryEntry {
470 expires_at_unix_ms: endpoint.live_until_unix_ms,
471 version,
472 key: ExpiringKey::Announcement(key.clone()),
473 }));
474 inner.announcements.insert(
475 key,
476 Versioned {
477 value: endpoint,
478 version,
479 },
480 );
481 Self::compact_expiry(&mut inner);
482 Ok(PutResult::Stored)
483 }
484
485 pub(crate) fn announcements_for_sync(
486 &self,
487 service_name: &str,
488 now_unix_ms: i64,
489 ) -> Vec<AnnouncedEndpoint> {
490 let mut inner = self.inner.lock();
491 Self::reap(&mut inner, now_unix_ms);
492 inner
493 .service_index
494 .get(service_name)
495 .into_iter()
496 .flatten()
497 .filter_map(|socket_kind| {
498 inner
499 .announcements
500 .get(&AnnouncementKey {
501 service_name: service_name.to_owned(),
502 socket_kind: socket_kind.clone(),
503 })
504 .map(|entry| entry.value.clone())
505 })
506 .filter(|entry| entry.is_live_at(now_unix_ms))
507 .collect()
508 }
509
510 fn all_announcements_sync(&self, now_unix_ms: i64) -> Vec<(String, Vec<AnnouncedEndpoint>)> {
511 let mut inner = self.inner.lock();
512 Self::reap(&mut inner, now_unix_ms);
513 inner
514 .service_index
515 .iter()
516 .filter_map(|(service_name, socket_kinds)| {
517 let endpoints: Vec<_> = socket_kinds
518 .iter()
519 .filter_map(|socket_kind| {
520 inner
521 .announcements
522 .get(&AnnouncementKey {
523 service_name: service_name.clone(),
524 socket_kind: socket_kind.clone(),
525 })
526 .map(|entry| entry.value.clone())
527 })
528 .filter(|entry| entry.is_live_at(now_unix_ms))
529 .collect();
530 (!endpoints.is_empty()).then(|| (service_name.clone(), endpoints))
531 })
532 .collect()
533 }
534
535 #[cfg(feature = "valkey")]
536 pub(crate) fn clear_announcements_sync(&self, service_name: &str) {
537 let mut inner = self.inner.lock();
538 let Some(socket_kinds) = inner.service_index.remove(service_name) else {
539 return;
540 };
541 for socket_kind in socket_kinds {
542 inner.announcements.remove(&AnnouncementKey {
543 service_name: service_name.to_owned(),
544 socket_kind,
545 });
546 }
547 Self::compact_expiry(&mut inner);
548 }
549
550 #[cfg(feature = "valkey")]
551 fn clear_liveness_sync(&self, node: &Did) {
552 let mut inner = self.inner.lock();
553 inner.liveness.remove(node.as_str());
554 Self::compact_expiry(&mut inner);
555 }
556
557 #[cfg(feature = "valkey")]
558 fn clear_entity_statement_sync(&self, issuer: &str) {
559 self.inner.lock().entity_statements.remove(issuer);
560 }
561
562 #[cfg(feature = "valkey")]
563 fn clear_envelope_keyset_sync(&self, service_did: &str) {
564 self.inner.lock().envelope_keysets.remove(service_did);
565 }
566}
567
568impl Default for MemoryStateStore {
569 fn default() -> Self {
570 Self::new(16_384, 16_384, 4_096)
571 }
572}
573
574#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
575#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
576impl DiscoveryStateStore for MemoryStateStore {
577 async fn put_announcement(
578 &self,
579 service_name: &str,
580 endpoint: AnnouncedEndpoint,
581 ) -> Result<PutResult> {
582 let now = unix_millis_now();
583 let Some(endpoint) = endpoint.receipt_lease(now) else {
584 return Ok(PutResult::IgnoredOlder);
585 };
586 self.put_announcement_at(service_name, endpoint, now)
587 }
588
589 async fn announcements_for(
590 &self,
591 service_name: &str,
592 now_unix_ms: i64,
593 ) -> Result<Vec<AnnouncedEndpoint>> {
594 Ok(self.announcements_for_sync(service_name, now_unix_ms))
595 }
596
597 async fn all_announcements(
598 &self,
599 now_unix_ms: i64,
600 ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>> {
601 Ok(self.all_announcements_sync(now_unix_ms))
602 }
603
604 async fn put_liveness(&self, node: &Did, value: LiveAllocatable) -> Result<PutResult> {
605 let mut inner = self.inner.lock();
606 Self::reap(&mut inner, unix_millis_now());
607 let node = node.as_str().to_owned();
608 if let Some(existing) = inner.liveness.get(&node) {
609 if existing.value.last_seen > value.last_seen
613 && existing.value.live_until_unix_ms >= value.live_until_unix_ms
614 {
615 return Ok(PutResult::IgnoredOlder);
616 }
617 } else if inner.liveness.len() >= self.liveness_capacity {
618 bail!("Discovery memory liveness capacity exhausted");
619 }
620 let version = Self::next_version(&mut inner);
621 inner.expiry.push(Reverse(ExpiryEntry {
622 expires_at_unix_ms: value.live_until_unix_ms,
623 version,
624 key: ExpiringKey::Liveness(node.clone()),
625 }));
626 inner.liveness.insert(node, Versioned { value, version });
627 Self::compact_expiry(&mut inner);
628 Ok(PutResult::Stored)
629 }
630
631 #[cfg(test)]
632 async fn liveness(&self, node: &Did, now_unix_ms: i64) -> Result<Option<LiveAllocatable>> {
633 let mut inner = self.inner.lock();
634 Self::reap(&mut inner, now_unix_ms);
635 Ok(inner
636 .liveness
637 .get(node.as_str())
638 .map(|entry| entry.value.clone())
639 .filter(|entry| entry.is_live_at(now_unix_ms)))
640 }
641
642 async fn all_liveness(&self, now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>> {
643 let mut inner = self.inner.lock();
644 Self::reap(&mut inner, now_unix_ms);
645 Ok(inner
646 .liveness
647 .iter()
648 .filter(|(_, entry)| entry.value.is_live_at(now_unix_ms))
649 .map(|(node, entry)| (Did::new(node.clone()), entry.value.clone()))
650 .collect())
651 }
652
653 async fn put_entity_statement(&self, issuer: &str, value: CachedEntityStatement) -> Result<()> {
654 let mut inner = self.inner.lock();
655 if !inner.entity_statements.contains_key(issuer)
656 && inner.entity_statements.len() >= self.artifact_capacity
657 {
658 if let Some(oldest) = inner
661 .entity_statements
662 .iter()
663 .min_by_key(|(name, value)| (value.fetched_at, *name))
664 .map(|(name, _)| name.clone())
665 {
666 inner.entity_statements.remove(&oldest);
667 }
668 }
669 inner.entity_statements.insert(issuer.to_owned(), value);
670 Ok(())
671 }
672
673 async fn entity_statement(&self, issuer: &str) -> Result<Option<CachedEntityStatement>> {
674 Ok(self.inner.lock().entity_statements.get(issuer).cloned())
675 }
676
677 async fn known_issuers(&self) -> Result<Vec<String>> {
678 Ok(self
679 .inner
680 .lock()
681 .entity_statements
682 .keys()
683 .cloned()
684 .collect())
685 }
686
687 async fn known_issuer_count(&self) -> Result<usize> {
688 Ok(self.inner.lock().entity_statements.len())
689 }
690
691 async fn put_envelope_keyset(
692 &self,
693 service_did: &str,
694 value: CachedEnvelopeKeyset,
695 ) -> Result<()> {
696 let mut inner = self.inner.lock();
697 if !inner.envelope_keysets.contains_key(service_did)
698 && inner.envelope_keysets.len() >= self.artifact_capacity
699 {
700 bail!("Discovery memory federation artifact capacity exhausted");
701 }
702 inner.envelope_keysets.insert(service_did.to_owned(), value);
703 Ok(())
704 }
705
706 async fn envelope_keyset(&self, service_did: &str) -> Result<Option<CachedEnvelopeKeyset>> {
707 Ok(self.inner.lock().envelope_keysets.get(service_did).cloned())
708 }
709}
710
711#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
712#[derive(Serialize, Deserialize)]
713struct StoredLiveness {
714 node: Did,
715 #[serde(default)]
716 shared_receipt: bool,
717 #[serde(flatten)]
718 value: LiveAllocatable,
719}
720
721#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
722#[derive(Clone)]
723struct ValkeyStateStore {
724 pool: fred::prelude::RedisPool,
725 _driver: Arc<tokio::sync::oneshot::Sender<()>>,
727 prefix: String,
728 announcement_capacity: usize,
729 liveness_capacity: usize,
730 artifact_capacity: usize,
731}
732
733#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
734impl ValkeyStateStore {
735 async fn connect(config: &ValkeyStateConfig) -> Result<Self> {
736 use anyhow::Context as _;
737 use fred::prelude::*;
738
739 anyhow::ensure!(
740 config.pool_size > 0,
741 "Discovery Valkey pool_size must be positive"
742 );
743 anyhow::ensure!(
744 !config.url.trim().is_empty(),
745 "Discovery Valkey URL must be configured explicitly"
746 );
747 anyhow::ensure!(
748 config.command_timeout_ms > 0,
749 "Discovery Valkey command_timeout_ms must be positive"
750 );
751 anyhow::ensure!(
752 config.announcement_capacity > 0
753 && config.liveness_capacity > 0
754 && config.artifact_capacity > 0,
755 "Discovery Valkey capacities must be positive"
756 );
757 hyprstream_rpc::transport::pq_provider::install_pq_crypto_provider()?;
760 let redis = RedisConfig::from_url(&config.url).context("invalid Discovery Valkey URL")?;
761 let mut builder = Builder::from_config(redis);
762 builder.with_performance_config(|performance| {
763 performance.default_command_timeout =
764 std::time::Duration::from_millis(config.command_timeout_ms);
765 });
766 let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
771 let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
772 let pool_size = config.pool_size;
773 let timeout = std::time::Duration::from_millis(config.command_timeout_ms);
774 std::thread::Builder::new()
775 .name("discovery-valkey".to_owned())
776 .spawn(move || {
777 let runtime = match tokio::runtime::Builder::new_current_thread()
778 .enable_all()
779 .build()
780 {
781 Ok(runtime) => runtime,
782 Err(error) => {
783 let _ = ready_tx.send(Err(anyhow::Error::from(error)));
784 return;
785 }
786 };
787 runtime.block_on(async move {
788 let connected: Result<RedisPool> = async {
789 let pool = builder.build_pool(pool_size)?;
790 pool.connect();
791 tokio::time::timeout(timeout, pool.wait_for_connect()).await??;
792 let _: String = pool.ping().await?;
793 Ok(pool)
794 }
795 .await;
796 match connected {
797 Ok(pool) => {
798 if ready_tx.send(Ok(pool.clone())).is_ok() {
799 let _ = shutdown_rx.await;
800 let _ = tokio::time::timeout(timeout, pool.quit()).await;
801 }
802 }
803 Err(error) => {
804 let _ = ready_tx.send(Err(error));
805 }
806 }
807 });
808 })
809 .context("start Discovery Valkey driver")?;
810 let pool = ready_rx
811 .await
812 .context("Discovery Valkey driver stopped during startup")??;
813 Ok(Self {
814 pool,
815 _driver: Arc::new(shutdown_tx),
816 prefix: format!(
818 "{}:{{discovery-state}}",
819 config.key_prefix.trim_end_matches(':')
820 ),
821 announcement_capacity: config.announcement_capacity,
822 liveness_capacity: config.liveness_capacity,
823 artifact_capacity: config.artifact_capacity,
824 })
825 }
826
827 fn digest(value: &str) -> String {
828 blake3::hash(value.as_bytes()).to_hex().to_string()
829 }
830
831 fn key(&self, suffix: &str) -> String {
832 format!("{}:{suffix}", self.prefix)
833 }
834
835 fn service_id(service_name: &str) -> String {
836 Self::digest(service_name)
837 }
838
839 fn announcement_key(&self, service_name: &str, socket_kind: &str) -> String {
840 format!(
841 "{}:announcement:{}:{}",
842 self.prefix,
843 Self::service_id(service_name),
844 Self::digest(socket_kind)
845 )
846 }
847
848 fn announcement_index(&self, service_name: &str) -> String {
849 format!(
850 "{}:announcement-index:{}",
851 self.prefix,
852 Self::service_id(service_name)
853 )
854 }
855
856 async fn revision(&self, key: String) -> Result<u64> {
857 use fred::prelude::*;
858 Ok(self.pool.get::<Option<u64>, _>(key).await?.unwrap_or(0))
859 }
860
861 #[cfg(test)]
865 async fn announcement_revision(&self) -> Result<u64> {
866 self.revision(self.key("announcement-global-revision"))
867 .await
868 }
869
870 #[cfg(test)]
871 async fn liveness_revision(&self) -> Result<u64> {
872 self.revision(self.key("liveness-global-revision")).await
873 }
874
875 const REAP_ANNOUNCEMENTS: &str = r#"
880local clock = redis.call('TIME')
881local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
882local function reap(expiry, services, names, revision)
883 local expired = redis.call('ZRANGEBYSCORE', expiry, '-inf', now)
884 if #expired > 0 then redis.call('INCR', revision) end
885 for _, key in ipairs(expired) do
886 local service = string.match(key, ':announcement:([^:]+):[^:]+$')
887 local index = string.gsub(key, ':announcement:[^:]+:[^:]+$', ':announcement-index:' .. service)
888 redis.call('DEL', key)
889 redis.call('SREM', index, key)
890 redis.call('ZREM', expiry, key)
891 if redis.call('SCARD', index) == 0 then
892 redis.call('SREM', services, service)
893 redis.call('HDEL', names, service)
894 end
895 end
896end
897"#;
898
899 async fn announcement_revision_at_shared_time(&self) -> Result<(u64, i64)> {
900 use fred::prelude::*;
901 self.pool.eval(
904 format!("{}\nreap(KEYS[1], KEYS[2], KEYS[3], KEYS[4])\nreturn {{redis.call('GET', KEYS[4]) or '0', now}}", Self::REAP_ANNOUNCEMENTS),
905 vec![self.key("announcement-expiry"), self.key("services"),
906 self.key("service-names"), self.key("announcement-global-revision")],
907 Vec::<String>::new(),
908 ).await.map_err(Into::into)
909 }
910
911 async fn entity_revision(&self) -> Result<u64> {
912 self.revision(self.key("entity-global-revision")).await
915 }
916
917 async fn envelope_revision(&self, service_did: &str) -> Result<u64> {
918 self.revision(format!(
919 "{}:envelope-revision:{}",
920 self.prefix,
921 Self::digest(service_did)
922 ))
923 .await
924 }
925
926 async fn announcements_for_inner(&self, service_name: &str) -> Result<Vec<AnnouncedEndpoint>> {
927 use fred::prelude::*;
928
929 const LIST: &str = r#"
933reap(KEYS[4], KEYS[2], KEYS[3], KEYS[5])
934local live = {}
935for _, key in ipairs(redis.call('SMEMBERS', KEYS[1])) do
936 local encoded = redis.call('GET', key)
937 if encoded then
938 local ok, value = pcall(cjson.decode, encoded)
939 if not ok then return redis.error_reply('corrupt Discovery announcement') end
940 if tonumber(value.live_until_unix_ms) > now and tonumber(value.expires_at_unix_ms) > now then
941 table.insert(live, encoded)
942 else
943 redis.call('INCR', KEYS[5])
944 redis.call('DEL', key)
945 redis.call('SREM', KEYS[1], key)
946 redis.call('ZREM', KEYS[4], key)
947 end
948 else
949 redis.call('SREM', KEYS[1], key)
950 redis.call('ZREM', KEYS[4], key)
951 end
952end
953if redis.call('SCARD', KEYS[1]) == 0 then
954 redis.call('SREM', KEYS[2], ARGV[1])
955 redis.call('HDEL', KEYS[3], ARGV[1])
956end
957return live
958"#;
959 let encoded: Vec<String> = self
960 .pool
961 .eval(
962 format!("{}{LIST}", Self::REAP_ANNOUNCEMENTS),
963 vec![
964 self.announcement_index(service_name),
965 self.key("services"),
966 self.key("service-names"),
967 self.key("announcement-expiry"),
968 self.key("announcement-global-revision"),
969 ],
970 vec![Self::service_id(service_name)],
971 )
972 .await?;
973 encoded
974 .into_iter()
975 .map(|value| serde_json::from_str(&value).map_err(Into::into))
976 .collect()
977 }
978}
979
980#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
981#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
982impl DiscoveryStateStore for ValkeyStateStore {
983 async fn put_announcement(
984 &self,
985 service_name: &str,
986 endpoint: AnnouncedEndpoint,
987 ) -> Result<PutResult> {
988 use fred::prelude::*;
989
990 const PUT: &str = r#"
991reap(KEYS[6], KEYS[3], KEYS[4], KEYS[5])
992local signed_expiry = tonumber(ARGV[3])
993local heartbeat_expiry = now + tonumber(ARGV[8])
994if signed_expiry == 0 then signed_expiry = heartbeat_expiry end
995local expiry = math.min(heartbeat_expiry, signed_expiry, tonumber(ARGV[4]))
996if expiry <= now then return 0 end
997local current = redis.call('GET', KEYS[1])
998if not current and redis.call('ZCARD', KEYS[6]) >= tonumber(ARGV[7]) then
999 return redis.error_reply('Discovery Valkey announcement capacity exhausted')
1000end
1001if current then
1002 local ok, decoded = pcall(cjson.decode, current)
1003 if not ok then return redis.error_reply('corrupt Discovery announcement') end
1004 local old_epoch = tonumber(decoded.accepted_state_epoch) or 0
1005 local old_exp = tonumber(decoded.expires_at_unix_ms) or 0
1006 local new_epoch = tonumber(ARGV[2])
1007 local new_exp = signed_expiry
1008 if old_epoch > new_epoch or (old_epoch == new_epoch and old_exp > new_exp) then
1009 return 0
1010 end
1011end
1012-- Only the effective lease (and legacy application expiry) is computed here.
1013-- Preserve all other JSON fields/arrays exactly as encoded by Rust, not cjson.
1014local encoded = string.sub(ARGV[1], 1, -2) .. ',"live_until_unix_ms":' .. string.format('%.0f', expiry)
1015if tonumber(ARGV[3]) == 0 then
1016 encoded = encoded .. ',"expires_at_unix_ms":' .. string.format('%.0f', signed_expiry)
1017end
1018encoded = encoded .. '}'
1019redis.call('SET', KEYS[1], encoded, 'PXAT', expiry)
1020redis.call('SADD', KEYS[2], KEYS[1])
1021redis.call('SADD', KEYS[3], ARGV[5])
1022redis.call('HSET', KEYS[4], ARGV[5], ARGV[6])
1023redis.call('INCR', KEYS[5])
1024redis.call('ZADD', KEYS[6], expiry, KEYS[1])
1025return 1
1026"#;
1027 let service_id = Self::service_id(service_name);
1028 let mut payload = serde_json::to_value(&endpoint)?;
1029 let fields = payload
1030 .as_object_mut()
1031 .ok_or_else(|| anyhow::anyhow!("announcement must encode as an object"))?;
1032 fields.remove("live_until_unix_ms");
1033 if endpoint.expires_at_unix_ms == 0 {
1034 fields.remove("expires_at_unix_ms");
1035 }
1036 let encoded = serde_json::to_string(&payload)?;
1037 let stored: i64 = self
1038 .pool
1039 .eval(
1040 format!("{}{PUT}", Self::REAP_ANNOUNCEMENTS),
1041 vec![
1042 self.announcement_key(service_name, &endpoint.socket_kind),
1043 self.announcement_index(service_name),
1044 self.key("services"),
1045 self.key("service-names"),
1046 self.key("announcement-global-revision"),
1047 self.key("announcement-expiry"),
1048 ],
1049 vec![
1050 encoded,
1051 endpoint.accepted_state_epoch.to_string(),
1052 endpoint.expires_at_unix_ms.to_string(),
1053 endpoint.live_until_unix_ms.to_string(),
1054 service_id,
1055 service_name.to_owned(),
1056 self.announcement_capacity.to_string(),
1057 ANNOUNCED_ENDPOINT_TTL.as_millis().to_string(),
1058 ],
1059 )
1060 .await?;
1061 Ok(if stored == 1 {
1062 PutResult::Stored
1063 } else {
1064 PutResult::IgnoredOlder
1065 })
1066 }
1067
1068 async fn announcements_for(
1069 &self,
1070 service_name: &str,
1071 _now_unix_ms: i64,
1072 ) -> Result<Vec<AnnouncedEndpoint>> {
1073 self.announcements_for_inner(service_name).await
1074 }
1075
1076 async fn all_announcements(
1077 &self,
1078 _now_unix_ms: i64,
1079 ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>> {
1080 use fred::prelude::*;
1081
1082 const LIST: &str = r#"
1086reap(KEYS[1], KEYS[2], KEYS[3], KEYS[4])
1087local values = {}
1088for _, key in ipairs(redis.call('ZRANGE', KEYS[1], 0, -1)) do
1089 local value = redis.call('GET', key)
1090 if value then
1091 local ok, decoded = pcall(cjson.decode, value)
1092 if not ok then return redis.error_reply('corrupt Discovery announcement') end
1093 if tonumber(decoded.live_until_unix_ms) > now and tonumber(decoded.expires_at_unix_ms) > now then
1094 local service = string.match(key, ':announcement:([^:]+):[^:]+$')
1095 local name = redis.call('HGET', KEYS[3], service)
1096 if not name then return redis.error_reply('missing Discovery service name') end
1097 table.insert(values, {name, value})
1098 end
1099 end
1100end
1101return values
1102"#;
1103 let encoded: Vec<(String, String)> = self
1104 .pool
1105 .eval(
1106 format!("{}{LIST}", Self::REAP_ANNOUNCEMENTS),
1107 vec![
1108 self.key("announcement-expiry"),
1109 self.key("services"),
1110 self.key("service-names"),
1111 self.key("announcement-global-revision"),
1112 ],
1113 Vec::<String>::new(),
1114 )
1115 .await?;
1116 let mut all: HashMap<String, Vec<AnnouncedEndpoint>> = HashMap::new();
1117 for (name, value) in encoded {
1118 let endpoint: AnnouncedEndpoint = serde_json::from_str(&value)?;
1119 all.entry(name).or_default().push(endpoint);
1120 }
1121 Ok(all.into_iter().collect())
1122 }
1123
1124 async fn put_liveness(&self, node: &Did, value: LiveAllocatable) -> Result<PutResult> {
1125 use fred::prelude::*;
1126
1127 const PUT: &str = r#"
1128local clock = redis.call('TIME')
1129local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
1130local current = redis.call('GET', KEYS[1])
1131redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now)
1132if not current and redis.call('ZCARD', KEYS[3]) >= tonumber(ARGV[3]) then
1133 return redis.error_reply('Discovery Valkey liveness capacity exhausted')
1134end
1135local expiry = now + tonumber(ARGV[2])
1136-- Preserve JSON arrays/numeric payload exactly: Lua cjson turns [] into {}.
1137local encoded = string.sub(ARGV[1], 1, -2) .. ',"last_seen":' .. string.format('%.0f', now)
1138 .. ',"live_until_unix_ms":' .. string.format('%.0f', expiry) .. '}'
1139redis.call('SET', KEYS[1], encoded, 'PXAT', expiry)
1140redis.call('INCR', KEYS[2])
1141redis.call('ZADD', KEYS[3], expiry, KEYS[1])
1142return 1
1143"#;
1144 let node_id = Self::digest(node.as_str());
1145 let stored: i64 = self
1146 .pool
1147 .eval(
1148 PUT,
1149 vec![
1150 format!("{}:liveness:{node_id}", self.prefix),
1151 self.key("liveness-global-revision"),
1152 self.key("liveness-expiry"),
1153 ],
1154 vec![
1155 serde_json::to_string(&serde_json::json!({
1156 "node": node, "allocatable": value.allocatable,
1157 "load_fraction": value.load_fraction, "shared_receipt": true,
1158 }))?,
1159 value
1160 .live_until_unix_ms
1161 .saturating_sub(value.last_seen)
1162 .clamp(1, 45_000)
1163 .to_string(),
1164 self.liveness_capacity.to_string(),
1165 ],
1166 )
1167 .await?;
1168 Ok(if stored == 1 {
1169 PutResult::Stored
1170 } else {
1171 PutResult::IgnoredOlder
1172 })
1173 }
1174
1175 #[cfg(test)]
1176 async fn liveness(&self, node: &Did, _now_unix_ms: i64) -> Result<Option<LiveAllocatable>> {
1177 use fred::prelude::*;
1178 let encoded: Option<String> = self
1179 .pool
1180 .get(format!(
1181 "{}:liveness:{}",
1182 self.prefix,
1183 Self::digest(node.as_str())
1184 ))
1185 .await?;
1186 Ok(encoded
1187 .map(|encoded| serde_json::from_str::<StoredLiveness>(&encoded))
1188 .transpose()?
1189 .filter(|record| record.shared_receipt)
1190 .map(|record| record.value))
1191 }
1192
1193 async fn all_liveness(&self, _now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>> {
1194 use fred::prelude::*;
1195 const LIST: &str = r#"
1199local clock = redis.call('TIME')
1200local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
1201redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
1202local values = {}
1203for _, key in ipairs(redis.call('ZRANGE', KEYS[1], 0, -1)) do
1204 local value = redis.call('GET', key)
1205 if value then
1206 local record = cjson.decode(value)
1207 if record.shared_receipt == true then
1208 table.insert(values, value)
1209 else
1210 -- Legacy replica-clock rows have no trustworthy shared lifetime. Retire
1211 -- only this volatile row; the next admitted heartbeat recreates it.
1212 redis.call('DEL', key)
1213 redis.call('ZREM', KEYS[1], key)
1214 redis.call('INCR', KEYS[2])
1215 end
1216 end
1217end
1218return values
1219"#;
1220 let encoded: Vec<String> = self
1221 .pool
1222 .eval(
1223 LIST,
1224 vec![
1225 self.key("liveness-expiry"),
1226 self.key("liveness-global-revision"),
1227 ],
1228 Vec::<String>::new(),
1229 )
1230 .await?;
1231 let mut live = Vec::with_capacity(encoded.len());
1232 for value in encoded {
1233 let record: StoredLiveness = serde_json::from_str(&value)?;
1234 live.push((record.node, record.value));
1235 }
1236 Ok(live)
1237 }
1238
1239 async fn put_entity_statement(&self, issuer: &str, value: CachedEntityStatement) -> Result<()> {
1240 use fred::prelude::*;
1241 const PUT: &str = r#"
1242if redis.call('EXISTS', KEYS[1]) == 0 and redis.call('SCARD', KEYS[2]) >= tonumber(ARGV[4]) then
1243 local oldest = redis.call('ZRANGE', KEYS[5], 0, 0)[1]
1244 if not oldest then return redis.error_reply('Discovery issuer cache index is inconsistent') end
1245 local oldkey = string.gsub(KEYS[1], ':entity:[^:]+$', ':entity:' .. oldest)
1246 redis.call('DEL', oldkey)
1247 redis.call('SREM', KEYS[2], oldest)
1248 redis.call('HDEL', KEYS[3], oldest)
1249 redis.call('ZREM', KEYS[5], oldest)
1250end
1251redis.call('SET', KEYS[1], ARGV[1])
1252redis.call('SADD', KEYS[2], ARGV[2])
1253redis.call('HSET', KEYS[3], ARGV[2], ARGV[3])
1254redis.call('INCR', KEYS[4])
1255redis.call('ZADD', KEYS[5], ARGV[5], ARGV[2])
1256return 1
1257"#;
1258 let id = Self::digest(issuer);
1259 let _: i64 = self
1260 .pool
1261 .eval(
1262 PUT,
1263 vec![
1264 format!("{}:entity:{id}", self.prefix),
1265 self.key("issuers"),
1266 self.key("issuer-names"),
1267 self.key("entity-global-revision"),
1268 self.key("issuer-fetched"),
1269 ],
1270 vec![
1271 serde_json::to_string(&value)?,
1272 id,
1273 issuer.to_owned(),
1274 self.artifact_capacity.to_string(),
1275 value.fetched_at.to_string(),
1276 ],
1277 )
1278 .await?;
1279 Ok(())
1280 }
1281
1282 async fn entity_statement(&self, issuer: &str) -> Result<Option<CachedEntityStatement>> {
1283 use fred::prelude::*;
1284 let encoded: Option<String> = self
1285 .pool
1286 .get(format!("{}:entity:{}", self.prefix, Self::digest(issuer)))
1287 .await?;
1288 encoded
1289 .map(|encoded| serde_json::from_str(&encoded).map_err(Into::into))
1290 .transpose()
1291 }
1292
1293 async fn known_issuers(&self) -> Result<Vec<String>> {
1294 use fred::prelude::*;
1295 Ok(self.pool.hvals(self.key("issuer-names")).await?)
1296 }
1297
1298 async fn known_issuer_count(&self) -> Result<usize> {
1299 use fred::prelude::*;
1300 Ok(self.pool.scard(self.key("issuers")).await?)
1301 }
1302
1303 async fn put_envelope_keyset(
1304 &self,
1305 service_did: &str,
1306 value: CachedEnvelopeKeyset,
1307 ) -> Result<()> {
1308 use fred::prelude::*;
1309 const PUT: &str = r#"
1310if redis.call('EXISTS', KEYS[1]) == 0 and redis.call('SCARD', KEYS[3]) >= tonumber(ARGV[2]) then
1311 return redis.error_reply('Discovery Valkey federation artifact capacity exhausted')
1312end
1313redis.call('SET', KEYS[1], ARGV[1])
1314redis.call('INCR', KEYS[2])
1315redis.call('SADD', KEYS[3], KEYS[1])
1316return 1
1317"#;
1318 let id = Self::digest(service_did);
1319 let _: i64 = self
1320 .pool
1321 .eval(
1322 PUT,
1323 vec![
1324 format!("{}:envelope:{id}", self.prefix),
1325 format!("{}:envelope-revision:{id}", self.prefix),
1326 self.key("envelopes"),
1327 ],
1328 vec![
1329 serde_json::to_string(&value)?,
1330 self.artifact_capacity.to_string(),
1331 ],
1332 )
1333 .await?;
1334 Ok(())
1335 }
1336
1337 async fn envelope_keyset(&self, service_did: &str) -> Result<Option<CachedEnvelopeKeyset>> {
1338 use fred::prelude::*;
1339 let encoded: Option<String> = self
1340 .pool
1341 .get(format!(
1342 "{}:envelope:{}",
1343 self.prefix,
1344 Self::digest(service_did)
1345 ))
1346 .await?;
1347 encoded
1348 .map(|encoded| serde_json::from_str(&encoded).map_err(Into::into))
1349 .transpose()
1350 }
1351}
1352
1353#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1354#[derive(Clone, Copy)]
1355struct ObservedRevision {
1356 revision: u64,
1357 valid_until_unix_ms: i64,
1358}
1359
1360#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1361struct TieredStateStore {
1362 memory: Arc<MemoryStateStore>,
1363 valkey: Arc<ValkeyStateStore>,
1364 l1_max_ttl_ms: i64,
1365 observed: Mutex<HashMap<String, ObservedRevision>>,
1366 operations: [tokio::sync::Mutex<()>; 64],
1370}
1371
1372#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1373impl TieredStateStore {
1374 fn new(
1375 memory: Arc<MemoryStateStore>,
1376 valkey: Arc<ValkeyStateStore>,
1377 l1_max_ttl_ms: u64,
1378 ) -> Self {
1379 Self {
1380 memory,
1381 valkey,
1382 l1_max_ttl_ms: i64::try_from(l1_max_ttl_ms).unwrap_or(i64::MAX),
1383 observed: Mutex::new(HashMap::new()),
1384 operations: std::array::from_fn(|_| tokio::sync::Mutex::new(())),
1385 }
1386 }
1387
1388 fn scope(kind: &str, value: &str) -> String {
1389 format!("{kind}:{value}")
1390 }
1391
1392 fn operation(&self, kind: &str, value: &str) -> &tokio::sync::Mutex<()> {
1393 let digest = blake3::hash(Self::scope(kind, value).as_bytes());
1394 &self.operations[usize::from(digest.as_bytes()[0]) % self.operations.len()]
1395 }
1396
1397 fn is_observed(&self, scope: &str, revision: u64, now_unix_ms: i64) -> bool {
1398 self.observed.lock().get(scope).is_some_and(|observed| {
1399 observed.revision == revision && now_unix_ms < observed.valid_until_unix_ms
1400 })
1401 }
1402
1403 fn observe(&self, scope: String, revision: u64, now_unix_ms: i64) {
1404 let mut observed = self.observed.lock();
1405 observed.retain(|_, value| now_unix_ms < value.valid_until_unix_ms);
1406 let capacity = self
1407 .memory
1408 .announcement_capacity
1409 .saturating_add(self.memory.liveness_capacity)
1410 .saturating_add(self.memory.artifact_capacity.saturating_mul(2));
1411 if observed.len() >= capacity {
1412 observed.clear();
1413 }
1414 observed.insert(
1415 scope,
1416 ObservedRevision {
1417 revision,
1418 valid_until_unix_ms: now_unix_ms.saturating_add(self.l1_max_ttl_ms),
1419 },
1420 );
1421 }
1422
1423 fn l1_announcement(
1424 &self,
1425 mut endpoint: AnnouncedEndpoint,
1426 now_unix_ms: i64,
1427 ) -> AnnouncedEndpoint {
1428 endpoint.live_until_unix_ms = endpoint
1429 .live_until_unix_ms
1430 .min(now_unix_ms.saturating_add(self.l1_max_ttl_ms));
1431 endpoint
1432 }
1433}
1434
1435#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1436#[async_trait]
1437impl DiscoveryStateStore for TieredStateStore {
1438 async fn put_announcement(
1439 &self,
1440 service_name: &str,
1441 endpoint: AnnouncedEndpoint,
1442 ) -> Result<PutResult> {
1443 let _operation = self.operation("announcement", service_name).lock().await;
1444 self.observed
1447 .lock()
1448 .remove(&Self::scope("announcement", service_name));
1449 self.memory.clear_announcements_sync(service_name);
1450 self.valkey.put_announcement(service_name, endpoint).await
1451 }
1452
1453 async fn announcements_for(
1454 &self,
1455 service_name: &str,
1456 _now_unix_ms: i64,
1457 ) -> Result<Vec<AnnouncedEndpoint>> {
1458 let _operation = self.operation("announcement", service_name).lock().await;
1459 let scope = Self::scope("announcement", service_name);
1460 let (revision, now_unix_ms) = self.valkey.announcement_revision_at_shared_time().await?;
1461 if self.is_observed(&scope, revision, now_unix_ms) {
1462 return self
1463 .memory
1464 .announcements_for(service_name, now_unix_ms)
1465 .await;
1466 }
1467 let values = self
1468 .valkey
1469 .announcements_for(service_name, now_unix_ms)
1470 .await?;
1471 self.observed.lock().remove(&scope);
1472 self.memory.clear_announcements_sync(service_name);
1473 for value in &values {
1476 if self
1477 .memory
1478 .put_announcement_at(
1479 service_name,
1480 self.l1_announcement(value.clone(), now_unix_ms),
1481 now_unix_ms,
1482 )
1483 .is_err()
1484 {
1485 self.memory.clear_announcements_sync(service_name);
1486 return Ok(values);
1487 }
1488 }
1489 if !values.is_empty() {
1490 self.observe(scope, revision, now_unix_ms);
1491 }
1492 Ok(values)
1493 }
1494
1495 async fn all_announcements(
1496 &self,
1497 now_unix_ms: i64,
1498 ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>> {
1499 self.valkey.all_announcements(now_unix_ms).await
1502 }
1503
1504 async fn put_liveness(&self, node: &Did, value: LiveAllocatable) -> Result<PutResult> {
1505 let _operation = self.operation("liveness", node.as_str()).lock().await;
1506 self.observed
1507 .lock()
1508 .remove(&Self::scope("liveness", node.as_str()));
1509 self.memory.clear_liveness_sync(node);
1510 self.valkey.put_liveness(node, value).await
1511 }
1512
1513 #[cfg(test)]
1514 async fn liveness(&self, node: &Did, now_unix_ms: i64) -> Result<Option<LiveAllocatable>> {
1515 let _operation = self.operation("liveness", node.as_str()).lock().await;
1516 self.valkey.liveness(node, now_unix_ms).await
1519 }
1520
1521 async fn all_liveness(&self, now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>> {
1522 self.valkey.all_liveness(now_unix_ms).await
1523 }
1524
1525 async fn put_entity_statement(&self, issuer: &str, value: CachedEntityStatement) -> Result<()> {
1526 let _operation = self.operation("entity", issuer).lock().await;
1527 self.observed.lock().remove(&Self::scope("entity", issuer));
1528 self.memory.clear_entity_statement_sync(issuer);
1529 self.valkey.put_entity_statement(issuer, value).await
1530 }
1531
1532 async fn entity_statement(&self, issuer: &str) -> Result<Option<CachedEntityStatement>> {
1533 let _operation = self.operation("entity", issuer).lock().await;
1534 let now = unix_millis_now();
1535 let scope = Self::scope("entity", issuer);
1536 let revision = self.valkey.entity_revision().await?;
1537 if self.is_observed(&scope, revision, now) {
1538 if let Some(value) = self.memory.entity_statement(issuer).await? {
1541 return Ok(Some(value));
1542 }
1543 }
1544 let value = self.valkey.entity_statement(issuer).await?;
1545 self.observed.lock().remove(&scope);
1546 self.memory.clear_entity_statement_sync(issuer);
1547 if let Some(value) = &value {
1548 if self
1549 .memory
1550 .put_entity_statement(issuer, value.clone())
1551 .await
1552 .is_ok()
1553 {
1554 self.observe(scope, revision, now);
1555 }
1556 }
1557 Ok(value)
1558 }
1559
1560 async fn known_issuers(&self) -> Result<Vec<String>> {
1561 self.valkey.known_issuers().await
1562 }
1563
1564 async fn known_issuer_count(&self) -> Result<usize> {
1565 self.valkey.known_issuer_count().await
1566 }
1567
1568 async fn put_envelope_keyset(
1569 &self,
1570 service_did: &str,
1571 value: CachedEnvelopeKeyset,
1572 ) -> Result<()> {
1573 let _operation = self.operation("envelope", service_did).lock().await;
1574 self.observed
1575 .lock()
1576 .remove(&Self::scope("envelope", service_did));
1577 self.memory.clear_envelope_keyset_sync(service_did);
1578 self.valkey.put_envelope_keyset(service_did, value).await
1579 }
1580
1581 async fn envelope_keyset(&self, service_did: &str) -> Result<Option<CachedEnvelopeKeyset>> {
1582 let _operation = self.operation("envelope", service_did).lock().await;
1583 let now = unix_millis_now();
1584 let scope = Self::scope("envelope", service_did);
1585 let revision = self.valkey.envelope_revision(service_did).await?;
1586 if self.is_observed(&scope, revision, now) {
1587 return self.memory.envelope_keyset(service_did).await;
1588 }
1589 let value = self.valkey.envelope_keyset(service_did).await?;
1590 self.observed.lock().remove(&scope);
1591 self.memory.clear_envelope_keyset_sync(service_did);
1592 if let Some(value) = &value {
1593 if self
1594 .memory
1595 .put_envelope_keyset(service_did, value.clone())
1596 .await
1597 .is_ok()
1598 {
1599 self.observe(scope, revision, now);
1600 }
1601 }
1602 Ok(value)
1603 }
1604}
1605
1606#[cfg(test)]
1607thread_local! {
1608 static TEST_REPLICA_TIME: std::cell::Cell<Option<i64>> = const { std::cell::Cell::new(None) };
1611}
1612
1613pub(crate) fn unix_millis_now() -> i64 {
1614 #[cfg(test)]
1615 if let Some(now) = TEST_REPLICA_TIME.with(std::cell::Cell::get) {
1616 return now;
1617 }
1618 use std::time::{SystemTime, UNIX_EPOCH};
1619 SystemTime::now()
1620 .duration_since(UNIX_EPOCH)
1621 .map(|duration| duration.as_millis() as i64)
1622 .unwrap_or(0)
1623}
1624
1625#[cfg(test)]
1626pub(crate) mod tests {
1627 #![allow(clippy::expect_used, clippy::indexing_slicing, clippy::unwrap_used)]
1628
1629 use super::*;
1630
1631 fn endpoint(kind: &str, epoch: u64, signed_expiry: i64, live_until: i64) -> AnnouncedEndpoint {
1632 AnnouncedEndpoint {
1633 socket_kind: kind.to_owned(),
1634 endpoint: format!("iroh://{kind}"),
1635 service_did: Did::new("did:at9p:test".to_owned()),
1636 capabilities: BTreeSet::from(["discovery".to_owned()]),
1637 accepted_state_digest: vec![7; 64],
1638 accepted_state_epoch: epoch,
1639 response_key_id: "did:at9p:test#response".to_owned(),
1640 request_kem_key_id: "did:at9p:test#kem".to_owned(),
1641 request_kem_recipient: vec![1, 2, 3],
1642 expires_at_unix_ms: signed_expiry,
1643 source_signer: [9; 32],
1644 live_until_unix_ms: live_until,
1645 }
1646 }
1647
1648 #[test]
1649 fn legacy_announcement_bearer_field_is_dropped_on_read_and_rewrite() {
1650 let now = unix_millis_now();
1651 let mut legacy = serde_json::to_value(endpoint("iroh", 1, now + 10_000, now + 50))
1652 .expect("serialize current endpoint fixture");
1653 legacy
1654 .as_object_mut()
1655 .expect("endpoint fixture is an object")
1656 .insert(
1657 "service_jwt".to_owned(),
1658 serde_json::json!("legacy-bearer-jwt"),
1659 );
1660
1661 let parsed: AnnouncedEndpoint =
1662 serde_json::from_value(legacy).expect("legacy endpoint remains readable");
1663 let rewritten = serde_json::to_string(&parsed).expect("rewrite endpoint");
1664 assert!(!rewritten.contains("service_jwt"));
1665 assert!(!rewritten.contains("legacy-bearer-jwt"));
1666 }
1667
1668 async fn assert_announcement_backend_contract(store: &dyn DiscoveryStateStore) {
1669 let now = unix_millis_now();
1670 store
1671 .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 50))
1672 .await
1673 .unwrap();
1674 assert_eq!(
1675 store.announcements_for("model", now).await.unwrap().len(),
1676 1
1677 );
1678 store
1679 .put_announcement("model", endpoint("iroh", 2, now + 10_000, now + 50))
1680 .await
1681 .unwrap();
1682 assert_eq!(
1683 store.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
1684 2
1685 );
1686
1687 tokio::time::sleep(std::time::Duration::from_millis(80)).await;
1688 assert!(store
1689 .announcements_for("model", unix_millis_now())
1690 .await
1691 .unwrap()
1692 .is_empty());
1693
1694 let now = unix_millis_now();
1695 store
1696 .put_announcement("policy", endpoint("iroh", 1, now + 10_000, now + 5_000))
1697 .await
1698 .unwrap();
1699 let error = store
1700 .put_announcement("other", endpoint("iroh", 1, now + 10_000, now + 5_000))
1701 .await
1702 .unwrap_err();
1703 assert!(error.to_string().contains("capacity exhausted"));
1704 }
1705
1706 #[tokio::test]
1707 async fn memory_satisfies_announcement_backend_contract() {
1708 assert_announcement_backend_contract(&MemoryStateStore::new(1, 1, 1)).await;
1709 }
1710
1711 async fn assert_receipt_ordered_liveness(store: &dyn DiscoveryStateStore) {
1712 let node = Did::new("did:web:clock-rollback.example".to_owned());
1713 let now = unix_millis_now();
1714 let mut value = LiveAllocatable {
1715 allocatable: vec![("cpu".to_owned(), "1".to_owned())],
1716 load_fraction: 0.9,
1717 last_seen: now + 86_400_000,
1719 live_until_unix_ms: now + 100,
1720 };
1721 store.put_liveness(&node, value.clone()).await.unwrap();
1722 value.last_seen = now;
1723 value.load_fraction = 0.2;
1724 value.allocatable[0].1 = "8".to_owned();
1725 value.live_until_unix_ms = now + 500;
1726 assert_eq!(
1727 store.put_liveness(&node, value.clone()).await.unwrap(),
1728 PutResult::Stored
1729 );
1730 assert_eq!(
1731 store.liveness(&node, now + 200).await.unwrap(),
1732 Some(value.clone())
1733 );
1734 let mut delayed = value.clone();
1735 delayed.last_seen = now - 1;
1736 delayed.live_until_unix_ms = now + 300;
1737 delayed.load_fraction = 1.0;
1738 assert_eq!(
1739 store.put_liveness(&node, delayed).await.unwrap(),
1740 PutResult::IgnoredOlder
1741 );
1742 assert_eq!(store.liveness(&node, now + 200).await.unwrap(), Some(value));
1743 assert!(store.liveness(&node, now + 501).await.unwrap().is_none());
1744
1745 store
1747 .put_announcement("clock", endpoint("iroh", 1, now + 50, now + 500))
1748 .await
1749 .unwrap();
1750 assert!(store
1751 .announcements_for("clock", now + 51)
1752 .await
1753 .unwrap()
1754 .is_empty());
1755 }
1756
1757 #[tokio::test]
1758 async fn memory_heartbeat_receipt_survives_clock_rollback() {
1759 assert_receipt_ordered_liveness(&MemoryStateStore::new(8, 8, 8)).await;
1760 }
1761
1762 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1763 #[tokio::test]
1764 async fn valkey_heartbeat_receipt_survives_clock_rollback() {
1765 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
1766 return;
1767 };
1768 use fred::prelude::*;
1769 let config = valkey_config(url, "rollback");
1770 let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
1771 let other = ValkeyStateStore::connect(&config).await.unwrap();
1772 let tiered = TieredStateStore::new(
1773 Arc::new(MemoryStateStore::new(8, 8, 8)),
1774 shared.clone(),
1775 1000,
1776 );
1777 let node = Did::new("did:web:skew.example".to_owned());
1778 let now = unix_millis_now();
1779 let mut value = LiveAllocatable {
1780 allocatable: vec![],
1781 load_fraction: 0.9,
1782 last_seen: now + 86_400_000,
1783 live_until_unix_ms: now + 86_400_400,
1784 };
1785 tiered.put_liveness(&node, value.clone()).await.unwrap();
1786 let first = tiered.liveness(&node, i64::MAX).await.unwrap().unwrap();
1787 assert!((first.last_seen - now).abs() < 5000);
1788 value.last_seen = now - 86_400_000;
1789 value.live_until_unix_ms = value.last_seen + 400;
1790 value.load_fraction = 0.2;
1791 value.allocatable = vec![("cpu".to_owned(), "8".to_owned())];
1792 assert_eq!(
1793 other.put_liveness(&node, value).await.unwrap(),
1794 PutResult::Stored
1795 );
1796 let latest = tiered.liveness(&node, i64::MAX).await.unwrap().unwrap();
1797 assert!(latest.last_seen >= first.last_seen);
1798 assert_eq!(latest.load_fraction, 0.2);
1799 assert_eq!(latest.allocatable[0].1, "8");
1800 assert_eq!(latest.live_until_unix_ms - latest.last_seen, 400);
1801 assert_eq!(tiered.all_liveness(i64::MAX).await.unwrap().len(), 1);
1802 let ttl: i64 = shared
1803 .pool
1804 .pttl(format!(
1805 "{}:liveness:{}",
1806 shared.prefix,
1807 ValkeyStateStore::digest(node.as_str())
1808 ))
1809 .await
1810 .unwrap();
1811 assert!(
1812 (1..=400).contains(&ttl),
1813 "expiry uses shared receipt, not ahead host time"
1814 );
1815 tokio::time::sleep(std::time::Duration::from_millis(450)).await;
1816 assert!(tiered.liveness(&node, i64::MIN).await.unwrap().is_none());
1817 assert!(tiered.all_liveness(i64::MIN).await.unwrap().is_empty());
1818 }
1819
1820 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1821 #[tokio::test]
1822 async fn rediss_uses_tls_and_rejects_plaintext_valkey() {
1823 use fred::prelude::RedisConfig;
1824 hyprstream_rpc::transport::install_pq_crypto_provider().unwrap();
1825 assert!(RedisConfig::from_url("rediss://localhost:6379")
1826 .unwrap()
1827 .tls
1828 .is_some());
1829 assert!(RedisConfig::from_url("redis://localhost:6379")
1830 .unwrap()
1831 .tls
1832 .is_none());
1833 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
1834 return;
1835 };
1836 let mut config = valkey_config(url.clone(), "tls");
1837 let plaintext = ValkeyStateStore::connect(&config).await.unwrap();
1838 assert!(plaintext
1839 .all_announcements(unix_millis_now())
1840 .await
1841 .unwrap()
1842 .is_empty());
1843 config.url = url.replacen("redis://", "rediss://", 1);
1844 let start = std::time::Instant::now();
1845 assert!(
1846 ValkeyStateStore::connect(&config).await.is_err(),
1847 "TLS must not downgrade to plaintext"
1848 );
1849 assert!(start.elapsed() < std::time::Duration::from_secs(3));
1850 }
1851
1852 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1853 #[tokio::test]
1854 async fn valkey_batched_listing_preserves_names_values_and_cleanup() {
1855 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
1856 return;
1857 };
1858 let mut config = valkey_config(url, "batched-list");
1859 config.announcement_capacity = 512;
1860 let writer = ValkeyStateStore::connect(&config).await.unwrap();
1861 let reader = TieredStateStore::new(
1862 Arc::new(MemoryStateStore::new(1, 1, 1)),
1863 Arc::new(ValkeyStateStore::connect(&config).await.unwrap()),
1864 10_000,
1865 );
1866 let now = unix_millis_now();
1867 let cohort_expiry = now + 5_000;
1868 for i in 0..256 {
1869 let name = format!("service:{i}:λ");
1870 for kind in ["iroh", "quic"] {
1871 writer
1872 .put_announcement(&name, endpoint(kind, 1, cohort_expiry, cohort_expiry))
1873 .await
1874 .unwrap();
1875 }
1876 }
1877 assert_eq!(
1878 reader
1879 .announcements_for("service:0:λ", now)
1880 .await
1881 .unwrap()
1882 .len(),
1883 2
1884 );
1885 let entries = reader.all_announcements(now).await.unwrap();
1886 assert_eq!(entries.len(), 256);
1887 assert!(entries
1888 .iter()
1889 .all(|(name, values)| name.starts_with("service:") && values.len() == 2));
1890 writer
1891 .put_announcement(
1892 "service:0:λ",
1893 endpoint("iroh", 2, now + 90_000, now + 90_000),
1894 )
1895 .await
1896 .unwrap();
1897 assert_eq!(
1901 reader.all_announcements(now + 60_001).await.unwrap().len(),
1902 256
1903 );
1904 tokio::time::sleep(std::time::Duration::from_millis(
1905 (cohort_expiry + 50 - unix_millis_now()).max(0) as u64,
1906 ))
1907 .await;
1908 let remaining = reader.all_announcements(now + 60_001).await.unwrap();
1909 assert_eq!(remaining.len(), 1);
1910 assert_eq!(remaining[0].0, "service:0:λ");
1911 assert_eq!(remaining[0].1.len(), 1);
1912 assert_eq!(remaining[0].1[0].accepted_state_epoch, 2);
1913 writer
1915 .put_announcement("new", endpoint("iroh", 1, now + 90_000, now + 90_000))
1916 .await
1917 .unwrap();
1918 assert_eq!(
1919 reader
1920 .announcements_for("service:0:λ", now + 60_001)
1921 .await
1922 .unwrap()[0]
1923 .accepted_state_epoch,
1924 2
1925 );
1926 assert_eq!(
1927 reader.all_announcements(now + 60_001).await.unwrap().len(),
1928 2
1929 );
1930 }
1931
1932 #[tokio::test]
1933 async fn memory_replaces_by_key_without_fleet_scan() {
1934 let store = MemoryStateStore::new(8, 8, 8);
1935 let now = unix_millis_now();
1936 store
1937 .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 1_000))
1938 .await
1939 .unwrap();
1940 store
1941 .put_announcement("model", endpoint("iroh", 2, now + 20_000, now + 2_000))
1942 .await
1943 .unwrap();
1944 let entries = store.announcements_for("model", now).await.unwrap();
1945 assert_eq!(entries.len(), 1);
1946 assert_eq!(entries[0].accepted_state_epoch, 2);
1947 }
1948
1949 #[tokio::test]
1950 async fn memory_rejects_delayed_older_announcement() {
1951 let store = MemoryStateStore::new(8, 8, 8);
1952 let now = unix_millis_now();
1953 store
1954 .put_announcement("model", endpoint("iroh", 3, now + 20_000, now + 2_000))
1955 .await
1956 .unwrap();
1957 assert_eq!(
1958 store
1959 .put_announcement("model", endpoint("iroh", 2, now + 30_000, now + 3_000))
1960 .await
1961 .unwrap(),
1962 PutResult::IgnoredOlder
1963 );
1964 assert_eq!(
1965 store.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
1966 3
1967 );
1968 }
1969
1970 #[tokio::test]
1971 async fn memory_expiry_removes_secondary_index_entry() {
1972 let store = MemoryStateStore::new(8, 8, 8);
1973 let now = unix_millis_now();
1974 store
1975 .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 1))
1976 .await
1977 .unwrap();
1978 assert!(store.all_announcements(now + 2).await.unwrap().is_empty());
1979 }
1980
1981 #[tokio::test]
1982 async fn memory_capacity_rejects_instead_of_evicting_live_reach() {
1983 let store = MemoryStateStore::new(1, 1, 1);
1984 let now = unix_millis_now();
1985 store
1986 .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 1_000))
1987 .await
1988 .unwrap();
1989 let error = store
1990 .put_announcement("policy", endpoint("iroh", 1, now + 10_000, now + 1_000))
1991 .await
1992 .unwrap_err();
1993 assert!(error.to_string().contains("capacity exhausted"));
1994 }
1995
1996 #[tokio::test]
1997 async fn active_active_rejects_isolated_memory() {
1998 let config = DiscoveryStateConfig {
1999 active_active: true,
2000 ..DiscoveryStateConfig::default()
2001 };
2002 let error = match DiscoveryState::connect(&config).await {
2003 Ok(_) => panic!("active-active memory configuration was admitted"),
2004 Err(error) => error,
2005 };
2006 assert!(error.to_string().contains("requires valkey or tiered"));
2007 }
2008
2009 #[tokio::test]
2010 async fn rejects_zero_memory_capacity_and_tiered_freshness() {
2011 let mut memory = DiscoveryStateConfig::default();
2012 memory.memory.announcement_capacity = 0;
2013 let error = match DiscoveryState::connect(&memory).await {
2014 Ok(_) => panic!("zero-capacity memory configuration was admitted"),
2015 Err(error) => error,
2016 };
2017 assert!(error.to_string().contains("capacities must be positive"));
2018
2019 let mut tiered = DiscoveryStateConfig {
2020 backend: DiscoveryStateBackend::Tiered,
2021 ..DiscoveryStateConfig::default()
2022 };
2023 tiered.tiered.l1_max_ttl_ms = 0;
2024 let error = match DiscoveryState::connect(&tiered).await {
2025 Ok(_) => panic!("zero-freshness tiered configuration was admitted"),
2026 Err(error) => error,
2027 };
2028 assert!(error.to_string().contains("l1_max_ttl_ms must be positive"));
2029 }
2030
2031 async fn assert_issuer_eviction_contract(store: &dyn DiscoveryStateStore) {
2032 for (issuer, fetched_at) in [
2033 ("old", 1),
2034 ("refresh", 2),
2035 ("keep", 3),
2036 ("refresh", 4),
2037 ("new", 5),
2038 ] {
2039 store
2040 .put_entity_statement(
2041 issuer,
2042 CachedEntityStatement {
2043 jwt: format!("inert:{issuer}:{fetched_at}"),
2044 fetched_at,
2045 },
2046 )
2047 .await
2048 .unwrap();
2049 }
2050 assert_eq!(store.known_issuer_count().await.unwrap(), 3);
2051 let mut names = store.known_issuers().await.unwrap();
2052 names.sort();
2053 assert_eq!(names, ["keep", "new", "refresh"]);
2054 assert!(store.entity_statement("old").await.unwrap().is_none());
2055 assert_eq!(
2056 store
2057 .entity_statement("refresh")
2058 .await
2059 .unwrap()
2060 .unwrap()
2061 .fetched_at,
2062 4
2063 );
2064 store
2065 .put_entity_statement(
2066 "old",
2067 CachedEntityStatement {
2068 jwt: "reinserted".to_owned(),
2069 fetched_at: 6,
2070 },
2071 )
2072 .await
2073 .unwrap();
2074 assert!(store.entity_statement("keep").await.unwrap().is_none());
2075 assert_eq!(
2076 store.entity_statement("old").await.unwrap().unwrap().jwt,
2077 "reinserted"
2078 );
2079 }
2080
2081 #[tokio::test]
2082 async fn memory_issuer_cache_evicts_oldest_and_accepts_reinsertion() {
2083 assert_issuer_eviction_contract(&MemoryStateStore::new(1, 1, 3)).await;
2084 }
2085
2086 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2087 #[tokio::test]
2088 async fn valkey_issuer_cache_churn_bounds_metadata_and_invalidates_other_l1() {
2089 use fred::prelude::*;
2090 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2091 return;
2092 };
2093 let mut config = valkey_config(url, "issuer-churn");
2094 config.artifact_capacity = 3;
2095 let writer = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2096 assert_issuer_eviction_contract(writer.as_ref()).await;
2097 let reader = TieredStateStore::new(
2098 Arc::new(MemoryStateStore::new(1, 1, 1)),
2099 Arc::new(ValkeyStateStore::connect(&config).await.unwrap()),
2100 60_000,
2101 );
2102 for issuer in ["old", "refresh", "old"] {
2105 assert!(reader.entity_statement(issuer).await.unwrap().is_some());
2106 }
2107 let generation = writer.entity_revision().await.unwrap();
2108 for i in 0..96 {
2109 writer
2110 .put_entity_statement(
2111 &format!("issuer-{i}"),
2112 CachedEntityStatement {
2113 jwt: format!("inert-{i}"),
2114 fetched_at: 100 + i,
2115 },
2116 )
2117 .await
2118 .unwrap();
2119 assert_eq!(writer.known_issuer_count().await.unwrap(), 3);
2120 }
2121 assert!(reader.entity_statement("old").await.unwrap().is_none());
2122 let counts: Vec<usize> = writer.pool.eval(
2123 "return {redis.call('SCARD', KEYS[1]), redis.call('HLEN', KEYS[2]), redis.call('ZCARD', KEYS[3]), #redis.call('KEYS', ARGV[1]), #redis.call('KEYS', ARGV[2])}",
2124 vec![writer.key("issuers"), writer.key("issuer-names"), writer.key("issuer-fetched")],
2125 vec![writer.key("*"), writer.key("*revision*")],
2126 ).await.unwrap();
2127 assert_eq!(counts, [3, 3, 3, 7, 1]);
2128 assert!(writer.entity_revision().await.unwrap() > generation);
2129 writer
2130 .put_entity_statement(
2131 "old",
2132 CachedEntityStatement {
2133 jwt: "new-generation".to_owned(),
2134 fetched_at: 1000,
2135 },
2136 )
2137 .await
2138 .unwrap();
2139 assert_eq!(
2140 reader.entity_statement("old").await.unwrap().unwrap().jwt,
2141 "new-generation"
2142 );
2143 for i in 0..3 {
2145 writer
2146 .put_entity_statement(
2147 &format!("replacement-{i}"),
2148 CachedEntityStatement {
2149 jwt: "other".to_owned(),
2150 fetched_at: 2000 + i,
2151 },
2152 )
2153 .await
2154 .unwrap();
2155 }
2156 writer
2157 .put_entity_statement(
2158 "old",
2159 CachedEntityStatement {
2160 jwt: "after-aba".to_owned(),
2161 fetched_at: 3000,
2162 },
2163 )
2164 .await
2165 .unwrap();
2166 assert_eq!(
2167 reader.entity_statement("old").await.unwrap().unwrap().jwt,
2168 "after-aba"
2169 );
2170 assert!(reader.observed.lock().len() <= 4);
2171 }
2172
2173 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2174 #[tokio::test]
2175 async fn valkey_issuer_listing_and_count_return_complete_bounded_cache() {
2176 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2177 return;
2178 };
2179 let mut config = valkey_config(url, "issuer-list-count");
2180 config.artifact_capacity = 128;
2181 let writer = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2182 let reader = TieredStateStore::new(
2183 Arc::new(MemoryStateStore::new(1, 1, 1)),
2184 writer.clone(),
2185 60_000,
2186 );
2187 let mut expected = Vec::new();
2188 for i in 0..256 {
2189 let name = format!("https://issuer:{i}:λ.example");
2190 writer
2191 .put_entity_statement(
2192 &name,
2193 CachedEntityStatement {
2194 jwt: "inert".to_owned(),
2195 fetched_at: i,
2196 },
2197 )
2198 .await
2199 .unwrap();
2200 if i >= 128 {
2201 expected.push(name);
2202 }
2203 }
2204 expected.sort();
2205 for _ in 0..2 {
2206 let mut actual = reader.known_issuers().await.unwrap();
2207 actual.sort();
2208 assert_eq!(actual, expected);
2209 assert_eq!(reader.known_issuer_count().await.unwrap(), 128);
2210 }
2211 assert!(reader.memory.inner.lock().entity_statements.is_empty());
2212 assert!(reader.observed.lock().is_empty());
2213 }
2214
2215 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2216 fn valkey_config(url: String, suffix: &str) -> ValkeyStateConfig {
2217 ValkeyStateConfig {
2218 url,
2219 key_prefix: format!(
2220 "hs-test-{}-{}-{suffix}",
2221 std::process::id(),
2222 unix_millis_now()
2223 ),
2224 pool_size: 2,
2225 announcement_capacity: 64,
2226 liveness_capacity: 64,
2227 artifact_capacity: 64,
2228 command_timeout_ms: 250,
2229 }
2230 }
2231
2232 pub(crate) struct ReplicaClock;
2233
2234 impl ReplicaClock {
2235 pub(crate) fn at(now: i64) -> Self {
2236 TEST_REPLICA_TIME.with(|clock| {
2237 assert!(clock.replace(Some(now)).is_none());
2238 });
2239 Self
2240 }
2241 }
2242
2243 impl Drop for ReplicaClock {
2244 fn drop(&mut self) {
2245 TEST_REPLICA_TIME.with(|clock| clock.set(None));
2246 }
2247 }
2248
2249 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2250 #[tokio::test]
2251 async fn valkey_announcement_lease_uses_receipt_and_preserves_payload() {
2252 use fred::prelude::*;
2253 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2254 return;
2255 };
2256 let store = ValkeyStateStore::connect(&valkey_config(url, "announcement-write-lease"))
2257 .await
2258 .unwrap();
2259 let now: i64 = store.pool.eval(
2260 "local t = redis.call('TIME'); return tonumber(t[1])*1000 + math.floor(tonumber(t[2])/1000)",
2261 Vec::<String>::new(), Vec::<String>::new(),
2262 ).await.unwrap();
2263 for skew in [-3_600_000, 3_600_000] {
2264 let name = format!("lease-{skew}");
2265 let mut value = endpoint("iroh", 1, now + 7_200_000, now + 7_200_000);
2266 value.capabilities.clear();
2267 value.response_key_id = "payload with \"live_until_unix_ms\":123 and λ".to_owned();
2268 value.source_signer = [255; 32];
2269 let mut expected = serde_json::to_value(&value).unwrap();
2270 expected
2271 .as_object_mut()
2272 .unwrap()
2273 .remove("live_until_unix_ms");
2274 let _clock = ReplicaClock::at(now + skew);
2275 assert_eq!(
2276 store.put_announcement(&name, value).await.unwrap(),
2277 PutResult::Stored
2278 );
2279 let rows: (String, i64, i64, i64) = store.pool.eval(
2280 "local t = redis.call('TIME'); return {redis.call('GET', KEYS[1]), redis.call('PEXPIRETIME', KEYS[1]), tonumber(redis.call('ZSCORE', KEYS[2], KEYS[1])), tonumber(t[1])*1000 + math.floor(tonumber(t[2])/1000)}",
2281 vec![store.announcement_key(&name, "iroh"), store.key("announcement-expiry")], Vec::<String>::new(),
2282 ).await.unwrap();
2283 let mut actual: serde_json::Value = serde_json::from_str(&rows.0).unwrap();
2284 let encoded_expiry = actual
2285 .as_object_mut()
2286 .unwrap()
2287 .remove("live_until_unix_ms")
2288 .unwrap()
2289 .as_i64()
2290 .unwrap();
2291 assert_eq!(
2292 actual, expected,
2293 "only the effective lease field may change"
2294 );
2295 assert_eq!(encoded_expiry, rows.1);
2296 assert_eq!(encoded_expiry, rows.2);
2297 assert!(
2298 (89_000..=90_000).contains(&(encoded_expiry - rows.3)),
2299 "bounded shared receipt lease, independent of replica skew"
2300 );
2301 assert_eq!(
2302 store
2303 .announcements_for(&name, now + skew)
2304 .await
2305 .unwrap()
2306 .len(),
2307 1
2308 );
2309 }
2310 }
2311
2312 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2313 #[tokio::test]
2314 async fn valkey_announcement_reaping_ignores_replica_clock_skew() {
2315 use fred::prelude::*;
2316
2317 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2318 return;
2319 };
2320 for operation in ["put", "point", "list", "tiered"] {
2324 let config = valkey_config(url.clone(), operation);
2325 let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2326 let replica = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2327 let tiered = TieredStateStore::new(
2328 Arc::new(MemoryStateStore::new(8, 8, 8)),
2329 replica.clone(),
2330 30_000,
2331 );
2332 let now: i64 = shared.pool.eval(
2333 "local t = redis.call('TIME'); return tonumber(t[1])*1000 + math.floor(tonumber(t[2])/1000)",
2334 Vec::<String>::new(), Vec::<String>::new(),
2335 ).await.unwrap();
2336 shared
2337 .put_announcement("anchor", endpoint("iroh", 1, now + 60_000, now + 30_000))
2338 .await
2339 .unwrap();
2340 if operation == "tiered" {
2341 assert_eq!(
2342 tiered.announcements_for("anchor", now).await.unwrap().len(),
2343 1
2344 );
2345 }
2346 let revision = shared.announcement_revision().await.unwrap();
2347 let ahead = now + 3_600_000;
2348 {
2349 let _clock = ReplicaClock::at(ahead);
2350 match operation {
2351 "put" => {
2352 replica
2353 .put_announcement(
2354 "writer",
2355 endpoint("iroh", 1, ahead + 60_000, ahead + 30_000),
2356 )
2357 .await
2358 .unwrap();
2359 }
2360 "point" => assert_eq!(
2361 replica
2362 .announcements_for("anchor", ahead)
2363 .await
2364 .unwrap()
2365 .len(),
2366 1
2367 ),
2368 "list" => assert_eq!(replica.all_announcements(ahead).await.unwrap().len(), 1),
2369 "tiered" => assert_eq!(
2370 tiered
2371 .announcements_for("anchor", ahead)
2372 .await
2373 .unwrap()
2374 .len(),
2375 1
2376 ),
2377 _ => unreachable!(),
2378 }
2379 }
2380 assert_eq!(
2381 shared.announcements_for("anchor", now).await.unwrap().len(),
2382 1,
2383 "{operation} deleted another replica's live announcement"
2384 );
2385 let exists: bool = shared
2386 .pool
2387 .exists(shared.announcement_key("anchor", "iroh"))
2388 .await
2389 .unwrap();
2390 assert!(exists);
2391 assert_eq!(
2392 shared.announcement_revision().await.unwrap(),
2393 revision + u64::from(operation == "put")
2394 );
2395 }
2396 }
2397
2398 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2399 #[tokio::test]
2400 async fn valkey_announcement_shared_expiry_rejects_behind_replica_revival() {
2401 use fred::prelude::*;
2402
2403 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2404 return;
2405 };
2406 let config = valkey_config(url, "announcement-expiry-clock");
2407 let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2408 let tiered = TieredStateStore::new(
2409 Arc::new(MemoryStateStore::new(8, 8, 8)),
2410 shared.clone(),
2411 30_000,
2412 );
2413 let now: i64 = shared.pool.eval(
2414 "local t = redis.call('TIME'); return tonumber(t[1])*1000 + math.floor(tonumber(t[2])/1000)",
2415 Vec::<String>::new(), Vec::<String>::new(),
2416 ).await.unwrap();
2417 let expiry = now + 300;
2420 for (name, signed, live) in [
2421 ("signed", expiry, now + 60_000),
2422 ("lease", now + 60_000, expiry),
2423 ] {
2424 shared
2425 .put_announcement(name, endpoint("iroh", 1, signed, live))
2426 .await
2427 .unwrap();
2428 let pxat: i64 = shared
2429 .pool
2430 .eval(
2431 "return redis.call('PEXPIRETIME', KEYS[1])",
2432 vec![shared.announcement_key(name, "iroh")],
2433 Vec::<String>::new(),
2434 )
2435 .await
2436 .unwrap();
2437 assert_eq!(
2438 pxat, expiry,
2439 "preserve the absolute authority/lease ceiling"
2440 );
2441 assert_eq!(tiered.announcements_for(name, now).await.unwrap().len(), 1);
2442 }
2443 let revision = shared.announcement_revision().await.unwrap();
2444 tokio::time::sleep(std::time::Duration::from_millis(400)).await;
2445 let behind = now - 3_600_000;
2446 let _clock = ReplicaClock::at(behind);
2447 for name in ["signed", "lease"] {
2448 assert!(tiered
2451 .announcements_for(name, behind)
2452 .await
2453 .unwrap()
2454 .is_empty());
2455 assert!(shared
2456 .announcements_for(name, behind)
2457 .await
2458 .unwrap()
2459 .is_empty());
2460 assert_eq!(
2461 shared
2462 .put_announcement(name, endpoint("iroh", 2, expiry, now + 60_000))
2463 .await
2464 .unwrap(),
2465 PutResult::IgnoredOlder
2466 );
2467 }
2468 assert!(shared.all_announcements(behind).await.unwrap().is_empty());
2469 assert!(shared.announcement_revision().await.unwrap() > revision);
2470 let metadata: Vec<usize> = shared.pool.eval(
2471 "return {redis.call('ZCARD', KEYS[1]), redis.call('SCARD', KEYS[2]), redis.call('HLEN', KEYS[3])}",
2472 vec![shared.key("announcement-expiry"), shared.key("services"), shared.key("service-names")], Vec::<String>::new(),
2473 ).await.unwrap();
2474 assert_eq!(
2475 metadata,
2476 vec![0, 0, 0],
2477 "expired writes must not resurrect secondary indexes"
2478 );
2479 }
2480
2481 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2482 #[tokio::test]
2483 async fn valkey_satisfies_announcement_backend_contract() {
2484 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2485 return;
2486 };
2487 let mut config = valkey_config(url, "contract");
2488 config.announcement_capacity = 1;
2489 let store = ValkeyStateStore::connect(&config).await.unwrap();
2490 assert_announcement_backend_contract(&store).await;
2491 }
2492
2493 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2494 #[tokio::test]
2495 async fn valkey_two_replicas_share_all_discovery_state() {
2496 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2497 return;
2498 };
2499 let config = valkey_config(url, "shared");
2500 let replica_a = ValkeyStateStore::connect(&config).await.unwrap();
2501 let replica_b = ValkeyStateStore::connect(&config).await.unwrap();
2502 let now = unix_millis_now();
2503
2504 replica_a
2505 .put_announcement("model", endpoint("iroh", 1, now + 30_000, now + 20_000))
2506 .await
2507 .unwrap();
2508 assert_eq!(
2509 replica_b
2510 .announcements_for("model", now)
2511 .await
2512 .unwrap()
2513 .len(),
2514 1
2515 );
2516
2517 replica_b
2518 .put_liveness(
2519 &Did::new("did:web:node.example".to_owned()),
2520 LiveAllocatable {
2521 allocatable: vec![("cpu".to_owned(), "8".to_owned())],
2522 load_fraction: 0.25,
2523 last_seen: now,
2524 live_until_unix_ms: now + 20_000,
2525 },
2526 )
2527 .await
2528 .unwrap();
2529 assert!(replica_a
2530 .liveness(&Did::new("did:web:node.example".to_owned()), now)
2531 .await
2532 .unwrap()
2533 .is_some());
2534
2535 replica_a
2536 .put_entity_statement(
2537 "https://issuer.example",
2538 CachedEntityStatement {
2539 jwt: "statement".to_owned(),
2540 fetched_at: now / 1_000,
2541 },
2542 )
2543 .await
2544 .unwrap();
2545 assert_eq!(
2546 replica_b
2547 .entity_statement("https://issuer.example")
2548 .await
2549 .unwrap()
2550 .unwrap()
2551 .jwt,
2552 "statement"
2553 );
2554
2555 replica_b
2556 .put_envelope_keyset(
2557 "did:web:service.example",
2558 CachedEnvelopeKeyset {
2559 cose_keyset_cbor: vec![1, 2, 3],
2560 fetched_at: now / 1_000,
2561 },
2562 )
2563 .await
2564 .unwrap();
2565 assert_eq!(
2566 replica_a
2567 .envelope_keyset("did:web:service.example")
2568 .await
2569 .unwrap()
2570 .unwrap()
2571 .cose_keyset_cbor,
2572 vec![1, 2, 3]
2573 );
2574
2575 let restarted = ValkeyStateStore::connect(&config).await.unwrap();
2578 assert_eq!(
2579 restarted
2580 .announcements_for("model", now)
2581 .await
2582 .unwrap()
2583 .len(),
2584 1
2585 );
2586 }
2587
2588 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2589 #[tokio::test]
2590 async fn valkey_rejects_delayed_old_update_and_expires_server_side() {
2591 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2592 return;
2593 };
2594 let config = valkey_config(url, "ordering");
2595 let store = ValkeyStateStore::connect(&config).await.unwrap();
2596 let now = unix_millis_now();
2597 store
2598 .put_announcement("model", endpoint("iroh", 4, now + 30_000, now + 20_000))
2599 .await
2600 .unwrap();
2601 assert_eq!(
2602 store
2603 .put_announcement("model", endpoint("iroh", 3, now + 40_000, now + 20_000))
2604 .await
2605 .unwrap(),
2606 PutResult::IgnoredOlder
2607 );
2608 assert_eq!(
2609 store.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
2610 4
2611 );
2612
2613 store
2614 .put_announcement("short", endpoint("iroh", 1, now + 10_000, now + 50))
2615 .await
2616 .unwrap();
2617 tokio::time::sleep(std::time::Duration::from_millis(80)).await;
2618 assert!(store
2619 .announcements_for("short", unix_millis_now())
2620 .await
2621 .unwrap()
2622 .is_empty());
2623 }
2624
2625 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2626 #[tokio::test]
2627 async fn tiered_revision_check_invalidates_other_replica_l1() {
2628 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2629 return;
2630 };
2631 let config = valkey_config(url, "tiered");
2632 let shared_a = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2633 let shared_b = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2634 let tier_a =
2635 TieredStateStore::new(Arc::new(MemoryStateStore::new(8, 8, 8)), shared_a, 10_000);
2636 let tier_b =
2637 TieredStateStore::new(Arc::new(MemoryStateStore::new(8, 8, 8)), shared_b, 10_000);
2638 let now = unix_millis_now();
2639 tier_a
2640 .put_announcement("model", endpoint("iroh", 1, now + 30_000, now + 20_000))
2641 .await
2642 .unwrap();
2643 assert_eq!(
2644 tier_b.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
2645 1
2646 );
2647 tier_a
2648 .put_announcement("model", endpoint("iroh", 2, now + 40_000, now + 20_000))
2649 .await
2650 .unwrap();
2651 assert_eq!(
2652 tier_b.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
2653 2,
2654 "L2 revision change must invalidate replica B's populated L1"
2655 );
2656 }
2657
2658 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2659 #[tokio::test]
2660 async fn tiered_does_not_resurrect_expired_l2_data() {
2661 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2662 return;
2663 };
2664 let config = valkey_config(url, "tiered-expiry");
2665 let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2666 let tiered =
2667 TieredStateStore::new(Arc::new(MemoryStateStore::new(8, 8, 8)), shared, 10_000);
2668 let now = unix_millis_now();
2669 tiered
2670 .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 50))
2671 .await
2672 .unwrap();
2673 assert_eq!(
2674 tiered.announcements_for("model", now).await.unwrap().len(),
2675 1
2676 );
2677
2678 tokio::time::sleep(std::time::Duration::from_millis(80)).await;
2679 assert!(tiered
2680 .announcements_for("model", unix_millis_now())
2681 .await
2682 .unwrap()
2683 .is_empty());
2684 }
2685
2686 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2687 #[tokio::test]
2688 async fn tiered_fails_closed_after_l2_client_shutdown() {
2689 use fred::prelude::ClientLike as _;
2690
2691 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2692 return;
2693 };
2694 let config = valkey_config(url, "tiered-outage");
2695 let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2696 let tiered = TieredStateStore::new(
2697 Arc::new(MemoryStateStore::new(8, 8, 8)),
2698 Arc::clone(&shared),
2699 10_000,
2700 );
2701 let now = unix_millis_now();
2702 tiered
2703 .put_announcement("model", endpoint("iroh", 1, now + 30_000, now + 20_000))
2704 .await
2705 .unwrap();
2706 assert_eq!(
2707 tiered.announcements_for("model", now).await.unwrap().len(),
2708 1
2709 );
2710
2711 shared.pool.quit().await.unwrap();
2712 let result = tokio::time::timeout(
2713 std::time::Duration::from_secs(2),
2714 tiered.announcements_for("model", unix_millis_now()),
2715 )
2716 .await
2717 .expect("configured command timeout must bound an L2 outage");
2718 assert!(
2719 result.is_err(),
2720 "tiered state must not return its populated L1"
2721 );
2722 }
2723 #[tokio::test]
2724 async fn memory_liveness_enumeration_excludes_expiry_and_remains_bounded() {
2725 let store = MemoryStateStore::new(1, 1, 1);
2726 let node = Did::new("did:web:live.example".to_owned());
2727 let now = unix_millis_now();
2728 for i in 0..100 {
2729 store
2730 .put_liveness(
2731 &node,
2732 LiveAllocatable {
2733 allocatable: vec![],
2734 load_fraction: 0.1,
2735 last_seen: now + i,
2736 live_until_unix_ms: now + 30_000,
2737 },
2738 )
2739 .await
2740 .unwrap();
2741 }
2742 assert_eq!(store.all_liveness(now).await.unwrap()[0].0, node);
2743 assert!(store.inner.lock().expiry.len() <= 2);
2744 assert!(store.all_liveness(now + 30_000).await.unwrap().is_empty());
2745 }
2746
2747 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2748 #[tokio::test]
2749 async fn valkey_driver_survives_temporary_bootstrap_runtime() {
2750 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2751 return;
2752 };
2753 for backend in [DiscoveryStateBackend::Valkey, DiscoveryStateBackend::Tiered] {
2754 let config = DiscoveryStateConfig {
2755 backend,
2756 active_active: true,
2757 valkey: valkey_config(url.clone(), &format!("bootstrap-{backend:?}")),
2758 ..DiscoveryStateConfig::default()
2759 };
2760 let state = std::thread::spawn(move || {
2763 tokio::runtime::Builder::new_current_thread()
2764 .enable_all()
2765 .build()
2766 .unwrap()
2767 .block_on(DiscoveryState::connect(&config))
2768 .unwrap()
2769 })
2770 .join()
2771 .unwrap();
2772 let store = state.into_inner();
2773 let now = unix_millis_now();
2774 store
2775 .put_announcement("model", endpoint("iroh", 1, now + 30_000, now + 20_000))
2776 .await
2777 .unwrap();
2778 assert_eq!(
2779 store.announcements_for("model", now).await.unwrap().len(),
2780 1
2781 );
2782 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2783 assert_eq!(
2784 store
2785 .all_announcements(unix_millis_now())
2786 .await
2787 .unwrap()
2788 .len(),
2789 1
2790 );
2791 }
2792 }
2793
2794 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2795 #[tokio::test]
2796 async fn tiered_scoped_operations_allow_unrelated_liveness_and_heartbeats_to_progress() {
2797 use futures::{stream, StreamExt};
2798 use std::time::Duration;
2799 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2800 return;
2801 };
2802 let shared = Arc::new(
2803 ValkeyStateStore::connect(&valkey_config(url, "striped-concurrency"))
2804 .await
2805 .unwrap(),
2806 );
2807 let tier = TieredStateStore::new(
2808 Arc::new(MemoryStateStore::new(64, 64, 64)),
2809 shared.clone(),
2810 60_000,
2811 );
2812 let now = unix_millis_now();
2813 let mut nodes: Vec<Did> = Vec::new();
2816 for i in 0..1024 {
2817 let node = Did::new(format!("did:web:concurrent-{i}.example"));
2818 if nodes.iter().all(|other| {
2819 !std::ptr::eq(
2820 tier.operation("liveness", other.as_str()),
2821 tier.operation("liveness", node.as_str()),
2822 )
2823 }) {
2824 nodes.push(node);
2825 }
2826 if nodes.len() == 16 {
2827 break;
2828 }
2829 }
2830 assert_eq!(nodes.len(), 16);
2831 for node in &nodes {
2832 shared
2833 .put_liveness(
2834 node,
2835 LiveAllocatable {
2836 allocatable: vec![],
2837 load_fraction: 0.1,
2838 last_seen: now,
2839 live_until_unix_ms: now + 60_000,
2840 },
2841 )
2842 .await
2843 .unwrap();
2844 }
2845 let stalled = tier.operation("liveness", nodes[0].as_str()).lock().await;
2849 let mut reads = stream::iter(nodes.iter().map(|node| {
2850 let tier = &tier;
2851 async move { (node, tier.liveness(node, now).await) }
2852 }))
2853 .buffer_unordered(16);
2854 tokio::time::timeout(Duration::from_secs(2), async {
2855 for _ in 0..15 {
2856 let (node, result) = reads.next().await.unwrap();
2857 assert_ne!(node, &nodes[0]);
2858 assert_eq!(result.unwrap().unwrap().load_fraction, 0.1);
2859 }
2860 })
2861 .await
2862 .expect("unrelated tiered liveness reads serialized behind a stalled scope");
2863 assert!(
2864 tokio::time::timeout(Duration::from_millis(20), reads.next())
2865 .await
2866 .is_err()
2867 );
2868
2869 let heartbeat = LiveAllocatable {
2870 allocatable: vec![],
2871 load_fraction: 0.8,
2872 last_seen: now + 1,
2873 live_until_unix_ms: now + 60_001,
2874 };
2875 tokio::time::timeout(
2876 Duration::from_secs(2),
2877 tier.put_liveness(&nodes[1], heartbeat.clone()),
2878 )
2879 .await
2880 .expect("unrelated heartbeat serialized behind a stalled scope")
2881 .unwrap();
2882 assert!(tokio::time::timeout(
2885 Duration::from_millis(20),
2886 tier.put_liveness(&nodes[0], heartbeat.clone())
2887 )
2888 .await
2889 .is_err());
2890 drop(reads);
2891 drop(stalled);
2892 assert_eq!(
2893 tier.liveness(&nodes[0], now)
2894 .await
2895 .unwrap()
2896 .unwrap()
2897 .load_fraction,
2898 0.1
2899 );
2900 tier.put_liveness(&nodes[0], heartbeat).await.unwrap();
2901 assert_eq!(
2902 tier.liveness(&nodes[0], now)
2903 .await
2904 .unwrap()
2905 .unwrap()
2906 .load_fraction,
2907 0.8
2908 );
2909 assert_eq!(
2910 tier.liveness(&nodes[1], now)
2911 .await
2912 .unwrap()
2913 .unwrap()
2914 .load_fraction,
2915 0.8
2916 );
2917 assert_eq!(tier.operations.len(), 64);
2918 }
2919
2920 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2921 #[tokio::test]
2922 async fn tiered_capacity_never_limits_authoritative_results_or_writes() {
2923 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2924 return;
2925 };
2926 let shared = Arc::new(
2927 ValkeyStateStore::connect(&valkey_config(url, "capacity"))
2928 .await
2929 .unwrap(),
2930 );
2931 let memory = Arc::new(MemoryStateStore::new(1, 1, 1));
2932 let tier = TieredStateStore::new(memory.clone(), shared, 10_000);
2933 let now = unix_millis_now();
2934 for name in ["a", "b", "c"] {
2935 tier.put_announcement(name, endpoint("iroh", 1, now + 30_000, now + 20_000))
2936 .await
2937 .unwrap();
2938 assert_eq!(tier.announcements_for(name, now).await.unwrap().len(), 1);
2939 let node = Did::new(format!("did:web:{name}.example"));
2940 tier.put_liveness(
2941 &node,
2942 LiveAllocatable {
2943 allocatable: vec![],
2944 load_fraction: 0.1,
2945 last_seen: now,
2946 live_until_unix_ms: now + 20_000,
2947 },
2948 )
2949 .await
2950 .unwrap();
2951 assert!(tier.liveness(&node, now).await.unwrap().is_some());
2952 tier.put_entity_statement(
2953 name,
2954 CachedEntityStatement {
2955 jwt: name.to_owned(),
2956 fetched_at: now,
2957 },
2958 )
2959 .await
2960 .unwrap();
2961 assert!(tier.entity_statement(name).await.unwrap().is_some());
2962 tier.put_envelope_keyset(
2963 name,
2964 CachedEnvelopeKeyset {
2965 cose_keyset_cbor: vec![1],
2966 fetched_at: now,
2967 },
2968 )
2969 .await
2970 .unwrap();
2971 assert!(tier.envelope_keyset(name).await.unwrap().is_some());
2972 }
2973 for _ in 0..2 {
2974 assert_eq!(tier.all_announcements(now).await.unwrap().len(), 3);
2975 assert_eq!(tier.all_liveness(now).await.unwrap().len(), 3);
2976 assert_eq!(tier.known_issuers().await.unwrap().len(), 3);
2977 }
2978 tier.put_announcement("a", endpoint("quic", 1, now + 30_000, now + 20_000))
2981 .await
2982 .unwrap();
2983 for i in 0..50 {
2984 assert_eq!(tier.announcements_for("a", now).await.unwrap().len(), 2);
2985 let absent = format!("missing-{i}");
2986 assert!(tier
2987 .announcements_for(&absent, now)
2988 .await
2989 .unwrap()
2990 .is_empty());
2991 assert!(tier.entity_statement(&absent).await.unwrap().is_none());
2992 assert!(tier.envelope_keyset(&absent).await.unwrap().is_none());
2993 assert!(tier
2994 .liveness(&Did::new(absent), now)
2995 .await
2996 .unwrap()
2997 .is_none());
2998 }
2999 let cache = memory.inner.lock();
3000 assert!(cache.announcements.len() <= 1);
3001 assert!(cache.liveness.len() <= 1);
3002 assert!(cache.entity_statements.len() <= 1);
3003 assert!(cache.envelope_keysets.len() <= 1);
3004 assert!(cache.expiry.len() <= 4);
3005 assert!(tier.observed.lock().len() <= 4);
3006 }
3007
3008 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
3009 #[tokio::test]
3010 async fn valkey_expired_identity_churn_bounds_metadata_and_listing_work() {
3011 use fred::prelude::*;
3012
3013 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
3014 return;
3015 };
3016 let mut config = valkey_config(url, "metadata-churn");
3017 config.announcement_capacity = 4;
3018 config.liveness_capacity = 4;
3019 let store = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
3020 let tier = TieredStateStore::new(
3021 Arc::new(MemoryStateStore::new(4, 4, 4)),
3022 store.clone(),
3023 30_000,
3024 );
3025 let now = unix_millis_now();
3026 let anchor = Did::new("did:web:anchor.example".to_owned());
3027 store
3028 .put_announcement("anchor", endpoint("iroh", 1, now + 30_000, now + 30_000))
3029 .await
3030 .unwrap();
3031 store
3032 .put_liveness(
3033 &anchor,
3034 LiveAllocatable {
3035 allocatable: vec![],
3036 load_fraction: 0.1,
3037 last_seen: now,
3038 live_until_unix_ms: now + 30_000,
3039 },
3040 )
3041 .await
3042 .unwrap();
3043 tier.announcements_for("anchor", now).await.unwrap();
3044 tier.liveness(&anchor, now).await.unwrap();
3045 let announcement_generation = store.announcement_revision().await.unwrap();
3046 let liveness_generation = store.liveness_revision().await.unwrap();
3047
3048 for cohort in 0..12 {
3052 let now = unix_millis_now();
3053 for slot in 0..3 {
3054 let name = format!("churn-{cohort}-{slot}");
3055 store
3056 .put_announcement(&name, endpoint("iroh", 1, now + 100, now + 100))
3057 .await
3058 .unwrap();
3059 store
3060 .put_liveness(
3061 &Did::new(format!("did:web:{name}.example")),
3062 LiveAllocatable {
3063 allocatable: vec![],
3064 load_fraction: 0.2,
3065 last_seen: now,
3066 live_until_unix_ms: now + 100,
3067 },
3068 )
3069 .await
3070 .unwrap();
3071 }
3072 let counts: Vec<usize> = store.pool.eval(
3075 "return {redis.call('SCARD', KEYS[1]), redis.call('HLEN', KEYS[2]), #redis.call('KEYS', ARGV[1]), #redis.call('KEYS', ARGV[2]), #redis.call('KEYS', ARGV[3])}",
3076 vec![store.key("services"), store.key("service-names")],
3077 vec![store.key("*"), store.key("announcement-index:*"), store.key("*revision*")],
3078 ).await.unwrap();
3079 assert!(
3081 counts[0] <= 4,
3082 "global listing work grew with history: {counts:?}"
3083 );
3084 assert!(counts[1] <= 4, "service names leaked: {counts:?}");
3085 assert!(counts[2] <= 3 * 4 + 6, "backend keys leaked: {counts:?}");
3086 assert!(counts[3] <= 4, "per-service indexes leaked: {counts:?}");
3087 assert_eq!(counts[4], 2, "only two global generations may persist");
3088 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
3089 }
3090 let now = unix_millis_now();
3091 let all = store.all_announcements(now).await.unwrap();
3092 assert_eq!(all.len(), 1);
3093 assert_eq!(all[0].0, "anchor");
3094 assert_eq!(store.all_liveness(now).await.unwrap().len(), 1);
3095 let count: usize = store.pool.scard(store.key("services")).await.unwrap();
3096 assert_eq!(count, 1, "expired cohort must leave no listing work");
3097 assert!(store.announcement_revision().await.unwrap() > announcement_generation);
3098 assert!(store.liveness_revision().await.unwrap() > liveness_generation);
3099
3100 store
3103 .put_announcement("anchor", endpoint("iroh", 2, now + 100, now + 100))
3104 .await
3105 .unwrap();
3106 store
3107 .put_liveness(
3108 &anchor,
3109 LiveAllocatable {
3110 allocatable: vec![],
3111 load_fraction: 0.5,
3112 last_seen: now,
3113 live_until_unix_ms: now + 100,
3114 },
3115 )
3116 .await
3117 .unwrap();
3118 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
3119 let now = unix_millis_now();
3120 assert!(store.all_announcements(now).await.unwrap().is_empty());
3121 assert!(store.all_liveness(now).await.unwrap().is_empty());
3122 let keys: Vec<String> = store
3123 .pool
3124 .eval(
3125 "return redis.call('KEYS', ARGV[1])",
3126 Vec::<String>::new(),
3127 vec![store.key("*")],
3128 )
3129 .await
3130 .unwrap();
3131 assert_eq!(
3132 keys.len(),
3133 2,
3134 "only persistent generations survive expiry: {keys:?}"
3135 );
3136 let expired_generation = store.announcement_revision().await.unwrap();
3137 let expired_liveness_generation = store.liveness_revision().await.unwrap();
3138 store
3139 .put_announcement("anchor", endpoint("iroh", 3, now + 30_000, now + 30_000))
3140 .await
3141 .unwrap();
3142 store
3143 .put_liveness(
3144 &anchor,
3145 LiveAllocatable {
3146 allocatable: vec![],
3147 load_fraction: 0.9,
3148 last_seen: now,
3149 live_until_unix_ms: now + 30_000,
3150 },
3151 )
3152 .await
3153 .unwrap();
3154 assert!(store.announcement_revision().await.unwrap() > expired_generation);
3155 assert!(store.liveness_revision().await.unwrap() > expired_liveness_generation);
3156 assert_eq!(
3157 tier.announcements_for("anchor", now).await.unwrap()[0].accepted_state_epoch,
3158 3
3159 );
3160 assert_eq!(
3161 tier.liveness(&anchor, now)
3162 .await
3163 .unwrap()
3164 .unwrap()
3165 .load_fraction,
3166 0.9
3167 );
3168 }
3169
3170 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
3171 #[tokio::test]
3172 async fn tiered_writes_invalidate_instead_of_tagging_caller_values() {
3173 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
3174 return;
3175 };
3176 let config = valkey_config(url, "write-race");
3177 let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
3178 let other = ValkeyStateStore::connect(&config).await.unwrap();
3179 let tier = TieredStateStore::new(Arc::new(MemoryStateStore::new(8, 8, 8)), shared, 10_000);
3180 let now = unix_millis_now();
3181 let node = Did::new("did:web:node.example".to_owned());
3182 for epoch in 1..=8 {
3185 let a = endpoint("iroh", epoch, now + 30_000, now + 20_000);
3186 let b = endpoint("iroh", epoch + 1, now + 30_000, now + 20_000);
3187 let (left, right) = tokio::join!(
3188 tier.put_announcement("model", a),
3189 other.put_announcement("model", b)
3190 );
3191 left.unwrap();
3192 right.unwrap();
3193 assert!(!tier.observed.lock().contains_key("announcement:model"));
3194 assert_eq!(
3195 tier.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
3196 epoch + 1
3197 );
3198 let a = LiveAllocatable {
3199 allocatable: vec![],
3200 load_fraction: 0.1,
3201 last_seen: now + epoch as i64,
3202 live_until_unix_ms: now + 20_000,
3203 };
3204 let mut b = a.clone();
3205 b.last_seen += 1;
3206 b.load_fraction = 0.9;
3207 let (left, right) = tokio::join!(
3208 tier.put_liveness(&node, a),
3209 other.put_liveness(&node, b.clone())
3210 );
3211 left.unwrap();
3212 right.unwrap();
3213 assert!(!tier
3214 .observed
3215 .lock()
3216 .contains_key(&TieredStateStore::scope("liveness", node.as_str())));
3217 assert_eq!(
3218 tier.liveness(&node, now)
3219 .await
3220 .unwrap()
3221 .unwrap()
3222 .load_fraction,
3223 other
3224 .liveness(&node, now)
3225 .await
3226 .unwrap()
3227 .unwrap()
3228 .load_fraction
3229 );
3230 let a = CachedEntityStatement {
3231 jwt: "a".to_owned(),
3232 fetched_at: now,
3233 };
3234 let b = CachedEntityStatement {
3235 jwt: "b".to_owned(),
3236 fetched_at: now,
3237 };
3238 let (left, right) = tokio::join!(
3239 tier.put_entity_statement("issuer", a),
3240 other.put_entity_statement("issuer", b)
3241 );
3242 left.unwrap();
3243 right.unwrap();
3244 assert!(!tier.observed.lock().contains_key("entity:issuer"));
3245 assert_eq!(
3246 tier.entity_statement("issuer").await.unwrap(),
3247 other.entity_statement("issuer").await.unwrap()
3248 );
3249 let a = CachedEnvelopeKeyset {
3250 cose_keyset_cbor: vec![1],
3251 fetched_at: now,
3252 };
3253 let b = CachedEnvelopeKeyset {
3254 cose_keyset_cbor: vec![2],
3255 fetched_at: now,
3256 };
3257 let (left, right) = tokio::join!(
3258 tier.put_envelope_keyset("service", a),
3259 other.put_envelope_keyset("service", b)
3260 );
3261 left.unwrap();
3262 right.unwrap();
3263 assert!(!tier.observed.lock().contains_key("envelope:service"));
3264 assert_eq!(
3265 tier.envelope_keyset("service").await.unwrap(),
3266 other.envelope_keyset("service").await.unwrap()
3267 );
3268 }
3269 }
3270 #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
3271 #[tokio::test]
3272 async fn valkey_legacy_liveness_encoding_fails_closed_until_heartbeat() {
3273 use fred::prelude::*;
3274 let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
3275 return;
3276 };
3277 let store = ValkeyStateStore::connect(&valkey_config(url, "legacy-liveness"))
3278 .await
3279 .unwrap();
3280 let node = Did::new("did:web:legacy-node.example".to_owned());
3281 let now = unix_millis_now();
3282 let value = LiveAllocatable {
3283 allocatable: vec![],
3284 load_fraction: 0.1,
3285 last_seen: now,
3286 live_until_unix_ms: now + 30_000,
3287 };
3288 store.put_liveness(&node, value.clone()).await.unwrap();
3289 let _: String = store
3291 .pool
3292 .eval(
3293 "return redis.call('SET', KEYS[1], ARGV[1], 'KEEPTTL')",
3294 vec![format!(
3295 "{}:liveness:{}",
3296 store.prefix,
3297 ValkeyStateStore::digest(node.as_str())
3298 )],
3299 vec![serde_json::to_string(&value).unwrap()],
3300 )
3301 .await
3302 .unwrap();
3303 assert!(store.all_liveness(now).await.unwrap().is_empty());
3304 store.put_liveness(&node, value).await.unwrap();
3305 assert_eq!(store.all_liveness(now).await.unwrap()[0].0, node);
3306 }
3307}