Skip to main content

hyprstream_discovery/
state_store.rs

1//! Volatile Discovery state behind a backend-neutral contract.
2//!
3//! The values in this store are never identity or policy authority. Callers
4//! validate signed artifacts before writes and re-check accepted-current state
5//! before use. The shared backend only makes short-lived reach and liveness
6//! observations coherent across Discovery replicas.
7
8use anyhow::{bail, Result};
9use async_trait::async_trait;
10use hyprstream_rpc::identity::Did;
11use parking_lot::Mutex;
12use serde::{Deserialize, Serialize};
13use std::cmp::Reverse;
14use std::collections::{BTreeSet, BinaryHeap, HashMap};
15use std::sync::Arc;
16
17pub(crate) const ANNOUNCED_ENDPOINT_TTL: std::time::Duration = std::time::Duration::from_secs(90);
18
19#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
20#[serde(rename_all = "kebab-case")]
21pub enum DiscoveryStateBackend {
22    #[default]
23    Memory,
24    Valkey,
25    Tiered,
26}
27
28#[derive(Clone, Debug, Serialize, Deserialize)]
29#[serde(default)]
30pub struct MemoryStateConfig {
31    pub announcement_capacity: usize,
32    pub liveness_capacity: usize,
33    pub artifact_capacity: usize,
34}
35
36impl Default for MemoryStateConfig {
37    fn default() -> Self {
38        Self {
39            announcement_capacity: 16_384,
40            liveness_capacity: 16_384,
41            artifact_capacity: 4_096,
42        }
43    }
44}
45
46#[derive(Clone, Debug, Serialize, Deserialize)]
47#[serde(default)]
48pub struct ValkeyStateConfig {
49    pub url: String,
50    pub key_prefix: String,
51    pub pool_size: usize,
52    pub announcement_capacity: usize,
53    pub liveness_capacity: usize,
54    pub artifact_capacity: usize,
55    /// Upper bound for a shared-state command. Required-HA callers receive an
56    /// error after this interval instead of waiting indefinitely or serving L1.
57    pub command_timeout_ms: u64,
58}
59
60impl Default for ValkeyStateConfig {
61    fn default() -> Self {
62        Self {
63            url: String::new(),
64            key_prefix: "hs".to_owned(),
65            pool_size: 8,
66            announcement_capacity: 65_536,
67            liveness_capacity: 65_536,
68            artifact_capacity: 16_384,
69            command_timeout_ms: 2_000,
70        }
71    }
72}
73
74#[derive(Clone, Debug, Serialize, Deserialize)]
75#[serde(default)]
76pub struct TieredStateConfig {
77    /// Maximum time an L1 value may be reused after its L2 revision was
78    /// observed. Every L1 hit still verifies the cheap L2 revision key.
79    pub l1_max_ttl_ms: u64,
80}
81
82impl Default for TieredStateConfig {
83    fn default() -> Self {
84        Self {
85            l1_max_ttl_ms: 1_000,
86        }
87    }
88}
89
90#[derive(Clone, Debug, Default, Serialize, Deserialize)]
91#[serde(default)]
92pub struct DiscoveryStateConfig {
93    pub backend: DiscoveryStateBackend,
94    /// Declares that more than one active Discovery replica may serve calls.
95    /// Such a deployment must select a shared backend.
96    pub active_active: bool,
97    pub memory: MemoryStateConfig,
98    pub valkey: ValkeyStateConfig,
99    pub tiered: TieredStateConfig,
100}
101
102/// Constructed backend handle accepted by [`crate::DiscoveryService`].
103#[derive(Clone)]
104pub struct DiscoveryState(Arc<dyn DiscoveryStateStore>);
105
106impl DiscoveryState {
107    pub async fn connect(config: &DiscoveryStateConfig) -> Result<Self> {
108        anyhow::ensure!(
109            config.memory.announcement_capacity > 0
110                && config.memory.liveness_capacity > 0
111                && config.memory.artifact_capacity > 0,
112            "Discovery memory capacities must be positive"
113        );
114        if config.active_active && config.backend == DiscoveryStateBackend::Memory {
115            bail!("discovery.state.active_active requires valkey or tiered backend");
116        }
117        if config.backend == DiscoveryStateBackend::Tiered {
118            anyhow::ensure!(
119                config.tiered.l1_max_ttl_ms > 0,
120                "Discovery tiered l1_max_ttl_ms must be positive"
121            );
122        }
123        let memory = || {
124            Arc::new(MemoryStateStore::new(
125                config.memory.announcement_capacity,
126                config.memory.liveness_capacity,
127                config.memory.artifact_capacity,
128            ))
129        };
130
131        match config.backend {
132            DiscoveryStateBackend::Memory => Ok(Self(memory())),
133            DiscoveryStateBackend::Valkey | DiscoveryStateBackend::Tiered => {
134                #[cfg(target_arch = "wasm32")]
135                {
136                    bail!("Valkey-backed Discovery state is unavailable on wasm32")
137                }
138                #[cfg(all(not(target_arch = "wasm32"), not(feature = "valkey")))]
139                {
140                    bail!("Valkey-backed Discovery state requires the valkey feature")
141                }
142                #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
143                {
144                    let valkey = Arc::new(ValkeyStateStore::connect(&config.valkey).await?);
145                    if config.backend == DiscoveryStateBackend::Valkey {
146                        Ok(Self(valkey))
147                    } else {
148                        Ok(Self(Arc::new(TieredStateStore::new(
149                            memory(),
150                            valkey,
151                            config.tiered.l1_max_ttl_ms,
152                        ))))
153                    }
154                }
155            }
156        }
157    }
158
159    pub(crate) fn into_inner(self) -> Arc<dyn DiscoveryStateStore> {
160        self.0
161    }
162}
163
164/// Endpoint data stored per announced entry.
165///
166/// The announcement's service JWT is verified before this value is created,
167/// but is deliberately not retained: it is a bearer credential, not endpoint
168/// resolution metadata. Older Valkey JSON rows may contain that now-unknown
169/// field; serde ignores it on read and never writes it back.
170#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
171pub(crate) struct AnnouncedEndpoint {
172    pub(crate) socket_kind: String,
173    pub(crate) endpoint: String,
174    pub(crate) service_did: Did,
175    pub(crate) capabilities: BTreeSet<String>,
176    pub(crate) accepted_state_digest: Vec<u8>,
177    pub(crate) accepted_state_epoch: u64,
178    pub(crate) response_key_id: String,
179    pub(crate) request_kem_key_id: String,
180    pub(crate) request_kem_recipient: Vec<u8>,
181    /// Signed/application expiry carried by the announcement. Zero on a
182    /// legacy write requests a backend-receipt-bounded application expiry.
183    pub(crate) expires_at_unix_ms: i64,
184    pub(crate) source_signer: [u8; 32],
185    /// On write: independently verified signed/current-state authority ceiling.
186    /// On read: effective lifetime, also capped by the backend's receipt + TTL.
187    pub(crate) live_until_unix_ms: i64,
188}
189
190impl AnnouncedEndpoint {
191    fn receipt_lease(mut self, now: i64) -> Option<Self> {
192        let heartbeat_expiry = now.saturating_add(ANNOUNCED_ENDPOINT_TTL.as_millis() as i64);
193        if self.expires_at_unix_ms == 0 {
194            self.expires_at_unix_ms = heartbeat_expiry;
195        }
196        self.live_until_unix_ms = heartbeat_expiry
197            .min(self.expires_at_unix_ms)
198            .min(self.live_until_unix_ms);
199        self.is_live_at(now).then_some(self)
200    }
201
202    pub(crate) fn is_live_at(&self, now_unix_ms: i64) -> bool {
203        now_unix_ms < self.live_until_unix_ms && now_unix_ms < self.expires_at_unix_ms
204    }
205
206    fn order(&self) -> (u64, i64) {
207        (self.accepted_state_epoch, self.expires_at_unix_ms)
208    }
209}
210
211#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
212pub(crate) struct LiveAllocatable {
213    pub(crate) allocatable: Vec<(String, String)>,
214    pub(crate) load_fraction: f32,
215    /// Server receipt time, never the reporting node's clock.
216    pub(crate) last_seen: i64,
217    pub(crate) live_until_unix_ms: i64,
218}
219
220impl LiveAllocatable {
221    fn is_live_at(&self, now_unix_ms: i64) -> bool {
222        now_unix_ms < self.live_until_unix_ms
223    }
224}
225
226#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
227pub(crate) struct CachedEntityStatement {
228    pub(crate) jwt: String,
229    pub(crate) fetched_at: i64,
230}
231
232#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
233pub(crate) struct CachedEnvelopeKeyset {
234    pub(crate) cose_keyset_cbor: Vec<u8>,
235    pub(crate) fetched_at: i64,
236}
237
238#[derive(Clone, Copy, Debug, PartialEq, Eq)]
239pub(crate) enum PutResult {
240    Stored,
241    IgnoredOlder,
242}
243
244/// Typed query surface needed by Discovery handlers and resolvers.
245///
246/// Native futures are `Send`; browser/embedded WASM uses the repository's
247/// established `?Send` convention. Implementations remain `Send + Sync` so a
248/// native service can share one store across handlers.
249#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
250#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
251pub(crate) trait DiscoveryStateStore: Send + Sync {
252    async fn put_announcement(
253        &self,
254        service_name: &str,
255        endpoint: AnnouncedEndpoint,
256    ) -> Result<PutResult>;
257    async fn announcements_for(
258        &self,
259        service_name: &str,
260        now_unix_ms: i64,
261    ) -> Result<Vec<AnnouncedEndpoint>>;
262    async fn all_announcements(
263        &self,
264        now_unix_ms: i64,
265    ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>>;
266
267    async fn put_liveness(&self, node: &Did, value: LiveAllocatable) -> Result<PutResult>;
268    #[cfg(test)]
269    async fn liveness(&self, node: &Did, now_unix_ms: i64) -> Result<Option<LiveAllocatable>>;
270    /// Bounded enumeration of live nodes from the authoritative backend.
271    async fn all_liveness(&self, now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>>;
272
273    async fn put_entity_statement(&self, issuer: &str, value: CachedEntityStatement) -> Result<()>;
274    async fn entity_statement(&self, issuer: &str) -> Result<Option<CachedEntityStatement>>;
275    async fn known_issuers(&self) -> Result<Vec<String>>;
276    async fn known_issuer_count(&self) -> Result<usize>;
277
278    async fn put_envelope_keyset(
279        &self,
280        service_did: &str,
281        value: CachedEnvelopeKeyset,
282    ) -> Result<()>;
283    async fn envelope_keyset(&self, service_did: &str) -> Result<Option<CachedEnvelopeKeyset>>;
284}
285
286#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
287struct AnnouncementKey {
288    service_name: String,
289    socket_kind: String,
290}
291
292#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
293enum ExpiringKey {
294    Announcement(AnnouncementKey),
295    Liveness(String),
296}
297
298#[derive(Clone, Debug, PartialEq, Eq)]
299struct ExpiryEntry {
300    expires_at_unix_ms: i64,
301    version: u64,
302    key: ExpiringKey,
303}
304
305impl Ord for ExpiryEntry {
306    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
307        self.expires_at_unix_ms
308            .cmp(&other.expires_at_unix_ms)
309            .then_with(|| self.version.cmp(&other.version))
310            .then_with(|| self.key.cmp(&other.key))
311    }
312}
313
314impl PartialOrd for ExpiryEntry {
315    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
316        Some(self.cmp(other))
317    }
318}
319
320#[derive(Clone)]
321struct Versioned<T> {
322    value: T,
323    version: u64,
324}
325
326struct MemoryInner {
327    announcements: HashMap<AnnouncementKey, Versioned<AnnouncedEndpoint>>,
328    service_index: HashMap<String, BTreeSet<String>>,
329    liveness: HashMap<String, Versioned<LiveAllocatable>>,
330    expiry: BinaryHeap<Reverse<ExpiryEntry>>,
331    entity_statements: HashMap<String, CachedEntityStatement>,
332    envelope_keysets: HashMap<String, CachedEnvelopeKeyset>,
333    next_version: u64,
334}
335
336/// Bounded in-process backend used by single-replica and WASM deployments.
337pub(crate) struct MemoryStateStore {
338    inner: Mutex<MemoryInner>,
339    announcement_capacity: usize,
340    liveness_capacity: usize,
341    artifact_capacity: usize,
342}
343
344impl MemoryStateStore {
345    pub(crate) fn new(
346        announcement_capacity: usize,
347        liveness_capacity: usize,
348        artifact_capacity: usize,
349    ) -> Self {
350        Self {
351            inner: Mutex::new(MemoryInner {
352                announcements: HashMap::new(),
353                service_index: HashMap::new(),
354                liveness: HashMap::new(),
355                expiry: BinaryHeap::new(),
356                entity_statements: HashMap::new(),
357                envelope_keysets: HashMap::new(),
358                next_version: 1,
359            }),
360            announcement_capacity,
361            liveness_capacity,
362            artifact_capacity,
363        }
364    }
365
366    pub(crate) fn production_default() -> Arc<dyn DiscoveryStateStore> {
367        Arc::new(Self::new(16_384, 16_384, 4_096))
368    }
369
370    fn next_version(inner: &mut MemoryInner) -> u64 {
371        let version = inner.next_version;
372        inner.next_version = inner.next_version.saturating_add(1);
373        version
374    }
375
376    // Replacement/declined L1 fills must not accumulate an unbounded heap of
377    // stale expiry versions while the actual maps remain capacity-bounded.
378    fn compact_expiry(inner: &mut MemoryInner) {
379        let limit = 2 * (inner.announcements.len() + inner.liveness.len());
380        if inner.expiry.len() > limit {
381            let MemoryInner {
382                announcements,
383                liveness,
384                expiry,
385                ..
386            } = inner;
387            expiry.retain(|Reverse(entry)| match &entry.key {
388                ExpiringKey::Announcement(key) => announcements
389                    .get(key)
390                    .is_some_and(|value| value.version == entry.version),
391                ExpiringKey::Liveness(node) => liveness
392                    .get(node)
393                    .is_some_and(|value| value.version == entry.version),
394            });
395        }
396    }
397
398    fn reap(inner: &mut MemoryInner, now_unix_ms: i64) {
399        while let Some(Reverse(head)) = inner.expiry.peek() {
400            if head.expires_at_unix_ms > now_unix_ms {
401                break;
402            }
403            let Some(Reverse(expired)) = inner.expiry.pop() else {
404                break;
405            };
406            match &expired.key {
407                ExpiringKey::Announcement(key) => {
408                    let remove = inner
409                        .announcements
410                        .get(key)
411                        .is_some_and(|entry| entry.version == expired.version);
412                    if remove {
413                        inner.announcements.remove(key);
414                        if let Some(kinds) = inner.service_index.get_mut(&key.service_name) {
415                            kinds.remove(&key.socket_kind);
416                            if kinds.is_empty() {
417                                inner.service_index.remove(&key.service_name);
418                            }
419                        }
420                    }
421                }
422                ExpiringKey::Liveness(node) => {
423                    let remove = inner
424                        .liveness
425                        .get(node)
426                        .is_some_and(|entry| entry.version == expired.version);
427                    if remove {
428                        inner.liveness.remove(node);
429                    }
430                }
431            }
432        }
433    }
434
435    #[cfg(test)]
436    pub(crate) fn put_announcement_sync(
437        &self,
438        service_name: &str,
439        endpoint: AnnouncedEndpoint,
440    ) -> Result<PutResult> {
441        self.put_announcement_at(service_name, endpoint, unix_millis_now())
442    }
443
444    fn put_announcement_at(
445        &self,
446        service_name: &str,
447        endpoint: AnnouncedEndpoint,
448        now_unix_ms: i64,
449    ) -> Result<PutResult> {
450        let mut inner = self.inner.lock();
451        Self::reap(&mut inner, now_unix_ms);
452        let key = AnnouncementKey {
453            service_name: service_name.to_owned(),
454            socket_kind: endpoint.socket_kind.clone(),
455        };
456        if let Some(existing) = inner.announcements.get(&key) {
457            if existing.value.order() > endpoint.order() {
458                return Ok(PutResult::IgnoredOlder);
459            }
460        } else if inner.announcements.len() >= self.announcement_capacity {
461            bail!("Discovery memory announcement capacity exhausted");
462        }
463        let version = Self::next_version(&mut inner);
464        inner
465            .service_index
466            .entry(service_name.to_owned())
467            .or_default()
468            .insert(endpoint.socket_kind.clone());
469        inner.expiry.push(Reverse(ExpiryEntry {
470            expires_at_unix_ms: endpoint.live_until_unix_ms,
471            version,
472            key: ExpiringKey::Announcement(key.clone()),
473        }));
474        inner.announcements.insert(
475            key,
476            Versioned {
477                value: endpoint,
478                version,
479            },
480        );
481        Self::compact_expiry(&mut inner);
482        Ok(PutResult::Stored)
483    }
484
485    pub(crate) fn announcements_for_sync(
486        &self,
487        service_name: &str,
488        now_unix_ms: i64,
489    ) -> Vec<AnnouncedEndpoint> {
490        let mut inner = self.inner.lock();
491        Self::reap(&mut inner, now_unix_ms);
492        inner
493            .service_index
494            .get(service_name)
495            .into_iter()
496            .flatten()
497            .filter_map(|socket_kind| {
498                inner
499                    .announcements
500                    .get(&AnnouncementKey {
501                        service_name: service_name.to_owned(),
502                        socket_kind: socket_kind.clone(),
503                    })
504                    .map(|entry| entry.value.clone())
505            })
506            .filter(|entry| entry.is_live_at(now_unix_ms))
507            .collect()
508    }
509
510    fn all_announcements_sync(&self, now_unix_ms: i64) -> Vec<(String, Vec<AnnouncedEndpoint>)> {
511        let mut inner = self.inner.lock();
512        Self::reap(&mut inner, now_unix_ms);
513        inner
514            .service_index
515            .iter()
516            .filter_map(|(service_name, socket_kinds)| {
517                let endpoints: Vec<_> = socket_kinds
518                    .iter()
519                    .filter_map(|socket_kind| {
520                        inner
521                            .announcements
522                            .get(&AnnouncementKey {
523                                service_name: service_name.clone(),
524                                socket_kind: socket_kind.clone(),
525                            })
526                            .map(|entry| entry.value.clone())
527                    })
528                    .filter(|entry| entry.is_live_at(now_unix_ms))
529                    .collect();
530                (!endpoints.is_empty()).then(|| (service_name.clone(), endpoints))
531            })
532            .collect()
533    }
534
535    #[cfg(feature = "valkey")]
536    pub(crate) fn clear_announcements_sync(&self, service_name: &str) {
537        let mut inner = self.inner.lock();
538        let Some(socket_kinds) = inner.service_index.remove(service_name) else {
539            return;
540        };
541        for socket_kind in socket_kinds {
542            inner.announcements.remove(&AnnouncementKey {
543                service_name: service_name.to_owned(),
544                socket_kind,
545            });
546        }
547        Self::compact_expiry(&mut inner);
548    }
549
550    #[cfg(feature = "valkey")]
551    fn clear_liveness_sync(&self, node: &Did) {
552        let mut inner = self.inner.lock();
553        inner.liveness.remove(node.as_str());
554        Self::compact_expiry(&mut inner);
555    }
556
557    #[cfg(feature = "valkey")]
558    fn clear_entity_statement_sync(&self, issuer: &str) {
559        self.inner.lock().entity_statements.remove(issuer);
560    }
561
562    #[cfg(feature = "valkey")]
563    fn clear_envelope_keyset_sync(&self, service_did: &str) {
564        self.inner.lock().envelope_keysets.remove(service_did);
565    }
566}
567
568impl Default for MemoryStateStore {
569    fn default() -> Self {
570        Self::new(16_384, 16_384, 4_096)
571    }
572}
573
574#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
575#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
576impl DiscoveryStateStore for MemoryStateStore {
577    async fn put_announcement(
578        &self,
579        service_name: &str,
580        endpoint: AnnouncedEndpoint,
581    ) -> Result<PutResult> {
582        let now = unix_millis_now();
583        let Some(endpoint) = endpoint.receipt_lease(now) else {
584            return Ok(PutResult::IgnoredOlder);
585        };
586        self.put_announcement_at(service_name, endpoint, now)
587    }
588
589    async fn announcements_for(
590        &self,
591        service_name: &str,
592        now_unix_ms: i64,
593    ) -> Result<Vec<AnnouncedEndpoint>> {
594        Ok(self.announcements_for_sync(service_name, now_unix_ms))
595    }
596
597    async fn all_announcements(
598        &self,
599        now_unix_ms: i64,
600    ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>> {
601        Ok(self.all_announcements_sync(now_unix_ms))
602    }
603
604    async fn put_liveness(&self, node: &Did, value: LiveAllocatable) -> Result<PutResult> {
605        let mut inner = self.inner.lock();
606        Self::reap(&mut inner, unix_millis_now());
607        let node = node.as_str().to_owned();
608        if let Some(existing) = inner.liveness.get(&node) {
609            // Both fields derive from receipt time. A newer receipt may
610            // shorten lifetime; a newer lifetime also supersedes legacy
611            // client-clock skew without waiting for the old value to expire.
612            if existing.value.last_seen > value.last_seen
613                && existing.value.live_until_unix_ms >= value.live_until_unix_ms
614            {
615                return Ok(PutResult::IgnoredOlder);
616            }
617        } else if inner.liveness.len() >= self.liveness_capacity {
618            bail!("Discovery memory liveness capacity exhausted");
619        }
620        let version = Self::next_version(&mut inner);
621        inner.expiry.push(Reverse(ExpiryEntry {
622            expires_at_unix_ms: value.live_until_unix_ms,
623            version,
624            key: ExpiringKey::Liveness(node.clone()),
625        }));
626        inner.liveness.insert(node, Versioned { value, version });
627        Self::compact_expiry(&mut inner);
628        Ok(PutResult::Stored)
629    }
630
631    #[cfg(test)]
632    async fn liveness(&self, node: &Did, now_unix_ms: i64) -> Result<Option<LiveAllocatable>> {
633        let mut inner = self.inner.lock();
634        Self::reap(&mut inner, now_unix_ms);
635        Ok(inner
636            .liveness
637            .get(node.as_str())
638            .map(|entry| entry.value.clone())
639            .filter(|entry| entry.is_live_at(now_unix_ms)))
640    }
641
642    async fn all_liveness(&self, now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>> {
643        let mut inner = self.inner.lock();
644        Self::reap(&mut inner, now_unix_ms);
645        Ok(inner
646            .liveness
647            .iter()
648            .filter(|(_, entry)| entry.value.is_live_at(now_unix_ms))
649            .map(|(node, entry)| (Did::new(node.clone()), entry.value.clone()))
650            .collect())
651    }
652
653    async fn put_entity_statement(&self, issuer: &str, value: CachedEntityStatement) -> Result<()> {
654        let mut inner = self.inner.lock();
655        if !inner.entity_statements.contains_key(issuer)
656            && inner.entity_statements.len() >= self.artifact_capacity
657        {
658            // These are inert artifacts, verified again at use. Retain the
659            // newest fetched entries without interpreting JWT expiry as trust.
660            if let Some(oldest) = inner
661                .entity_statements
662                .iter()
663                .min_by_key(|(name, value)| (value.fetched_at, *name))
664                .map(|(name, _)| name.clone())
665            {
666                inner.entity_statements.remove(&oldest);
667            }
668        }
669        inner.entity_statements.insert(issuer.to_owned(), value);
670        Ok(())
671    }
672
673    async fn entity_statement(&self, issuer: &str) -> Result<Option<CachedEntityStatement>> {
674        Ok(self.inner.lock().entity_statements.get(issuer).cloned())
675    }
676
677    async fn known_issuers(&self) -> Result<Vec<String>> {
678        Ok(self
679            .inner
680            .lock()
681            .entity_statements
682            .keys()
683            .cloned()
684            .collect())
685    }
686
687    async fn known_issuer_count(&self) -> Result<usize> {
688        Ok(self.inner.lock().entity_statements.len())
689    }
690
691    async fn put_envelope_keyset(
692        &self,
693        service_did: &str,
694        value: CachedEnvelopeKeyset,
695    ) -> Result<()> {
696        let mut inner = self.inner.lock();
697        if !inner.envelope_keysets.contains_key(service_did)
698            && inner.envelope_keysets.len() >= self.artifact_capacity
699        {
700            bail!("Discovery memory federation artifact capacity exhausted");
701        }
702        inner.envelope_keysets.insert(service_did.to_owned(), value);
703        Ok(())
704    }
705
706    async fn envelope_keyset(&self, service_did: &str) -> Result<Option<CachedEnvelopeKeyset>> {
707        Ok(self.inner.lock().envelope_keysets.get(service_did).cloned())
708    }
709}
710
711#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
712#[derive(Serialize, Deserialize)]
713struct StoredLiveness {
714    node: Did,
715    #[serde(default)]
716    shared_receipt: bool,
717    #[serde(flatten)]
718    value: LiveAllocatable,
719}
720
721#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
722#[derive(Clone)]
723struct ValkeyStateStore {
724    pool: fred::prelude::RedisPool,
725    // The last store owner closes this channel, stopping the driver runtime.
726    _driver: Arc<tokio::sync::oneshot::Sender<()>>,
727    prefix: String,
728    announcement_capacity: usize,
729    liveness_capacity: usize,
730    artifact_capacity: usize,
731}
732
733#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
734impl ValkeyStateStore {
735    async fn connect(config: &ValkeyStateConfig) -> Result<Self> {
736        use anyhow::Context as _;
737        use fred::prelude::*;
738
739        anyhow::ensure!(
740            config.pool_size > 0,
741            "Discovery Valkey pool_size must be positive"
742        );
743        anyhow::ensure!(
744            !config.url.trim().is_empty(),
745            "Discovery Valkey URL must be configured explicitly"
746        );
747        anyhow::ensure!(
748            config.command_timeout_ms > 0,
749            "Discovery Valkey command_timeout_ms must be positive"
750        );
751        anyhow::ensure!(
752            config.announcement_capacity > 0
753                && config.liveness_capacity > 0
754                && config.artifact_capacity > 0,
755            "Discovery Valkey capacities must be positive"
756        );
757        // Fred's URL parser constructs a rustls config. Initialize the existing
758        // external TLS policy first, including in standalone state-store use.
759        hyprstream_rpc::transport::pq_provider::install_pq_crypto_provider()?;
760        let redis = RedisConfig::from_url(&config.url).context("invalid Discovery Valkey URL")?;
761        let mut builder = Builder::from_config(redis);
762        builder.with_performance_config(|performance| {
763            performance.default_command_timeout =
764                std::time::Duration::from_millis(config.command_timeout_ms);
765        });
766        // fred spawns drivers on the current runtime. Give them a runtime
767        // owned by this store, independent of a temporary factory/bootstrap
768        // runtime and of the caller's executor. Closing the last owner also
769        // shuts down the runtime; cancelled/failed connects cannot leak it.
770        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
771        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
772        let pool_size = config.pool_size;
773        let timeout = std::time::Duration::from_millis(config.command_timeout_ms);
774        std::thread::Builder::new()
775            .name("discovery-valkey".to_owned())
776            .spawn(move || {
777                let runtime = match tokio::runtime::Builder::new_current_thread()
778                    .enable_all()
779                    .build()
780                {
781                    Ok(runtime) => runtime,
782                    Err(error) => {
783                        let _ = ready_tx.send(Err(anyhow::Error::from(error)));
784                        return;
785                    }
786                };
787                runtime.block_on(async move {
788                    let connected: Result<RedisPool> = async {
789                        let pool = builder.build_pool(pool_size)?;
790                        pool.connect();
791                        tokio::time::timeout(timeout, pool.wait_for_connect()).await??;
792                        let _: String = pool.ping().await?;
793                        Ok(pool)
794                    }
795                    .await;
796                    match connected {
797                        Ok(pool) => {
798                            if ready_tx.send(Ok(pool.clone())).is_ok() {
799                                let _ = shutdown_rx.await;
800                                let _ = tokio::time::timeout(timeout, pool.quit()).await;
801                            }
802                        }
803                        Err(error) => {
804                            let _ = ready_tx.send(Err(error));
805                        }
806                    }
807                });
808            })
809            .context("start Discovery Valkey driver")?;
810        let pool = ready_rx
811            .await
812            .context("Discovery Valkey driver stopped during startup")??;
813        Ok(Self {
814            pool,
815            _driver: Arc::new(shutdown_tx),
816            // One hash tag keeps every Lua transaction in one cluster slot.
817            prefix: format!(
818                "{}:{{discovery-state}}",
819                config.key_prefix.trim_end_matches(':')
820            ),
821            announcement_capacity: config.announcement_capacity,
822            liveness_capacity: config.liveness_capacity,
823            artifact_capacity: config.artifact_capacity,
824        })
825    }
826
827    fn digest(value: &str) -> String {
828        blake3::hash(value.as_bytes()).to_hex().to_string()
829    }
830
831    fn key(&self, suffix: &str) -> String {
832        format!("{}:{suffix}", self.prefix)
833    }
834
835    fn service_id(service_name: &str) -> String {
836        Self::digest(service_name)
837    }
838
839    fn announcement_key(&self, service_name: &str, socket_kind: &str) -> String {
840        format!(
841            "{}:announcement:{}:{}",
842            self.prefix,
843            Self::service_id(service_name),
844            Self::digest(socket_kind)
845        )
846    }
847
848    fn announcement_index(&self, service_name: &str) -> String {
849        format!(
850            "{}:announcement-index:{}",
851            self.prefix,
852            Self::service_id(service_name)
853        )
854    }
855
856    async fn revision(&self, key: String) -> Result<u64> {
857        use fred::prelude::*;
858        Ok(self.pool.get::<Option<u64>, _>(key).await?.unwrap_or(0))
859    }
860
861    // Persistent family-wide generations bound revision storage independently
862    // of identity churn. Never expire/reset these counters: recreating a scope
863    // must not reuse a revision still attached to another replica's L1 value.
864    #[cfg(test)]
865    async fn announcement_revision(&self) -> Result<u64> {
866        self.revision(self.key("announcement-global-revision"))
867            .await
868    }
869
870    #[cfg(test)]
871    async fn liveness_revision(&self) -> Result<u64> {
872        self.revision(self.key("liveness-global-revision")).await
873    }
874
875    // Reap values and ALL secondary metadata in the same transaction as the
876    // capacity check/read. The expiry index contains at most the live capacity
877    // plus the last expired cohort; no historical service-name scan is needed.
878    // All derived keys retain the deployment's cluster hash tag.
879    const REAP_ANNOUNCEMENTS: &str = r#"
880local clock = redis.call('TIME')
881local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
882local function reap(expiry, services, names, revision)
883  local expired = redis.call('ZRANGEBYSCORE', expiry, '-inf', now)
884  if #expired > 0 then redis.call('INCR', revision) end
885  for _, key in ipairs(expired) do
886    local service = string.match(key, ':announcement:([^:]+):[^:]+$')
887    local index = string.gsub(key, ':announcement:[^:]+:[^:]+$', ':announcement-index:' .. service)
888    redis.call('DEL', key)
889    redis.call('SREM', index, key)
890    redis.call('ZREM', expiry, key)
891    if redis.call('SCARD', index) == 0 then
892      redis.call('SREM', services, service)
893      redis.call('HDEL', names, service)
894    end
895  end
896end
897"#;
898
899    async fn announcement_revision_at_shared_time(&self) -> Result<(u64, i64)> {
900        use fred::prelude::*;
901        // Cache hits need the same expiry clock as L2. Reap before reading the
902        // generation so an expired L2 value cannot remain valid in another L1.
903        self.pool.eval(
904            format!("{}\nreap(KEYS[1], KEYS[2], KEYS[3], KEYS[4])\nreturn {{redis.call('GET', KEYS[4]) or '0', now}}", Self::REAP_ANNOUNCEMENTS),
905            vec![self.key("announcement-expiry"), self.key("services"),
906                self.key("service-names"), self.key("announcement-global-revision")],
907            Vec::<String>::new(),
908        ).await.map_err(Into::into)
909    }
910
911    async fn entity_revision(&self) -> Result<u64> {
912        // Survives eviction/reinsertion: no per-issuer tombstones and no ABA
913        // when another replica still has an evicted artifact in L1.
914        self.revision(self.key("entity-global-revision")).await
915    }
916
917    async fn envelope_revision(&self, service_did: &str) -> Result<u64> {
918        self.revision(format!(
919            "{}:envelope-revision:{}",
920            self.prefix,
921            Self::digest(service_did)
922        ))
923        .await
924    }
925
926    async fn announcements_for_inner(&self, service_name: &str) -> Result<Vec<AnnouncedEndpoint>> {
927        use fred::prelude::*;
928
929        // Cleanup must share the transaction with reads: a later writer may
930        // refresh the same key, so GET followed by a separate DEL/SREM can
931        // delete its newer value or remove its live index membership.
932        const LIST: &str = r#"
933reap(KEYS[4], KEYS[2], KEYS[3], KEYS[5])
934local live = {}
935for _, key in ipairs(redis.call('SMEMBERS', KEYS[1])) do
936  local encoded = redis.call('GET', key)
937  if encoded then
938    local ok, value = pcall(cjson.decode, encoded)
939    if not ok then return redis.error_reply('corrupt Discovery announcement') end
940    if tonumber(value.live_until_unix_ms) > now and tonumber(value.expires_at_unix_ms) > now then
941      table.insert(live, encoded)
942    else
943      redis.call('INCR', KEYS[5])
944      redis.call('DEL', key)
945      redis.call('SREM', KEYS[1], key)
946      redis.call('ZREM', KEYS[4], key)
947    end
948  else
949    redis.call('SREM', KEYS[1], key)
950    redis.call('ZREM', KEYS[4], key)
951  end
952end
953if redis.call('SCARD', KEYS[1]) == 0 then
954  redis.call('SREM', KEYS[2], ARGV[1])
955  redis.call('HDEL', KEYS[3], ARGV[1])
956end
957return live
958"#;
959        let encoded: Vec<String> = self
960            .pool
961            .eval(
962                format!("{}{LIST}", Self::REAP_ANNOUNCEMENTS),
963                vec![
964                    self.announcement_index(service_name),
965                    self.key("services"),
966                    self.key("service-names"),
967                    self.key("announcement-expiry"),
968                    self.key("announcement-global-revision"),
969                ],
970                vec![Self::service_id(service_name)],
971            )
972            .await?;
973        encoded
974            .into_iter()
975            .map(|value| serde_json::from_str(&value).map_err(Into::into))
976            .collect()
977    }
978}
979
980#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
981#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
982impl DiscoveryStateStore for ValkeyStateStore {
983    async fn put_announcement(
984        &self,
985        service_name: &str,
986        endpoint: AnnouncedEndpoint,
987    ) -> Result<PutResult> {
988        use fred::prelude::*;
989
990        const PUT: &str = r#"
991reap(KEYS[6], KEYS[3], KEYS[4], KEYS[5])
992local signed_expiry = tonumber(ARGV[3])
993local heartbeat_expiry = now + tonumber(ARGV[8])
994if signed_expiry == 0 then signed_expiry = heartbeat_expiry end
995local expiry = math.min(heartbeat_expiry, signed_expiry, tonumber(ARGV[4]))
996if expiry <= now then return 0 end
997local current = redis.call('GET', KEYS[1])
998if not current and redis.call('ZCARD', KEYS[6]) >= tonumber(ARGV[7]) then
999  return redis.error_reply('Discovery Valkey announcement capacity exhausted')
1000end
1001if current then
1002  local ok, decoded = pcall(cjson.decode, current)
1003  if not ok then return redis.error_reply('corrupt Discovery announcement') end
1004  local old_epoch = tonumber(decoded.accepted_state_epoch) or 0
1005  local old_exp = tonumber(decoded.expires_at_unix_ms) or 0
1006  local new_epoch = tonumber(ARGV[2])
1007  local new_exp = signed_expiry
1008  if old_epoch > new_epoch or (old_epoch == new_epoch and old_exp > new_exp) then
1009    return 0
1010  end
1011end
1012-- Only the effective lease (and legacy application expiry) is computed here.
1013-- Preserve all other JSON fields/arrays exactly as encoded by Rust, not cjson.
1014local encoded = string.sub(ARGV[1], 1, -2) .. ',"live_until_unix_ms":' .. string.format('%.0f', expiry)
1015if tonumber(ARGV[3]) == 0 then
1016  encoded = encoded .. ',"expires_at_unix_ms":' .. string.format('%.0f', signed_expiry)
1017end
1018encoded = encoded .. '}'
1019redis.call('SET', KEYS[1], encoded, 'PXAT', expiry)
1020redis.call('SADD', KEYS[2], KEYS[1])
1021redis.call('SADD', KEYS[3], ARGV[5])
1022redis.call('HSET', KEYS[4], ARGV[5], ARGV[6])
1023redis.call('INCR', KEYS[5])
1024redis.call('ZADD', KEYS[6], expiry, KEYS[1])
1025return 1
1026"#;
1027        let service_id = Self::service_id(service_name);
1028        let mut payload = serde_json::to_value(&endpoint)?;
1029        let fields = payload
1030            .as_object_mut()
1031            .ok_or_else(|| anyhow::anyhow!("announcement must encode as an object"))?;
1032        fields.remove("live_until_unix_ms");
1033        if endpoint.expires_at_unix_ms == 0 {
1034            fields.remove("expires_at_unix_ms");
1035        }
1036        let encoded = serde_json::to_string(&payload)?;
1037        let stored: i64 = self
1038            .pool
1039            .eval(
1040                format!("{}{PUT}", Self::REAP_ANNOUNCEMENTS),
1041                vec![
1042                    self.announcement_key(service_name, &endpoint.socket_kind),
1043                    self.announcement_index(service_name),
1044                    self.key("services"),
1045                    self.key("service-names"),
1046                    self.key("announcement-global-revision"),
1047                    self.key("announcement-expiry"),
1048                ],
1049                vec![
1050                    encoded,
1051                    endpoint.accepted_state_epoch.to_string(),
1052                    endpoint.expires_at_unix_ms.to_string(),
1053                    endpoint.live_until_unix_ms.to_string(),
1054                    service_id,
1055                    service_name.to_owned(),
1056                    self.announcement_capacity.to_string(),
1057                    ANNOUNCED_ENDPOINT_TTL.as_millis().to_string(),
1058                ],
1059            )
1060            .await?;
1061        Ok(if stored == 1 {
1062            PutResult::Stored
1063        } else {
1064            PutResult::IgnoredOlder
1065        })
1066    }
1067
1068    async fn announcements_for(
1069        &self,
1070        service_name: &str,
1071        _now_unix_ms: i64,
1072    ) -> Result<Vec<AnnouncedEndpoint>> {
1073        self.announcements_for_inner(service_name).await
1074    }
1075
1076    async fn all_announcements(
1077        &self,
1078        _now_unix_ms: i64,
1079    ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>> {
1080        use fred::prelude::*;
1081
1082        // One transaction returns names and values from the capacity-bounded
1083        // expiry index. Reap and refresh still serialize, and a listing never
1084        // labels an L1 snapshot or performs round trips per service.
1085        const LIST: &str = r#"
1086reap(KEYS[1], KEYS[2], KEYS[3], KEYS[4])
1087local values = {}
1088for _, key in ipairs(redis.call('ZRANGE', KEYS[1], 0, -1)) do
1089  local value = redis.call('GET', key)
1090  if value then
1091    local ok, decoded = pcall(cjson.decode, value)
1092    if not ok then return redis.error_reply('corrupt Discovery announcement') end
1093    if tonumber(decoded.live_until_unix_ms) > now and tonumber(decoded.expires_at_unix_ms) > now then
1094      local service = string.match(key, ':announcement:([^:]+):[^:]+$')
1095      local name = redis.call('HGET', KEYS[3], service)
1096      if not name then return redis.error_reply('missing Discovery service name') end
1097      table.insert(values, {name, value})
1098    end
1099  end
1100end
1101return values
1102"#;
1103        let encoded: Vec<(String, String)> = self
1104            .pool
1105            .eval(
1106                format!("{}{LIST}", Self::REAP_ANNOUNCEMENTS),
1107                vec![
1108                    self.key("announcement-expiry"),
1109                    self.key("services"),
1110                    self.key("service-names"),
1111                    self.key("announcement-global-revision"),
1112                ],
1113                Vec::<String>::new(),
1114            )
1115            .await?;
1116        let mut all: HashMap<String, Vec<AnnouncedEndpoint>> = HashMap::new();
1117        for (name, value) in encoded {
1118            let endpoint: AnnouncedEndpoint = serde_json::from_str(&value)?;
1119            all.entry(name).or_default().push(endpoint);
1120        }
1121        Ok(all.into_iter().collect())
1122    }
1123
1124    async fn put_liveness(&self, node: &Did, value: LiveAllocatable) -> Result<PutResult> {
1125        use fred::prelude::*;
1126
1127        const PUT: &str = r#"
1128local clock = redis.call('TIME')
1129local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
1130local current = redis.call('GET', KEYS[1])
1131redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now)
1132if not current and redis.call('ZCARD', KEYS[3]) >= tonumber(ARGV[3]) then
1133  return redis.error_reply('Discovery Valkey liveness capacity exhausted')
1134end
1135local expiry = now + tonumber(ARGV[2])
1136-- Preserve JSON arrays/numeric payload exactly: Lua cjson turns [] into {}.
1137local encoded = string.sub(ARGV[1], 1, -2) .. ',"last_seen":' .. string.format('%.0f', now)
1138  .. ',"live_until_unix_ms":' .. string.format('%.0f', expiry) .. '}'
1139redis.call('SET', KEYS[1], encoded, 'PXAT', expiry)
1140redis.call('INCR', KEYS[2])
1141redis.call('ZADD', KEYS[3], expiry, KEYS[1])
1142return 1
1143"#;
1144        let node_id = Self::digest(node.as_str());
1145        let stored: i64 = self
1146            .pool
1147            .eval(
1148                PUT,
1149                vec![
1150                    format!("{}:liveness:{node_id}", self.prefix),
1151                    self.key("liveness-global-revision"),
1152                    self.key("liveness-expiry"),
1153                ],
1154                vec![
1155                    serde_json::to_string(&serde_json::json!({
1156                        "node": node, "allocatable": value.allocatable,
1157                        "load_fraction": value.load_fraction, "shared_receipt": true,
1158                    }))?,
1159                    value
1160                        .live_until_unix_ms
1161                        .saturating_sub(value.last_seen)
1162                        .clamp(1, 45_000)
1163                        .to_string(),
1164                    self.liveness_capacity.to_string(),
1165                ],
1166            )
1167            .await?;
1168        Ok(if stored == 1 {
1169            PutResult::Stored
1170        } else {
1171            PutResult::IgnoredOlder
1172        })
1173    }
1174
1175    #[cfg(test)]
1176    async fn liveness(&self, node: &Did, _now_unix_ms: i64) -> Result<Option<LiveAllocatable>> {
1177        use fred::prelude::*;
1178        let encoded: Option<String> = self
1179            .pool
1180            .get(format!(
1181                "{}:liveness:{}",
1182                self.prefix,
1183                Self::digest(node.as_str())
1184            ))
1185            .await?;
1186        Ok(encoded
1187            .map(|encoded| serde_json::from_str::<StoredLiveness>(&encoded))
1188            .transpose()?
1189            .filter(|record| record.shared_receipt)
1190            .map(|record| record.value))
1191    }
1192
1193    async fn all_liveness(&self, _now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>> {
1194        use fred::prelude::*;
1195        // The expiry index is capacity-bounded on every write. Enumerate it
1196        // atomically with the values so concurrent expiry/replacement cannot
1197        // leave a process-local candidate seed out of sync with shared state.
1198        const LIST: &str = r#"
1199local clock = redis.call('TIME')
1200local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
1201redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
1202local values = {}
1203for _, key in ipairs(redis.call('ZRANGE', KEYS[1], 0, -1)) do
1204  local value = redis.call('GET', key)
1205  if value then
1206    local record = cjson.decode(value)
1207    if record.shared_receipt == true then
1208      table.insert(values, value)
1209    else
1210      -- Legacy replica-clock rows have no trustworthy shared lifetime. Retire
1211      -- only this volatile row; the next admitted heartbeat recreates it.
1212      redis.call('DEL', key)
1213      redis.call('ZREM', KEYS[1], key)
1214      redis.call('INCR', KEYS[2])
1215    end
1216  end
1217end
1218return values
1219"#;
1220        let encoded: Vec<String> = self
1221            .pool
1222            .eval(
1223                LIST,
1224                vec![
1225                    self.key("liveness-expiry"),
1226                    self.key("liveness-global-revision"),
1227                ],
1228                Vec::<String>::new(),
1229            )
1230            .await?;
1231        let mut live = Vec::with_capacity(encoded.len());
1232        for value in encoded {
1233            let record: StoredLiveness = serde_json::from_str(&value)?;
1234            live.push((record.node, record.value));
1235        }
1236        Ok(live)
1237    }
1238
1239    async fn put_entity_statement(&self, issuer: &str, value: CachedEntityStatement) -> Result<()> {
1240        use fred::prelude::*;
1241        const PUT: &str = r#"
1242if redis.call('EXISTS', KEYS[1]) == 0 and redis.call('SCARD', KEYS[2]) >= tonumber(ARGV[4]) then
1243  local oldest = redis.call('ZRANGE', KEYS[5], 0, 0)[1]
1244  if not oldest then return redis.error_reply('Discovery issuer cache index is inconsistent') end
1245  local oldkey = string.gsub(KEYS[1], ':entity:[^:]+$', ':entity:' .. oldest)
1246  redis.call('DEL', oldkey)
1247  redis.call('SREM', KEYS[2], oldest)
1248  redis.call('HDEL', KEYS[3], oldest)
1249  redis.call('ZREM', KEYS[5], oldest)
1250end
1251redis.call('SET', KEYS[1], ARGV[1])
1252redis.call('SADD', KEYS[2], ARGV[2])
1253redis.call('HSET', KEYS[3], ARGV[2], ARGV[3])
1254redis.call('INCR', KEYS[4])
1255redis.call('ZADD', KEYS[5], ARGV[5], ARGV[2])
1256return 1
1257"#;
1258        let id = Self::digest(issuer);
1259        let _: i64 = self
1260            .pool
1261            .eval(
1262                PUT,
1263                vec![
1264                    format!("{}:entity:{id}", self.prefix),
1265                    self.key("issuers"),
1266                    self.key("issuer-names"),
1267                    self.key("entity-global-revision"),
1268                    self.key("issuer-fetched"),
1269                ],
1270                vec![
1271                    serde_json::to_string(&value)?,
1272                    id,
1273                    issuer.to_owned(),
1274                    self.artifact_capacity.to_string(),
1275                    value.fetched_at.to_string(),
1276                ],
1277            )
1278            .await?;
1279        Ok(())
1280    }
1281
1282    async fn entity_statement(&self, issuer: &str) -> Result<Option<CachedEntityStatement>> {
1283        use fred::prelude::*;
1284        let encoded: Option<String> = self
1285            .pool
1286            .get(format!("{}:entity:{}", self.prefix, Self::digest(issuer)))
1287            .await?;
1288        encoded
1289            .map(|encoded| serde_json::from_str(&encoded).map_err(Into::into))
1290            .transpose()
1291    }
1292
1293    async fn known_issuers(&self) -> Result<Vec<String>> {
1294        use fred::prelude::*;
1295        Ok(self.pool.hvals(self.key("issuer-names")).await?)
1296    }
1297
1298    async fn known_issuer_count(&self) -> Result<usize> {
1299        use fred::prelude::*;
1300        Ok(self.pool.scard(self.key("issuers")).await?)
1301    }
1302
1303    async fn put_envelope_keyset(
1304        &self,
1305        service_did: &str,
1306        value: CachedEnvelopeKeyset,
1307    ) -> Result<()> {
1308        use fred::prelude::*;
1309        const PUT: &str = r#"
1310if redis.call('EXISTS', KEYS[1]) == 0 and redis.call('SCARD', KEYS[3]) >= tonumber(ARGV[2]) then
1311  return redis.error_reply('Discovery Valkey federation artifact capacity exhausted')
1312end
1313redis.call('SET', KEYS[1], ARGV[1])
1314redis.call('INCR', KEYS[2])
1315redis.call('SADD', KEYS[3], KEYS[1])
1316return 1
1317"#;
1318        let id = Self::digest(service_did);
1319        let _: i64 = self
1320            .pool
1321            .eval(
1322                PUT,
1323                vec![
1324                    format!("{}:envelope:{id}", self.prefix),
1325                    format!("{}:envelope-revision:{id}", self.prefix),
1326                    self.key("envelopes"),
1327                ],
1328                vec![
1329                    serde_json::to_string(&value)?,
1330                    self.artifact_capacity.to_string(),
1331                ],
1332            )
1333            .await?;
1334        Ok(())
1335    }
1336
1337    async fn envelope_keyset(&self, service_did: &str) -> Result<Option<CachedEnvelopeKeyset>> {
1338        use fred::prelude::*;
1339        let encoded: Option<String> = self
1340            .pool
1341            .get(format!(
1342                "{}:envelope:{}",
1343                self.prefix,
1344                Self::digest(service_did)
1345            ))
1346            .await?;
1347        encoded
1348            .map(|encoded| serde_json::from_str(&encoded).map_err(Into::into))
1349            .transpose()
1350    }
1351}
1352
1353#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1354#[derive(Clone, Copy)]
1355struct ObservedRevision {
1356    revision: u64,
1357    valid_until_unix_ms: i64,
1358}
1359
1360#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1361struct TieredStateStore {
1362    memory: Arc<MemoryStateStore>,
1363    valkey: Arc<ValkeyStateStore>,
1364    l1_max_ttl_ms: i64,
1365    observed: Mutex<HashMap<String, ObservedRevision>>,
1366    // Serialize cache contents/revision labels within a scope, while unrelated
1367    // network reads and heartbeats can use the pool concurrently. Fixed storage
1368    // avoids accumulating a mutex for every historical identity.
1369    operations: [tokio::sync::Mutex<()>; 64],
1370}
1371
1372#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1373impl TieredStateStore {
1374    fn new(
1375        memory: Arc<MemoryStateStore>,
1376        valkey: Arc<ValkeyStateStore>,
1377        l1_max_ttl_ms: u64,
1378    ) -> Self {
1379        Self {
1380            memory,
1381            valkey,
1382            l1_max_ttl_ms: i64::try_from(l1_max_ttl_ms).unwrap_or(i64::MAX),
1383            observed: Mutex::new(HashMap::new()),
1384            operations: std::array::from_fn(|_| tokio::sync::Mutex::new(())),
1385        }
1386    }
1387
1388    fn scope(kind: &str, value: &str) -> String {
1389        format!("{kind}:{value}")
1390    }
1391
1392    fn operation(&self, kind: &str, value: &str) -> &tokio::sync::Mutex<()> {
1393        let digest = blake3::hash(Self::scope(kind, value).as_bytes());
1394        &self.operations[usize::from(digest.as_bytes()[0]) % self.operations.len()]
1395    }
1396
1397    fn is_observed(&self, scope: &str, revision: u64, now_unix_ms: i64) -> bool {
1398        self.observed.lock().get(scope).is_some_and(|observed| {
1399            observed.revision == revision && now_unix_ms < observed.valid_until_unix_ms
1400        })
1401    }
1402
1403    fn observe(&self, scope: String, revision: u64, now_unix_ms: i64) {
1404        let mut observed = self.observed.lock();
1405        observed.retain(|_, value| now_unix_ms < value.valid_until_unix_ms);
1406        let capacity = self
1407            .memory
1408            .announcement_capacity
1409            .saturating_add(self.memory.liveness_capacity)
1410            .saturating_add(self.memory.artifact_capacity.saturating_mul(2));
1411        if observed.len() >= capacity {
1412            observed.clear();
1413        }
1414        observed.insert(
1415            scope,
1416            ObservedRevision {
1417                revision,
1418                valid_until_unix_ms: now_unix_ms.saturating_add(self.l1_max_ttl_ms),
1419            },
1420        );
1421    }
1422
1423    fn l1_announcement(
1424        &self,
1425        mut endpoint: AnnouncedEndpoint,
1426        now_unix_ms: i64,
1427    ) -> AnnouncedEndpoint {
1428        endpoint.live_until_unix_ms = endpoint
1429            .live_until_unix_ms
1430            .min(now_unix_ms.saturating_add(self.l1_max_ttl_ms));
1431        endpoint
1432    }
1433}
1434
1435#[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1436#[async_trait]
1437impl DiscoveryStateStore for TieredStateStore {
1438    async fn put_announcement(
1439        &self,
1440        service_name: &str,
1441        endpoint: AnnouncedEndpoint,
1442    ) -> Result<PutResult> {
1443        let _operation = self.operation("announcement", service_name).lock().await;
1444        // Invalidate before the command, including on ambiguous write errors
1445        // or cancellation. Never tag a caller's value with a later revision.
1446        self.observed
1447            .lock()
1448            .remove(&Self::scope("announcement", service_name));
1449        self.memory.clear_announcements_sync(service_name);
1450        self.valkey.put_announcement(service_name, endpoint).await
1451    }
1452
1453    async fn announcements_for(
1454        &self,
1455        service_name: &str,
1456        _now_unix_ms: i64,
1457    ) -> Result<Vec<AnnouncedEndpoint>> {
1458        let _operation = self.operation("announcement", service_name).lock().await;
1459        let scope = Self::scope("announcement", service_name);
1460        let (revision, now_unix_ms) = self.valkey.announcement_revision_at_shared_time().await?;
1461        if self.is_observed(&scope, revision, now_unix_ms) {
1462            return self
1463                .memory
1464                .announcements_for(service_name, now_unix_ms)
1465                .await;
1466        }
1467        let values = self
1468            .valkey
1469            .announcements_for(service_name, now_unix_ms)
1470            .await?;
1471        self.observed.lock().remove(&scope);
1472        self.memory.clear_announcements_sync(service_name);
1473        // L1 admission is optional; only a complete nonempty result is marked
1474        // observed. Oversized scopes are served from L2 without partial hits.
1475        for value in &values {
1476            if self
1477                .memory
1478                .put_announcement_at(
1479                    service_name,
1480                    self.l1_announcement(value.clone(), now_unix_ms),
1481                    now_unix_ms,
1482                )
1483                .is_err()
1484            {
1485                self.memory.clear_announcements_sync(service_name);
1486                return Ok(values);
1487            }
1488        }
1489        if !values.is_empty() {
1490            self.observe(scope, revision, now_unix_ms);
1491        }
1492        Ok(values)
1493    }
1494
1495    async fn all_announcements(
1496        &self,
1497        now_unix_ms: i64,
1498    ) -> Result<Vec<(String, Vec<AnnouncedEndpoint>)>> {
1499        // A bounded L1 is not a complete replica of L2. Listings always read
1500        // authoritative indexes and cannot invalidate point-cache snapshots.
1501        self.valkey.all_announcements(now_unix_ms).await
1502    }
1503
1504    async fn put_liveness(&self, node: &Did, value: LiveAllocatable) -> Result<PutResult> {
1505        let _operation = self.operation("liveness", node.as_str()).lock().await;
1506        self.observed
1507            .lock()
1508            .remove(&Self::scope("liveness", node.as_str()));
1509        self.memory.clear_liveness_sync(node);
1510        self.valkey.put_liveness(node, value).await
1511    }
1512
1513    #[cfg(test)]
1514    async fn liveness(&self, node: &Did, now_unix_ms: i64) -> Result<Option<LiveAllocatable>> {
1515        let _operation = self.operation("liveness", node.as_str()).lock().await;
1516        // One authoritative GET is cheaper than revision + value on a miss,
1517        // and leaves expiry entirely on the shared clock, even with host skew.
1518        self.valkey.liveness(node, now_unix_ms).await
1519    }
1520
1521    async fn all_liveness(&self, now_unix_ms: i64) -> Result<Vec<(Did, LiveAllocatable)>> {
1522        self.valkey.all_liveness(now_unix_ms).await
1523    }
1524
1525    async fn put_entity_statement(&self, issuer: &str, value: CachedEntityStatement) -> Result<()> {
1526        let _operation = self.operation("entity", issuer).lock().await;
1527        self.observed.lock().remove(&Self::scope("entity", issuer));
1528        self.memory.clear_entity_statement_sync(issuer);
1529        self.valkey.put_entity_statement(issuer, value).await
1530    }
1531
1532    async fn entity_statement(&self, issuer: &str) -> Result<Option<CachedEntityStatement>> {
1533        let _operation = self.operation("entity", issuer).lock().await;
1534        let now = unix_millis_now();
1535        let scope = Self::scope("entity", issuer);
1536        let revision = self.valkey.entity_revision().await?;
1537        if self.is_observed(&scope, revision, now) {
1538            // A fill of a different issuer can evict this L1 entry without a
1539            // shared write. Missing L1 data is a miss, never cached absence.
1540            if let Some(value) = self.memory.entity_statement(issuer).await? {
1541                return Ok(Some(value));
1542            }
1543        }
1544        let value = self.valkey.entity_statement(issuer).await?;
1545        self.observed.lock().remove(&scope);
1546        self.memory.clear_entity_statement_sync(issuer);
1547        if let Some(value) = &value {
1548            if self
1549                .memory
1550                .put_entity_statement(issuer, value.clone())
1551                .await
1552                .is_ok()
1553            {
1554                self.observe(scope, revision, now);
1555            }
1556        }
1557        Ok(value)
1558    }
1559
1560    async fn known_issuers(&self) -> Result<Vec<String>> {
1561        self.valkey.known_issuers().await
1562    }
1563
1564    async fn known_issuer_count(&self) -> Result<usize> {
1565        self.valkey.known_issuer_count().await
1566    }
1567
1568    async fn put_envelope_keyset(
1569        &self,
1570        service_did: &str,
1571        value: CachedEnvelopeKeyset,
1572    ) -> Result<()> {
1573        let _operation = self.operation("envelope", service_did).lock().await;
1574        self.observed
1575            .lock()
1576            .remove(&Self::scope("envelope", service_did));
1577        self.memory.clear_envelope_keyset_sync(service_did);
1578        self.valkey.put_envelope_keyset(service_did, value).await
1579    }
1580
1581    async fn envelope_keyset(&self, service_did: &str) -> Result<Option<CachedEnvelopeKeyset>> {
1582        let _operation = self.operation("envelope", service_did).lock().await;
1583        let now = unix_millis_now();
1584        let scope = Self::scope("envelope", service_did);
1585        let revision = self.valkey.envelope_revision(service_did).await?;
1586        if self.is_observed(&scope, revision, now) {
1587            return self.memory.envelope_keyset(service_did).await;
1588        }
1589        let value = self.valkey.envelope_keyset(service_did).await?;
1590        self.observed.lock().remove(&scope);
1591        self.memory.clear_envelope_keyset_sync(service_did);
1592        if let Some(value) = &value {
1593            if self
1594                .memory
1595                .put_envelope_keyset(service_did, value.clone())
1596                .await
1597                .is_ok()
1598            {
1599                self.observe(scope, revision, now);
1600            }
1601        }
1602        Ok(value)
1603    }
1604}
1605
1606#[cfg(test)]
1607thread_local! {
1608    // Used only by current-thread clock-skew tests; never changes the host or
1609    // the clock of another concurrently running test/Valkey driver thread.
1610    static TEST_REPLICA_TIME: std::cell::Cell<Option<i64>> = const { std::cell::Cell::new(None) };
1611}
1612
1613pub(crate) fn unix_millis_now() -> i64 {
1614    #[cfg(test)]
1615    if let Some(now) = TEST_REPLICA_TIME.with(std::cell::Cell::get) {
1616        return now;
1617    }
1618    use std::time::{SystemTime, UNIX_EPOCH};
1619    SystemTime::now()
1620        .duration_since(UNIX_EPOCH)
1621        .map(|duration| duration.as_millis() as i64)
1622        .unwrap_or(0)
1623}
1624
1625#[cfg(test)]
1626pub(crate) mod tests {
1627    #![allow(clippy::expect_used, clippy::indexing_slicing, clippy::unwrap_used)]
1628
1629    use super::*;
1630
1631    fn endpoint(kind: &str, epoch: u64, signed_expiry: i64, live_until: i64) -> AnnouncedEndpoint {
1632        AnnouncedEndpoint {
1633            socket_kind: kind.to_owned(),
1634            endpoint: format!("iroh://{kind}"),
1635            service_did: Did::new("did:at9p:test".to_owned()),
1636            capabilities: BTreeSet::from(["discovery".to_owned()]),
1637            accepted_state_digest: vec![7; 64],
1638            accepted_state_epoch: epoch,
1639            response_key_id: "did:at9p:test#response".to_owned(),
1640            request_kem_key_id: "did:at9p:test#kem".to_owned(),
1641            request_kem_recipient: vec![1, 2, 3],
1642            expires_at_unix_ms: signed_expiry,
1643            source_signer: [9; 32],
1644            live_until_unix_ms: live_until,
1645        }
1646    }
1647
1648    #[test]
1649    fn legacy_announcement_bearer_field_is_dropped_on_read_and_rewrite() {
1650        let now = unix_millis_now();
1651        let mut legacy = serde_json::to_value(endpoint("iroh", 1, now + 10_000, now + 50))
1652            .expect("serialize current endpoint fixture");
1653        legacy
1654            .as_object_mut()
1655            .expect("endpoint fixture is an object")
1656            .insert(
1657                "service_jwt".to_owned(),
1658                serde_json::json!("legacy-bearer-jwt"),
1659            );
1660
1661        let parsed: AnnouncedEndpoint =
1662            serde_json::from_value(legacy).expect("legacy endpoint remains readable");
1663        let rewritten = serde_json::to_string(&parsed).expect("rewrite endpoint");
1664        assert!(!rewritten.contains("service_jwt"));
1665        assert!(!rewritten.contains("legacy-bearer-jwt"));
1666    }
1667
1668    async fn assert_announcement_backend_contract(store: &dyn DiscoveryStateStore) {
1669        let now = unix_millis_now();
1670        store
1671            .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 50))
1672            .await
1673            .unwrap();
1674        assert_eq!(
1675            store.announcements_for("model", now).await.unwrap().len(),
1676            1
1677        );
1678        store
1679            .put_announcement("model", endpoint("iroh", 2, now + 10_000, now + 50))
1680            .await
1681            .unwrap();
1682        assert_eq!(
1683            store.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
1684            2
1685        );
1686
1687        tokio::time::sleep(std::time::Duration::from_millis(80)).await;
1688        assert!(store
1689            .announcements_for("model", unix_millis_now())
1690            .await
1691            .unwrap()
1692            .is_empty());
1693
1694        let now = unix_millis_now();
1695        store
1696            .put_announcement("policy", endpoint("iroh", 1, now + 10_000, now + 5_000))
1697            .await
1698            .unwrap();
1699        let error = store
1700            .put_announcement("other", endpoint("iroh", 1, now + 10_000, now + 5_000))
1701            .await
1702            .unwrap_err();
1703        assert!(error.to_string().contains("capacity exhausted"));
1704    }
1705
1706    #[tokio::test]
1707    async fn memory_satisfies_announcement_backend_contract() {
1708        assert_announcement_backend_contract(&MemoryStateStore::new(1, 1, 1)).await;
1709    }
1710
1711    async fn assert_receipt_ordered_liveness(store: &dyn DiscoveryStateStore) {
1712        let node = Did::new("did:web:clock-rollback.example".to_owned());
1713        let now = unix_millis_now();
1714        let mut value = LiveAllocatable {
1715            allocatable: vec![("cpu".to_owned(), "1".to_owned())],
1716            load_fraction: 0.9,
1717            // Legacy client-clock record must not poison subsequent receipts.
1718            last_seen: now + 86_400_000,
1719            live_until_unix_ms: now + 100,
1720        };
1721        store.put_liveness(&node, value.clone()).await.unwrap();
1722        value.last_seen = now;
1723        value.load_fraction = 0.2;
1724        value.allocatable[0].1 = "8".to_owned();
1725        value.live_until_unix_ms = now + 500;
1726        assert_eq!(
1727            store.put_liveness(&node, value.clone()).await.unwrap(),
1728            PutResult::Stored
1729        );
1730        assert_eq!(
1731            store.liveness(&node, now + 200).await.unwrap(),
1732            Some(value.clone())
1733        );
1734        let mut delayed = value.clone();
1735        delayed.last_seen = now - 1;
1736        delayed.live_until_unix_ms = now + 300;
1737        delayed.load_fraction = 1.0;
1738        assert_eq!(
1739            store.put_liveness(&node, delayed).await.unwrap(),
1740            PutResult::IgnoredOlder
1741        );
1742        assert_eq!(store.liveness(&node, now + 200).await.unwrap(), Some(value));
1743        assert!(store.liveness(&node, now + 501).await.unwrap().is_none());
1744
1745        // Liveness refreshes do not renew a signed announcement's authority.
1746        store
1747            .put_announcement("clock", endpoint("iroh", 1, now + 50, now + 500))
1748            .await
1749            .unwrap();
1750        assert!(store
1751            .announcements_for("clock", now + 51)
1752            .await
1753            .unwrap()
1754            .is_empty());
1755    }
1756
1757    #[tokio::test]
1758    async fn memory_heartbeat_receipt_survives_clock_rollback() {
1759        assert_receipt_ordered_liveness(&MemoryStateStore::new(8, 8, 8)).await;
1760    }
1761
1762    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1763    #[tokio::test]
1764    async fn valkey_heartbeat_receipt_survives_clock_rollback() {
1765        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
1766            return;
1767        };
1768        use fred::prelude::*;
1769        let config = valkey_config(url, "rollback");
1770        let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
1771        let other = ValkeyStateStore::connect(&config).await.unwrap();
1772        let tiered = TieredStateStore::new(
1773            Arc::new(MemoryStateStore::new(8, 8, 8)),
1774            shared.clone(),
1775            1000,
1776        );
1777        let node = Did::new("did:web:skew.example".to_owned());
1778        let now = unix_millis_now();
1779        let mut value = LiveAllocatable {
1780            allocatable: vec![],
1781            load_fraction: 0.9,
1782            last_seen: now + 86_400_000,
1783            live_until_unix_ms: now + 86_400_400,
1784        };
1785        tiered.put_liveness(&node, value.clone()).await.unwrap();
1786        let first = tiered.liveness(&node, i64::MAX).await.unwrap().unwrap();
1787        assert!((first.last_seen - now).abs() < 5000);
1788        value.last_seen = now - 86_400_000;
1789        value.live_until_unix_ms = value.last_seen + 400;
1790        value.load_fraction = 0.2;
1791        value.allocatable = vec![("cpu".to_owned(), "8".to_owned())];
1792        assert_eq!(
1793            other.put_liveness(&node, value).await.unwrap(),
1794            PutResult::Stored
1795        );
1796        let latest = tiered.liveness(&node, i64::MAX).await.unwrap().unwrap();
1797        assert!(latest.last_seen >= first.last_seen);
1798        assert_eq!(latest.load_fraction, 0.2);
1799        assert_eq!(latest.allocatable[0].1, "8");
1800        assert_eq!(latest.live_until_unix_ms - latest.last_seen, 400);
1801        assert_eq!(tiered.all_liveness(i64::MAX).await.unwrap().len(), 1);
1802        let ttl: i64 = shared
1803            .pool
1804            .pttl(format!(
1805                "{}:liveness:{}",
1806                shared.prefix,
1807                ValkeyStateStore::digest(node.as_str())
1808            ))
1809            .await
1810            .unwrap();
1811        assert!(
1812            (1..=400).contains(&ttl),
1813            "expiry uses shared receipt, not ahead host time"
1814        );
1815        tokio::time::sleep(std::time::Duration::from_millis(450)).await;
1816        assert!(tiered.liveness(&node, i64::MIN).await.unwrap().is_none());
1817        assert!(tiered.all_liveness(i64::MIN).await.unwrap().is_empty());
1818    }
1819
1820    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1821    #[tokio::test]
1822    async fn rediss_uses_tls_and_rejects_plaintext_valkey() {
1823        use fred::prelude::RedisConfig;
1824        hyprstream_rpc::transport::install_pq_crypto_provider().unwrap();
1825        assert!(RedisConfig::from_url("rediss://localhost:6379")
1826            .unwrap()
1827            .tls
1828            .is_some());
1829        assert!(RedisConfig::from_url("redis://localhost:6379")
1830            .unwrap()
1831            .tls
1832            .is_none());
1833        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
1834            return;
1835        };
1836        let mut config = valkey_config(url.clone(), "tls");
1837        let plaintext = ValkeyStateStore::connect(&config).await.unwrap();
1838        assert!(plaintext
1839            .all_announcements(unix_millis_now())
1840            .await
1841            .unwrap()
1842            .is_empty());
1843        config.url = url.replacen("redis://", "rediss://", 1);
1844        let start = std::time::Instant::now();
1845        assert!(
1846            ValkeyStateStore::connect(&config).await.is_err(),
1847            "TLS must not downgrade to plaintext"
1848        );
1849        assert!(start.elapsed() < std::time::Duration::from_secs(3));
1850    }
1851
1852    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
1853    #[tokio::test]
1854    async fn valkey_batched_listing_preserves_names_values_and_cleanup() {
1855        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
1856            return;
1857        };
1858        let mut config = valkey_config(url, "batched-list");
1859        config.announcement_capacity = 512;
1860        let writer = ValkeyStateStore::connect(&config).await.unwrap();
1861        let reader = TieredStateStore::new(
1862            Arc::new(MemoryStateStore::new(1, 1, 1)),
1863            Arc::new(ValkeyStateStore::connect(&config).await.unwrap()),
1864            10_000,
1865        );
1866        let now = unix_millis_now();
1867        let cohort_expiry = now + 5_000;
1868        for i in 0..256 {
1869            let name = format!("service:{i}:λ");
1870            for kind in ["iroh", "quic"] {
1871                writer
1872                    .put_announcement(&name, endpoint(kind, 1, cohort_expiry, cohort_expiry))
1873                    .await
1874                    .unwrap();
1875            }
1876        }
1877        assert_eq!(
1878            reader
1879                .announcements_for("service:0:λ", now)
1880                .await
1881                .unwrap()
1882                .len(),
1883            2
1884        );
1885        let entries = reader.all_announcements(now).await.unwrap();
1886        assert_eq!(entries.len(), 256);
1887        assert!(entries
1888            .iter()
1889            .all(|(name, values)| name.starts_with("service:") && values.len() == 2));
1890        writer
1891            .put_announcement(
1892                "service:0:λ",
1893                endpoint("iroh", 2, now + 90_000, now + 90_000),
1894            )
1895            .await
1896            .unwrap();
1897        // Caller time cannot expire shared values. Keep the original name,
1898        // value, index cleanup, and capacity assertions, but wait for the
1899        // cohort's real server-side lease instead of advancing a replica.
1900        assert_eq!(
1901            reader.all_announcements(now + 60_001).await.unwrap().len(),
1902            256
1903        );
1904        tokio::time::sleep(std::time::Duration::from_millis(
1905            (cohort_expiry + 50 - unix_millis_now()).max(0) as u64,
1906        ))
1907        .await;
1908        let remaining = reader.all_announcements(now + 60_001).await.unwrap();
1909        assert_eq!(remaining.len(), 1);
1910        assert_eq!(remaining[0].0, "service:0:λ");
1911        assert_eq!(remaining[0].1.len(), 1);
1912        assert_eq!(remaining[0].1[0].accepted_state_epoch, 2);
1913        // Reaping restores live capacity and cannot populate/poison point L1.
1914        writer
1915            .put_announcement("new", endpoint("iroh", 1, now + 90_000, now + 90_000))
1916            .await
1917            .unwrap();
1918        assert_eq!(
1919            reader
1920                .announcements_for("service:0:λ", now + 60_001)
1921                .await
1922                .unwrap()[0]
1923                .accepted_state_epoch,
1924            2
1925        );
1926        assert_eq!(
1927            reader.all_announcements(now + 60_001).await.unwrap().len(),
1928            2
1929        );
1930    }
1931
1932    #[tokio::test]
1933    async fn memory_replaces_by_key_without_fleet_scan() {
1934        let store = MemoryStateStore::new(8, 8, 8);
1935        let now = unix_millis_now();
1936        store
1937            .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 1_000))
1938            .await
1939            .unwrap();
1940        store
1941            .put_announcement("model", endpoint("iroh", 2, now + 20_000, now + 2_000))
1942            .await
1943            .unwrap();
1944        let entries = store.announcements_for("model", now).await.unwrap();
1945        assert_eq!(entries.len(), 1);
1946        assert_eq!(entries[0].accepted_state_epoch, 2);
1947    }
1948
1949    #[tokio::test]
1950    async fn memory_rejects_delayed_older_announcement() {
1951        let store = MemoryStateStore::new(8, 8, 8);
1952        let now = unix_millis_now();
1953        store
1954            .put_announcement("model", endpoint("iroh", 3, now + 20_000, now + 2_000))
1955            .await
1956            .unwrap();
1957        assert_eq!(
1958            store
1959                .put_announcement("model", endpoint("iroh", 2, now + 30_000, now + 3_000))
1960                .await
1961                .unwrap(),
1962            PutResult::IgnoredOlder
1963        );
1964        assert_eq!(
1965            store.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
1966            3
1967        );
1968    }
1969
1970    #[tokio::test]
1971    async fn memory_expiry_removes_secondary_index_entry() {
1972        let store = MemoryStateStore::new(8, 8, 8);
1973        let now = unix_millis_now();
1974        store
1975            .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 1))
1976            .await
1977            .unwrap();
1978        assert!(store.all_announcements(now + 2).await.unwrap().is_empty());
1979    }
1980
1981    #[tokio::test]
1982    async fn memory_capacity_rejects_instead_of_evicting_live_reach() {
1983        let store = MemoryStateStore::new(1, 1, 1);
1984        let now = unix_millis_now();
1985        store
1986            .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 1_000))
1987            .await
1988            .unwrap();
1989        let error = store
1990            .put_announcement("policy", endpoint("iroh", 1, now + 10_000, now + 1_000))
1991            .await
1992            .unwrap_err();
1993        assert!(error.to_string().contains("capacity exhausted"));
1994    }
1995
1996    #[tokio::test]
1997    async fn active_active_rejects_isolated_memory() {
1998        let config = DiscoveryStateConfig {
1999            active_active: true,
2000            ..DiscoveryStateConfig::default()
2001        };
2002        let error = match DiscoveryState::connect(&config).await {
2003            Ok(_) => panic!("active-active memory configuration was admitted"),
2004            Err(error) => error,
2005        };
2006        assert!(error.to_string().contains("requires valkey or tiered"));
2007    }
2008
2009    #[tokio::test]
2010    async fn rejects_zero_memory_capacity_and_tiered_freshness() {
2011        let mut memory = DiscoveryStateConfig::default();
2012        memory.memory.announcement_capacity = 0;
2013        let error = match DiscoveryState::connect(&memory).await {
2014            Ok(_) => panic!("zero-capacity memory configuration was admitted"),
2015            Err(error) => error,
2016        };
2017        assert!(error.to_string().contains("capacities must be positive"));
2018
2019        let mut tiered = DiscoveryStateConfig {
2020            backend: DiscoveryStateBackend::Tiered,
2021            ..DiscoveryStateConfig::default()
2022        };
2023        tiered.tiered.l1_max_ttl_ms = 0;
2024        let error = match DiscoveryState::connect(&tiered).await {
2025            Ok(_) => panic!("zero-freshness tiered configuration was admitted"),
2026            Err(error) => error,
2027        };
2028        assert!(error.to_string().contains("l1_max_ttl_ms must be positive"));
2029    }
2030
2031    async fn assert_issuer_eviction_contract(store: &dyn DiscoveryStateStore) {
2032        for (issuer, fetched_at) in [
2033            ("old", 1),
2034            ("refresh", 2),
2035            ("keep", 3),
2036            ("refresh", 4),
2037            ("new", 5),
2038        ] {
2039            store
2040                .put_entity_statement(
2041                    issuer,
2042                    CachedEntityStatement {
2043                        jwt: format!("inert:{issuer}:{fetched_at}"),
2044                        fetched_at,
2045                    },
2046                )
2047                .await
2048                .unwrap();
2049        }
2050        assert_eq!(store.known_issuer_count().await.unwrap(), 3);
2051        let mut names = store.known_issuers().await.unwrap();
2052        names.sort();
2053        assert_eq!(names, ["keep", "new", "refresh"]);
2054        assert!(store.entity_statement("old").await.unwrap().is_none());
2055        assert_eq!(
2056            store
2057                .entity_statement("refresh")
2058                .await
2059                .unwrap()
2060                .unwrap()
2061                .fetched_at,
2062            4
2063        );
2064        store
2065            .put_entity_statement(
2066                "old",
2067                CachedEntityStatement {
2068                    jwt: "reinserted".to_owned(),
2069                    fetched_at: 6,
2070                },
2071            )
2072            .await
2073            .unwrap();
2074        assert!(store.entity_statement("keep").await.unwrap().is_none());
2075        assert_eq!(
2076            store.entity_statement("old").await.unwrap().unwrap().jwt,
2077            "reinserted"
2078        );
2079    }
2080
2081    #[tokio::test]
2082    async fn memory_issuer_cache_evicts_oldest_and_accepts_reinsertion() {
2083        assert_issuer_eviction_contract(&MemoryStateStore::new(1, 1, 3)).await;
2084    }
2085
2086    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2087    #[tokio::test]
2088    async fn valkey_issuer_cache_churn_bounds_metadata_and_invalidates_other_l1() {
2089        use fred::prelude::*;
2090        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2091            return;
2092        };
2093        let mut config = valkey_config(url, "issuer-churn");
2094        config.artifact_capacity = 3;
2095        let writer = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2096        assert_issuer_eviction_contract(writer.as_ref()).await;
2097        let reader = TieredStateStore::new(
2098            Arc::new(MemoryStateStore::new(1, 1, 1)),
2099            Arc::new(ValkeyStateStore::connect(&config).await.unwrap()),
2100            60_000,
2101        );
2102        // A smaller L1 must refetch entries evicted by other fills even if L2
2103        // has not changed. Previously an observed-but-missing entry returned None.
2104        for issuer in ["old", "refresh", "old"] {
2105            assert!(reader.entity_statement(issuer).await.unwrap().is_some());
2106        }
2107        let generation = writer.entity_revision().await.unwrap();
2108        for i in 0..96 {
2109            writer
2110                .put_entity_statement(
2111                    &format!("issuer-{i}"),
2112                    CachedEntityStatement {
2113                        jwt: format!("inert-{i}"),
2114                        fetched_at: 100 + i,
2115                    },
2116                )
2117                .await
2118                .unwrap();
2119            assert_eq!(writer.known_issuer_count().await.unwrap(), 3);
2120        }
2121        assert!(reader.entity_statement("old").await.unwrap().is_none());
2122        let counts: Vec<usize> = writer.pool.eval(
2123            "return {redis.call('SCARD', KEYS[1]), redis.call('HLEN', KEYS[2]), redis.call('ZCARD', KEYS[3]), #redis.call('KEYS', ARGV[1]), #redis.call('KEYS', ARGV[2])}",
2124            vec![writer.key("issuers"), writer.key("issuer-names"), writer.key("issuer-fetched")],
2125            vec![writer.key("*"), writer.key("*revision*")],
2126        ).await.unwrap();
2127        assert_eq!(counts, [3, 3, 3, 7, 1]);
2128        assert!(writer.entity_revision().await.unwrap() > generation);
2129        writer
2130            .put_entity_statement(
2131                "old",
2132                CachedEntityStatement {
2133                    jwt: "new-generation".to_owned(),
2134                    fetched_at: 1000,
2135                },
2136            )
2137            .await
2138            .unwrap();
2139        assert_eq!(
2140            reader.entity_statement("old").await.unwrap().unwrap().jwt,
2141            "new-generation"
2142        );
2143        // Retain a positive old snapshot across eviction AND reinsertion (ABA).
2144        for i in 0..3 {
2145            writer
2146                .put_entity_statement(
2147                    &format!("replacement-{i}"),
2148                    CachedEntityStatement {
2149                        jwt: "other".to_owned(),
2150                        fetched_at: 2000 + i,
2151                    },
2152                )
2153                .await
2154                .unwrap();
2155        }
2156        writer
2157            .put_entity_statement(
2158                "old",
2159                CachedEntityStatement {
2160                    jwt: "after-aba".to_owned(),
2161                    fetched_at: 3000,
2162                },
2163            )
2164            .await
2165            .unwrap();
2166        assert_eq!(
2167            reader.entity_statement("old").await.unwrap().unwrap().jwt,
2168            "after-aba"
2169        );
2170        assert!(reader.observed.lock().len() <= 4);
2171    }
2172
2173    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2174    #[tokio::test]
2175    async fn valkey_issuer_listing_and_count_return_complete_bounded_cache() {
2176        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2177            return;
2178        };
2179        let mut config = valkey_config(url, "issuer-list-count");
2180        config.artifact_capacity = 128;
2181        let writer = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2182        let reader = TieredStateStore::new(
2183            Arc::new(MemoryStateStore::new(1, 1, 1)),
2184            writer.clone(),
2185            60_000,
2186        );
2187        let mut expected = Vec::new();
2188        for i in 0..256 {
2189            let name = format!("https://issuer:{i}:λ.example");
2190            writer
2191                .put_entity_statement(
2192                    &name,
2193                    CachedEntityStatement {
2194                        jwt: "inert".to_owned(),
2195                        fetched_at: i,
2196                    },
2197                )
2198                .await
2199                .unwrap();
2200            if i >= 128 {
2201                expected.push(name);
2202            }
2203        }
2204        expected.sort();
2205        for _ in 0..2 {
2206            let mut actual = reader.known_issuers().await.unwrap();
2207            actual.sort();
2208            assert_eq!(actual, expected);
2209            assert_eq!(reader.known_issuer_count().await.unwrap(), 128);
2210        }
2211        assert!(reader.memory.inner.lock().entity_statements.is_empty());
2212        assert!(reader.observed.lock().is_empty());
2213    }
2214
2215    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2216    fn valkey_config(url: String, suffix: &str) -> ValkeyStateConfig {
2217        ValkeyStateConfig {
2218            url,
2219            key_prefix: format!(
2220                "hs-test-{}-{}-{suffix}",
2221                std::process::id(),
2222                unix_millis_now()
2223            ),
2224            pool_size: 2,
2225            announcement_capacity: 64,
2226            liveness_capacity: 64,
2227            artifact_capacity: 64,
2228            command_timeout_ms: 250,
2229        }
2230    }
2231
2232    pub(crate) struct ReplicaClock;
2233
2234    impl ReplicaClock {
2235        pub(crate) fn at(now: i64) -> Self {
2236            TEST_REPLICA_TIME.with(|clock| {
2237                assert!(clock.replace(Some(now)).is_none());
2238            });
2239            Self
2240        }
2241    }
2242
2243    impl Drop for ReplicaClock {
2244        fn drop(&mut self) {
2245            TEST_REPLICA_TIME.with(|clock| clock.set(None));
2246        }
2247    }
2248
2249    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2250    #[tokio::test]
2251    async fn valkey_announcement_lease_uses_receipt_and_preserves_payload() {
2252        use fred::prelude::*;
2253        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2254            return;
2255        };
2256        let store = ValkeyStateStore::connect(&valkey_config(url, "announcement-write-lease"))
2257            .await
2258            .unwrap();
2259        let now: i64 = store.pool.eval(
2260            "local t = redis.call('TIME'); return tonumber(t[1])*1000 + math.floor(tonumber(t[2])/1000)",
2261            Vec::<String>::new(), Vec::<String>::new(),
2262        ).await.unwrap();
2263        for skew in [-3_600_000, 3_600_000] {
2264            let name = format!("lease-{skew}");
2265            let mut value = endpoint("iroh", 1, now + 7_200_000, now + 7_200_000);
2266            value.capabilities.clear();
2267            value.response_key_id = "payload with \"live_until_unix_ms\":123 and λ".to_owned();
2268            value.source_signer = [255; 32];
2269            let mut expected = serde_json::to_value(&value).unwrap();
2270            expected
2271                .as_object_mut()
2272                .unwrap()
2273                .remove("live_until_unix_ms");
2274            let _clock = ReplicaClock::at(now + skew);
2275            assert_eq!(
2276                store.put_announcement(&name, value).await.unwrap(),
2277                PutResult::Stored
2278            );
2279            let rows: (String, i64, i64, i64) = store.pool.eval(
2280                "local t = redis.call('TIME'); return {redis.call('GET', KEYS[1]), redis.call('PEXPIRETIME', KEYS[1]), tonumber(redis.call('ZSCORE', KEYS[2], KEYS[1])), tonumber(t[1])*1000 + math.floor(tonumber(t[2])/1000)}",
2281                vec![store.announcement_key(&name, "iroh"), store.key("announcement-expiry")], Vec::<String>::new(),
2282            ).await.unwrap();
2283            let mut actual: serde_json::Value = serde_json::from_str(&rows.0).unwrap();
2284            let encoded_expiry = actual
2285                .as_object_mut()
2286                .unwrap()
2287                .remove("live_until_unix_ms")
2288                .unwrap()
2289                .as_i64()
2290                .unwrap();
2291            assert_eq!(
2292                actual, expected,
2293                "only the effective lease field may change"
2294            );
2295            assert_eq!(encoded_expiry, rows.1);
2296            assert_eq!(encoded_expiry, rows.2);
2297            assert!(
2298                (89_000..=90_000).contains(&(encoded_expiry - rows.3)),
2299                "bounded shared receipt lease, independent of replica skew"
2300            );
2301            assert_eq!(
2302                store
2303                    .announcements_for(&name, now + skew)
2304                    .await
2305                    .unwrap()
2306                    .len(),
2307                1
2308            );
2309        }
2310    }
2311
2312    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2313    #[tokio::test]
2314    async fn valkey_announcement_reaping_ignores_replica_clock_skew() {
2315        use fred::prelude::*;
2316
2317        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2318            return;
2319        };
2320        // Each entry point gets its own still-live anchor. The clock guard
2321        // also affects put_announcement's implicit host-clock call before the
2322        // repair, not just the explicit timestamp arguments on read methods.
2323        for operation in ["put", "point", "list", "tiered"] {
2324            let config = valkey_config(url.clone(), operation);
2325            let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2326            let replica = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2327            let tiered = TieredStateStore::new(
2328                Arc::new(MemoryStateStore::new(8, 8, 8)),
2329                replica.clone(),
2330                30_000,
2331            );
2332            let now: i64 = shared.pool.eval(
2333                "local t = redis.call('TIME'); return tonumber(t[1])*1000 + math.floor(tonumber(t[2])/1000)",
2334                Vec::<String>::new(), Vec::<String>::new(),
2335            ).await.unwrap();
2336            shared
2337                .put_announcement("anchor", endpoint("iroh", 1, now + 60_000, now + 30_000))
2338                .await
2339                .unwrap();
2340            if operation == "tiered" {
2341                assert_eq!(
2342                    tiered.announcements_for("anchor", now).await.unwrap().len(),
2343                    1
2344                );
2345            }
2346            let revision = shared.announcement_revision().await.unwrap();
2347            let ahead = now + 3_600_000;
2348            {
2349                let _clock = ReplicaClock::at(ahead);
2350                match operation {
2351                    "put" => {
2352                        replica
2353                            .put_announcement(
2354                                "writer",
2355                                endpoint("iroh", 1, ahead + 60_000, ahead + 30_000),
2356                            )
2357                            .await
2358                            .unwrap();
2359                    }
2360                    "point" => assert_eq!(
2361                        replica
2362                            .announcements_for("anchor", ahead)
2363                            .await
2364                            .unwrap()
2365                            .len(),
2366                        1
2367                    ),
2368                    "list" => assert_eq!(replica.all_announcements(ahead).await.unwrap().len(), 1),
2369                    "tiered" => assert_eq!(
2370                        tiered
2371                            .announcements_for("anchor", ahead)
2372                            .await
2373                            .unwrap()
2374                            .len(),
2375                        1
2376                    ),
2377                    _ => unreachable!(),
2378                }
2379            }
2380            assert_eq!(
2381                shared.announcements_for("anchor", now).await.unwrap().len(),
2382                1,
2383                "{operation} deleted another replica's live announcement"
2384            );
2385            let exists: bool = shared
2386                .pool
2387                .exists(shared.announcement_key("anchor", "iroh"))
2388                .await
2389                .unwrap();
2390            assert!(exists);
2391            assert_eq!(
2392                shared.announcement_revision().await.unwrap(),
2393                revision + u64::from(operation == "put")
2394            );
2395        }
2396    }
2397
2398    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2399    #[tokio::test]
2400    async fn valkey_announcement_shared_expiry_rejects_behind_replica_revival() {
2401        use fred::prelude::*;
2402
2403        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2404            return;
2405        };
2406        let config = valkey_config(url, "announcement-expiry-clock");
2407        let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2408        let tiered = TieredStateStore::new(
2409            Arc::new(MemoryStateStore::new(8, 8, 8)),
2410            shared.clone(),
2411            30_000,
2412        );
2413        let now: i64 = shared.pool.eval(
2414            "local t = redis.call('TIME'); return tonumber(t[1])*1000 + math.floor(tonumber(t[2])/1000)",
2415            Vec::<String>::new(), Vec::<String>::new(),
2416        ).await.unwrap();
2417        // Either signed/accepted authority or the shorter effective heartbeat
2418        // lease can expire first; neither becomes a new receipt-relative lease.
2419        let expiry = now + 300;
2420        for (name, signed, live) in [
2421            ("signed", expiry, now + 60_000),
2422            ("lease", now + 60_000, expiry),
2423        ] {
2424            shared
2425                .put_announcement(name, endpoint("iroh", 1, signed, live))
2426                .await
2427                .unwrap();
2428            let pxat: i64 = shared
2429                .pool
2430                .eval(
2431                    "return redis.call('PEXPIRETIME', KEYS[1])",
2432                    vec![shared.announcement_key(name, "iroh")],
2433                    Vec::<String>::new(),
2434                )
2435                .await
2436                .unwrap();
2437            assert_eq!(
2438                pxat, expiry,
2439                "preserve the absolute authority/lease ceiling"
2440            );
2441            assert_eq!(tiered.announcements_for(name, now).await.unwrap().len(), 1);
2442        }
2443        let revision = shared.announcement_revision().await.unwrap();
2444        tokio::time::sleep(std::time::Duration::from_millis(400)).await;
2445        let behind = now - 3_600_000;
2446        let _clock = ReplicaClock::at(behind);
2447        for name in ["signed", "lease"] {
2448            // Exercise an already-populated L1 before any separate listing
2449            // reaps L2 or invalidates its revision for us.
2450            assert!(tiered
2451                .announcements_for(name, behind)
2452                .await
2453                .unwrap()
2454                .is_empty());
2455            assert!(shared
2456                .announcements_for(name, behind)
2457                .await
2458                .unwrap()
2459                .is_empty());
2460            assert_eq!(
2461                shared
2462                    .put_announcement(name, endpoint("iroh", 2, expiry, now + 60_000))
2463                    .await
2464                    .unwrap(),
2465                PutResult::IgnoredOlder
2466            );
2467        }
2468        assert!(shared.all_announcements(behind).await.unwrap().is_empty());
2469        assert!(shared.announcement_revision().await.unwrap() > revision);
2470        let metadata: Vec<usize> = shared.pool.eval(
2471            "return {redis.call('ZCARD', KEYS[1]), redis.call('SCARD', KEYS[2]), redis.call('HLEN', KEYS[3])}",
2472            vec![shared.key("announcement-expiry"), shared.key("services"), shared.key("service-names")], Vec::<String>::new(),
2473        ).await.unwrap();
2474        assert_eq!(
2475            metadata,
2476            vec![0, 0, 0],
2477            "expired writes must not resurrect secondary indexes"
2478        );
2479    }
2480
2481    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2482    #[tokio::test]
2483    async fn valkey_satisfies_announcement_backend_contract() {
2484        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2485            return;
2486        };
2487        let mut config = valkey_config(url, "contract");
2488        config.announcement_capacity = 1;
2489        let store = ValkeyStateStore::connect(&config).await.unwrap();
2490        assert_announcement_backend_contract(&store).await;
2491    }
2492
2493    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2494    #[tokio::test]
2495    async fn valkey_two_replicas_share_all_discovery_state() {
2496        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2497            return;
2498        };
2499        let config = valkey_config(url, "shared");
2500        let replica_a = ValkeyStateStore::connect(&config).await.unwrap();
2501        let replica_b = ValkeyStateStore::connect(&config).await.unwrap();
2502        let now = unix_millis_now();
2503
2504        replica_a
2505            .put_announcement("model", endpoint("iroh", 1, now + 30_000, now + 20_000))
2506            .await
2507            .unwrap();
2508        assert_eq!(
2509            replica_b
2510                .announcements_for("model", now)
2511                .await
2512                .unwrap()
2513                .len(),
2514            1
2515        );
2516
2517        replica_b
2518            .put_liveness(
2519                &Did::new("did:web:node.example".to_owned()),
2520                LiveAllocatable {
2521                    allocatable: vec![("cpu".to_owned(), "8".to_owned())],
2522                    load_fraction: 0.25,
2523                    last_seen: now,
2524                    live_until_unix_ms: now + 20_000,
2525                },
2526            )
2527            .await
2528            .unwrap();
2529        assert!(replica_a
2530            .liveness(&Did::new("did:web:node.example".to_owned()), now)
2531            .await
2532            .unwrap()
2533            .is_some());
2534
2535        replica_a
2536            .put_entity_statement(
2537                "https://issuer.example",
2538                CachedEntityStatement {
2539                    jwt: "statement".to_owned(),
2540                    fetched_at: now / 1_000,
2541                },
2542            )
2543            .await
2544            .unwrap();
2545        assert_eq!(
2546            replica_b
2547                .entity_statement("https://issuer.example")
2548                .await
2549                .unwrap()
2550                .unwrap()
2551                .jwt,
2552            "statement"
2553        );
2554
2555        replica_b
2556            .put_envelope_keyset(
2557                "did:web:service.example",
2558                CachedEnvelopeKeyset {
2559                    cose_keyset_cbor: vec![1, 2, 3],
2560                    fetched_at: now / 1_000,
2561                },
2562            )
2563            .await
2564            .unwrap();
2565        assert_eq!(
2566            replica_a
2567                .envelope_keyset("did:web:service.example")
2568                .await
2569                .unwrap()
2570                .unwrap()
2571                .cose_keyset_cbor,
2572            vec![1, 2, 3]
2573        );
2574
2575        // A newly connected replica sees the same state after process-local
2576        // state is discarded.
2577        let restarted = ValkeyStateStore::connect(&config).await.unwrap();
2578        assert_eq!(
2579            restarted
2580                .announcements_for("model", now)
2581                .await
2582                .unwrap()
2583                .len(),
2584            1
2585        );
2586    }
2587
2588    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2589    #[tokio::test]
2590    async fn valkey_rejects_delayed_old_update_and_expires_server_side() {
2591        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2592            return;
2593        };
2594        let config = valkey_config(url, "ordering");
2595        let store = ValkeyStateStore::connect(&config).await.unwrap();
2596        let now = unix_millis_now();
2597        store
2598            .put_announcement("model", endpoint("iroh", 4, now + 30_000, now + 20_000))
2599            .await
2600            .unwrap();
2601        assert_eq!(
2602            store
2603                .put_announcement("model", endpoint("iroh", 3, now + 40_000, now + 20_000))
2604                .await
2605                .unwrap(),
2606            PutResult::IgnoredOlder
2607        );
2608        assert_eq!(
2609            store.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
2610            4
2611        );
2612
2613        store
2614            .put_announcement("short", endpoint("iroh", 1, now + 10_000, now + 50))
2615            .await
2616            .unwrap();
2617        tokio::time::sleep(std::time::Duration::from_millis(80)).await;
2618        assert!(store
2619            .announcements_for("short", unix_millis_now())
2620            .await
2621            .unwrap()
2622            .is_empty());
2623    }
2624
2625    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2626    #[tokio::test]
2627    async fn tiered_revision_check_invalidates_other_replica_l1() {
2628        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2629            return;
2630        };
2631        let config = valkey_config(url, "tiered");
2632        let shared_a = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2633        let shared_b = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2634        let tier_a =
2635            TieredStateStore::new(Arc::new(MemoryStateStore::new(8, 8, 8)), shared_a, 10_000);
2636        let tier_b =
2637            TieredStateStore::new(Arc::new(MemoryStateStore::new(8, 8, 8)), shared_b, 10_000);
2638        let now = unix_millis_now();
2639        tier_a
2640            .put_announcement("model", endpoint("iroh", 1, now + 30_000, now + 20_000))
2641            .await
2642            .unwrap();
2643        assert_eq!(
2644            tier_b.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
2645            1
2646        );
2647        tier_a
2648            .put_announcement("model", endpoint("iroh", 2, now + 40_000, now + 20_000))
2649            .await
2650            .unwrap();
2651        assert_eq!(
2652            tier_b.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
2653            2,
2654            "L2 revision change must invalidate replica B's populated L1"
2655        );
2656    }
2657
2658    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2659    #[tokio::test]
2660    async fn tiered_does_not_resurrect_expired_l2_data() {
2661        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2662            return;
2663        };
2664        let config = valkey_config(url, "tiered-expiry");
2665        let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2666        let tiered =
2667            TieredStateStore::new(Arc::new(MemoryStateStore::new(8, 8, 8)), shared, 10_000);
2668        let now = unix_millis_now();
2669        tiered
2670            .put_announcement("model", endpoint("iroh", 1, now + 10_000, now + 50))
2671            .await
2672            .unwrap();
2673        assert_eq!(
2674            tiered.announcements_for("model", now).await.unwrap().len(),
2675            1
2676        );
2677
2678        tokio::time::sleep(std::time::Duration::from_millis(80)).await;
2679        assert!(tiered
2680            .announcements_for("model", unix_millis_now())
2681            .await
2682            .unwrap()
2683            .is_empty());
2684    }
2685
2686    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2687    #[tokio::test]
2688    async fn tiered_fails_closed_after_l2_client_shutdown() {
2689        use fred::prelude::ClientLike as _;
2690
2691        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2692            return;
2693        };
2694        let config = valkey_config(url, "tiered-outage");
2695        let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
2696        let tiered = TieredStateStore::new(
2697            Arc::new(MemoryStateStore::new(8, 8, 8)),
2698            Arc::clone(&shared),
2699            10_000,
2700        );
2701        let now = unix_millis_now();
2702        tiered
2703            .put_announcement("model", endpoint("iroh", 1, now + 30_000, now + 20_000))
2704            .await
2705            .unwrap();
2706        assert_eq!(
2707            tiered.announcements_for("model", now).await.unwrap().len(),
2708            1
2709        );
2710
2711        shared.pool.quit().await.unwrap();
2712        let result = tokio::time::timeout(
2713            std::time::Duration::from_secs(2),
2714            tiered.announcements_for("model", unix_millis_now()),
2715        )
2716        .await
2717        .expect("configured command timeout must bound an L2 outage");
2718        assert!(
2719            result.is_err(),
2720            "tiered state must not return its populated L1"
2721        );
2722    }
2723    #[tokio::test]
2724    async fn memory_liveness_enumeration_excludes_expiry_and_remains_bounded() {
2725        let store = MemoryStateStore::new(1, 1, 1);
2726        let node = Did::new("did:web:live.example".to_owned());
2727        let now = unix_millis_now();
2728        for i in 0..100 {
2729            store
2730                .put_liveness(
2731                    &node,
2732                    LiveAllocatable {
2733                        allocatable: vec![],
2734                        load_fraction: 0.1,
2735                        last_seen: now + i,
2736                        live_until_unix_ms: now + 30_000,
2737                    },
2738                )
2739                .await
2740                .unwrap();
2741        }
2742        assert_eq!(store.all_liveness(now).await.unwrap()[0].0, node);
2743        assert!(store.inner.lock().expiry.len() <= 2);
2744        assert!(store.all_liveness(now + 30_000).await.unwrap().is_empty());
2745    }
2746
2747    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2748    #[tokio::test]
2749    async fn valkey_driver_survives_temporary_bootstrap_runtime() {
2750        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2751            return;
2752        };
2753        for backend in [DiscoveryStateBackend::Valkey, DiscoveryStateBackend::Tiered] {
2754            let config = DiscoveryStateConfig {
2755                backend,
2756                active_active: true,
2757                valkey: valkey_config(url.clone(), &format!("bootstrap-{backend:?}")),
2758                ..DiscoveryStateConfig::default()
2759            };
2760            // Match the synchronous production factory, including exiting its
2761            // thread and dropping its current-thread runtime before any use.
2762            let state = std::thread::spawn(move || {
2763                tokio::runtime::Builder::new_current_thread()
2764                    .enable_all()
2765                    .build()
2766                    .unwrap()
2767                    .block_on(DiscoveryState::connect(&config))
2768                    .unwrap()
2769            })
2770            .join()
2771            .unwrap();
2772            let store = state.into_inner();
2773            let now = unix_millis_now();
2774            store
2775                .put_announcement("model", endpoint("iroh", 1, now + 30_000, now + 20_000))
2776                .await
2777                .unwrap();
2778            assert_eq!(
2779                store.announcements_for("model", now).await.unwrap().len(),
2780                1
2781            );
2782            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2783            assert_eq!(
2784                store
2785                    .all_announcements(unix_millis_now())
2786                    .await
2787                    .unwrap()
2788                    .len(),
2789                1
2790            );
2791        }
2792    }
2793
2794    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2795    #[tokio::test]
2796    async fn tiered_scoped_operations_allow_unrelated_liveness_and_heartbeats_to_progress() {
2797        use futures::{stream, StreamExt};
2798        use std::time::Duration;
2799        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2800            return;
2801        };
2802        let shared = Arc::new(
2803            ValkeyStateStore::connect(&valkey_config(url, "striped-concurrency"))
2804                .await
2805                .unwrap(),
2806        );
2807        let tier = TieredStateStore::new(
2808            Arc::new(MemoryStateStore::new(64, 64, 64)),
2809            shared.clone(),
2810            60_000,
2811        );
2812        let now = unix_millis_now();
2813        // Pick sixteen distinct stripes so the regression is deterministic,
2814        // including when the hash happens to collide for adjacent node names.
2815        let mut nodes: Vec<Did> = Vec::new();
2816        for i in 0..1024 {
2817            let node = Did::new(format!("did:web:concurrent-{i}.example"));
2818            if nodes.iter().all(|other| {
2819                !std::ptr::eq(
2820                    tier.operation("liveness", other.as_str()),
2821                    tier.operation("liveness", node.as_str()),
2822                )
2823            }) {
2824                nodes.push(node);
2825            }
2826            if nodes.len() == 16 {
2827                break;
2828            }
2829        }
2830        assert_eq!(nodes.len(), 16);
2831        for node in &nodes {
2832            shared
2833                .put_liveness(
2834                    node,
2835                    LiveAllocatable {
2836                        allocatable: vec![],
2837                        load_fraction: 0.1,
2838                        last_seen: now,
2839                        live_until_unix_ms: now + 60_000,
2840                    },
2841                )
2842                .await
2843                .unwrap();
2844        }
2845        // Hold one scope exactly as a delayed read/write does across its L2
2846        // awaits. A store-wide mutex prevents every other operation below from
2847        // finishing; striped ordering lets all fifteen real Valkey reads pass.
2848        let stalled = tier.operation("liveness", nodes[0].as_str()).lock().await;
2849        let mut reads = stream::iter(nodes.iter().map(|node| {
2850            let tier = &tier;
2851            async move { (node, tier.liveness(node, now).await) }
2852        }))
2853        .buffer_unordered(16);
2854        tokio::time::timeout(Duration::from_secs(2), async {
2855            for _ in 0..15 {
2856                let (node, result) = reads.next().await.unwrap();
2857                assert_ne!(node, &nodes[0]);
2858                assert_eq!(result.unwrap().unwrap().load_fraction, 0.1);
2859            }
2860        })
2861        .await
2862        .expect("unrelated tiered liveness reads serialized behind a stalled scope");
2863        assert!(
2864            tokio::time::timeout(Duration::from_millis(20), reads.next())
2865                .await
2866                .is_err()
2867        );
2868
2869        let heartbeat = LiveAllocatable {
2870            allocatable: vec![],
2871            load_fraction: 0.8,
2872            last_seen: now + 1,
2873            live_until_unix_ms: now + 60_001,
2874        };
2875        tokio::time::timeout(
2876            Duration::from_secs(2),
2877            tier.put_liveness(&nodes[1], heartbeat.clone()),
2878        )
2879        .await
2880        .expect("unrelated heartbeat serialized behind a stalled scope")
2881        .unwrap();
2882        // Same-scope writes still wait. Cancelling a queued operation must not
2883        // corrupt the cache or leave the stripe permanently locked.
2884        assert!(tokio::time::timeout(
2885            Duration::from_millis(20),
2886            tier.put_liveness(&nodes[0], heartbeat.clone())
2887        )
2888        .await
2889        .is_err());
2890        drop(reads);
2891        drop(stalled);
2892        assert_eq!(
2893            tier.liveness(&nodes[0], now)
2894                .await
2895                .unwrap()
2896                .unwrap()
2897                .load_fraction,
2898            0.1
2899        );
2900        tier.put_liveness(&nodes[0], heartbeat).await.unwrap();
2901        assert_eq!(
2902            tier.liveness(&nodes[0], now)
2903                .await
2904                .unwrap()
2905                .unwrap()
2906                .load_fraction,
2907            0.8
2908        );
2909        assert_eq!(
2910            tier.liveness(&nodes[1], now)
2911                .await
2912                .unwrap()
2913                .unwrap()
2914                .load_fraction,
2915            0.8
2916        );
2917        assert_eq!(tier.operations.len(), 64);
2918    }
2919
2920    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
2921    #[tokio::test]
2922    async fn tiered_capacity_never_limits_authoritative_results_or_writes() {
2923        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
2924            return;
2925        };
2926        let shared = Arc::new(
2927            ValkeyStateStore::connect(&valkey_config(url, "capacity"))
2928                .await
2929                .unwrap(),
2930        );
2931        let memory = Arc::new(MemoryStateStore::new(1, 1, 1));
2932        let tier = TieredStateStore::new(memory.clone(), shared, 10_000);
2933        let now = unix_millis_now();
2934        for name in ["a", "b", "c"] {
2935            tier.put_announcement(name, endpoint("iroh", 1, now + 30_000, now + 20_000))
2936                .await
2937                .unwrap();
2938            assert_eq!(tier.announcements_for(name, now).await.unwrap().len(), 1);
2939            let node = Did::new(format!("did:web:{name}.example"));
2940            tier.put_liveness(
2941                &node,
2942                LiveAllocatable {
2943                    allocatable: vec![],
2944                    load_fraction: 0.1,
2945                    last_seen: now,
2946                    live_until_unix_ms: now + 20_000,
2947                },
2948            )
2949            .await
2950            .unwrap();
2951            assert!(tier.liveness(&node, now).await.unwrap().is_some());
2952            tier.put_entity_statement(
2953                name,
2954                CachedEntityStatement {
2955                    jwt: name.to_owned(),
2956                    fetched_at: now,
2957                },
2958            )
2959            .await
2960            .unwrap();
2961            assert!(tier.entity_statement(name).await.unwrap().is_some());
2962            tier.put_envelope_keyset(
2963                name,
2964                CachedEnvelopeKeyset {
2965                    cose_keyset_cbor: vec![1],
2966                    fetched_at: now,
2967                },
2968            )
2969            .await
2970            .unwrap();
2971            assert!(tier.envelope_keyset(name).await.unwrap().is_some());
2972        }
2973        for _ in 0..2 {
2974            assert_eq!(tier.all_announcements(now).await.unwrap().len(), 3);
2975            assert_eq!(tier.all_liveness(now).await.unwrap().len(), 3);
2976            assert_eq!(tier.known_issuers().await.unwrap().len(), 3);
2977        }
2978        // A single service can also be larger than L1. Never mark a partial
2979        // result as observed, including after repeated declined cache fills.
2980        tier.put_announcement("a", endpoint("quic", 1, now + 30_000, now + 20_000))
2981            .await
2982            .unwrap();
2983        for i in 0..50 {
2984            assert_eq!(tier.announcements_for("a", now).await.unwrap().len(), 2);
2985            let absent = format!("missing-{i}");
2986            assert!(tier
2987                .announcements_for(&absent, now)
2988                .await
2989                .unwrap()
2990                .is_empty());
2991            assert!(tier.entity_statement(&absent).await.unwrap().is_none());
2992            assert!(tier.envelope_keyset(&absent).await.unwrap().is_none());
2993            assert!(tier
2994                .liveness(&Did::new(absent), now)
2995                .await
2996                .unwrap()
2997                .is_none());
2998        }
2999        let cache = memory.inner.lock();
3000        assert!(cache.announcements.len() <= 1);
3001        assert!(cache.liveness.len() <= 1);
3002        assert!(cache.entity_statements.len() <= 1);
3003        assert!(cache.envelope_keysets.len() <= 1);
3004        assert!(cache.expiry.len() <= 4);
3005        assert!(tier.observed.lock().len() <= 4);
3006    }
3007
3008    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
3009    #[tokio::test]
3010    async fn valkey_expired_identity_churn_bounds_metadata_and_listing_work() {
3011        use fred::prelude::*;
3012
3013        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
3014            return;
3015        };
3016        let mut config = valkey_config(url, "metadata-churn");
3017        config.announcement_capacity = 4;
3018        config.liveness_capacity = 4;
3019        let store = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
3020        let tier = TieredStateStore::new(
3021            Arc::new(MemoryStateStore::new(4, 4, 4)),
3022            store.clone(),
3023            30_000,
3024        );
3025        let now = unix_millis_now();
3026        let anchor = Did::new("did:web:anchor.example".to_owned());
3027        store
3028            .put_announcement("anchor", endpoint("iroh", 1, now + 30_000, now + 30_000))
3029            .await
3030            .unwrap();
3031        store
3032            .put_liveness(
3033                &anchor,
3034                LiveAllocatable {
3035                    allocatable: vec![],
3036                    load_fraction: 0.1,
3037                    last_seen: now,
3038                    live_until_unix_ms: now + 30_000,
3039                },
3040            )
3041            .await
3042            .unwrap();
3043        tier.announcements_for("anchor", now).await.unwrap();
3044        tier.liveness(&anchor, now).await.unwrap();
3045        let announcement_generation = store.announcement_revision().await.unwrap();
3046        let liveness_generation = store.liveness_revision().await.unwrap();
3047
3048        // No historical service is ever individually read. Each cohort expires
3049        // server-side before the next one; 36 distinct identities exceed the
3050        // live capacity nine times while the anchor's L1 snapshot stays warm.
3051        for cohort in 0..12 {
3052            let now = unix_millis_now();
3053            for slot in 0..3 {
3054                let name = format!("churn-{cohort}-{slot}");
3055                store
3056                    .put_announcement(&name, endpoint("iroh", 1, now + 100, now + 100))
3057                    .await
3058                    .unwrap();
3059                store
3060                    .put_liveness(
3061                        &Did::new(format!("did:web:{name}.example")),
3062                        LiveAllocatable {
3063                            allocatable: vec![],
3064                            load_fraction: 0.2,
3065                            last_seen: now,
3066                            live_until_unix_ms: now + 100,
3067                        },
3068                    )
3069                    .await
3070                    .unwrap();
3071            }
3072            // Inspect the backing namespace BEFORE any listing can clean it.
3073            // KEYS is test-only, confined to this test's unique prefix.
3074            let counts: Vec<usize> = store.pool.eval(
3075                "return {redis.call('SCARD', KEYS[1]), redis.call('HLEN', KEYS[2]), #redis.call('KEYS', ARGV[1]), #redis.call('KEYS', ARGV[2]), #redis.call('KEYS', ARGV[3])}",
3076                vec![store.key("services"), store.key("service-names")],
3077                vec![store.key("*"), store.key("announcement-index:*"), store.key("*revision*")],
3078            ).await.unwrap();
3079            // services is exactly the worklist consumed by all_announcements.
3080            assert!(
3081                counts[0] <= 4,
3082                "global listing work grew with history: {counts:?}"
3083            );
3084            assert!(counts[1] <= 4, "service names leaked: {counts:?}");
3085            assert!(counts[2] <= 3 * 4 + 6, "backend keys leaked: {counts:?}");
3086            assert!(counts[3] <= 4, "per-service indexes leaked: {counts:?}");
3087            assert_eq!(counts[4], 2, "only two global generations may persist");
3088            tokio::time::sleep(std::time::Duration::from_millis(150)).await;
3089        }
3090        let now = unix_millis_now();
3091        let all = store.all_announcements(now).await.unwrap();
3092        assert_eq!(all.len(), 1);
3093        assert_eq!(all[0].0, "anchor");
3094        assert_eq!(store.all_liveness(now).await.unwrap().len(), 1);
3095        let count: usize = store.pool.scard(store.key("services")).await.unwrap();
3096        assert_eq!(count, 1, "expired cohort must leave no listing work");
3097        assert!(store.announcement_revision().await.unwrap() > announcement_generation);
3098        assert!(store.liveness_revision().await.unwrap() > liveness_generation);
3099
3100        // Replace the still-cached anchor with a short-lived value on L2, then
3101        // expire and recreate the SAME scopes. Revisions must not reset/ABA.
3102        store
3103            .put_announcement("anchor", endpoint("iroh", 2, now + 100, now + 100))
3104            .await
3105            .unwrap();
3106        store
3107            .put_liveness(
3108                &anchor,
3109                LiveAllocatable {
3110                    allocatable: vec![],
3111                    load_fraction: 0.5,
3112                    last_seen: now,
3113                    live_until_unix_ms: now + 100,
3114                },
3115            )
3116            .await
3117            .unwrap();
3118        tokio::time::sleep(std::time::Duration::from_millis(150)).await;
3119        let now = unix_millis_now();
3120        assert!(store.all_announcements(now).await.unwrap().is_empty());
3121        assert!(store.all_liveness(now).await.unwrap().is_empty());
3122        let keys: Vec<String> = store
3123            .pool
3124            .eval(
3125                "return redis.call('KEYS', ARGV[1])",
3126                Vec::<String>::new(),
3127                vec![store.key("*")],
3128            )
3129            .await
3130            .unwrap();
3131        assert_eq!(
3132            keys.len(),
3133            2,
3134            "only persistent generations survive expiry: {keys:?}"
3135        );
3136        let expired_generation = store.announcement_revision().await.unwrap();
3137        let expired_liveness_generation = store.liveness_revision().await.unwrap();
3138        store
3139            .put_announcement("anchor", endpoint("iroh", 3, now + 30_000, now + 30_000))
3140            .await
3141            .unwrap();
3142        store
3143            .put_liveness(
3144                &anchor,
3145                LiveAllocatable {
3146                    allocatable: vec![],
3147                    load_fraction: 0.9,
3148                    last_seen: now,
3149                    live_until_unix_ms: now + 30_000,
3150                },
3151            )
3152            .await
3153            .unwrap();
3154        assert!(store.announcement_revision().await.unwrap() > expired_generation);
3155        assert!(store.liveness_revision().await.unwrap() > expired_liveness_generation);
3156        assert_eq!(
3157            tier.announcements_for("anchor", now).await.unwrap()[0].accepted_state_epoch,
3158            3
3159        );
3160        assert_eq!(
3161            tier.liveness(&anchor, now)
3162                .await
3163                .unwrap()
3164                .unwrap()
3165                .load_fraction,
3166            0.9
3167        );
3168    }
3169
3170    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
3171    #[tokio::test]
3172    async fn tiered_writes_invalidate_instead_of_tagging_caller_values() {
3173        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
3174            return;
3175        };
3176        let config = valkey_config(url, "write-race");
3177        let shared = Arc::new(ValkeyStateStore::connect(&config).await.unwrap());
3178        let other = ValkeyStateStore::connect(&config).await.unwrap();
3179        let tier = TieredStateStore::new(Arc::new(MemoryStateStore::new(8, 8, 8)), shared, 10_000);
3180        let now = unix_millis_now();
3181        let node = Did::new("did:web:node.example".to_owned());
3182        // Two writers contend on all value families. An older announcement or
3183        // heartbeat may be fenced; unordered artifacts follow completion order.
3184        for epoch in 1..=8 {
3185            let a = endpoint("iroh", epoch, now + 30_000, now + 20_000);
3186            let b = endpoint("iroh", epoch + 1, now + 30_000, now + 20_000);
3187            let (left, right) = tokio::join!(
3188                tier.put_announcement("model", a),
3189                other.put_announcement("model", b)
3190            );
3191            left.unwrap();
3192            right.unwrap();
3193            assert!(!tier.observed.lock().contains_key("announcement:model"));
3194            assert_eq!(
3195                tier.announcements_for("model", now).await.unwrap()[0].accepted_state_epoch,
3196                epoch + 1
3197            );
3198            let a = LiveAllocatable {
3199                allocatable: vec![],
3200                load_fraction: 0.1,
3201                last_seen: now + epoch as i64,
3202                live_until_unix_ms: now + 20_000,
3203            };
3204            let mut b = a.clone();
3205            b.last_seen += 1;
3206            b.load_fraction = 0.9;
3207            let (left, right) = tokio::join!(
3208                tier.put_liveness(&node, a),
3209                other.put_liveness(&node, b.clone())
3210            );
3211            left.unwrap();
3212            right.unwrap();
3213            assert!(!tier
3214                .observed
3215                .lock()
3216                .contains_key(&TieredStateStore::scope("liveness", node.as_str())));
3217            assert_eq!(
3218                tier.liveness(&node, now)
3219                    .await
3220                    .unwrap()
3221                    .unwrap()
3222                    .load_fraction,
3223                other
3224                    .liveness(&node, now)
3225                    .await
3226                    .unwrap()
3227                    .unwrap()
3228                    .load_fraction
3229            );
3230            let a = CachedEntityStatement {
3231                jwt: "a".to_owned(),
3232                fetched_at: now,
3233            };
3234            let b = CachedEntityStatement {
3235                jwt: "b".to_owned(),
3236                fetched_at: now,
3237            };
3238            let (left, right) = tokio::join!(
3239                tier.put_entity_statement("issuer", a),
3240                other.put_entity_statement("issuer", b)
3241            );
3242            left.unwrap();
3243            right.unwrap();
3244            assert!(!tier.observed.lock().contains_key("entity:issuer"));
3245            assert_eq!(
3246                tier.entity_statement("issuer").await.unwrap(),
3247                other.entity_statement("issuer").await.unwrap()
3248            );
3249            let a = CachedEnvelopeKeyset {
3250                cose_keyset_cbor: vec![1],
3251                fetched_at: now,
3252            };
3253            let b = CachedEnvelopeKeyset {
3254                cose_keyset_cbor: vec![2],
3255                fetched_at: now,
3256            };
3257            let (left, right) = tokio::join!(
3258                tier.put_envelope_keyset("service", a),
3259                other.put_envelope_keyset("service", b)
3260            );
3261            left.unwrap();
3262            right.unwrap();
3263            assert!(!tier.observed.lock().contains_key("envelope:service"));
3264            assert_eq!(
3265                tier.envelope_keyset("service").await.unwrap(),
3266                other.envelope_keyset("service").await.unwrap()
3267            );
3268        }
3269    }
3270    #[cfg(all(not(target_arch = "wasm32"), feature = "valkey"))]
3271    #[tokio::test]
3272    async fn valkey_legacy_liveness_encoding_fails_closed_until_heartbeat() {
3273        use fred::prelude::*;
3274        let Ok(url) = std::env::var("HYPRSTREAM_TEST_VALKEY_URL") else {
3275            return;
3276        };
3277        let store = ValkeyStateStore::connect(&valkey_config(url, "legacy-liveness"))
3278            .await
3279            .unwrap();
3280        let node = Did::new("did:web:legacy-node.example".to_owned());
3281        let now = unix_millis_now();
3282        let value = LiveAllocatable {
3283            allocatable: vec![],
3284            load_fraction: 0.1,
3285            last_seen: now,
3286            live_until_unix_ms: now + 30_000,
3287        };
3288        store.put_liveness(&node, value.clone()).await.unwrap();
3289        // Reproduce the pre-repair wire value, which had no enumerable DID.
3290        let _: String = store
3291            .pool
3292            .eval(
3293                "return redis.call('SET', KEYS[1], ARGV[1], 'KEEPTTL')",
3294                vec![format!(
3295                    "{}:liveness:{}",
3296                    store.prefix,
3297                    ValkeyStateStore::digest(node.as_str())
3298                )],
3299                vec![serde_json::to_string(&value).unwrap()],
3300            )
3301            .await
3302            .unwrap();
3303        assert!(store.all_liveness(now).await.unwrap().is_empty());
3304        store.put_liveness(&node, value).await.unwrap();
3305        assert_eq!(store.all_liveness(now).await.unwrap()[0].0, node);
3306    }
3307}