Skip to main content

hyprstream_core/services/
at9p_renewal.rs

1//! In-daemon at9p service-identity renewal timer (design O-Ia).
2//!
3//! The registry daemon is the deployment's sole read-write record-store
4//! owner, so it hosts one periodic task that renews every DEPLOYMENT service
5//! identity itself (O-Ia) — the accepted states whose `#service` entries
6//! name daemon services (`hyprstream_service::get_factory`), not generic
7//! at9p records admitted through the public RPC — reusing the boot
8//! provisioner's mint core
9//! ([`crate::cli::deployment_bootstrap::provision_one`]) against the daemon's
10//! LIVE `PdsRecordStore`/`At9pStateIngest` handles. It must never call
11//! `provision_services` as-is — that re-opens the store (self-conflicting on
12//! the RocksDB directory LOCK the daemon already holds) and re-runs
13//! `authenticate_local_deployment_registry` per invocation, which would drag
14//! the ≤1 h credential / 30 d delegation dependency back into every tick.
15//! The deployment credential/UCAN is checked exactly once per process start;
16//! admission itself needs only the pre-committed on-volume chain keys and the
17//! daemon clock, so a running process renews for its lifetime while restarts
18//! still gate on the credential (unchanged by this timer).
19//!
20//! Trust property (review note): signing sibling services' successors
21//! routinely from the long-lived registry process differs in process context
22//! — not in authority — from the offline exclusive-window mint. The signing
23//! material is identical (each service's on-volume key, pre-committed in
24//! `next_key_commitments`), and every admission invariant still holds:
25//! key mismatch refuses (never substitutes), terminal identity refuses, the
26//! same key carries forward, successors are `epoch+1`, expiry is evaluated
27//! on the daemon's clock with no skew tolerance, and the commit is the same
28//! guarded CAS as the RPC path.
29//!
30//! Every tick also re-exports the roster projection
31//! (`<data>/config/native-network/roster.json` by default) through the same
32//! readback-verified atomic writer as the boot command, so host-side
33//! monitoring that flags a roster whose `generated_at` ages past a day reads
34//! fresh expiries — and a stale roster means this loop died, not that
35//! nothing was due. The boot-time `roster.candidate.json` → `config.toml`
36//! native-config projection stays boot-scoped host tooling: runtime consumers
37//! (native announcements) re-project from the store, never from that file.
38//!
39//! The same writer tick refreshes an idle PDS model-record head from its
40//! authenticated sealed ES256 generation. This is independent of at9p chain
41//! renewal and preserves the record pointer/unsigned commit; the two attempts
42//! cannot starve each other on an unrelated failure.
43
44use std::collections::BTreeMap;
45use std::path::{Path, PathBuf};
46use std::sync::Arc;
47
48use anyhow::{Context, Result};
49use chrono::{DateTime, Utc};
50use ed25519_dalek::SigningKey;
51use hyprstream_pds::at9p_duplicity::AcceptedAt9pState;
52
53use crate::auth::identity_store::{load_existing_service_signing_key, SecretsProfile};
54use crate::cli::deployment_bootstrap::{export_verified_roster_snapshot, provision_one};
55use crate::services::discovery::{At9pStateIngest, PdsPublisher, PdsRecordStore};
56
57/// Default roster projection refreshed every tick: the promoted manifest the
58/// host-side renewal monitor reads, under the deployment's shared config
59/// tree. The boot unit's `--roster-export` candidate flow is separate.
60pub(crate) fn default_roster_export_path() -> Option<PathBuf> {
61    hyprstream_rpc::paths::data_dir().map(|data| {
62        data.join("config")
63            .join("native-network")
64            .join("roster.json")
65    })
66}
67
68/// Spawn the renewal timer on the current tokio runtime (the registry
69/// service's process runtime).
70///
71/// The task holds the publisher by [`Weak`] reference: a strong handle would
72/// pin the store's RocksDB directory LOCK after the service stops and block
73/// an in-process registry restart (the same lifetime discipline as the ES256
74/// promotion hook). When the publisher drops, the next tick observes the
75/// dead weak reference and the task exits.
76///
77/// The first check runs immediately (a daemon starting against states
78/// already past their half-TTL window — e.g. after downtime — catches up
79/// instead of waiting a full interval); subsequent checks run once per
80/// `check_interval` with `MissedTickBehavior::Skip` (the JWT key-rotation
81/// task idiom). Tick failures are logged and retried on the next interval.
82/// Dropping the returned handle detaches (does not cancel) the task.
83pub(crate) fn spawn_at9p_renewal_task(
84    publisher: Arc<PdsPublisher>,
85    secrets_dir: PathBuf,
86    check_interval: std::time::Duration,
87    valid_for_seconds: i64,
88    roster_export: Option<PathBuf>,
89) -> tokio::task::JoinHandle<()> {
90    let publisher = Arc::downgrade(&publisher);
91    tokio::spawn(async move {
92        let mut interval = tokio::time::interval(check_interval);
93        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
94        // The first interval tick completes immediately; consume it so the
95        // loop below always waits a full interval after each check.
96        interval.tick().await;
97        loop {
98            let Some(publisher) = publisher.upgrade() else {
99                tracing::info!(
100                    "at9p service-identity renewal: registry publisher dropped; timer exiting"
101                );
102                break;
103            };
104            // The tick is fully synchronous RocksDB/Postgres + ML-DSA work;
105            // run it on the blocking pool (the `ingestAt9pCandidate` handler
106            // uses spawn_blocking for the same ingest operations) instead of
107            // stalling an async worker — the immediate startup tick or a due
108            // renewal would otherwise block unrelated registry requests.
109            let tick_publisher = Arc::clone(&publisher);
110            let tick_secrets = secrets_dir.clone();
111            let tick_roster = roster_export.clone();
112            let tick = tokio::task::spawn_blocking(move || {
113                run_renewal_tick(
114                    tick_publisher.as_ref(),
115                    &tick_secrets,
116                    valid_for_seconds,
117                    tick_roster.as_deref(),
118                    Utc::now(),
119                )
120            })
121            .await;
122            match tick {
123                Ok(Ok(())) => {}
124                Ok(Err(error)) => {
125                    tracing::warn!(
126                        %error,
127                        "at9p service-identity renewal tick failed; retrying next interval"
128                    );
129                }
130                Err(join_error) => {
131                    // The blocking task itself failed (panic); retried on the
132                    // next interval like any tick error.
133                    let error = anyhow::anyhow!(join_error);
134                    tracing::warn!(
135                        %error,
136                        "at9p service-identity renewal tick task failed; retrying next interval"
137                    );
138                }
139            }
140            drop(publisher);
141            interval.tick().await;
142        }
143    })
144}
145
146/// One renewal tick against the daemon's live publisher handles.
147pub(crate) fn run_renewal_tick(
148    publisher: &PdsPublisher,
149    secrets_dir: &Path,
150    valid_for_seconds: i64,
151    roster_export: Option<&Path>,
152    now: DateTime<Utc>,
153) -> Result<()> {
154    // The same writer owns idle model-record heads. Resolve its authenticated
155    // sealed generation even when an unrelated service renewal fails; likewise
156    // a head-refresh error must not starve service identity renewal.
157    let head_refresh = publisher
158        .resign_head()
159        .map(|_| ())
160        .context("refreshing idle PDS head from the active generation");
161    let store = publisher.at9p_record_store();
162    let renewal = publisher
163        .at9p_state_ingest()
164        .context("at9p renewal requires the daemon-owned state ingest")
165        .and_then(|ingest| {
166            renewal_tick(
167                store,
168                ingest,
169                secrets_dir,
170                valid_for_seconds,
171                roster_export,
172                now,
173            )
174        });
175    head_refresh.and(renewal)
176}
177
178/// One renewal tick against explicit handles (the testable core): renew
179/// every due identity, then re-export the roster projection. The export is
180/// all-or-nothing by construction: a member that fails verification fails
181/// the whole snapshot rather than emitting a partial manifest that would
182/// hide a sick member from host-side monitoring.
183pub(crate) fn renewal_tick(
184    store: &PdsRecordStore,
185    ingest: &At9pStateIngest,
186    secrets_dir: &Path,
187    valid_for_seconds: i64,
188    roster_export: Option<&Path>,
189    now: DateTime<Utc>,
190) -> Result<()> {
191    let admitted =
192        renew_due_service_identities(store, ingest, secrets_dir, valid_for_seconds, now)?;
193    if let Some(path) = roster_export {
194        export_verified_roster_snapshot(store, &admitted, path)?;
195    }
196    Ok(())
197}
198
199/// Renew every accepted at9p service identity whose remaining life is below
200/// the half-TTL window. The renewal set is the intersection of the accepted
201/// states with the daemon's own service vocabulary
202/// (`hyprstream_service::get_factory` — the same membership test the offline
203/// provisioner's `validate_service_roster` applies): the store may also hold
204/// generic at9p records admitted through the public `ingestAt9pCandidate`
205/// RPC (arbitrary `#id` service entries that are NOT deployment identities),
206/// and those must neither be renewed nor abort the tick — a foreign entry
207/// with no credentials directory would otherwise fail every tick and starve
208/// the real deployment identities past expiry, where renewal is impossible.
209/// For the deployment's own services, keys are loaded, never generated: a
210/// missing sibling key on the shared credentials volume is an error naming
211/// the service, retried on the next tick. Inside the half-TTL window
212/// `provision_one` is a verified no-op returning the current state unchanged.
213pub(crate) fn renew_due_service_identities(
214    store: &PdsRecordStore,
215    ingest: &At9pStateIngest,
216    secrets_dir: &Path,
217    valid_for_seconds: i64,
218    now: DateTime<Utc>,
219) -> Result<Vec<(String, SigningKey, AcceptedAt9pState)>> {
220    let states = store.accepted_at9p_states()?;
221    let mut service_epochs: BTreeMap<String, u64> = BTreeMap::new();
222    for state in &states {
223        for entry in &state.current.services {
224            let Some(name) = entry.id.strip_prefix('#') else {
225                continue;
226            };
227            if hyprstream_service::get_factory(name).is_none() {
228                tracing::debug!(
229                    service = name,
230                    "accepted at9p state is not a deployment service identity; not renewed"
231                );
232                continue;
233            }
234            service_epochs.insert(entry.id.clone(), state.epoch);
235        }
236    }
237    // BTreeMap iteration is deterministic ('#' sorts before alphanumerics).
238    // Failure isolation: a service whose key is missing/corrupt/mismatched or
239    // whose identity is terminal must not starve the identities ordered
240    // after it — the deterministic order would otherwise skip the SAME later
241    // services on every retry until they expire, where renewal is impossible.
242    // Every service is attempted; the first error is reported (with the
243    // failure count) only after the loop, and the roster export below still
244    // fails closed because a failed service cannot be verified.
245    let mut admitted = Vec::with_capacity(service_epochs.len());
246    let mut failures: Vec<(String, anyhow::Error)> = Vec::new();
247    for (service_id, epoch_before) in &service_epochs {
248        let name = service_id.strip_prefix('#').unwrap_or(service_id);
249        let attempt = (|| -> Result<(SigningKey, AcceptedAt9pState)> {
250            let key = load_existing_service_signing_key(
251                secrets_dir,
252                name,
253                SecretsProfile::SharedDirectory,
254            )
255            .with_context(|| format!("loading existing signing key for service {name}"))?;
256            let state = provision_one(store, ingest, name, &key, now, valid_for_seconds)
257                .with_context(|| format!("renewing service identity for {name}"))?;
258            Ok((key, state))
259        })();
260        match attempt {
261            Ok((key, state)) => {
262                if state.epoch > *epoch_before {
263                    tracing::info!(
264                        service = name,
265                        did = %state.did,
266                        epoch = state.epoch,
267                        "at9p service identity renewed"
268                    );
269                }
270                admitted.push((name.to_owned(), key, state));
271            }
272            Err(error) => {
273                tracing::warn!(
274                    service = name,
275                    %error,
276                    "at9p service-identity renewal failed for this service; continuing"
277                );
278                failures.push((name.to_owned(), error));
279            }
280        }
281    }
282    if !failures.is_empty() {
283        let failure_count = failures.len();
284        let failed = failures
285            .iter()
286            .map(|(name, _)| name.as_str())
287            .collect::<Vec<_>>()
288            .join(", ");
289        let (_, error) = failures.swap_remove(0);
290        return Err(error).with_context(|| {
291            format!(
292                "{} of {} service identities failed renewal ({}); the healthy \
293                 identities were still renewed",
294                failure_count,
295                service_epochs.len(),
296                failed
297            )
298        });
299    }
300    Ok(admitted)
301}
302
303#[cfg(test)]
304mod tests {
305    #![allow(clippy::expect_used, clippy::unwrap_used)]
306
307    use super::*;
308    use chrono::Duration;
309
310    fn fixture() -> Result<(
311        tempfile::TempDir,
312        Arc<PdsRecordStore>,
313        At9pStateIngest,
314        SigningKey,
315    )> {
316        let dir = tempfile::tempdir()?;
317        let acceptance = SigningKey::from_bytes(&[0x61; 32]);
318        let store = Arc::new(
319            PdsRecordStore::open(dir.path())?
320                .with_at9p_acceptance_identity(acceptance.verifying_key()),
321        );
322        let audit = hyprstream_rpc::node_identity::derive_purpose_key(
323            &acceptance,
324            "hyprstream-at9p-audit-ed25519-v1",
325        );
326        let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&audit);
327        let ingest = At9pStateIngest::open(
328            Arc::clone(&store),
329            &dir.path().join("audit.wal"),
330            acceptance,
331            audit,
332            pq,
333        )?;
334        Ok((dir, store, ingest, SigningKey::from_bytes(&[0x62; 32])))
335    }
336
337    fn write_service_key(dir: &Path, name: &str, key: &SigningKey) -> Result<()> {
338        crate::auth::identity_store::write_secret(&dir.join(name), "signing-key", &key.to_bytes())?;
339        Ok(())
340    }
341
342    fn sorted_states(store: &PdsRecordStore) -> Result<Vec<AcceptedAt9pState>> {
343        let mut states = store.accepted_at9p_states()?;
344        states.sort_by(|a, b| a.did.cmp(&b.did));
345        Ok(states)
346    }
347
348    /// A live-shaped roster: two identities provisioned at T0 with `ttl`
349    /// seconds, their keys on the shared credentials volume.
350    #[allow(clippy::type_complexity)]
351    fn provisioned_fixture(
352        ttl: i64,
353    ) -> Result<(
354        tempfile::TempDir,
355        Arc<PdsRecordStore>,
356        At9pStateIngest,
357        tempfile::TempDir,
358        SigningKey,
359        DateTime<Utc>,
360    )> {
361        let (dir, store, ingest, key) = fixture()?;
362        let event_key = SigningKey::from_bytes(&[0x63; 32]);
363        let now = Utc::now();
364        provision_one(&store, &ingest, "model", &key, now, ttl)?;
365        provision_one(&store, &ingest, "event", &event_key, now, ttl)?;
366        let secrets = tempfile::tempdir()?;
367        write_service_key(secrets.path(), "model", &key)?;
368        write_service_key(secrets.path(), "event", &event_key)?;
369        Ok((dir, store, ingest, secrets, key, now))
370    }
371
372    #[test]
373    fn renewal_tick_is_noop_inside_half_ttl_window() -> Result<()> {
374        let (_dir, store, ingest, secrets, _key, now) = provisioned_fixture(600)?;
375        let out = tempfile::tempdir()?;
376        let roster = out.path().join("roster.json");
377        let before = sorted_states(&store)?;
378        renewal_tick(&store, &ingest, secrets.path(), 600, Some(&roster), now)?;
379        let after = sorted_states(&store)?;
380        assert_eq!(after.len(), before.len());
381        for (state_before, state_after) in before.iter().zip(after.iter()) {
382            assert_eq!(state_before.did, state_after.did);
383            assert_eq!(state_before.epoch, state_after.epoch);
384            assert_eq!(state_before.head_digest, state_after.head_digest);
385        }
386        // The roster projection is still exported every tick (loop-liveness
387        // signal for host-side monitoring), with both verified members.
388        let parsed: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&roster)?)?;
389        assert_eq!(parsed["schema"], "hyprstream/verified-service-roster@1");
390        assert_eq!(parsed["services"].as_array().expect("services").len(), 2);
391        Ok(())
392    }
393
394    #[test]
395    fn missing_sealed_model_head_authority_does_not_starve_service_renewal() -> Result<()> {
396        let (_dir, store, ingest, secrets, _key, now) = provisioned_fixture(600)?;
397        let state = tempfile::tempdir()?;
398        let did = hyprstream_rpc::did_key::ed25519_to_did_key(&[7u8; 32]);
399        let initial = PdsPublisher::new(
400            Arc::clone(&store),
401            did.clone(),
402            p256::ecdsa::SigningKey::random(&mut rand::rngs::OsRng),
403        );
404        initial.publish(
405            "00000000-0000-4000-8000-000000000123",
406            "3f1d3a63258e132cd06314b736445a8bab13b9bd",
407        )?;
408        let publisher = PdsPublisher::with_generation_source(
409            Arc::clone(&store),
410            did,
411            Arc::new(crate::auth::SealedHeadEs256Source::new(
412                state.path(),
413                secrets.path(),
414            )),
415        )
416        .with_at9p_state_ingest(ingest);
417        let before = sorted_states(&store)?;
418        let out = tempfile::tempdir()?;
419        let roster = out.path().join("roster.json");
420        let error = run_renewal_tick(
421            &publisher,
422            secrets.path(),
423            900,
424            Some(&roster),
425            now + Duration::seconds(301),
426        )
427        .expect_err("sealed model signing authority is absent");
428        assert!(error.to_string().contains("refreshing idle PDS head"));
429        let after = sorted_states(&store)?;
430        assert_eq!(before.len(), after.len());
431        for (prior, renewed) in before.iter().zip(after.iter()) {
432            assert_eq!(prior.did, renewed.did);
433            assert_eq!(renewed.epoch, prior.epoch + 1);
434        }
435        let projected: serde_json::Value = serde_json::from_slice(&std::fs::read(roster)?)?;
436        assert_eq!(
437            projected["services"]
438                .as_array()
439                .expect("roster services")
440                .len(),
441            after.len()
442        );
443        Ok(())
444    }
445
446    #[test]
447    fn renewal_tick_extends_same_did_at_epoch_plus_one_with_later_expiry() -> Result<()> {
448        let (_dir, store, ingest, secrets, _key, now) = provisioned_fixture(600)?;
449        let out = tempfile::tempdir()?;
450        let roster = out.path().join("roster.json");
451        let first = sorted_states(&store)?;
452        // Past the half-TTL window (600/2 = 300 s) both identities renew.
453        let later = now + Duration::seconds(301);
454        renewal_tick(&store, &ingest, secrets.path(), 900, Some(&roster), later)?;
455        let renewed = sorted_states(&store)?;
456        assert_eq!(renewed.len(), first.len());
457        for (state_before, state_after) in first.iter().zip(renewed.iter()) {
458            assert_eq!(
459                state_after.did, state_before.did,
460                "renewal must never churn DIDs"
461            );
462            assert_eq!(state_after.epoch, state_before.epoch + 1);
463            assert_ne!(state_after.head_digest, state_before.head_digest);
464            let expiry_before = state_before
465                .expires_at
466                .as_deref()
467                .expect("bounded expiry before");
468            let expiry_after = state_after
469                .expires_at
470                .as_deref()
471                .expect("bounded expiry after");
472            assert!(expiry_after > expiry_before, "expiry must move later");
473            // Same key carried forward: the subject key set is unchanged and
474            // therefore still pre-committed for the next renewal.
475            assert_eq!(
476                state_after.current.subject_keys,
477                state_before.current.subject_keys
478            );
479        }
480        // The exported roster tracks the renewed readback, not the inputs
481        // (keyed by DID: the manifest order is the admitted order).
482        let parsed: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&roster)?)?;
483        let services = parsed["services"].as_array().expect("services");
484        assert_eq!(services.len(), 2);
485        let by_did: std::collections::HashMap<&str, &serde_json::Value> = services
486            .iter()
487            .map(|entry| (entry["did"].as_str().expect("did"), entry))
488            .collect();
489        for state in &renewed {
490            let entry = by_did[state.did.as_str()];
491            assert_eq!(entry["epoch"], serde_json::Value::from(state.epoch));
492            assert_eq!(
493                entry["expires_at"],
494                state.expires_at.as_deref().expect("expiry")
495            );
496        }
497        // A second tick inside the new half-TTL window is a verified no-op.
498        let epoch_snapshot: Vec<u64> = renewed.iter().map(|state| state.epoch).collect();
499        renewal_tick(
500            &store,
501            &ingest,
502            secrets.path(),
503            900,
504            Some(&roster),
505            later + Duration::seconds(60),
506        )?;
507        assert_eq!(
508            sorted_states(&store)?
509                .into_iter()
510                .map(|state| state.epoch)
511                .collect::<Vec<_>>(),
512            epoch_snapshot
513        );
514        Ok(())
515    }
516
517    /// Local cross-repository fixture: real temporary store admission and
518    /// renewal, followed by the current publisher image's native projector.
519    /// The projector source path is supplied by the operator; no live store,
520    /// clock, service, or credential is used.
521    #[test]
522    #[ignore = "requires HYPRSTREAM_NATIVE_PROJECTOR_SOURCE from the Ingest checkout"]
523    fn disposable_renewal_projects_real_verified_roster() -> Result<()> {
524        let projector = std::env::var_os("HYPRSTREAM_NATIVE_PROJECTOR_SOURCE")
525            .context("set HYPRSTREAM_NATIVE_PROJECTOR_SOURCE to the Ingest native projector")?;
526        let (_store_dir, store, ingest, _) = fixture()?;
527        let secrets = tempfile::tempdir()?;
528        let names = [
529            "event",
530            "discovery",
531            "policy",
532            "registry",
533            "streams",
534            "model",
535            "oai",
536            "oauth",
537            "mcp",
538        ];
539        let now = Utc::now();
540        for (index, name) in names.iter().enumerate() {
541            let key = SigningKey::from_bytes(&[(0x70 + index) as u8; 32]);
542            provision_one(&store, &ingest, name, &key, now, 600)?;
543            if *name == "policy" {
544                // The shared-directory profile keeps policy's root/CA key
545                // flat; every other service uses its own subdirectory.
546                crate::auth::identity_store::write_secret(
547                    secrets.path(),
548                    "signing-key",
549                    &key.to_bytes(),
550                )?;
551            } else {
552                write_service_key(secrets.path(), name, &key)?;
553            }
554        }
555        let before = sorted_states(&store)?;
556        assert_eq!(before.len(), names.len());
557
558        let output = tempfile::tempdir()?;
559        let roster = output.path().join("roster.json");
560        let config = output.path().join("config.toml");
561        renewal_tick(
562            &store,
563            &ingest,
564            secrets.path(),
565            900,
566            Some(&roster),
567            now + Duration::seconds(301),
568        )?;
569        let after = sorted_states(&store)?;
570        assert_eq!(after.len(), names.len());
571        for (old, new) in before.iter().zip(&after) {
572            assert!(new.did == old.did, "renewal changed a fixture DID");
573            assert_eq!(new.epoch, old.epoch + 1);
574            assert!(
575                new.current.subject_keys == old.current.subject_keys,
576                "renewal changed a fixture key"
577            );
578            assert!(new.expires_at > old.expires_at);
579        }
580        let manifest: serde_json::Value = serde_json::from_slice(&std::fs::read(&roster)?)?;
581        let entries = manifest["services"].as_array().context("roster services")?;
582        assert_eq!(entries.len(), names.len());
583        for entry in entries {
584            let name = entry["service"].as_str().context("roster service name")?;
585            let service_id = format!("#{name}");
586            let state = after
587                .iter()
588                .find(|state| {
589                    state
590                        .current
591                        .services
592                        .iter()
593                        .any(|svc| svc.id == service_id)
594                })
595                .context("roster service missing from store readback")?;
596            assert!(
597                entry["did"].as_str() == Some(state.did.as_str()),
598                "roster DID mismatch"
599            );
600            assert_eq!(entry["epoch"].as_u64(), Some(state.epoch));
601            assert!(
602                entry["accepted_head_digest"].as_str()
603                    == Some(hex::encode(state.head_digest).as_str()),
604                "roster head digest mismatch"
605            );
606        }
607
608        let script = r#"
609import datetime as dt
610import importlib.util
611import json
612import pathlib
613import sys
614import tomllib
615
616sys.dont_write_bytecode = True
617spec = importlib.util.spec_from_file_location('native_projector_fixture', sys.argv[1])
618native = importlib.util.module_from_spec(spec)
619spec.loader.exec_module(native)
620roster = json.loads(pathlib.Path(sys.argv[2]).read_text())
621assert {member['service'] for member in roster['services']} == set(native.SERVICES)
622generated = dt.datetime.fromisoformat(roster['generated_at'].replace('Z', '+00:00'))
623contents = native.project(roster, generated)
624target = pathlib.Path(sys.argv[3])
625native.write_config(target, contents)
626assert tomllib.loads(target.read_text())['registry']['at9p_renewal']['valid_for_secs'] == 7776000
627"#;
628        let projection = std::process::Command::new("python3")
629            .arg("-c")
630            .arg(script)
631            .arg(projector)
632            .arg(&roster)
633            .arg(&config)
634            .output()?;
635        assert!(
636            projection.status.success(),
637            "disposable native projector fixture failed"
638        );
639        Ok(())
640    }
641
642    #[test]
643    fn renewal_tick_refuses_mismatched_key_without_substituting_identity() -> Result<()> {
644        let (_dir, store, ingest, secrets, key, now) = provisioned_fixture(600)?;
645        // Replace the on-volume key: the tick must refuse (never silently
646        // treat the service as absent or mint a new DID for the new key).
647        let wrong = SigningKey::from_bytes(&[0x77; 32]);
648        write_service_key(secrets.path(), "model", &wrong)?;
649        let out = tempfile::tempdir()?;
650        let roster = out.path().join("roster.json");
651        let later = now + Duration::seconds(301);
652        let error = renewal_tick(&store, &ingest, secrets.path(), 900, Some(&roster), later)
653            .expect_err("key mismatch must fail the tick");
654        assert!(
655            format!("{error:#}").contains("hybrid key does not match"),
656            "unexpected error: {error:#}"
657        );
658        // The refused service is untouched: same DID, same epoch, and the
659        // accepted key is still the original (the refusal never substitutes).
660        let model = sorted_states(&store)?
661            .into_iter()
662            .find(|state| state.current.services.iter().any(|e| e.id == "#model"))
663            .expect("model state");
664        assert_eq!(model.epoch, 1);
665        assert!(model
666            .current
667            .subject_keys
668            .iter()
669            .any(|k| { k.ed25519_pub.as_slice() == key.verifying_key().to_bytes().as_slice() }));
670        // All-or-nothing export: the failed tick publishes no roster.
671        assert!(!roster.exists());
672        Ok(())
673    }
674
675    #[test]
676    fn renewal_tick_continues_past_a_failing_service() -> Result<()> {
677        let (_dir, store, ingest, secrets, _key, now) = provisioned_fixture(600)?;
678        // #event orders before #model and its key is replaced: the loop must
679        // still attempt #model (deterministic order would otherwise starve
680        // every later identity on every retry until it expires unrenewably).
681        let wrong = SigningKey::from_bytes(&[0x76; 32]);
682        write_service_key(secrets.path(), "event", &wrong)?;
683        let out = tempfile::tempdir()?;
684        let roster = out.path().join("roster.json");
685        let later = now + Duration::seconds(301);
686        let error = renewal_tick(&store, &ingest, secrets.path(), 900, Some(&roster), later)
687            .expect_err("failing service must still fail the tick");
688        let chained = format!("{error:#}");
689        assert!(chained.contains("hybrid key does not match"), "{chained}");
690        assert!(
691            chained.contains("healthy identities were still renewed"),
692            "summary must say the healthy services were renewed: {chained}"
693        );
694        let states = sorted_states(&store)?;
695        let model = states
696            .iter()
697            .find(|state| state.current.services.iter().any(|e| e.id == "#model"))
698            .expect("model state");
699        assert_eq!(model.epoch, 2, "model must renew past the failing event");
700        let event = states
701            .iter()
702            .find(|state| state.current.services.iter().any(|e| e.id == "#event"))
703            .expect("event state");
704        assert_eq!(event.epoch, 1, "event refusal leaves it untouched");
705        // The export still fails closed.
706        assert!(!roster.exists());
707        Ok(())
708    }
709
710    #[test]
711    fn renewal_tick_export_survives_a_stale_predictable_temp_sibling() -> Result<()> {
712        let (_dir, store, ingest, secrets, _key, now) = provisioned_fixture(86_400)?;
713        let out = tempfile::tempdir()?;
714        let roster = out.path().join("roster.json");
715        // A prior writer crashed after creating the PREDICTABLE temp name
716        // this process would use (same pid — e.g. PID 1 in a container): the
717        // periodic export's attempt-unique name must not collide with it.
718        let stale = out
719            .path()
720            .join(format!(".roster.json.tmp-{}", std::process::id()));
721        std::fs::write(&stale, b"interrupted earlier attempt")?;
722        renewal_tick(&store, &ingest, secrets.path(), 86_400, Some(&roster), now)?;
723        let parsed: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&roster)?)?;
724        assert_eq!(parsed["services"].as_array().expect("services").len(), 2);
725        assert_eq!(
726            std::fs::read(&stale)?,
727            b"interrupted earlier attempt",
728            "the foreign leftover is never cleanup scope"
729        );
730        Ok(())
731    }
732
733    #[test]
734    fn renewal_tick_never_generates_missing_service_keys() -> Result<()> {
735        let (_dir, store, ingest, secrets, _key, now) = provisioned_fixture(600)?;
736        // Remove one sibling's key from the shared credentials volume.
737        std::fs::remove_file(secrets.path().join("event").join("signing-key"))?;
738        let out = tempfile::tempdir()?;
739        let roster = out.path().join("roster.json");
740        let later = now + Duration::seconds(301);
741        let error = renewal_tick(&store, &ingest, secrets.path(), 900, Some(&roster), later)
742            .expect_err("missing key must fail the tick");
743        assert!(
744            format!("{error:#}").contains("event"),
745            "error must name the service: {error:#}"
746        );
747        // Nothing was generated for the missing service.
748        assert!(!secrets.path().join("event").join("signing-key").exists());
749        assert_eq!(sorted_states(&store)?.len(), 2);
750        assert!(!roster.exists());
751        Ok(())
752    }
753
754    #[test]
755    fn renewal_tick_skips_foreign_at9p_records_without_aborting() -> Result<()> {
756        let (_dir, store, ingest, secrets, _key, now) = provisioned_fixture(600)?;
757        // Admit a generic at9p capsule (public-RPC shape) whose service id is
758        // NOT a deployment service: it has no credentials directory and must
759        // neither be renewed nor fail the tick — a hard failure here would
760        // starve the deployment identities past expiry on every retry.
761        use hyprstream_pds::at9p::{
762            CapsuleBody, HybridKeyPair, ServiceEndpoint, ServiceEntry, ServiceType, Transport,
763        };
764        use hyprstream_pds::at9p_sign::sign_capsule;
765        use hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes;
766        let foreign = SigningKey::from_bytes(&[0x78; 32]);
767        let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&foreign);
768        let pair = HybridKeyPair::new(
769            foreign.verifying_key().to_bytes().to_vec(),
770            ml_dsa_sk_to_vk_bytes(&pq),
771        )?;
772        let endpoint = ServiceEndpoint::new(
773            Transport::Iroh,
774            format!("iroh://{}", hex::encode([0x79; 32])),
775        )?;
776        let service = ServiceEntry::new("#ns", ServiceType::NinePExport, endpoint)?;
777        let body = CapsuleBody::new(vec![pair], vec![service])?;
778        let capsule = sign_capsule(body, &foreign, &pq)?;
779        let foreign_did = format!("did:at9p:{}", capsule.cid512()?);
780        let foreign_state = ingest.ingest_genesis(&foreign_did, &capsule.to_dag_cbor()?)?;
781        assert_eq!(foreign_state.epoch, 0);
782        // Past the deployment half-TTL: both deployment identities renew and
783        // the roster exports exactly the two deployment members.
784        let out = tempfile::tempdir()?;
785        let roster = out.path().join("roster.json");
786        let later = now + Duration::seconds(301);
787        renewal_tick(&store, &ingest, secrets.path(), 900, Some(&roster), later)?;
788        let states = sorted_states(&store)?;
789        assert_eq!(states.len(), 3);
790        let foreign_after = states
791            .iter()
792            .find(|state| state.did == foreign_did)
793            .expect("foreign record still accepted");
794        assert_eq!(foreign_after.epoch, 0, "foreign record must not be renewed");
795        let parsed: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&roster)?)?;
796        let services = parsed["services"].as_array().expect("services");
797        assert_eq!(services.len(), 2);
798        assert!(
799            services
800                .iter()
801                .all(|entry| entry["did"].as_str() != Some(foreign_did.as_str())),
802            "foreign record must not appear in the deployment roster"
803        );
804        Ok(())
805    }
806
807    #[test]
808    fn renewal_tick_refuses_terminal_identity() -> Result<()> {
809        // 24 h base TTL so the sibling #event identity stays fresh across the
810        // whole scenario; #model is made terminal below.
811        let (_dir, store, ingest, secrets, key, now) = provisioned_fixture(86_400)?;
812        // Advance #model to a terminal successor (empty next_key_commitments)
813        // through the real ingest path, then require the tick to refuse once
814        // the terminal state ages past the half-TTL window.
815        use hyprstream_pds::at9p::CapsuleBody;
816        use hyprstream_pds::at9p_sign::sign_update_record;
817        let state = sorted_states(&store)?
818            .into_iter()
819            .find(|state| state.current.services.iter().any(|e| e.id == "#model"))
820            .expect("model state");
821        let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&key);
822        let mut body = CapsuleBody::new(
823            state.current.subject_keys.clone(),
824            state.current.services.clone(),
825        )?;
826        body.next_key_commitments = vec![]; // terminal: no authorized successor
827        let signed_at = now + Duration::seconds(60);
828        let expires = (signed_at + Duration::seconds(7200))
829            .to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
830        let update = sign_update_record(
831            state.subject_cid512.clone(),
832            state.epoch + 1,
833            state.head_digest,
834            body,
835            expires,
836            &key,
837            &pq,
838        )?;
839        ingest.ingest_successor(
840            &state.did,
841            &update.to_dag_cbor()?,
842            &signed_at.to_rfc3339_opts(chrono::SecondsFormat::Secs, true),
843        )?;
844        // Remaining life 400 s < half of the tick's 900 s TTL: renewal is
845        // due and must refuse on the terminal invariant while still fresh.
846        let tick_at = signed_at + Duration::seconds(6800);
847        let out = tempfile::tempdir()?;
848        let roster = out.path().join("roster.json");
849        let error = renewal_tick(&store, &ingest, secrets.path(), 900, Some(&roster), tick_at)
850            .expect_err("terminal identity must fail the tick");
851        assert!(
852            format!("{error:#}").contains("no authorized successor"),
853            "unexpected error: {error:#}"
854        );
855        assert!(!roster.exists());
856        Ok(())
857    }
858
859    #[tokio::test]
860    async fn renewal_task_exits_when_publisher_drops() -> Result<()> {
861        // The store directory must outlive the task's final tick.
862        let (_dir, store, ingest, key) = fixture()?;
863        let now = Utc::now();
864        provision_one(&store, &ingest, "model", &key, now, 600)?;
865        let secrets = tempfile::tempdir()?;
866        write_service_key(secrets.path(), "model", &key)?;
867        let p256 = p256::ecdsa::SigningKey::from_slice(&[0x71; 32])?;
868        let publisher = Arc::new(
869            PdsPublisher::new(Arc::clone(&store), "did:key:test".to_owned(), p256)
870                .with_at9p_state_ingest(ingest),
871        );
872        let roster = tempfile::tempdir()?;
873        let handle = spawn_at9p_renewal_task(
874            Arc::clone(&publisher),
875            secrets.keep(),
876            std::time::Duration::from_millis(10),
877            900,
878            Some(roster.keep().join("roster.json")),
879        );
880        // Dropping the last strong publisher reference must end the task on a
881        // subsequent tick: a surviving task would pin the store's RocksDB
882        // directory LOCK and block an in-process registry restart.
883        drop(publisher);
884        let exited = tokio::time::timeout(std::time::Duration::from_secs(30), handle).await;
885        assert!(
886            exited.is_ok(),
887            "renewal task must exit after the publisher drops"
888        );
889        Ok(())
890    }
891}