Skip to main content

hyprstream_core/account/
renewal.rs

1//! Renewal, monitoring, and the expiry alarm for the account-zone wildcard
2//! certificate (epic #1158, ticket #1162 / A3).
3//!
4//! One wildcard cert authenticates **every** account host for a deployment, so
5//! one cert failing to renew breaks **all** account resolution — a total-outage
6//! surface. Per the ticket, the alarm **ships with the feature**, not as a
7//! follow-up.
8//!
9//! What this module owns:
10//! - [`CertRenewer`]: on a schedule, asks the [`WildcardCertIssuer`](super::tls::WildcardCertIssuer)
11//!   to refresh the cert and pushes the result into a [`CertHandle`](super::tls::CertHandle).
12//! - [`CertHealth`]/[`CertHealthMonitor`]: computes a coarse health state
13//!   (Ok / ExpiringSoon / Critical / Expired / Unissued / Unprovisioned) from
14//!   the current cert's NotAfter and the last issuance outcome, and surfaces it
15//!   loudly via `tracing` — the alarm. The health state is also exposed as a
16//!   `pub` field so a metrics/event plane can scrape it.
17//!
18//! `CertRenewer` uses the same trait seams as issuance; it does not depend on a
19//! specific ACME client or DNS backend.
20
21use std::sync::Arc;
22use std::time::Duration;
23
24use time::OffsetDateTime;
25use tokio::sync::watch;
26use tracing::{error, info, warn};
27
28use super::config::AccountZoneConfig;
29use super::dns::DnsProvider;
30use super::tls::{CertHandle, IssuanceError, IssuedCert, WildcardCertIssuer};
31
32/// Coarse health classification for the account-zone cert. The alarm state.
33#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub enum CertHealth {
35    /// No deployment-wide outage risk; cert is healthy and far from expiry.
36    Ok,
37    /// Cert is within `warn_lead` of expiry (default 30d). Renewal should be
38    /// imminent. Informational.
39    ExpiringSoon,
40    /// Cert is within `critical_lead` of expiry (default 7d) **or** the last
41    /// issuance failed. The deployment is at acute outage risk.
42    Critical,
43    /// Cert's NotAfter is in the past — account resolution is failing now.
44    Expired,
45    /// No cert has been issued yet.
46    Unissued,
47    /// The deployment has not provisioned DNS-01 issuance (zone/credential
48    /// missing). Account minting is disabled by design; this is the sane-degrade
49    /// state, surfaced so monitoring can distinguish "intentionally off" from
50    /// "broken".
51    Unprovisioned,
52}
53
54impl CertHealth {
55    /// Human-readable label for metrics/logs.
56    pub fn label(self) -> &'static str {
57        match self {
58            CertHealth::Ok => "ok",
59            CertHealth::ExpiringSoon => "expiring_soon",
60            CertHealth::Critical => "critical",
61            CertHealth::Expired => "expired",
62            CertHealth::Unissued => "unissued",
63            CertHealth::Unprovisioned => "unprovisioned",
64        }
65    }
66}
67
68/// Tunable lead times for the alarm.
69#[derive(Debug, Clone, Copy)]
70pub struct AlarmThresholds {
71    /// Warn when the cert is within this duration of expiry.
72    pub warn_lead: Duration,
73    /// Raise to Critical when within this duration of expiry.
74    pub critical_lead: Duration,
75}
76
77impl Default for AlarmThresholds {
78    fn default() -> Self {
79        // Let's Encrypt certs are 90 days; warn at 30d, critical at 7d.
80        Self {
81            warn_lead: Duration::from_secs(30 * 24 * 3600),
82            critical_lead: Duration::from_secs(7 * 24 * 3600),
83        }
84    }
85}
86
87/// Computes the alarm state from the current cert and the last issuance result.
88///
89/// Pure function; unit-testable without time mocking fixtures beyond
90/// [`OffsetDateTime`].
91pub fn classify(
92    handle: &CertHandle,
93    last_result: Option<&Result<IssuedCert, IssuanceError>>,
94    now: OffsetDateTime,
95    thresholds: &AlarmThresholds,
96    unprovisioned: bool,
97) -> CertHealth {
98    if unprovisioned {
99        return CertHealth::Unprovisioned;
100    }
101    // A recent issuance failure is always at least Critical.
102    let last_failed = matches!(last_result, Some(Err(_)));
103    let Some(not_after) = handle.not_after() else {
104        return if last_failed {
105            CertHealth::Critical
106        } else {
107            CertHealth::Unissued
108        };
109    };
110    let remaining = not_after - now;
111    if remaining.is_negative() {
112        return CertHealth::Expired;
113    }
114    let dur = Duration::try_from(remaining).unwrap_or(Duration::ZERO);
115    // A failed renewal attempt is always at least Critical, regardless of how
116    // much validity remains (the doc promises "within critical_lead OR last
117    // issuance failed" → Critical). Checking `last_failed` first honors the
118    // OR-semantics and avoids silently downgrading to ExpiringSoon when a
119    // renewal has already failed inside the warn band.
120    if last_failed || dur < thresholds.critical_lead {
121        CertHealth::Critical
122    } else if dur < thresholds.warn_lead {
123        CertHealth::ExpiringSoon
124    } else {
125        CertHealth::Ok
126    }
127}
128
129/// A monitor that recomputes [`CertHealth`] and logs it loudly when not `Ok`.
130#[derive(Debug, Clone)]
131pub struct CertHealthMonitor {
132    thresholds: AlarmThresholds,
133}
134
135impl CertHealthMonitor {
136    pub fn new(thresholds: AlarmThresholds) -> Self {
137        Self { thresholds }
138    }
139
140    /// Recompute and emit the alarm. Returns the state so callers (metrics
141    /// plane, readiness probes) can record it.
142    pub fn observe(
143        &self,
144        handle: &CertHandle,
145        last_result: Option<&Result<IssuedCert, IssuanceError>>,
146        unprovisioned: bool,
147    ) -> CertHealth {
148        let now = OffsetDateTime::now_utc();
149        let state = classify(handle, last_result, now, &self.thresholds, unprovisioned);
150        match state {
151            CertHealth::Ok | CertHealth::Unprovisioned => {
152                tracing::debug!(
153                    account_tls_health = state.label(),
154                    "account-zone wildcard TLS health: {}", state.label()
155                );
156            }
157            CertHealth::ExpiringSoon => warn!(
158                account_tls_health = state.label(),
159                "account-zone wildcard certificate expiring soon — renewal should be imminent"
160            ),
161            CertHealth::Critical => error!(
162                account_tls_health = state.label(),
163                "account-zone wildcard certificate CRITICAL — one cert failing breaks ALL account resolution for this deployment"
164            ),
165            CertHealth::Expired => error!(
166                account_tls_health = state.label(),
167                "account-zone wildcard certificate EXPIRED — account resolution is failing now"
168            ),
169            CertHealth::Unissued => info!(
170                account_tls_health = state.label(),
171                "account-zone wildcard certificate not yet issued"
172            ),
173        }
174        state
175    }
176}
177
178/// Background renewal + alarm loop for the account-zone wildcard cert.
179///
180/// Construct with [`CertRenewer::new`]; spawn [`CertRenewer::run`] on the
181/// service runtime. On each tick it:
182/// 1. checks provisioning (sane-degrade if unprovisioned — no panic),
183/// 2. asks the issuer for a fresh cert (only when due, see `renew_lead`),
184/// 3. pushes the result into the [`CertHandle`] via the `sender`,
185/// 4. runs the alarm.
186///
187/// The `sender` is the write half of a [`CertHandle::channel`](super::tls::CertHandle::channel)
188/// pair; the serving/monitoring layer holds the read half. `handle` is a
189/// `CertHandle` whose receiver mirrors the same channel so the renewer can read
190/// `not_after` for the "due?" decision.
191pub struct CertRenewer {
192    cfg: AccountZoneConfig,
193    issuer: Arc<dyn WildcardCertIssuer>,
194    dns: Arc<dyn DnsProvider>,
195    sender: watch::Sender<Option<IssuedCert>>,
196    handle: CertHandle,
197    thresholds: AlarmThresholds,
198    /// Renew when the cert has less than this remaining validity.
199    renew_lead: Duration,
200    /// How often to wake up and re-evaluate (between renewals).
201    tick: Duration,
202}
203
204impl CertRenewer {
205    pub fn new(
206        cfg: AccountZoneConfig,
207        issuer: Arc<dyn WildcardCertIssuer>,
208        dns: Arc<dyn DnsProvider>,
209        sender: watch::Sender<Option<IssuedCert>>,
210        handle: CertHandle,
211    ) -> Self {
212        Self {
213            cfg,
214            issuer,
215            dns,
216            sender,
217            handle,
218            thresholds: AlarmThresholds::default(),
219            renew_lead: Duration::from_secs(30 * 24 * 3600),
220            tick: Duration::from_secs(3600),
221        }
222    }
223
224    /// Override the renew lead time (default 30d).
225    pub fn with_renew_lead(mut self, lead: Duration) -> Self {
226        self.renew_lead = lead;
227        self
228    }
229
230    /// Override the alarm thresholds.
231    pub fn with_thresholds(mut self, t: AlarmThresholds) -> Self {
232        self.thresholds = t;
233        self
234    }
235
236    /// Override the re-evaluation tick (default 1h).
237    pub fn with_tick(mut self, tick: Duration) -> Self {
238        self.tick = tick;
239        self
240    }
241
242    /// Run the renewal + alarm loop until the process shuts down.
243    ///
244    /// Each iteration sleeps `tick`, then evaluates whether a renewal is due
245    /// (no cert, or cert within `renew_lead` of expiry) and, if so, asks the
246    /// issuer. Provisioning failures are reported via the alarm but do not
247    /// terminate the loop — the process keeps serving whatever it has.
248    pub async fn run(self) {
249        let monitor = CertHealthMonitor::new(self.thresholds);
250        // Sane-degrade: when unprovisioned, log once and skip issuance forever.
251        // We never synthesize a zone; the `Option<AccountZone>` stays `None` and
252        // the issuance block is gated on it.
253        let zone = match super::tls::require_provisioned(&self.cfg) {
254            Ok(z) => Some(z),
255            Err(e) => {
256                warn!("account-zone wildcard TLS not provisioned: {e}");
257                None
258            }
259        };
260        let unprovisioned = zone.is_none();
261        let mut last_result: Option<Result<IssuedCert, IssuanceError>> = None;
262
263        loop {
264            if let Some(zone) = zone.as_ref() {
265                let due = match self.handle.not_after() {
266                    None => true,
267                    Some(na) => {
268                        let remaining = na - OffsetDateTime::now_utc();
269                        let dur = Duration::try_from(remaining).unwrap_or(Duration::ZERO);
270                        dur < self.renew_lead
271                    }
272                };
273                if due {
274                    let res = self.issuer.issue(zone, &self.cfg, self.dns.as_ref()).await;
275                    match &res {
276                        Ok(c) => {
277                            info!(
278                                san = %c.san,
279                                not_after = %c.not_after,
280                                "account-zone wildcard certificate (re)issued"
281                            );
282                            // best_effort: if all receivers dropped, no-one to serve.
283                            let _ = self.sender.send(Some(c.clone()));
284                        }
285                        Err(e) => {
286                            error!(
287                                error = %e,
288                                "account-zone wildcard certificate renewal FAILED — outage risk for ALL account resolution"
289                            );
290                        }
291                    }
292                    last_result = Some(res);
293                }
294            }
295            monitor.observe(&self.handle, last_result.as_ref(), unprovisioned);
296            // Drop the borrowed reference before sleeping.
297            tokio::time::sleep(self.tick).await;
298        }
299    }
300}
301
302#[cfg(test)]
303#[allow(clippy::unwrap_used)]
304mod tests {
305    use super::*;
306    use crate::account::config::AccountZoneConfig;
307    use crate::account::dns::UnconfiguredDnsProvider;
308    use crate::account::tls::NullWildcardCertIssuer;
309
310    fn handle_with_cert(not_after: OffsetDateTime) -> CertHandle {
311        let (tx, h) = CertHandle::channel();
312        let _ = tx.send(Some(IssuedCert {
313            cert_der: vec![],
314            chain_der: vec![],
315            key_der: zeroize::Zeroizing::new(vec![]),
316            not_after,
317            san: "*.acct.example.com".to_owned(),
318        }));
319        h
320    }
321
322    #[test]
323    fn classify_states() {
324        let t = AlarmThresholds::default();
325        let now = OffsetDateTime::now_utc();
326        let h = handle_with_cert(now + time::Duration::days(60));
327        assert_eq!(classify(&h, None, now, &t, false), CertHealth::Ok);
328        let h = handle_with_cert(now + time::Duration::days(20));
329        assert_eq!(classify(&h, None, now, &t, false), CertHealth::ExpiringSoon);
330        let h = handle_with_cert(now + time::Duration::days(3));
331        assert_eq!(classify(&h, None, now, &t, false), CertHealth::Critical);
332        let h = handle_with_cert(now - time::Duration::days(1));
333        assert_eq!(classify(&h, None, now, &t, false), CertHealth::Expired);
334
335        // Unissued vs unprovisioned
336        let (tx, empty) = CertHandle::channel();
337        drop(tx);
338        assert_eq!(classify(&empty, None, now, &t, false), CertHealth::Unissued);
339        assert_eq!(
340            classify(&empty, None, now, &t, true),
341            CertHealth::Unprovisioned
342        );
343
344        // Last failure escalates a still-valid cert to Critical.
345        let h = handle_with_cert(now + time::Duration::days(60));
346        let err: Result<IssuedCert, IssuanceError> = Err(IssuanceError::Unprovisioned("x".into()));
347        assert_eq!(
348            classify(&h, Some(&err), now, &t, false),
349            CertHealth::Critical
350        );
351
352        // Regression (#1182 review): a failed renewal inside the warn band must
353        // NOT be downgraded to ExpiringSoon — the documented OR-semantics say a
354        // failed issuance is always at least Critical.
355        let h = handle_with_cert(now + time::Duration::days(20)); // ExpiringSoon band
356        assert_eq!(classify(&h, None, now, &t, false), CertHealth::ExpiringSoon);
357        assert_eq!(
358            classify(&h, Some(&err), now, &t, false),
359            CertHealth::Critical,
360            "failed renewal in the warn band must escalate to Critical, not ExpiringSoon"
361        );
362    }
363
364    #[tokio::test]
365    async fn renewer_degrades_when_unprovisioned() {
366        // No zone → sane-degrade; the constructor + a single tick should not panic.
367        let cfg = AccountZoneConfig::default();
368        let issuer = Arc::new(NullWildcardCertIssuer) as Arc<dyn WildcardCertIssuer>;
369        let dns = Arc::new(UnconfiguredDnsProvider::new()) as Arc<dyn DnsProvider>;
370        let (sender, handle) = CertHandle::channel();
371        let renewer =
372            CertRenewer::new(cfg, issuer, dns, sender, handle).with_tick(Duration::from_millis(10));
373        // Run briefly then drop; we only assert it does not panic on an
374        // unprovisioned deployment.
375        let task = tokio::spawn(async move {
376            renewer.run().await;
377        });
378        tokio::time::sleep(Duration::from_millis(30)).await;
379        task.abort();
380    }
381}