Skip to main content

hyprstream_core/systemd/
health.rs

1//! Health-based watchdog integration
2//!
3//! Provides a health checker that only pings the systemd watchdog when
4//! the service is healthy. If health checks fail, the watchdog won't be
5//! pinged and systemd will eventually restart the service.
6//!
7//! # Architecture
8//!
9//! ```text
10//! ┌─────────────────────────────────────────────┐
11//! │              HealthChecker                  │
12//! │  ┌─────────────────────────────────────┐   │
13//! │  │  check_zmq_connectivity()           │   │
14//! │  │  check_http_endpoint()              │   │
15//! │  │  check_memory_pressure()            │   │
16//! │  └─────────────────────────────────────┘   │
17//! │                    │                        │
18//! │                    ▼                        │
19//! │  ┌─────────────────────────────────────┐   │
20//! │  │  is_healthy() -> bool               │   │
21//! │  │  (all checks must pass)             │   │
22//! │  └─────────────────────────────────────┘   │
23//! └─────────────────────────────────────────────┘
24//!                      │
25//!                      ▼
26//! ┌─────────────────────────────────────────────┐
27//! │           run_watchdog()                    │
28//! │  if is_healthy() { SystemdNotifier::watchdog() }
29//! └─────────────────────────────────────────────┘
30//! ```
31//!
32//! # Usage
33//!
34//! ```ignore
35//! use hyprstream_core::systemd::health::{HealthChecker, run_watchdog};
36//!
37//! let checker = HealthChecker::new();
38//!
39//! // Spawn watchdog task (runs until shutdown)
40//! tokio::spawn(run_watchdog(checker));
41//! ```
42
43use super::SystemdNotifier;
44use std::time::Duration;
45use tokio::time::timeout;
46use tracing::{debug, info, warn};
47
48/// Health check configuration
49#[derive(Debug, Clone)]
50pub struct HealthConfig {
51    /// Timeout for individual health checks
52    pub check_timeout: Duration,
53
54    /// HTTP endpoint to check (if any)
55    pub http_endpoint: Option<String>,
56
57    /// Memory pressure threshold (0.0 - 1.0)
58    /// If memory usage exceeds this, service is unhealthy
59    pub memory_threshold: f64,
60}
61
62impl Default for HealthConfig {
63    fn default() -> Self {
64        Self {
65            check_timeout: Duration::from_secs(5),
66            http_endpoint: None,
67            memory_threshold: 0.95, // 95% memory usage threshold
68        }
69    }
70}
71
72/// Health checker for watchdog integration
73///
74/// Performs health checks and reports overall service health.
75/// Only healthy services should ping the watchdog.
76pub struct HealthChecker {
77    config: HealthConfig,
78}
79
80impl HealthChecker {
81    /// Create a new health checker with default configuration
82    pub fn new() -> Self {
83        Self {
84            config: HealthConfig::default(),
85        }
86    }
87
88    /// Create a new health checker with custom configuration
89    pub fn with_config(config: HealthConfig) -> Self {
90        Self { config }
91    }
92
93    /// Set the HTTP endpoint to check
94    pub fn with_http_endpoint(mut self, endpoint: String) -> Self {
95        self.config.http_endpoint = Some(endpoint);
96        self
97    }
98
99    /// Aggregate health check - returns true only if ALL checks pass
100    ///
101    /// Currently checks:
102    /// 1. HTTP endpoint (if configured)
103    /// 2. Memory pressure
104    ///
105    /// # Returns
106    ///
107    /// `true` if all health checks pass, `false` otherwise.
108    pub async fn is_healthy(&self) -> bool {
109        // 1. Check HTTP endpoint if configured
110        let http_ok = if self.config.http_endpoint.is_some() {
111            self.check_http_endpoint().await
112        } else {
113            true
114        };
115
116        // 2. Check memory pressure
117        let memory_ok = self.check_memory_pressure();
118
119        let healthy = http_ok && memory_ok;
120
121        if !healthy {
122            warn!(
123                "Health check failed: http={}, memory={}",
124                http_ok, memory_ok
125            );
126        } else {
127            debug!("Health check passed");
128        }
129
130        healthy
131    }
132
133    /// Check HTTP endpoint health
134    ///
135    /// Attempts a simple TCP connect to the configured HTTP endpoint.
136    async fn check_http_endpoint(&self) -> bool {
137        let endpoint = match &self.config.http_endpoint {
138            Some(e) => e,
139            None => return true,
140        };
141
142        let result = timeout(self.config.check_timeout, async {
143            // Try to connect to the endpoint
144            match tokio::net::TcpStream::connect(endpoint).await {
145                Ok(_) => {
146                    debug!("HTTP endpoint {} is reachable", endpoint);
147                    true
148                }
149                Err(e) => {
150                    warn!("HTTP endpoint {} unreachable: {}", endpoint, e);
151                    false
152                }
153            }
154        })
155        .await;
156
157        match result {
158            Ok(healthy) => healthy,
159            Err(_) => {
160                warn!("HTTP health check timed out for {}", endpoint);
161                false
162            }
163        }
164    }
165
166    /// Check memory pressure
167    ///
168    /// Reads /proc/meminfo to check if memory usage is within acceptable limits.
169    fn check_memory_pressure(&self) -> bool {
170        #[cfg(target_os = "linux")]
171        {
172            // Read /proc/meminfo
173            let meminfo = match std::fs::read_to_string("/proc/meminfo") {
174                Ok(content) => content,
175                Err(e) => {
176                    warn!("Failed to read /proc/meminfo: {}", e);
177                    return true; // Don't fail health check if we can't read meminfo
178                }
179            };
180
181            let mut mem_total: u64 = 0;
182            let mut mem_available: u64 = 0;
183
184            for line in meminfo.lines() {
185                if line.starts_with("MemTotal:") {
186                    mem_total = parse_meminfo_value(line);
187                } else if line.starts_with("MemAvailable:") {
188                    mem_available = parse_meminfo_value(line);
189                }
190            }
191
192            if mem_total == 0 {
193                return true; // Can't determine, assume healthy
194            }
195
196            let usage_ratio = 1.0 - (mem_available as f64 / mem_total as f64);
197
198            if usage_ratio > self.config.memory_threshold {
199                warn!(
200                    "Memory pressure high: {:.1}% used (threshold: {:.1}%)",
201                    usage_ratio * 100.0,
202                    self.config.memory_threshold * 100.0
203                );
204                false
205            } else {
206                debug!(
207                    "Memory usage: {:.1}% (threshold: {:.1}%)",
208                    usage_ratio * 100.0,
209                    self.config.memory_threshold * 100.0
210                );
211                true
212            }
213        }
214
215        #[cfg(not(target_os = "linux"))]
216        {
217            // Can't check memory pressure on non-Linux
218            true
219        }
220    }
221}
222
223impl Default for HealthChecker {
224    fn default() -> Self {
225        Self::new()
226    }
227}
228
229/// Parse a value from /proc/meminfo (e.g., "MemTotal:       16384000 kB")
230#[cfg(target_os = "linux")]
231fn parse_meminfo_value(line: &str) -> u64 {
232    line.split_whitespace()
233        .nth(1)
234        .and_then(|v| v.parse::<u64>().ok())
235        .unwrap_or(0)
236}
237
238/// Run the watchdog task
239///
240/// Periodically checks health and pings the watchdog only when healthy.
241/// If health checks fail, the watchdog won't be pinged and systemd will
242/// eventually restart the service after `WatchdogSec` timeout.
243///
244/// # Arguments
245///
246/// * `checker` - Health checker instance
247///
248/// # Example
249///
250/// ```ignore
251/// let checker = HealthChecker::new()
252///     .with_http_endpoint("127.0.0.1:3000".to_owned());
253///
254/// // Spawn as background task
255/// tokio::spawn(run_watchdog(checker));
256/// ```
257pub async fn run_watchdog(checker: HealthChecker) {
258    let Some(interval) = SystemdNotifier::watchdog_interval() else {
259        debug!("Watchdog not enabled (WATCHDOG_USEC not set)");
260        return;
261    };
262
263    info!("Watchdog enabled, pinging every {:?}", interval);
264
265    loop {
266        tokio::time::sleep(interval).await;
267
268        if checker.is_healthy().await {
269            SystemdNotifier::watchdog();
270        } else {
271            warn!("Health check failed, skipping watchdog ping");
272            // Systemd will restart us after WatchdogSec timeout
273        }
274    }
275}
276
277/// Run watchdog with a shutdown signal
278///
279/// Same as `run_watchdog` but can be stopped via a shutdown receiver.
280///
281/// # Arguments
282///
283/// * `checker` - Health checker instance
284/// * `shutdown` - Receiver that signals when to stop
285pub async fn run_watchdog_with_shutdown(
286    checker: HealthChecker,
287    mut shutdown: tokio::sync::broadcast::Receiver<()>,
288) {
289    let Some(interval) = SystemdNotifier::watchdog_interval() else {
290        debug!("Watchdog not enabled (WATCHDOG_USEC not set)");
291        return;
292    };
293
294    info!("Watchdog enabled, pinging every {:?}", interval);
295
296    loop {
297        tokio::select! {
298            _ = tokio::time::sleep(interval) => {
299                if checker.is_healthy().await {
300                    SystemdNotifier::watchdog();
301                } else {
302                    warn!("Health check failed, skipping watchdog ping");
303                }
304            }
305            _ = shutdown.recv() => {
306                info!("Watchdog task shutting down");
307                break;
308            }
309        }
310    }
311}
312
313#[cfg(test)]
314#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
315mod tests {
316    use super::*;
317
318    #[test]
319    fn test_health_config_default() {
320        let config = HealthConfig::default();
321        assert_eq!(config.check_timeout, Duration::from_secs(5));
322        assert!(config.http_endpoint.is_none());
323        assert!((config.memory_threshold - 0.95).abs() < f64::EPSILON);
324    }
325
326    #[test]
327    fn test_health_checker_default() {
328        let checker = HealthChecker::new();
329        assert!(checker.config.http_endpoint.is_none());
330    }
331
332    #[test]
333    fn test_health_checker_with_http() {
334        let checker = HealthChecker::new().with_http_endpoint("127.0.0.1:3000".to_owned());
335        assert_eq!(
336            checker.config.http_endpoint,
337            Some("127.0.0.1:3000".to_owned())
338        );
339    }
340
341    #[tokio::test]
342    async fn test_is_healthy_no_http() {
343        // Without HTTP endpoint, should check only memory
344        let checker = HealthChecker::new();
345        let healthy = checker.is_healthy().await;
346        // Should pass unless system is under severe memory pressure
347        println!("Health check result: {}", healthy);
348    }
349
350    #[test]
351    fn test_memory_check() {
352        let checker = HealthChecker::new();
353        let result = checker.check_memory_pressure();
354        // Should return true unless system is under pressure
355        println!("Memory check result: {}", result);
356    }
357
358    #[cfg(target_os = "linux")]
359    #[test]
360    fn test_parse_meminfo_value() {
361        assert_eq!(parse_meminfo_value("MemTotal:       16384000 kB"), 16384000);
362        assert_eq!(parse_meminfo_value("MemAvailable:   8192000 kB"), 8192000);
363        assert_eq!(parse_meminfo_value("Invalid line"), 0);
364    }
365}