Skip to main content

hyprstream_core/inference/
response.rs

1//! Response types for the inference service.
2//!
3//! Provides types for handling streaming generation responses
4//! and generation statistics.
5
6use futures::Stream;
7use std::pin::Pin;
8use std::task::{Context, Poll};
9use tokio::sync::{mpsc, oneshot};
10
11use super::client::InferenceError;
12use crate::config::FinishReason;
13use crate::runtime::generation_metrics::GenerationQualityMetrics;
14
15/// Handle for receiving streaming generation output.
16///
17/// This type wraps the channels used to communicate streaming
18/// generation results from the service to the client.
19pub struct StreamHandle {
20    /// Receiver for text chunks.
21    receiver: mpsc::Receiver<Result<String, InferenceError>>,
22    /// Receiver for final statistics (sent once after stream ends).
23    stats_receiver: Option<oneshot::Receiver<StreamStats>>,
24}
25
26impl StreamHandle {
27    /// Create a new stream handle.
28    pub fn new(
29        receiver: mpsc::Receiver<Result<String, InferenceError>>,
30        stats_receiver: oneshot::Receiver<StreamStats>,
31    ) -> Self {
32        Self {
33            receiver,
34            stats_receiver: Some(stats_receiver),
35        }
36    }
37
38    /// Get the next text chunk.
39    pub async fn next(&mut self) -> Option<Result<String, InferenceError>> {
40        self.receiver.recv().await
41    }
42
43    /// Convert to an async Stream.
44    pub fn into_stream(self) -> impl Stream<Item = Result<String, InferenceError>> {
45        StreamHandleStream { handle: self }
46    }
47
48    /// Wait for final statistics (call after stream exhausted).
49    ///
50    /// This consumes self since stats are only available after generation completes.
51    pub async fn stats(mut self) -> Result<StreamStats, InferenceError> {
52        // Drain any remaining chunks
53        while self.receiver.recv().await.is_some() {}
54
55        self.stats_receiver
56            .take()
57            .ok_or_else(|| InferenceError::channel("Stats already consumed"))?
58            .await
59            .map_err(|_| InferenceError::channel("Stats channel closed"))
60    }
61}
62
63/// Wrapper to implement Stream trait for StreamHandle.
64struct StreamHandleStream {
65    handle: StreamHandle,
66}
67
68impl Stream for StreamHandleStream {
69    type Item = Result<String, InferenceError>;
70
71    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
72        Pin::new(&mut self.handle.receiver).poll_recv(cx)
73    }
74}
75
76/// Statistics from a streaming generation.
77#[derive(Debug, Clone)]
78pub struct StreamStats {
79    /// Number of tokens generated.
80    pub tokens_generated: usize,
81    /// Total generation time in milliseconds.
82    pub generation_time_ms: u64,
83    /// Tokens per second rate.
84    pub tokens_per_second: f32,
85    /// Why generation finished.
86    pub finish_reason: Option<FinishReason>,
87    /// Quality metrics (if captured).
88    pub quality_metrics: Option<GenerationQualityMetrics>,
89}
90
91impl Default for StreamStats {
92    fn default() -> Self {
93        Self {
94            tokens_generated: 0,
95            generation_time_ms: 0,
96            tokens_per_second: 0.0,
97            finish_reason: None,
98            quality_metrics: None,
99        }
100    }
101}
102
103impl StreamStats {
104    /// Create stats from generation statistics.
105    pub fn from_generation_stats(stats: &crate::runtime::GenerationStats) -> Self {
106        Self {
107            tokens_generated: stats.tokens_generated,
108            generation_time_ms: stats.generation_time_ms,
109            tokens_per_second: stats.tokens_per_second,
110            finish_reason: stats.finish_reason.clone(),
111            quality_metrics: stats.quality_metrics,
112        }
113    }
114}
115
116#[cfg(test)]
117#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
118mod tests {
119    use super::*;
120
121    #[tokio::test]
122    async fn test_stream_handle_basic() {
123        let (chunk_tx, chunk_rx) = mpsc::channel(32);
124        let (stats_tx, stats_rx) = oneshot::channel();
125
126        let mut handle = StreamHandle::new(chunk_rx, stats_rx);
127
128        // Send some chunks
129        chunk_tx
130            .send(Ok("Hello".to_owned()))
131            .await
132            .expect("test: send chunk");
133        chunk_tx
134            .send(Ok(" World".to_owned()))
135            .await
136            .expect("test: send chunk");
137        drop(chunk_tx); // Close channel
138
139        // Receive chunks
140        assert_eq!(handle.next().await, Some(Ok("Hello".to_owned())));
141        assert_eq!(handle.next().await, Some(Ok(" World".to_owned())));
142        assert_eq!(handle.next().await, None);
143
144        // Send stats
145        let _ = stats_tx.send(StreamStats::default());
146
147        // Get stats
148        let stats = handle.stats().await.expect("test: get stats");
149        assert_eq!(stats.tokens_generated, 0);
150    }
151}