hyprstream_core/inference/
response.rs1use futures::Stream;
7use std::pin::Pin;
8use std::task::{Context, Poll};
9use tokio::sync::{mpsc, oneshot};
10
11use super::client::InferenceError;
12use crate::config::FinishReason;
13use crate::runtime::generation_metrics::GenerationQualityMetrics;
14
15pub struct StreamHandle {
20 receiver: mpsc::Receiver<Result<String, InferenceError>>,
22 stats_receiver: Option<oneshot::Receiver<StreamStats>>,
24}
25
26impl StreamHandle {
27 pub fn new(
29 receiver: mpsc::Receiver<Result<String, InferenceError>>,
30 stats_receiver: oneshot::Receiver<StreamStats>,
31 ) -> Self {
32 Self {
33 receiver,
34 stats_receiver: Some(stats_receiver),
35 }
36 }
37
38 pub async fn next(&mut self) -> Option<Result<String, InferenceError>> {
40 self.receiver.recv().await
41 }
42
43 pub fn into_stream(self) -> impl Stream<Item = Result<String, InferenceError>> {
45 StreamHandleStream { handle: self }
46 }
47
48 pub async fn stats(mut self) -> Result<StreamStats, InferenceError> {
52 while self.receiver.recv().await.is_some() {}
54
55 self.stats_receiver
56 .take()
57 .ok_or_else(|| InferenceError::channel("Stats already consumed"))?
58 .await
59 .map_err(|_| InferenceError::channel("Stats channel closed"))
60 }
61}
62
63struct StreamHandleStream {
65 handle: StreamHandle,
66}
67
68impl Stream for StreamHandleStream {
69 type Item = Result<String, InferenceError>;
70
71 fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
72 Pin::new(&mut self.handle.receiver).poll_recv(cx)
73 }
74}
75
76#[derive(Debug, Clone)]
78pub struct StreamStats {
79 pub tokens_generated: usize,
81 pub generation_time_ms: u64,
83 pub tokens_per_second: f32,
85 pub finish_reason: Option<FinishReason>,
87 pub quality_metrics: Option<GenerationQualityMetrics>,
89}
90
91impl Default for StreamStats {
92 fn default() -> Self {
93 Self {
94 tokens_generated: 0,
95 generation_time_ms: 0,
96 tokens_per_second: 0.0,
97 finish_reason: None,
98 quality_metrics: None,
99 }
100 }
101}
102
103impl StreamStats {
104 pub fn from_generation_stats(stats: &crate::runtime::GenerationStats) -> Self {
106 Self {
107 tokens_generated: stats.tokens_generated,
108 generation_time_ms: stats.generation_time_ms,
109 tokens_per_second: stats.tokens_per_second,
110 finish_reason: stats.finish_reason.clone(),
111 quality_metrics: stats.quality_metrics,
112 }
113 }
114}
115
116#[cfg(test)]
117#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
118mod tests {
119 use super::*;
120
121 #[tokio::test]
122 async fn test_stream_handle_basic() {
123 let (chunk_tx, chunk_rx) = mpsc::channel(32);
124 let (stats_tx, stats_rx) = oneshot::channel();
125
126 let mut handle = StreamHandle::new(chunk_rx, stats_rx);
127
128 chunk_tx
130 .send(Ok("Hello".to_owned()))
131 .await
132 .expect("test: send chunk");
133 chunk_tx
134 .send(Ok(" World".to_owned()))
135 .await
136 .expect("test: send chunk");
137 drop(chunk_tx); assert_eq!(handle.next().await, Some(Ok("Hello".to_owned())));
141 assert_eq!(handle.next().await, Some(Ok(" World".to_owned())));
142 assert_eq!(handle.next().await, None);
143
144 let _ = stats_tx.send(StreamStats::default());
146
147 let stats = handle.stats().await.expect("test: get stats");
149 assert_eq!(stats.tokens_generated, 0);
150 }
151}