hyprstream_inference/runtime/
token_metrics.rs1#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
39mod metered {
40 use once_cell::sync::Lazy;
41 use opentelemetry::{metrics::Meter, KeyValue};
42
43 const METER_NAME: &str = "hyprstream.inference";
45 const METRIC_TOKENS_TOTAL: &str = "inference_tokens_total";
46 const METRIC_REQUEST_TOKENS: &str = "inference_request_tokens";
47 const METRIC_SPECULATIVE_TOKENS: &str = "inference_speculative_tokens_total";
48 const UNIT: &str = "{token}";
50
51 const KIND_PROMPT: &str = "prompt";
52 const KIND_GENERATED: &str = "generated";
53 const KIND_ACCEPTED: &str = "accepted";
54 const KIND_REJECTED: &str = "rejected";
55
56 fn tenant_attr(tenant: Option<u64>) -> KeyValue {
58 match tenant {
59 Some(h) => KeyValue::new("tenant", format!("tenant:{h:016x}")),
60 None => KeyValue::new("tenant", "anonymous"),
61 }
62 }
63
64 fn counter_attrs(kind: &str, model: &str, tenant: Option<u64>) -> [KeyValue; 3] {
66 [
67 KeyValue::new("kind", kind.to_owned()),
68 KeyValue::new("model", model.to_owned()),
69 tenant_attr(tenant),
70 ]
71 }
72
73 fn request_attrs(model: &str, tenant: Option<u64>) -> [KeyValue; 2] {
75 [
76 KeyValue::new("model", model.to_owned()),
77 tenant_attr(tenant),
78 ]
79 }
80
81 pub struct TokenBurnMeter {
89 tokens_total: opentelemetry::metrics::Counter<u64>,
90 request_tokens: opentelemetry::metrics::Histogram<u64>,
91 speculative_tokens: opentelemetry::metrics::Counter<u64>,
92 }
93
94 impl TokenBurnMeter {
95 pub fn global() -> Self {
97 Self::from_meter(opentelemetry::global::meter(METER_NAME))
98 }
99
100 pub fn from_meter(meter: Meter) -> Self {
103 Self {
104 tokens_total: meter
105 .u64_counter(METRIC_TOKENS_TOTAL)
106 .with_unit(UNIT)
107 .with_description(
108 "Inference tokens consumed (prompt + generated), by kind/model/tenant",
109 )
110 .build(),
111 request_tokens: meter
112 .u64_histogram(METRIC_REQUEST_TOKENS)
113 .with_unit(UNIT)
114 .with_description("Total tokens per inference request (prompt + generated)")
115 .build(),
116 speculative_tokens: meter
117 .u64_counter(METRIC_SPECULATIVE_TOKENS)
118 .with_unit(UNIT)
119 .with_description(
120 "MTP self-speculative draft outcomes (accepted + rejected), by kind/model/tenant",
121 )
122 .build(),
123 }
124 }
125
126 pub fn record_prompt(&self, model: &str, tenant: Option<u64>, count: u64) {
128 if count == 0 {
129 return;
130 }
131 self.tokens_total
132 .add(count, &counter_attrs(KIND_PROMPT, model, tenant));
133 }
134
135 pub fn record_generated(&self, model: &str, tenant: Option<u64>, count: u64) {
137 if count == 0 {
138 return;
139 }
140 self.tokens_total
141 .add(count, &counter_attrs(KIND_GENERATED, model, tenant));
142 }
143
144 pub fn record_request_total(&self, model: &str, tenant: Option<u64>, total: u64) {
146 self.request_tokens
149 .record(total, &request_attrs(model, tenant));
150 }
151
152 pub fn record_speculative(
157 &self,
158 model: &str,
159 tenant: Option<u64>,
160 accepted: u64,
161 rejected: u64,
162 ) {
163 if accepted > 0 {
164 self.speculative_tokens
165 .add(accepted, &counter_attrs(KIND_ACCEPTED, model, tenant));
166 }
167 if rejected > 0 {
168 self.speculative_tokens
169 .add(rejected, &counter_attrs(KIND_REJECTED, model, tenant));
170 }
171 }
172 }
173
174 static GLOBAL_METER: Lazy<TokenBurnMeter> = Lazy::new(TokenBurnMeter::global);
181
182 pub fn global_meter() -> &'static TokenBurnMeter {
184 &GLOBAL_METER
185 }
186}
187
188#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
189pub use metered::{global_meter, TokenBurnMeter};
190
191#[cfg(not(all(feature = "otel", not(target_arch = "wasm32"))))]
197pub struct TokenBurnMeter;
198
199#[cfg(not(all(feature = "otel", not(target_arch = "wasm32"))))]
200impl TokenBurnMeter {
201 pub fn global() -> Self {
202 Self
203 }
204 pub fn record_prompt(&self, _model: &str, _tenant: Option<u64>, _count: u64) {}
205 pub fn record_generated(&self, _model: &str, _tenant: Option<u64>, _count: u64) {}
206 pub fn record_request_total(&self, _model: &str, _tenant: Option<u64>, _total: u64) {}
207 pub fn record_speculative(
208 &self,
209 _model: &str,
210 _tenant: Option<u64>,
211 _accepted: u64,
212 _rejected: u64,
213 ) {
214 }
215}
216
217#[cfg(not(all(feature = "otel", not(target_arch = "wasm32"))))]
219pub fn global_meter() -> &'static TokenBurnMeter {
220 static NOOP_METER: TokenBurnMeter = TokenBurnMeter;
221 &NOOP_METER
222}
223
224pub fn opaque_tenant_id(tenant: &Option<String>) -> Option<u64> {
239 use std::collections::hash_map::RandomState;
240 use std::hash::BuildHasher;
241 use std::sync::OnceLock;
242
243 static TENANT_HASH_KEY: OnceLock<RandomState> = OnceLock::new();
244
245 let subject = tenant.as_ref()?;
246 Some(
247 TENANT_HASH_KEY
248 .get_or_init(RandomState::new)
249 .hash_one(subject),
250 )
251}
252
253#[cfg(all(test, feature = "otel", not(target_arch = "wasm32")))]
254mod tests {
255 use super::*;
256 use opentelemetry::metrics::MeterProvider;
257 use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
258 use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};
259
260 fn attr_set<'a, I: Iterator<Item = &'a opentelemetry::KeyValue>>(attrs: I) -> Vec<String> {
263 let mut v: Vec<String> = attrs.map(|kv| format!("{}={}", kv.key, kv.value)).collect();
264 v.sort();
265 v
266 }
267
268 #[test]
272 fn token_burn_meter_records_prompt_and_generated() -> Result<(), Box<dyn std::error::Error>> {
273 let exporter = InMemoryMetricExporter::default();
274 let provider = SdkMeterProvider::builder()
275 .with_reader(PeriodicReader::builder(exporter.clone()).build())
276 .build();
277 let meter = provider.meter("hyprstream.inference");
278 let m = TokenBurnMeter::from_meter(meter);
279
280 let tenant_hash = opaque_tenant_id(&Some("did:web:acme.example".to_owned()));
281 assert!(tenant_hash.is_some(), "tenant hashes to an opaque id");
282 let expected_tenant = format!("tenant=tenant:{:016x}", tenant_hash.unwrap_or(0));
283 m.record_prompt("test-model", tenant_hash, 10);
284 m.record_generated("test-model", tenant_hash, 7);
285 m.record_request_total("test-model", tenant_hash, 17);
286
287 provider.force_flush()?;
288 let collected = exporter.get_finished_metrics()?;
289
290 let mut counter_points: Vec<(Vec<String>, u64)> = Vec::new();
291 let mut counter_unit = String::new();
292 let mut histogram_points: Vec<(Vec<String>, u64, u64)> = Vec::new();
293 let mut histogram_unit = String::new();
294 let mut any_attrs = String::new();
295 for rm in collected {
296 for sm in rm.scope_metrics() {
297 for metric in sm.metrics() {
298 match metric.data() {
299 AggregatedMetrics::U64(MetricData::Sum(s)) => {
300 assert_eq!(metric.name(), "inference_tokens_total");
301 counter_unit = metric.unit().to_owned();
302 for dp in s.data_points() {
303 let attrs = attr_set(dp.attributes());
304 any_attrs.push_str(&attrs.join(","));
305 counter_points.push((attrs, dp.value()));
306 }
307 }
308 AggregatedMetrics::U64(MetricData::Histogram(h)) => {
309 assert_eq!(metric.name(), "inference_request_tokens");
310 histogram_unit = metric.unit().to_owned();
311 for dp in h.data_points() {
312 let attrs = attr_set(dp.attributes());
313 any_attrs.push_str(&attrs.join(","));
314 histogram_points.push((attrs, dp.count(), dp.sum()));
315 }
316 }
317 _ => {}
318 }
319 }
320 }
321 }
322
323 assert_eq!(counter_unit, "{token}", "counter unit");
325 assert_eq!(histogram_unit, "{token}", "histogram unit");
326
327 counter_points.sort();
331 let expected_prompt_attrs = vec![
332 "kind=prompt".to_owned(),
333 "model=test-model".to_owned(),
334 expected_tenant.clone(),
335 ];
336 let expected_generated_attrs = vec![
337 "kind=generated".to_owned(),
338 "model=test-model".to_owned(),
339 expected_tenant.clone(),
340 ];
341 assert_eq!(
342 counter_points,
343 vec![(expected_generated_attrs, 7), (expected_prompt_attrs, 10),],
344 "counter points must be exactly prompt=10 and generated=7 with exact attrs"
345 );
346
347 let expected_histogram_attrs = vec!["model=test-model".to_owned(), expected_tenant];
350 assert_eq!(
351 histogram_points,
352 vec![(expected_histogram_attrs, 1, 17)],
353 "histogram must carry one request of 17 tokens with (model, tenant) attrs only"
354 );
355
356 assert!(
358 !any_attrs.contains("acme.example"),
359 "raw tenant subject leaked into metric attrs: {any_attrs}"
360 );
361 Ok(())
362 }
363
364 #[test]
365 fn opaque_tenant_id_is_stable_and_anonymous_when_absent() {
366 let a = opaque_tenant_id(&Some("did:web:acme.example".to_owned()));
367 let b = opaque_tenant_id(&Some("did:web:acme.example".to_owned()));
368 assert_eq!(a, b, "same subject must hash to the same opaque id");
369
370 let other = opaque_tenant_id(&Some("did:web:other.example".to_owned()));
371 assert_ne!(a, other, "different subjects must hash differently");
372
373 assert_eq!(
374 opaque_tenant_id(&None),
375 None,
376 "absent tenant yields no opaque id"
377 );
378 }
379
380 #[test]
381 fn zero_prompt_is_a_noop() -> Result<(), Box<dyn std::error::Error>> {
382 let exporter = InMemoryMetricExporter::default();
384 let provider = SdkMeterProvider::builder()
385 .with_reader(PeriodicReader::builder(exporter.clone()).build())
386 .build();
387 let meter = provider.meter("hyprstream.inference");
388 let m = TokenBurnMeter::from_meter(meter);
389 m.record_prompt("test-model", None, 0);
390 m.record_generated("test-model", None, 0);
391 provider.force_flush()?;
392
393 let mut any_counter: u64 = 0;
394 for rm in exporter.get_finished_metrics()? {
395 for sm in rm.scope_metrics() {
396 for metric in sm.metrics() {
397 if let AggregatedMetrics::U64(MetricData::Sum(s)) = metric.data() {
398 for dp in s.data_points() {
399 any_counter += dp.value();
400 }
401 }
402 }
403 }
404 }
405 assert_eq!(
406 any_counter, 0,
407 "zero-count records must not land on the counter"
408 );
409 Ok(())
410 }
411}