Skip to main content

hyprstream_inference/runtime/
token_metrics.rs

1//! OpenTelemetry token-burn metering for inference.
2//!
3//! Token burn — prompt tokens consumed plus generated tokens produced per
4//! request — is accounting- and ops-relevant, so it is emitted as OpenTelemetry
5//! **metrics** rather than process logs (#1253, #1261). The authoritative spend
6//! ledger lives in #1264; the instruments here are the metering/ops signal.
7//!
8//! # Instruments
9//!
10//! - `inference_tokens_total` — `Counter<u64>` (unit `{token}`, UCUM custom-unit
11//!   form), cumulative tokens consumed, with attributes `kind` =
12//!   `prompt` | `generated`, `model`, and an **opaque keyed-hash** tenant id.
13//! - `inference_request_tokens` — `Histogram<u64>` (unit `{token}`), total tokens
14//!   (prompt + generated) per request, with `model` / `tenant` attributes only.
15//! - `inference_speculative_tokens_total` — `Counter<u64>` (unit `{token}`),
16//!   MTP self-speculative draft outcomes, with attribute `kind` =
17//!   `accepted` | `rejected` plus `model` / opaque tenant. Integer draft counts
18//!   only — never token IDs or text.
19//!
20//! # Security
21//!
22//! The raw request subject is **never** used as a metric attribute: metrics are
23//! aggregatable across tenants and must not carry tenant-identifying content. The
24//! tenant is reduced to an opaque keyed hash via [`opaque_tenant_id`] before it
25//! becomes an attribute. Prompt text, token text, generated text, and token IDs
26//! are likewise never in metrics — only integer counts. See the no-leak canary
27//! test in [`crate::runtime::torch_engine`] which guards the same property for
28//! logs.
29//!
30//! # Platform gating
31//!
32//! The opentelemetry dependency family is not wasm32-compatible, so the crates
33//! are target-gated in `Cargo.toml` and the instruments here compile only for
34//! `all(feature = "otel", not(target_arch = "wasm32"))`. Everywhere else a no-op
35//! [`TokenBurnMeter`] keeps the call sites compiling — the wasm/browser client
36//! pays no metric cost and non-`otel` builds meter nothing.
37
38#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
39mod metered {
40    use once_cell::sync::Lazy;
41    use opentelemetry::{metrics::Meter, KeyValue};
42
43    /// OTel meter scope name for inference token-burn instruments.
44    const METER_NAME: &str = "hyprstream.inference";
45    const METRIC_TOKENS_TOTAL: &str = "inference_tokens_total";
46    const METRIC_REQUEST_TOKENS: &str = "inference_request_tokens";
47    const METRIC_SPECULATIVE_TOKENS: &str = "inference_speculative_tokens_total";
48    /// UCUM custom-unit form for token counts (OTel convention: `{unit}`).
49    const UNIT: &str = "{token}";
50
51    const KIND_PROMPT: &str = "prompt";
52    const KIND_GENERATED: &str = "generated";
53    const KIND_ACCEPTED: &str = "accepted";
54    const KIND_REJECTED: &str = "rejected";
55
56    /// The tenant attribute: opaque keyed-hash id, never the raw subject.
57    fn tenant_attr(tenant: Option<u64>) -> KeyValue {
58        match tenant {
59            Some(h) => KeyValue::new("tenant", format!("tenant:{h:016x}")),
60            None => KeyValue::new("tenant", "anonymous"),
61        }
62    }
63
64    /// (kind, model, opaque-tenant) attribute set for the counter.
65    fn counter_attrs(kind: &str, model: &str, tenant: Option<u64>) -> [KeyValue; 3] {
66        [
67            KeyValue::new("kind", kind.to_owned()),
68            KeyValue::new("model", model.to_owned()),
69            tenant_attr(tenant),
70        ]
71    }
72
73    /// (model, opaque-tenant) attribute set for the per-request histogram.
74    fn request_attrs(model: &str, tenant: Option<u64>) -> [KeyValue; 2] {
75        [
76            KeyValue::new("model", model.to_owned()),
77            tenant_attr(tenant),
78        ]
79    }
80
81    /// Token-burn instruments built from an OTel [`Meter`].
82    ///
83    /// When constructed via [`TokenBurnMeter::global`] the process-global OTel
84    /// meter is used, which is a no-op meter unless an SDK meter provider has been
85    /// registered (e.g. by `init_telemetry`). Recording is therefore always safe:
86    /// it never blocks or panics, and silently does nothing when no exporter is
87    /// configured or the `otel` feature is off.
88    pub struct TokenBurnMeter {
89        tokens_total: opentelemetry::metrics::Counter<u64>,
90        request_tokens: opentelemetry::metrics::Histogram<u64>,
91        speculative_tokens: opentelemetry::metrics::Counter<u64>,
92    }
93
94    impl TokenBurnMeter {
95        /// Build from the process-global OTel meter (no-op when no provider set).
96        pub fn global() -> Self {
97            Self::from_meter(opentelemetry::global::meter(METER_NAME))
98        }
99
100        /// Build from an explicit meter — the test seam, used with a local SDK
101        /// meter provider so recording can be asserted hermetically.
102        pub fn from_meter(meter: Meter) -> Self {
103            Self {
104                tokens_total: meter
105                    .u64_counter(METRIC_TOKENS_TOTAL)
106                    .with_unit(UNIT)
107                    .with_description(
108                        "Inference tokens consumed (prompt + generated), by kind/model/tenant",
109                    )
110                    .build(),
111                request_tokens: meter
112                    .u64_histogram(METRIC_REQUEST_TOKENS)
113                    .with_unit(UNIT)
114                    .with_description("Total tokens per inference request (prompt + generated)")
115                    .build(),
116                speculative_tokens: meter
117                    .u64_counter(METRIC_SPECULATIVE_TOKENS)
118                    .with_unit(UNIT)
119                    .with_description(
120                        "MTP self-speculative draft outcomes (accepted + rejected), by kind/model/tenant",
121                    )
122                    .build(),
123            }
124        }
125
126        /// Record prompt tokens consumed by tokenizing a request.
127        pub fn record_prompt(&self, model: &str, tenant: Option<u64>, count: u64) {
128            if count == 0 {
129                return;
130            }
131            self.tokens_total
132                .add(count, &counter_attrs(KIND_PROMPT, model, tenant));
133        }
134
135        /// Record generated tokens produced by a completed request.
136        pub fn record_generated(&self, model: &str, tenant: Option<u64>, count: u64) {
137            if count == 0 {
138                return;
139            }
140            self.tokens_total
141                .add(count, &counter_attrs(KIND_GENERATED, model, tenant));
142        }
143
144        /// Record the per-request token total (prompt + generated) on the histogram.
145        pub fn record_request_total(&self, model: &str, tenant: Option<u64>, total: u64) {
146            // Record even when total == 0 so an empty-generation request is still
147            // observable as a sample; the counter carries the non-zero breakdown.
148            self.request_tokens
149                .record(total, &request_attrs(model, tenant));
150        }
151
152        /// Record MTP self-speculative draft outcomes for a completed request.
153        ///
154        /// Integer draft counts only — never token IDs or text (#1253). Each
155        /// kind is skipped when zero so non-speculative requests add no points.
156        pub fn record_speculative(
157            &self,
158            model: &str,
159            tenant: Option<u64>,
160            accepted: u64,
161            rejected: u64,
162        ) {
163            if accepted > 0 {
164                self.speculative_tokens
165                    .add(accepted, &counter_attrs(KIND_ACCEPTED, model, tenant));
166            }
167            if rejected > 0 {
168                self.speculative_tokens
169                    .add(rejected, &counter_attrs(KIND_REJECTED, model, tenant));
170            }
171        }
172    }
173
174    /// Process-global instrument bundle, built **once** on first recording — not
175    /// per request (#1261 review: instrument construction must not be hot-path
176    /// work). Built lazily so a provider installed at startup (`init_telemetry`)
177    /// is already in place; if recording ever raced ahead of provider install,
178    /// the global meter defers and connects the instruments once the provider is
179    /// set, so no points are lost.
180    static GLOBAL_METER: Lazy<TokenBurnMeter> = Lazy::new(TokenBurnMeter::global);
181
182    /// Shared process-global token-burn instruments.
183    pub fn global_meter() -> &'static TokenBurnMeter {
184        &GLOBAL_METER
185    }
186}
187
188#[cfg(all(feature = "otel", not(target_arch = "wasm32")))]
189pub use metered::{global_meter, TokenBurnMeter};
190
191/// No-op instruments when the `otel` feature is disabled or the target is wasm32
192/// (the opentelemetry crates are not wasm-compatible and are target-gated out).
193///
194/// Keeps the recording call sites compiling without an OTel SDK so non-`otel`
195/// and wasm builds meter nothing without conditionalizing every call site.
196#[cfg(not(all(feature = "otel", not(target_arch = "wasm32"))))]
197pub struct TokenBurnMeter;
198
199#[cfg(not(all(feature = "otel", not(target_arch = "wasm32"))))]
200impl TokenBurnMeter {
201    pub fn global() -> Self {
202        Self
203    }
204    pub fn record_prompt(&self, _model: &str, _tenant: Option<u64>, _count: u64) {}
205    pub fn record_generated(&self, _model: &str, _tenant: Option<u64>, _count: u64) {}
206    pub fn record_request_total(&self, _model: &str, _tenant: Option<u64>, _total: u64) {}
207    pub fn record_speculative(
208        &self,
209        _model: &str,
210        _tenant: Option<u64>,
211        _accepted: u64,
212        _rejected: u64,
213    ) {
214    }
215}
216
217/// Shared no-op instruments for non-`otel` / wasm32 builds.
218#[cfg(not(all(feature = "otel", not(target_arch = "wasm32"))))]
219pub fn global_meter() -> &'static TokenBurnMeter {
220    static NOOP_METER: TokenBurnMeter = TokenBurnMeter;
221    &NOOP_METER
222}
223
224/// Reduce a raw tenant subject to an opaque 64-bit id for metric attrs and
225/// tenant-safe logs.
226///
227/// The tenant subject is never carried verbatim in metrics or logs (metrics are
228/// aggregatable across tenants and must not leak identity). The hash is a
229/// SipHash keyed with **process-random** keys (`RandomState`, drawn from the OS
230/// RNG): an unkeyed hash such as `DefaultHasher::new()` is offline-enumerable —
231/// anyone can precompute the ids of guessed low-entropy subjects (#1261 review).
232/// With a random key the id is stable within the process but cannot be
233/// precomputed or correlated across deployments. Returns `None` when no tenant
234/// is in scope.
235///
236/// This is pure std and available regardless of the `otel` feature or target so
237/// it can be unit-tested hermetically and reused by tenant-safe logging.
238pub fn opaque_tenant_id(tenant: &Option<String>) -> Option<u64> {
239    use std::collections::hash_map::RandomState;
240    use std::hash::BuildHasher;
241    use std::sync::OnceLock;
242
243    static TENANT_HASH_KEY: OnceLock<RandomState> = OnceLock::new();
244
245    let subject = tenant.as_ref()?;
246    Some(
247        TENANT_HASH_KEY
248            .get_or_init(RandomState::new)
249            .hash_one(subject),
250    )
251}
252
253#[cfg(all(test, feature = "otel", not(target_arch = "wasm32")))]
254mod tests {
255    use super::*;
256    use opentelemetry::metrics::MeterProvider;
257    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
258    use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};
259
260    /// Render a data point's attributes as a sorted `k=v` list for exact
261    /// set comparisons.
262    fn attr_set<'a, I: Iterator<Item = &'a opentelemetry::KeyValue>>(attrs: I) -> Vec<String> {
263        let mut v: Vec<String> = attrs.map(|kv| format!("{}={}", kv.key, kv.value)).collect();
264        v.sort();
265        v
266    }
267
268    /// Drive the meter through a real SDK pipeline (in-memory exporter) and assert
269    /// token burn lands on the right instruments with the exact attribute contract
270    /// and opaque, never-raw tenant attrs.
271    #[test]
272    fn token_burn_meter_records_prompt_and_generated() -> Result<(), Box<dyn std::error::Error>> {
273        let exporter = InMemoryMetricExporter::default();
274        let provider = SdkMeterProvider::builder()
275            .with_reader(PeriodicReader::builder(exporter.clone()).build())
276            .build();
277        let meter = provider.meter("hyprstream.inference");
278        let m = TokenBurnMeter::from_meter(meter);
279
280        let tenant_hash = opaque_tenant_id(&Some("did:web:acme.example".to_owned()));
281        assert!(tenant_hash.is_some(), "tenant hashes to an opaque id");
282        let expected_tenant = format!("tenant=tenant:{:016x}", tenant_hash.unwrap_or(0));
283        m.record_prompt("test-model", tenant_hash, 10);
284        m.record_generated("test-model", tenant_hash, 7);
285        m.record_request_total("test-model", tenant_hash, 17);
286
287        provider.force_flush()?;
288        let collected = exporter.get_finished_metrics()?;
289
290        let mut counter_points: Vec<(Vec<String>, u64)> = Vec::new();
291        let mut counter_unit = String::new();
292        let mut histogram_points: Vec<(Vec<String>, u64, u64)> = Vec::new();
293        let mut histogram_unit = String::new();
294        let mut any_attrs = String::new();
295        for rm in collected {
296            for sm in rm.scope_metrics() {
297                for metric in sm.metrics() {
298                    match metric.data() {
299                        AggregatedMetrics::U64(MetricData::Sum(s)) => {
300                            assert_eq!(metric.name(), "inference_tokens_total");
301                            counter_unit = metric.unit().to_owned();
302                            for dp in s.data_points() {
303                                let attrs = attr_set(dp.attributes());
304                                any_attrs.push_str(&attrs.join(","));
305                                counter_points.push((attrs, dp.value()));
306                            }
307                        }
308                        AggregatedMetrics::U64(MetricData::Histogram(h)) => {
309                            assert_eq!(metric.name(), "inference_request_tokens");
310                            histogram_unit = metric.unit().to_owned();
311                            for dp in h.data_points() {
312                                let attrs = attr_set(dp.attributes());
313                                any_attrs.push_str(&attrs.join(","));
314                                histogram_points.push((attrs, dp.count(), dp.sum()));
315                            }
316                        }
317                        _ => {}
318                    }
319                }
320            }
321        }
322
323        // Units use the UCUM custom-unit form.
324        assert_eq!(counter_unit, "{token}", "counter unit");
325        assert_eq!(histogram_unit, "{token}", "histogram unit");
326
327        // Counter: exactly two points — kind=prompt (10) and kind=generated (7) —
328        // each with the exact (kind, model, tenant) attribute set. The SDK does
329        // not guarantee data-point order, so sort before comparing.
330        counter_points.sort();
331        let expected_prompt_attrs = vec![
332            "kind=prompt".to_owned(),
333            "model=test-model".to_owned(),
334            expected_tenant.clone(),
335        ];
336        let expected_generated_attrs = vec![
337            "kind=generated".to_owned(),
338            "model=test-model".to_owned(),
339            expected_tenant.clone(),
340        ];
341        assert_eq!(
342            counter_points,
343            vec![(expected_generated_attrs, 7), (expected_prompt_attrs, 10),],
344            "counter points must be exactly prompt=10 and generated=7 with exact attrs"
345        );
346
347        // Histogram: exactly one point (count 1, sum 17) with ONLY the
348        // (model, tenant) attribute set — no kind.
349        let expected_histogram_attrs = vec!["model=test-model".to_owned(), expected_tenant];
350        assert_eq!(
351            histogram_points,
352            vec![(expected_histogram_attrs, 1, 17)],
353            "histogram must carry one request of 17 tokens with (model, tenant) attrs only"
354        );
355
356        // The raw tenant subject must never appear in metric attributes.
357        assert!(
358            !any_attrs.contains("acme.example"),
359            "raw tenant subject leaked into metric attrs: {any_attrs}"
360        );
361        Ok(())
362    }
363
364    #[test]
365    fn opaque_tenant_id_is_stable_and_anonymous_when_absent() {
366        let a = opaque_tenant_id(&Some("did:web:acme.example".to_owned()));
367        let b = opaque_tenant_id(&Some("did:web:acme.example".to_owned()));
368        assert_eq!(a, b, "same subject must hash to the same opaque id");
369
370        let other = opaque_tenant_id(&Some("did:web:other.example".to_owned()));
371        assert_ne!(a, other, "different subjects must hash differently");
372
373        assert_eq!(
374            opaque_tenant_id(&None),
375            None,
376            "absent tenant yields no opaque id"
377        );
378    }
379
380    #[test]
381    fn zero_prompt_is_a_noop() -> Result<(), Box<dyn std::error::Error>> {
382        // A zero count should not add a data point to the counter.
383        let exporter = InMemoryMetricExporter::default();
384        let provider = SdkMeterProvider::builder()
385            .with_reader(PeriodicReader::builder(exporter.clone()).build())
386            .build();
387        let meter = provider.meter("hyprstream.inference");
388        let m = TokenBurnMeter::from_meter(meter);
389        m.record_prompt("test-model", None, 0);
390        m.record_generated("test-model", None, 0);
391        provider.force_flush()?;
392
393        let mut any_counter: u64 = 0;
394        for rm in exporter.get_finished_metrics()? {
395            for sm in rm.scope_metrics() {
396                for metric in sm.metrics() {
397                    if let AggregatedMetrics::U64(MetricData::Sum(s)) = metric.data() {
398                        for dp in s.data_points() {
399                            any_counter += dp.value();
400                        }
401                    }
402                }
403            }
404        }
405        assert_eq!(
406            any_counter, 0,
407            "zero-count records must not land on the counter"
408        );
409        Ok(())
410    }
411}