Skip to main content

hyprstream_decision/
arrow.rs

1//! Arrow emission: the columnar contract for decision batches.
2//!
3//! For a set of questions, [`DecisionSchema`] emits (in question authoring order, then
4//! the version triple):
5//!
6//! | column | type | nullability | content |
7//! |---|---|---|---|
8//! | `{qid}.probabilities` | `fixed_size_list<f32; N>` | nullable | one probability per label, canonical order; null = abstained |
9//! | `{qid}.label` | `utf8` | nullable | argmax label (D6 tie-break); **null = abstained** |
10//! | `{qid}.conformal_set` | `list<utf8>` | nullable | conformal prediction set — **reserved in v1**, may be entirely unpopulated |
11//! | `schema_version` | `utf8` | non-null | question-set schema version |
12//! | `model_version` | `utf8` | non-null | resolved model id |
13//! | `calib_version` | `utf8` | nullable | calibration-fit version; null = uncalibrated |
14//!
15//! N (cardinality) is 2 for noul (`["false", "true"]`), the option count for choice, the
16//! level count for score.
17//!
18//! ## Contract rules
19//!
20//! - **Label lists live in Arrow *field* metadata** on the probabilities column
21//!   (`jev.kind`, `jev.labels` as a JSON array) — never in schema metadata, which this
22//!   crate always emits empty. Calibration parameters are likewise never schema content;
23//!   they are rows in the P0.3 metrics tables.
24//! - **Label-set evolution is schema evolution and fails loudly**:
25//!   [`DecisionSchema::check_evolution`] rejects any change to an existing question's kind
26//!   or label set — and any flip of conformal-set emission, which changes the emitted
27//!   column set; the remedy is minting a new schema version (the `schema_version`
28//!   triple column), which is what prepared statements pin (P3.5).
29//! - Question ids must be identifier-safe (`[A-Za-z_][A-Za-z0-9_]*`) for the columnar
30//!   surface so emitted columns compose unquoted in SQL (the ADBC path). Wire-level ids
31//!   stay opaque; the constraint applies at schema construction, not at authoring.
32
33use std::collections::HashMap;
34use std::sync::Arc;
35
36use arrow_array::builder::{FixedSizeListBuilder, Float32Builder, ListBuilder, StringBuilder};
37use arrow_array::{ArrayRef, RecordBatch, StringArray};
38use arrow_schema::{DataType, Field, Schema};
39use thiserror::Error;
40
41use crate::answer::{AnswerValue, QuestionAnswer, VersionTriple};
42use crate::confidence::{self, PRODUCER_SUM_TOLERANCE};
43use crate::spec::{QuestionKind, QuestionSet, QuestionSpec};
44
45/// Field-metadata key carrying the question kind (`noul` / `choice` / `score`).
46pub const FIELD_META_KIND: &str = "jev.kind";
47/// Field-metadata key carrying the label list as a JSON array in canonical order.
48pub const FIELD_META_LABELS: &str = "jev.labels";
49
50/// Column name of the question-set schema version (version triple).
51pub const SCHEMA_VERSION_COLUMN: &str = "schema_version";
52/// Column name of the resolved model id (version triple).
53pub const MODEL_VERSION_COLUMN: &str = "model_version";
54/// Column name of the calibration-fit version (version triple; null = uncalibrated).
55pub const CALIB_VERSION_COLUMN: &str = "calib_version";
56
57/// Why a set of questions cannot back an Arrow schema.
58#[derive(Debug, Clone, PartialEq, Error)]
59pub enum ArrowSchemaError {
60    /// A schema needs at least one question.
61    #[error("a decision schema needs at least one question")]
62    Empty,
63    /// A question id is not identifier-safe for the columnar surface.
64    #[error(
65        "question id `{0}` is not identifier-safe ([A-Za-z_][A-Za-z0-9_]*); \
66         wire ids stay opaque, but the Arrow surface needs unquoted SQL composition"
67    )]
68    InvalidIdentifier(String),
69    /// Two questions share an id.
70    #[error("duplicate question id `{0}`")]
71    DuplicateQuestionId(String),
72    /// A question id collides with a reserved version-triple column.
73    #[error("question id `{0}` collides with a reserved version-triple column name")]
74    ReservedColumnName(String),
75}
76
77/// One changed question in a failed label-set evolution check.
78#[derive(Debug, Clone, PartialEq)]
79pub struct LabelChange {
80    /// The question whose contract changed.
81    pub question_id: String,
82    /// Kind in the prior schema.
83    pub old_kind: QuestionKind,
84    /// Kind in the new schema.
85    pub new_kind: QuestionKind,
86    /// Labels in the prior schema.
87    pub old_labels: Vec<String>,
88    /// Labels in the new schema.
89    pub new_labels: Vec<String>,
90}
91
92/// Label-set evolution on an existing question — schema evolution fails loudly.
93#[derive(Debug, Clone, PartialEq, Error)]
94#[error(
95    "label-set evolution requires a new schema version; changed questions: {}",
96    .changes.iter().map(|change| change.question_id.as_str()).collect::<Vec<_>>().join(", ")
97)]
98pub struct LabelEvolutionError {
99    /// Every question whose kind or label set changed.
100    pub changes: Vec<LabelChange>,
101}
102
103/// Why a schema generation pair fails the evolution check.
104///
105/// Covers everything [`DecisionSchema::fingerprint`] treats as schema content, so the
106/// two drift detectors can never disagree: an evolution the fingerprint would see is an
107/// evolution this check rejects.
108#[derive(Debug, Clone, PartialEq, Error)]
109pub enum EvolutionError {
110    /// An existing question's kind or label set changed.
111    #[error(transparent)]
112    LabelSetChanged(#[from] LabelEvolutionError),
113    /// The conformal-set column flag flipped. The emitted Arrow schema gains or loses a
114    /// column per question, breaking positional consumers and prepared statements, so
115    /// this is schema evolution even though no question changed.
116    #[error(
117        "conformal-set emission flipped ({old_conformal_set} -> {new_conformal_set}); \
118         the emitted column set changes, so this requires a new schema version"
119    )]
120    EmissionShapeChanged {
121        /// Prior schema's flag.
122        old_conformal_set: bool,
123        /// New schema's flag.
124        new_conformal_set: bool,
125    },
126}
127
128/// Compatible differences between two schema generations.
129#[derive(Debug, Clone, Default, PartialEq, Eq)]
130pub struct EvolutionReport {
131    /// Question ids present only in the new schema.
132    pub added: Vec<String>,
133    /// Question ids present only in the prior schema.
134    pub removed: Vec<String>,
135}
136
137/// Why a row cannot be encoded into a batch under this schema.
138#[derive(Debug, Error)]
139pub enum BatchError {
140    /// A row is missing an answer (even an abstention) for a declared question.
141    #[error(
142        "row {row} is missing an answer for question `{question_id}` (abstain explicitly instead)"
143    )]
144    MissingAnswer {
145        /// Offending row index.
146        row: usize,
147        /// The unanswered question.
148        question_id: String,
149    },
150    /// A row answers a question the schema does not declare.
151    #[error("row {row} answers unknown question `{question_id}`")]
152    UnknownQuestion {
153        /// Offending row index.
154        row: usize,
155        /// The unknown question id.
156        question_id: String,
157    },
158    /// The answer kind does not match the question kind.
159    #[error("row {row} question `{question_id}`: answer kind {got} does not match question kind {expected}")]
160    KindMismatch {
161        /// Offending row index.
162        row: usize,
163        /// The question.
164        question_id: String,
165        /// The question's kind.
166        expected: QuestionKind,
167        /// The answer's kind.
168        got: QuestionKind,
169    },
170    /// The probability vector length does not equal the question cardinality.
171    #[error("row {row} question `{question_id}`: {got} probabilities for a cardinality-{expected} question")]
172    CardinalityMismatch {
173        /// Offending row index.
174        row: usize,
175        /// The question.
176        question_id: String,
177        /// The question's cardinality.
178        expected: usize,
179        /// The vector length supplied.
180        got: usize,
181    },
182    /// The distribution violates the producer tolerance (D5).
183    #[error("row {0} question `{1}`: {2}")]
184    InvalidDistribution(usize, String, #[source] confidence::DistributionError),
185    /// An abstained answer carries a conformal set.
186    #[error(
187        "row {row} question `{question_id}`: an abstained answer cannot carry a conformal set"
188    )]
189    AbstainedWithConformalSet {
190        /// Offending row index.
191        row: usize,
192        /// The question.
193        question_id: String,
194    },
195    /// A conformal set names a label outside the question's label set.
196    #[error("row {row} question `{question_id}`: conformal set member `{label}` is not one of the question's labels")]
197    UnknownConformalLabel {
198        /// Offending row index.
199        row: usize,
200        /// The question.
201        question_id: String,
202        /// The offending label.
203        label: String,
204    },
205    /// Arrow rejected the assembled batch (internal invariant violation).
206    #[error("arrow batch assembly failed: {0}")]
207    Arrow(#[from] arrow_schema::ArrowError),
208}
209
210/// One row of a decision batch: answers keyed by question id.
211#[derive(Debug, Clone, Default, PartialEq)]
212pub struct AnswerRow {
213    /// Per-question answers. Every declared question must be present — abstain
214    /// explicitly ([`QuestionAnswer::abstained`]) rather than omitting the key.
215    pub answers: std::collections::BTreeMap<String, QuestionAnswer>,
216}
217
218/// The Arrow-emission view over a set of questions. Cheap to clone (the questions are
219/// shared); the Arrow schema itself is rebuilt on demand so an emitted schema can never
220/// drift from the IR.
221#[derive(Debug, Clone)]
222pub struct DecisionSchema {
223    questions: Vec<QuestionSpec>,
224    emit_conformal_set: bool,
225}
226
227impl DecisionSchema {
228    /// Build from validated questions. The conformal-set column is emitted by default
229    /// (reserved in v1); see [`DecisionSchema::without_conformal_set`].
230    pub fn new(questions: Vec<QuestionSpec>) -> Result<Self, ArrowSchemaError> {
231        if questions.is_empty() {
232            return Err(ArrowSchemaError::Empty);
233        }
234        let mut seen: Vec<&str> = Vec::with_capacity(questions.len());
235        for question in &questions {
236            let id = question.id.as_str();
237            if !is_identifier_safe(id) {
238                return Err(ArrowSchemaError::InvalidIdentifier(question.id.clone()));
239            }
240            if matches!(
241                id,
242                SCHEMA_VERSION_COLUMN | MODEL_VERSION_COLUMN | CALIB_VERSION_COLUMN
243            ) {
244                return Err(ArrowSchemaError::ReservedColumnName(question.id.clone()));
245            }
246            if seen.contains(&id) {
247                return Err(ArrowSchemaError::DuplicateQuestionId(question.id.clone()));
248            }
249            seen.push(id);
250        }
251        Ok(Self {
252            questions,
253            emit_conformal_set: true,
254        })
255    }
256
257    /// Build from an authored question set (the state is not part of the columnar
258    /// contract).
259    pub fn from_question_set(set: &QuestionSet) -> Result<Self, ArrowSchemaError> {
260        Self::new(set.questions.clone())
261    }
262
263    /// Omit the reserved conformal-set columns (v1 consumers that do not read them).
264    pub fn without_conformal_set(mut self) -> Self {
265        self.emit_conformal_set = false;
266        self
267    }
268
269    /// The questions backing this schema, in canonical order.
270    pub fn questions(&self) -> &[QuestionSpec] {
271        &self.questions
272    }
273
274    /// The Arrow schema. Schema metadata is always empty by construction: labels live in
275    /// field metadata, calibration parameters in P0.3 tables.
276    pub fn arrow_schema(&self) -> Schema {
277        let mut fields = Vec::with_capacity(self.questions.len() * 3 + 3);
278        for question in &self.questions {
279            let mut metadata = HashMap::new();
280            metadata.insert(
281                FIELD_META_KIND.to_owned(),
282                question.kind.as_str().to_owned(),
283            );
284            metadata.insert(
285                FIELD_META_LABELS.to_owned(),
286                labels_metadata(&question.labels()),
287            );
288            // Inner field nullable=true matches the FixedSizeListBuilder default
289            // (repo convention, hyprstream-metrics); individual components are never
290            // null in practice — abstention nulls the whole list.
291            let cardinality = i32::try_from(question.cardinality()).unwrap_or(i32::MAX);
292            let probabilities = Field::new(
293                format!("{}.probabilities", question.id),
294                DataType::FixedSizeList(
295                    Arc::new(Field::new("item", DataType::Float32, true)),
296                    cardinality,
297                ),
298                true,
299            )
300            .with_metadata(metadata);
301            fields.push(probabilities);
302            fields.push(Field::new(
303                format!("{}.label", question.id),
304                DataType::Utf8,
305                true,
306            ));
307            if self.emit_conformal_set {
308                fields.push(Field::new(
309                    format!("{}.conformal_set", question.id),
310                    DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
311                    true,
312                ));
313            }
314        }
315        fields.push(Field::new(SCHEMA_VERSION_COLUMN, DataType::Utf8, false));
316        fields.push(Field::new(MODEL_VERSION_COLUMN, DataType::Utf8, false));
317        fields.push(Field::new(CALIB_VERSION_COLUMN, DataType::Utf8, true));
318        Schema::new(fields)
319    }
320
321    /// A stable content fingerprint over (id, kind, labels) per question plus the
322    /// conformal-set flag. Registries persist this to detect drift; the hex string is
323    /// safe to embed in P0.3 rows.
324    pub fn fingerprint(&self) -> String {
325        let mut hasher = blake3::Hasher::new();
326        hasher.update(b"hyprstream-decision schema v1\x00");
327        hasher.update(&[self.emit_conformal_set as u8]);
328        for question in &self.questions {
329            hasher.update(question.id.as_bytes());
330            hasher.update(b"\x00");
331            hasher.update(question.kind.as_str().as_bytes());
332            hasher.update(b"\x00");
333            for label in question.labels() {
334                hasher.update(label.as_bytes());
335                hasher.update(b"\x00");
336            }
337            hasher.update(b"\x01");
338        }
339        hasher.finalize().to_hex().to_string()
340    }
341
342    /// Fail-loud schema evolution check. Adding or removing questions is compatible
343    /// (reported); changing an existing question's kind or label set — or flipping
344    /// conformal-set emission, which changes the emitted column set — is not: mint a new
345    /// schema version instead. The rejected inputs are exactly the ones
346    /// [`DecisionSchema::fingerprint`] distinguishes.
347    pub fn check_evolution(old: &Self, new: &Self) -> Result<EvolutionReport, EvolutionError> {
348        if old.emit_conformal_set != new.emit_conformal_set {
349            return Err(EvolutionError::EmissionShapeChanged {
350                old_conformal_set: old.emit_conformal_set,
351                new_conformal_set: new.emit_conformal_set,
352            });
353        }
354        let mut changes = Vec::new();
355        for old_question in &old.questions {
356            if let Some(new_question) = new.questions.iter().find(|q| q.id == old_question.id) {
357                let old_labels = old_question.labels();
358                let new_labels = new_question.labels();
359                if old_question.kind != new_question.kind || old_labels != new_labels {
360                    changes.push(LabelChange {
361                        question_id: old_question.id.clone(),
362                        old_kind: old_question.kind,
363                        new_kind: new_question.kind,
364                        old_labels,
365                        new_labels,
366                    });
367                }
368            }
369        }
370        if !changes.is_empty() {
371            return Err(LabelEvolutionError { changes }.into());
372        }
373        let report = EvolutionReport {
374            added: new
375                .questions
376                .iter()
377                .filter(|q| !old.questions.iter().any(|old_q| old_q.id == q.id))
378                .map(|q| q.id.clone())
379                .collect(),
380            removed: old
381                .questions
382                .iter()
383                .filter(|q| !new.questions.iter().any(|new_q| new_q.id == q.id))
384                .map(|q| q.id.clone())
385                .collect(),
386        };
387        Ok(report)
388    }
389
390    /// Encode rows into a record batch under this schema. Every row must carry an answer
391    /// (possibly abstained) for every declared question; distributions are validated
392    /// against the producer tolerance (D5).
393    pub fn build_batch(
394        &self,
395        version: &VersionTriple,
396        rows: &[AnswerRow],
397    ) -> Result<RecordBatch, BatchError> {
398        let row_count = rows.len();
399        for (row_index, row) in rows.iter().enumerate() {
400            for key in row.answers.keys() {
401                if !self.questions.iter().any(|question| &question.id == key) {
402                    return Err(BatchError::UnknownQuestion {
403                        row: row_index,
404                        question_id: key.clone(),
405                    });
406                }
407            }
408        }
409
410        let mut columns: Vec<ArrayRef> = Vec::with_capacity(self.questions.len() * 3 + 3);
411
412        for question in &self.questions {
413            let labels = question.labels();
414            let cardinality = i32::try_from(labels.len()).unwrap_or(i32::MAX);
415            let mut probabilities = FixedSizeListBuilder::new(Float32Builder::new(), cardinality);
416            let mut label_builder = StringBuilder::new();
417            let mut conformal_builder = ListBuilder::new(StringBuilder::new());
418
419            for (row_index, row) in rows.iter().enumerate() {
420                let answer =
421                    row.answers
422                        .get(&question.id)
423                        .ok_or_else(|| BatchError::MissingAnswer {
424                            row: row_index,
425                            question_id: question.id.clone(),
426                        })?;
427                match (&answer.value, &answer.conformal_set) {
428                    (None, Some(_)) => {
429                        return Err(BatchError::AbstainedWithConformalSet {
430                            row: row_index,
431                            question_id: question.id.clone(),
432                        });
433                    }
434                    (None, None) => {
435                        // Abstention: null probabilities + null label.
436                        for _ in 0..cardinality {
437                            probabilities.values().append_value(0.0);
438                        }
439                        probabilities.append(false);
440                        label_builder.append_null();
441                        conformal_builder.append_null();
442                    }
443                    (Some(value), conformal_set) => {
444                        let distribution = validated_distribution(question, value, row_index)?;
445                        for &p in &distribution {
446                            probabilities.values().append_value(p);
447                        }
448                        probabilities.append(true);
449                        let argmax = confidence::argmax_index(&distribution).unwrap_or(0);
450                        label_builder.append_value(&labels[argmax]);
451                        match conformal_set {
452                            None => conformal_builder.append_null(),
453                            Some(set) => {
454                                for member in set {
455                                    if !labels.contains(member) {
456                                        return Err(BatchError::UnknownConformalLabel {
457                                            row: row_index,
458                                            question_id: question.id.clone(),
459                                            label: member.clone(),
460                                        });
461                                    }
462                                    conformal_builder.values().append_value(member);
463                                }
464                                conformal_builder.append(true);
465                            }
466                        }
467                    }
468                }
469            }
470
471            columns.push(Arc::new(probabilities.finish()));
472            columns.push(Arc::new(label_builder.finish()));
473            if self.emit_conformal_set {
474                columns.push(Arc::new(conformal_builder.finish()));
475            }
476        }
477
478        columns.push(Arc::new(StringArray::from_iter(std::iter::repeat_n(
479            Some(version.schema.as_str()),
480            row_count,
481        ))));
482        columns.push(Arc::new(StringArray::from_iter(std::iter::repeat_n(
483            Some(version.model.as_str()),
484            row_count,
485        ))));
486        columns.push(Arc::new(StringArray::from_iter(std::iter::repeat_n(
487            version.calib.as_deref(),
488            row_count,
489        ))));
490
491        Ok(RecordBatch::try_new(
492            Arc::new(self.arrow_schema()),
493            columns,
494        )?)
495    }
496}
497
498/// Validate kind/cardinality/distribution and return the canonical-order distribution.
499fn validated_distribution(
500    question: &QuestionSpec,
501    value: &AnswerValue,
502    row: usize,
503) -> Result<Vec<f32>, BatchError> {
504    let answer_kind = match value {
505        AnswerValue::Noul { .. } => QuestionKind::Noul,
506        AnswerValue::Choice { .. } => QuestionKind::Choice,
507        AnswerValue::Score { .. } => QuestionKind::Score,
508    };
509    if answer_kind != question.kind {
510        return Err(BatchError::KindMismatch {
511            row,
512            question_id: question.id.clone(),
513            expected: question.kind,
514            got: answer_kind,
515        });
516    }
517    let distribution = value.probabilities();
518    let cardinality = question.cardinality();
519    if distribution.len() != cardinality {
520        return Err(BatchError::CardinalityMismatch {
521            row,
522            question_id: question.id.clone(),
523            expected: cardinality,
524            got: distribution.len(),
525        });
526    }
527    confidence::check_distribution(&distribution, PRODUCER_SUM_TOLERANCE)
528        .map_err(|error| BatchError::InvalidDistribution(row, question.id.clone(), error))?;
529    Ok(distribution)
530}
531
532/// The `jev.labels` field-metadata value: a compact JSON array of label strings.
533fn labels_metadata(labels: &[String]) -> String {
534    let mut out = String::from("[");
535    for (index, label) in labels.iter().enumerate() {
536        if index > 0 {
537            out.push(',');
538        }
539        out.push_str(&serde_json::to_string(label).unwrap_or_else(|_| "\"\"".to_owned()));
540    }
541    out.push(']');
542    out
543}
544
545fn is_identifier_safe(id: &str) -> bool {
546    let mut chars = id.chars();
547    match chars.next() {
548        Some(first) if first.is_ascii_alphabetic() || first == '_' => {}
549        _ => return false,
550    }
551    chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_')
552}