1use std::collections::HashMap;
34use std::sync::Arc;
35
36use arrow_array::builder::{FixedSizeListBuilder, Float32Builder, ListBuilder, StringBuilder};
37use arrow_array::{ArrayRef, RecordBatch, StringArray};
38use arrow_schema::{DataType, Field, Schema};
39use thiserror::Error;
40
41use crate::answer::{AnswerValue, QuestionAnswer, VersionTriple};
42use crate::confidence::{self, PRODUCER_SUM_TOLERANCE};
43use crate::spec::{QuestionKind, QuestionSet, QuestionSpec};
44
45pub const FIELD_META_KIND: &str = "jev.kind";
47pub const FIELD_META_LABELS: &str = "jev.labels";
49
50pub const SCHEMA_VERSION_COLUMN: &str = "schema_version";
52pub const MODEL_VERSION_COLUMN: &str = "model_version";
54pub const CALIB_VERSION_COLUMN: &str = "calib_version";
56
57#[derive(Debug, Clone, PartialEq, Error)]
59pub enum ArrowSchemaError {
60 #[error("a decision schema needs at least one question")]
62 Empty,
63 #[error(
65 "question id `{0}` is not identifier-safe ([A-Za-z_][A-Za-z0-9_]*); \
66 wire ids stay opaque, but the Arrow surface needs unquoted SQL composition"
67 )]
68 InvalidIdentifier(String),
69 #[error("duplicate question id `{0}`")]
71 DuplicateQuestionId(String),
72 #[error("question id `{0}` collides with a reserved version-triple column name")]
74 ReservedColumnName(String),
75}
76
77#[derive(Debug, Clone, PartialEq)]
79pub struct LabelChange {
80 pub question_id: String,
82 pub old_kind: QuestionKind,
84 pub new_kind: QuestionKind,
86 pub old_labels: Vec<String>,
88 pub new_labels: Vec<String>,
90}
91
92#[derive(Debug, Clone, PartialEq, Error)]
94#[error(
95 "label-set evolution requires a new schema version; changed questions: {}",
96 .changes.iter().map(|change| change.question_id.as_str()).collect::<Vec<_>>().join(", ")
97)]
98pub struct LabelEvolutionError {
99 pub changes: Vec<LabelChange>,
101}
102
103#[derive(Debug, Clone, PartialEq, Error)]
109pub enum EvolutionError {
110 #[error(transparent)]
112 LabelSetChanged(#[from] LabelEvolutionError),
113 #[error(
117 "conformal-set emission flipped ({old_conformal_set} -> {new_conformal_set}); \
118 the emitted column set changes, so this requires a new schema version"
119 )]
120 EmissionShapeChanged {
121 old_conformal_set: bool,
123 new_conformal_set: bool,
125 },
126}
127
128#[derive(Debug, Clone, Default, PartialEq, Eq)]
130pub struct EvolutionReport {
131 pub added: Vec<String>,
133 pub removed: Vec<String>,
135}
136
137#[derive(Debug, Error)]
139pub enum BatchError {
140 #[error(
142 "row {row} is missing an answer for question `{question_id}` (abstain explicitly instead)"
143 )]
144 MissingAnswer {
145 row: usize,
147 question_id: String,
149 },
150 #[error("row {row} answers unknown question `{question_id}`")]
152 UnknownQuestion {
153 row: usize,
155 question_id: String,
157 },
158 #[error("row {row} question `{question_id}`: answer kind {got} does not match question kind {expected}")]
160 KindMismatch {
161 row: usize,
163 question_id: String,
165 expected: QuestionKind,
167 got: QuestionKind,
169 },
170 #[error("row {row} question `{question_id}`: {got} probabilities for a cardinality-{expected} question")]
172 CardinalityMismatch {
173 row: usize,
175 question_id: String,
177 expected: usize,
179 got: usize,
181 },
182 #[error("row {0} question `{1}`: {2}")]
184 InvalidDistribution(usize, String, #[source] confidence::DistributionError),
185 #[error(
187 "row {row} question `{question_id}`: an abstained answer cannot carry a conformal set"
188 )]
189 AbstainedWithConformalSet {
190 row: usize,
192 question_id: String,
194 },
195 #[error("row {row} question `{question_id}`: conformal set member `{label}` is not one of the question's labels")]
197 UnknownConformalLabel {
198 row: usize,
200 question_id: String,
202 label: String,
204 },
205 #[error("arrow batch assembly failed: {0}")]
207 Arrow(#[from] arrow_schema::ArrowError),
208}
209
210#[derive(Debug, Clone, Default, PartialEq)]
212pub struct AnswerRow {
213 pub answers: std::collections::BTreeMap<String, QuestionAnswer>,
216}
217
218#[derive(Debug, Clone)]
222pub struct DecisionSchema {
223 questions: Vec<QuestionSpec>,
224 emit_conformal_set: bool,
225}
226
227impl DecisionSchema {
228 pub fn new(questions: Vec<QuestionSpec>) -> Result<Self, ArrowSchemaError> {
231 if questions.is_empty() {
232 return Err(ArrowSchemaError::Empty);
233 }
234 let mut seen: Vec<&str> = Vec::with_capacity(questions.len());
235 for question in &questions {
236 let id = question.id.as_str();
237 if !is_identifier_safe(id) {
238 return Err(ArrowSchemaError::InvalidIdentifier(question.id.clone()));
239 }
240 if matches!(
241 id,
242 SCHEMA_VERSION_COLUMN | MODEL_VERSION_COLUMN | CALIB_VERSION_COLUMN
243 ) {
244 return Err(ArrowSchemaError::ReservedColumnName(question.id.clone()));
245 }
246 if seen.contains(&id) {
247 return Err(ArrowSchemaError::DuplicateQuestionId(question.id.clone()));
248 }
249 seen.push(id);
250 }
251 Ok(Self {
252 questions,
253 emit_conformal_set: true,
254 })
255 }
256
257 pub fn from_question_set(set: &QuestionSet) -> Result<Self, ArrowSchemaError> {
260 Self::new(set.questions.clone())
261 }
262
263 pub fn without_conformal_set(mut self) -> Self {
265 self.emit_conformal_set = false;
266 self
267 }
268
269 pub fn questions(&self) -> &[QuestionSpec] {
271 &self.questions
272 }
273
274 pub fn arrow_schema(&self) -> Schema {
277 let mut fields = Vec::with_capacity(self.questions.len() * 3 + 3);
278 for question in &self.questions {
279 let mut metadata = HashMap::new();
280 metadata.insert(
281 FIELD_META_KIND.to_owned(),
282 question.kind.as_str().to_owned(),
283 );
284 metadata.insert(
285 FIELD_META_LABELS.to_owned(),
286 labels_metadata(&question.labels()),
287 );
288 let cardinality = i32::try_from(question.cardinality()).unwrap_or(i32::MAX);
292 let probabilities = Field::new(
293 format!("{}.probabilities", question.id),
294 DataType::FixedSizeList(
295 Arc::new(Field::new("item", DataType::Float32, true)),
296 cardinality,
297 ),
298 true,
299 )
300 .with_metadata(metadata);
301 fields.push(probabilities);
302 fields.push(Field::new(
303 format!("{}.label", question.id),
304 DataType::Utf8,
305 true,
306 ));
307 if self.emit_conformal_set {
308 fields.push(Field::new(
309 format!("{}.conformal_set", question.id),
310 DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
311 true,
312 ));
313 }
314 }
315 fields.push(Field::new(SCHEMA_VERSION_COLUMN, DataType::Utf8, false));
316 fields.push(Field::new(MODEL_VERSION_COLUMN, DataType::Utf8, false));
317 fields.push(Field::new(CALIB_VERSION_COLUMN, DataType::Utf8, true));
318 Schema::new(fields)
319 }
320
321 pub fn fingerprint(&self) -> String {
325 let mut hasher = blake3::Hasher::new();
326 hasher.update(b"hyprstream-decision schema v1\x00");
327 hasher.update(&[self.emit_conformal_set as u8]);
328 for question in &self.questions {
329 hasher.update(question.id.as_bytes());
330 hasher.update(b"\x00");
331 hasher.update(question.kind.as_str().as_bytes());
332 hasher.update(b"\x00");
333 for label in question.labels() {
334 hasher.update(label.as_bytes());
335 hasher.update(b"\x00");
336 }
337 hasher.update(b"\x01");
338 }
339 hasher.finalize().to_hex().to_string()
340 }
341
342 pub fn check_evolution(old: &Self, new: &Self) -> Result<EvolutionReport, EvolutionError> {
348 if old.emit_conformal_set != new.emit_conformal_set {
349 return Err(EvolutionError::EmissionShapeChanged {
350 old_conformal_set: old.emit_conformal_set,
351 new_conformal_set: new.emit_conformal_set,
352 });
353 }
354 let mut changes = Vec::new();
355 for old_question in &old.questions {
356 if let Some(new_question) = new.questions.iter().find(|q| q.id == old_question.id) {
357 let old_labels = old_question.labels();
358 let new_labels = new_question.labels();
359 if old_question.kind != new_question.kind || old_labels != new_labels {
360 changes.push(LabelChange {
361 question_id: old_question.id.clone(),
362 old_kind: old_question.kind,
363 new_kind: new_question.kind,
364 old_labels,
365 new_labels,
366 });
367 }
368 }
369 }
370 if !changes.is_empty() {
371 return Err(LabelEvolutionError { changes }.into());
372 }
373 let report = EvolutionReport {
374 added: new
375 .questions
376 .iter()
377 .filter(|q| !old.questions.iter().any(|old_q| old_q.id == q.id))
378 .map(|q| q.id.clone())
379 .collect(),
380 removed: old
381 .questions
382 .iter()
383 .filter(|q| !new.questions.iter().any(|new_q| new_q.id == q.id))
384 .map(|q| q.id.clone())
385 .collect(),
386 };
387 Ok(report)
388 }
389
390 pub fn build_batch(
394 &self,
395 version: &VersionTriple,
396 rows: &[AnswerRow],
397 ) -> Result<RecordBatch, BatchError> {
398 let row_count = rows.len();
399 for (row_index, row) in rows.iter().enumerate() {
400 for key in row.answers.keys() {
401 if !self.questions.iter().any(|question| &question.id == key) {
402 return Err(BatchError::UnknownQuestion {
403 row: row_index,
404 question_id: key.clone(),
405 });
406 }
407 }
408 }
409
410 let mut columns: Vec<ArrayRef> = Vec::with_capacity(self.questions.len() * 3 + 3);
411
412 for question in &self.questions {
413 let labels = question.labels();
414 let cardinality = i32::try_from(labels.len()).unwrap_or(i32::MAX);
415 let mut probabilities = FixedSizeListBuilder::new(Float32Builder::new(), cardinality);
416 let mut label_builder = StringBuilder::new();
417 let mut conformal_builder = ListBuilder::new(StringBuilder::new());
418
419 for (row_index, row) in rows.iter().enumerate() {
420 let answer =
421 row.answers
422 .get(&question.id)
423 .ok_or_else(|| BatchError::MissingAnswer {
424 row: row_index,
425 question_id: question.id.clone(),
426 })?;
427 match (&answer.value, &answer.conformal_set) {
428 (None, Some(_)) => {
429 return Err(BatchError::AbstainedWithConformalSet {
430 row: row_index,
431 question_id: question.id.clone(),
432 });
433 }
434 (None, None) => {
435 for _ in 0..cardinality {
437 probabilities.values().append_value(0.0);
438 }
439 probabilities.append(false);
440 label_builder.append_null();
441 conformal_builder.append_null();
442 }
443 (Some(value), conformal_set) => {
444 let distribution = validated_distribution(question, value, row_index)?;
445 for &p in &distribution {
446 probabilities.values().append_value(p);
447 }
448 probabilities.append(true);
449 let argmax = confidence::argmax_index(&distribution).unwrap_or(0);
450 label_builder.append_value(&labels[argmax]);
451 match conformal_set {
452 None => conformal_builder.append_null(),
453 Some(set) => {
454 for member in set {
455 if !labels.contains(member) {
456 return Err(BatchError::UnknownConformalLabel {
457 row: row_index,
458 question_id: question.id.clone(),
459 label: member.clone(),
460 });
461 }
462 conformal_builder.values().append_value(member);
463 }
464 conformal_builder.append(true);
465 }
466 }
467 }
468 }
469 }
470
471 columns.push(Arc::new(probabilities.finish()));
472 columns.push(Arc::new(label_builder.finish()));
473 if self.emit_conformal_set {
474 columns.push(Arc::new(conformal_builder.finish()));
475 }
476 }
477
478 columns.push(Arc::new(StringArray::from_iter(std::iter::repeat_n(
479 Some(version.schema.as_str()),
480 row_count,
481 ))));
482 columns.push(Arc::new(StringArray::from_iter(std::iter::repeat_n(
483 Some(version.model.as_str()),
484 row_count,
485 ))));
486 columns.push(Arc::new(StringArray::from_iter(std::iter::repeat_n(
487 version.calib.as_deref(),
488 row_count,
489 ))));
490
491 Ok(RecordBatch::try_new(
492 Arc::new(self.arrow_schema()),
493 columns,
494 )?)
495 }
496}
497
498fn validated_distribution(
500 question: &QuestionSpec,
501 value: &AnswerValue,
502 row: usize,
503) -> Result<Vec<f32>, BatchError> {
504 let answer_kind = match value {
505 AnswerValue::Noul { .. } => QuestionKind::Noul,
506 AnswerValue::Choice { .. } => QuestionKind::Choice,
507 AnswerValue::Score { .. } => QuestionKind::Score,
508 };
509 if answer_kind != question.kind {
510 return Err(BatchError::KindMismatch {
511 row,
512 question_id: question.id.clone(),
513 expected: question.kind,
514 got: answer_kind,
515 });
516 }
517 let distribution = value.probabilities();
518 let cardinality = question.cardinality();
519 if distribution.len() != cardinality {
520 return Err(BatchError::CardinalityMismatch {
521 row,
522 question_id: question.id.clone(),
523 expected: cardinality,
524 got: distribution.len(),
525 });
526 }
527 confidence::check_distribution(&distribution, PRODUCER_SUM_TOLERANCE)
528 .map_err(|error| BatchError::InvalidDistribution(row, question.id.clone(), error))?;
529 Ok(distribution)
530}
531
532fn labels_metadata(labels: &[String]) -> String {
534 let mut out = String::from("[");
535 for (index, label) in labels.iter().enumerate() {
536 if index > 0 {
537 out.push(',');
538 }
539 out.push_str(&serde_json::to_string(label).unwrap_or_else(|_| "\"\"".to_owned()));
540 }
541 out.push(']');
542 out
543}
544
545fn is_identifier_safe(id: &str) -> bool {
546 let mut chars = id.chars();
547 match chars.next() {
548 Some(first) if first.is_ascii_alphabetic() || first == '_' => {}
549 _ => return false,
550 }
551 chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_')
552}