Skip to main content

hyprstream_bench/
item.rs

1//! One benchmark item: a jev-1 question spec, its state, and the verifiable
2//! truth — plus the canonical byte form the frozen manifest hashes.
3
4use std::fmt::Write as _;
5
6use hyprstream_decision::{ChoiceOption, Entry, QuestionBody, QuestionKind, QuestionSpec};
7
8use crate::family::{Family, Stratum};
9
10/// One score-bearing, verifiable-outcome item.
11///
12/// `truth` is the index of the correct label in the question's canonical
13/// label order (`["false","true"]` for noul, option order for choice, level
14/// order for score). The truth procedure is mechanical — every item is
15/// generated together with its answer, immune to teacher-pretraining
16/// contamination by construction.
17#[derive(Debug, Clone, PartialEq)]
18pub struct Item {
19    /// Stable item id: `vob1-<family>-<stratum>-<kind>-<seed:016x>`.
20    pub id: String,
21    /// Question family.
22    pub family: Family,
23    /// Difficulty/robustness stratum.
24    pub stratum: Stratum,
25    /// Permutation group: all cyclic rotations of one choice item share this
26    /// id (the base item's id). Base items are their own group.
27    pub group: String,
28    /// The raw seed this item was generated from (reproducibility handle).
29    pub seed: u64,
30    /// Shared state the question refers to.
31    pub state: Entry,
32    /// The jev-1 question spec (`question.id == item.id`).
33    pub question: QuestionSpec,
34    /// Index of the correct label in canonical label order.
35    pub truth: usize,
36}
37
38impl Item {
39    /// Build a noul item.
40    pub fn noul(
41        family: Family,
42        stratum: Stratum,
43        seed: u64,
44        state: Entry,
45        instructions: impl Into<String>,
46        truth: bool,
47    ) -> Self {
48        let id = item_id(family, stratum, QuestionKind::Noul, seed);
49        let question = QuestionSpec {
50            id: id.clone(),
51            kind: QuestionKind::Noul,
52            instructions: Some(Entry::Str(instructions.into())),
53            body: QuestionBody::Noul { criteria: None },
54        };
55        Self {
56            group: id.clone(),
57            id,
58            family,
59            stratum,
60            seed,
61            state,
62            question,
63            truth: usize::from(truth),
64        }
65    }
66
67    /// Build a choice item. `correct` is the index of the correct option.
68    pub fn choice(
69        family: Family,
70        stratum: Stratum,
71        seed: u64,
72        state: Entry,
73        instructions: impl Into<String>,
74        options: Vec<String>,
75        correct: usize,
76    ) -> Self {
77        assert!(options.len() >= 2, "jev-1 choice requires >= 2 options");
78        assert!(correct < options.len(), "truth index out of range");
79        let id = item_id(family, stratum, QuestionKind::Choice, seed);
80        let question = QuestionSpec {
81            id: id.clone(),
82            kind: QuestionKind::Choice,
83            instructions: Some(Entry::Str(instructions.into())),
84            body: QuestionBody::Choice {
85                options: options
86                    .into_iter()
87                    .map(|name| ChoiceOption { name, rubric: None })
88                    .collect(),
89            },
90        };
91        Self {
92            group: id.clone(),
93            id,
94            family,
95            stratum,
96            seed,
97            state,
98            question,
99            truth: correct,
100        }
101    }
102
103    /// Build a score item over `levels` ordered rubric texts.
104    pub fn score(
105        family: Family,
106        stratum: Stratum,
107        seed: u64,
108        state: Entry,
109        instructions: impl Into<String>,
110        levels: usize,
111        correct_level: usize,
112    ) -> Self {
113        assert!(levels >= 2, "jev-1 score requires >= 2 levels");
114        assert!(correct_level < levels, "truth level out of range");
115        let id = item_id(family, stratum, QuestionKind::Score, seed);
116        let question = QuestionSpec {
117            id: id.clone(),
118            kind: QuestionKind::Score,
119            instructions: Some(Entry::Str(instructions.into())),
120            body: QuestionBody::Score {
121                levels: vec![None; levels],
122            },
123        };
124        Self {
125            group: id.clone(),
126            id,
127            family,
128            stratum,
129            seed,
130            state,
131            question,
132            truth: correct_level,
133        }
134    }
135
136    /// The correct label string in canonical label order.
137    pub fn truth_label(&self) -> String {
138        self.question
139            .labels()
140            .get(self.truth)
141            .cloned()
142            .unwrap_or_default()
143    }
144
145    /// One-hot truth distribution over the canonical labels.
146    pub fn truth_distribution(&self) -> Vec<f32> {
147        let mut distribution = vec![0.0; self.question.cardinality()];
148        distribution[self.truth] = 1.0;
149        distribution
150    }
151
152    /// The cyclic permutation of a **choice** item (CircularEval stratum).
153    ///
154    /// Options rotate left by `rotation` positions; the truth index rotates
155    /// with them so the same option stays correct. Returns `None` for
156    /// non-choice questions (score levels are ordered — permutation would
157    /// change the question — and noul has no options) and for rotation 0.
158    pub fn cyclic_permutation(&self, rotation: usize) -> Option<Item> {
159        let QuestionBody::Choice { options } = &self.question.body else {
160            return None;
161        };
162        let n = options.len();
163        let rotation = rotation % n;
164        if rotation == 0 || !self.stratum.is_base() {
165            return None;
166        }
167        let rotated: Vec<ChoiceOption> = (0..n)
168            .map(|i| options[(i + rotation) % n].clone())
169            .collect();
170        let mut item = self.clone();
171        item.stratum = Stratum::Permuted {
172            rotation: rotation as u32,
173        };
174        item.id = format!("{}-p{rotation}", self.id);
175        item.group = self.id.clone();
176        item.truth = (self.truth + n - rotation) % n;
177        item.question = QuestionSpec {
178            id: item.id.clone(),
179            kind: QuestionKind::Choice,
180            instructions: self.question.instructions.clone(),
181            body: QuestionBody::Choice { options: rotated },
182        };
183        Some(item)
184    }
185
186    /// Canonical byte form — the exact input the manifest hashes.
187    ///
188    /// `FROZEN (vob-item/1)`: length-prefixed fields, one per line, so the
189    /// hash is independent of any serializer's key order or float formatting.
190    /// Changing anything here invalidates every published item hash.
191    pub fn canonical_bytes(&self) -> Vec<u8> {
192        let mut out = String::new();
193        out.push_str("vob-item/1\n");
194        write_field(&mut out, "id", &self.id);
195        write_field(&mut out, "family", self.family.as_str());
196        write_field(&mut out, "stratum", &self.stratum.as_str());
197        write_field(&mut out, "group", &self.group);
198        write_field(&mut out, "seed", &format!("{:016x}", self.seed));
199        write_field(&mut out, "state", &self.state.canonical_text());
200        write_field(&mut out, "kind", self.question.kind.as_str());
201        match &self.question.instructions {
202            Some(entry) => write_field(&mut out, "instructions", &entry.canonical_text()),
203            None => write_field(&mut out, "instructions", ""),
204        }
205        let labels = self.question.labels();
206        write_field(&mut out, "cardinality", &labels.len().to_string());
207        for (index, label) in labels.iter().enumerate() {
208            write_field(&mut out, &format!("label.{index}"), label);
209        }
210        if let QuestionBody::Choice { options } = &self.question.body {
211            for (index, option) in options.iter().enumerate() {
212                let rubric = option
213                    .rubric
214                    .as_ref()
215                    .map(Entry::canonical_text)
216                    .unwrap_or_default();
217                write_field(&mut out, &format!("rubric.{index}"), &rubric);
218            }
219        }
220        write_field(&mut out, "truth", &self.truth.to_string());
221        out.into_bytes()
222    }
223
224    /// blake3 hex digest of [`Self::canonical_bytes`] — the manifest hash.
225    pub fn hash(&self) -> String {
226        blake3::hash(&self.canonical_bytes()).to_hex().to_string()
227    }
228}
229
230/// The item-id shape. Ids are identifier-safe (they double as jev-1 question
231/// ids, which flow into Arrow field names).
232fn item_id(family: Family, stratum: Stratum, kind: QuestionKind, seed: u64) -> String {
233    format!(
234        "vob1-{}-{}-{}-{seed:016x}",
235        family.as_str(),
236        stratum.as_str(),
237        kind.as_str()
238    )
239}
240
241fn write_field(out: &mut String, name: &str, value: &str) {
242    // Length prefix in bytes so values may contain any character (including
243    // newlines and colons) without ambiguity.
244    let _ = write!(out, "{}={}:{};", name, value.len(), value);
245    out.push('\n');
246}
247
248#[cfg(test)]
249#[allow(clippy::unwrap_used, clippy::expect_used)]
250mod tests {
251    use super::*;
252
253    fn sample_choice() -> Item {
254        Item::choice(
255            Family::Arith,
256            Stratum::Clean,
257            7,
258            Entry::Str("A crate holds 4 apples.".into()),
259            "How many apples?",
260            vec!["3".into(), "4".into(), "5".into()],
261            1,
262        )
263    }
264
265    #[test]
266    fn permutation_rotates_truth_with_options() {
267        let item = sample_choice();
268        for rotation in 1..3 {
269            let rotated = item.cyclic_permutation(rotation).unwrap();
270            assert_eq!(rotated.truth_label(), item.truth_label());
271            assert_eq!(rotated.group, item.id);
272            assert_ne!(rotated.question.labels(), item.question.labels());
273        }
274        assert!(item.cyclic_permutation(0).is_none());
275        assert!(item.cyclic_permutation(3).is_none());
276    }
277
278    #[test]
279    fn score_and_noul_are_not_permutable() {
280        let score = Item::score(
281            Family::Arith,
282            Stratum::Clean,
283            7,
284            Entry::Null,
285            "How large?",
286            4,
287            2,
288        );
289        assert!(score.cyclic_permutation(1).is_none());
290        let noul = Item::noul(
291            Family::Arith,
292            Stratum::Clean,
293            7,
294            Entry::Null,
295            "Is it 4?",
296            true,
297        );
298        assert!(noul.cyclic_permutation(1).is_none());
299    }
300
301    #[test]
302    fn hash_is_stable() {
303        let item = sample_choice();
304        // Regression lock: pinned golden digest (blake3 over canonical bytes).
305        assert_eq!(
306            item.hash(),
307            "764f41e7ebd698307176671b7114369ff009d4659989bcaba8ef3e1858630344"
308        );
309    }
310}