hyprstream_bench/family.rs
1//! Question families, holdout designations, and difficulty strata.
2//!
3//! The family taxonomy is the unit of the **family-level holdout firewall**
4//! (plan v1.6): families designated [`Designation::Gate`] are *never
5//! synthesized into training data* — P1.3's synthesis run consumes the frozen
6//! manifest and must exclude them, and the P1.4 leg-(e) zero-shot transfer
7//! gate measures the model on exactly these families. Designation is decided
8//! NOW, at benchmark freeze, not after seeing results.
9
10use std::fmt;
11
12/// A question family: a cluster of procedurally generated, mechanically
13/// verifiable question shapes sharing one truth procedure.
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
15pub enum Family {
16 /// Arithmetic word problems (add/subtract/multi-step counts, unit prices).
17 Arith,
18 /// JSON well-formedness, field extraction, and repair identification.
19 JsonCheck,
20 /// Unit conversion (length, mass, temperature) with numeric outcomes.
21 UnitConv,
22 /// Date arithmetic and event ordering. **Held out (gate).**
23 Temporal,
24 /// Set-logic syllogisms (All/Some/None chains). **Held out (gate).**
25 Syllogism,
26}
27
28/// Whether a family may be synthesized into training data.
29#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30pub enum Designation {
31 /// Usable for training synthesis and evaluation.
32 Open,
33 /// **Firewalled**: never synthesized into training data; reserved for the
34 /// zero-shot transfer gate (P1.4 leg e) and public reporting.
35 Gate,
36}
37
38impl Designation {
39 /// Wire form (`"open"` / `"gate"`).
40 pub fn as_str(self) -> &'static str {
41 match self {
42 Self::Open => "open",
43 Self::Gate => "gate",
44 }
45 }
46}
47
48impl fmt::Display for Designation {
49 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
50 f.write_str(self.as_str())
51 }
52}
53
54impl Family {
55 /// Every family, in canonical (manifest) order.
56 pub const ALL: [Family; 5] = [
57 Family::Arith,
58 Family::JsonCheck,
59 Family::UnitConv,
60 Family::Temporal,
61 Family::Syllogism,
62 ];
63
64 /// Stable family id used in item ids and the manifest.
65 pub fn as_str(self) -> &'static str {
66 match self {
67 Self::Arith => "arith",
68 Self::JsonCheck => "jsoncheck",
69 Self::UnitConv => "unitconv",
70 Self::Temporal => "temporal",
71 Self::Syllogism => "syllogism",
72 }
73 }
74
75 /// Parse a family id.
76 pub fn from_id(id: &str) -> Option<Self> {
77 match id {
78 "arith" => Some(Self::Arith),
79 "jsoncheck" => Some(Self::JsonCheck),
80 "unitconv" => Some(Self::UnitConv),
81 "temporal" => Some(Self::Temporal),
82 "syllogism" => Some(Self::Syllogism),
83 _ => None,
84 }
85 }
86
87 /// The firewall designation. **Frozen at the vob-1.0 manifest freeze**;
88 /// P1.3 treats the manifest, not this code, as the consumed artifact.
89 pub fn designation(self) -> Designation {
90 match self {
91 Self::Arith | Self::JsonCheck | Self::UnitConv => Designation::Open,
92 Self::Temporal | Self::Syllogism => Designation::Gate,
93 }
94 }
95}
96
97impl fmt::Display for Family {
98 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
99 f.write_str(self.as_str())
100 }
101}
102
103/// The difficulty/robustness stratum an item belongs to. Reported per stratum
104/// by the P0.2 measurement protocol.
105#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
106pub enum Stratum {
107 /// Base form of the item.
108 Clean,
109 /// Adversarial near-miss form: near-miss arithmetic (±1, ×10 slips,
110 /// transposed digits as distractors/claims), near-miss JSON (trailing
111 /// commas, unquoted keys, single quotes), close distractors.
112 NearMiss,
113 /// Cyclic-permutation stratum (CircularEval precedent, S6b1): the same
114 /// choice item with options rotated by `rotation` positions; truth rotates
115 /// with the options. Applied to **choice** items only — score levels are
116 /// ordered (index = level is the semantics), so permuting them would
117 /// change the question, not the presentation. Noul has no options.
118 Permuted {
119 /// Cyclic rotation applied, in `1..cardinality`.
120 rotation: u32,
121 },
122}
123
124impl Stratum {
125 /// Wire form (`clean`, `nearmiss`, `perm-2`, ...).
126 pub fn as_str(&self) -> String {
127 match self {
128 Self::Clean => "clean".to_owned(),
129 Self::NearMiss => "nearmiss".to_owned(),
130 Self::Permuted { rotation } => format!("perm-{rotation}"),
131 }
132 }
133
134 /// Whether this stratum is a base form (not a permutation expansion).
135 pub fn is_base(&self) -> bool {
136 !matches!(self, Self::Permuted { .. })
137 }
138}
139
140impl fmt::Display for Stratum {
141 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
142 f.write_str(&self.as_str())
143 }
144}
145
146#[cfg(test)]
147mod tests {
148 use super::*;
149
150 #[test]
151 fn designations_roundtrip_and_gate_set_is_frozen() {
152 // The gate set is the P1.4 leg-(e) measurement surface; changing it
153 // after freeze invalidates the firewall. Lock it here.
154 let gates: Vec<Family> = Family::ALL
155 .iter()
156 .copied()
157 .filter(|family| family.designation() == Designation::Gate)
158 .collect();
159 assert_eq!(gates, vec![Family::Temporal, Family::Syllogism]);
160 }
161
162 #[test]
163 fn family_ids_roundtrip() {
164 for family in Family::ALL {
165 assert_eq!(Family::from_id(family.as_str()), Some(family));
166 }
167 }
168}