Skip to main content

hyprstream_bench/
family.rs

1//! Question families, holdout designations, and difficulty strata.
2//!
3//! The family taxonomy is the unit of the **family-level holdout firewall**
4//! (plan v1.6): families designated [`Designation::Gate`] are *never
5//! synthesized into training data* — P1.3's synthesis run consumes the frozen
6//! manifest and must exclude them, and the P1.4 leg-(e) zero-shot transfer
7//! gate measures the model on exactly these families. Designation is decided
8//! NOW, at benchmark freeze, not after seeing results.
9
10use std::fmt;
11
12/// A question family: a cluster of procedurally generated, mechanically
13/// verifiable question shapes sharing one truth procedure.
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
15pub enum Family {
16    /// Arithmetic word problems (add/subtract/multi-step counts, unit prices).
17    Arith,
18    /// JSON well-formedness, field extraction, and repair identification.
19    JsonCheck,
20    /// Unit conversion (length, mass, temperature) with numeric outcomes.
21    UnitConv,
22    /// Date arithmetic and event ordering. **Held out (gate).**
23    Temporal,
24    /// Set-logic syllogisms (All/Some/None chains). **Held out (gate).**
25    Syllogism,
26}
27
28/// Whether a family may be synthesized into training data.
29#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30pub enum Designation {
31    /// Usable for training synthesis and evaluation.
32    Open,
33    /// **Firewalled**: never synthesized into training data; reserved for the
34    /// zero-shot transfer gate (P1.4 leg e) and public reporting.
35    Gate,
36}
37
38impl Designation {
39    /// Wire form (`"open"` / `"gate"`).
40    pub fn as_str(self) -> &'static str {
41        match self {
42            Self::Open => "open",
43            Self::Gate => "gate",
44        }
45    }
46}
47
48impl fmt::Display for Designation {
49    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
50        f.write_str(self.as_str())
51    }
52}
53
54impl Family {
55    /// Every family, in canonical (manifest) order.
56    pub const ALL: [Family; 5] = [
57        Family::Arith,
58        Family::JsonCheck,
59        Family::UnitConv,
60        Family::Temporal,
61        Family::Syllogism,
62    ];
63
64    /// Stable family id used in item ids and the manifest.
65    pub fn as_str(self) -> &'static str {
66        match self {
67            Self::Arith => "arith",
68            Self::JsonCheck => "jsoncheck",
69            Self::UnitConv => "unitconv",
70            Self::Temporal => "temporal",
71            Self::Syllogism => "syllogism",
72        }
73    }
74
75    /// Parse a family id.
76    pub fn from_id(id: &str) -> Option<Self> {
77        match id {
78            "arith" => Some(Self::Arith),
79            "jsoncheck" => Some(Self::JsonCheck),
80            "unitconv" => Some(Self::UnitConv),
81            "temporal" => Some(Self::Temporal),
82            "syllogism" => Some(Self::Syllogism),
83            _ => None,
84        }
85    }
86
87    /// The firewall designation. **Frozen at the vob-1.0 manifest freeze**;
88    /// P1.3 treats the manifest, not this code, as the consumed artifact.
89    pub fn designation(self) -> Designation {
90        match self {
91            Self::Arith | Self::JsonCheck | Self::UnitConv => Designation::Open,
92            Self::Temporal | Self::Syllogism => Designation::Gate,
93        }
94    }
95}
96
97impl fmt::Display for Family {
98    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
99        f.write_str(self.as_str())
100    }
101}
102
103/// The difficulty/robustness stratum an item belongs to. Reported per stratum
104/// by the P0.2 measurement protocol.
105#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
106pub enum Stratum {
107    /// Base form of the item.
108    Clean,
109    /// Adversarial near-miss form: near-miss arithmetic (±1, ×10 slips,
110    /// transposed digits as distractors/claims), near-miss JSON (trailing
111    /// commas, unquoted keys, single quotes), close distractors.
112    NearMiss,
113    /// Cyclic-permutation stratum (CircularEval precedent, S6b1): the same
114    /// choice item with options rotated by `rotation` positions; truth rotates
115    /// with the options. Applied to **choice** items only — score levels are
116    /// ordered (index = level is the semantics), so permuting them would
117    /// change the question, not the presentation. Noul has no options.
118    Permuted {
119        /// Cyclic rotation applied, in `1..cardinality`.
120        rotation: u32,
121    },
122}
123
124impl Stratum {
125    /// Wire form (`clean`, `nearmiss`, `perm-2`, ...).
126    pub fn as_str(&self) -> String {
127        match self {
128            Self::Clean => "clean".to_owned(),
129            Self::NearMiss => "nearmiss".to_owned(),
130            Self::Permuted { rotation } => format!("perm-{rotation}"),
131        }
132    }
133
134    /// Whether this stratum is a base form (not a permutation expansion).
135    pub fn is_base(&self) -> bool {
136        !matches!(self, Self::Permuted { .. })
137    }
138}
139
140impl fmt::Display for Stratum {
141    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
142        f.write_str(&self.as_str())
143    }
144}
145
146#[cfg(test)]
147mod tests {
148    use super::*;
149
150    #[test]
151    fn designations_roundtrip_and_gate_set_is_frozen() {
152        // The gate set is the P1.4 leg-(e) measurement surface; changing it
153        // after freeze invalidates the firewall. Lock it here.
154        let gates: Vec<Family> = Family::ALL
155            .iter()
156            .copied()
157            .filter(|family| family.designation() == Designation::Gate)
158            .collect();
159        assert_eq!(gates, vec![Family::Temporal, Family::Syllogism]);
160    }
161
162    #[test]
163    fn family_ids_roundtrip() {
164        for family in Family::ALL {
165            assert_eq!(Family::from_id(family.as_str()), Some(family));
166        }
167    }
168}