hyprstream_decision/serialize.rs
1//! The ONE canonical serialization of a decision request (pinned per S6b1).
2//!
3//! This module is a **frozen contract**: meaning-preserving format changes swing model
4//! accuracy by up to 76 pp (Sclar et al., ICLR 2024), so the byte-level layout below does
5//! not change without a new schema version. The question encoder (P1.1) trains on exactly
6//! this form and the serving path emits exactly this form.
7//!
8//! ## Layout
9//!
10//! ```text
11//! <|state|>\n{state}\n
12//! <|question|>\n{instructions}\n (instructions line omitted when absent)
13//! noul: <|rubric|>\ntrue\n{true criteria}\n (each rubric block only if present)
14//! <|rubric|>\nfalse\n{false criteria}\n
15//! <|anchor|>\n
16//! choice: <|option|>\n{name}\n{rubric}\n<|anchor|>\n (rubric line omitted when null)
17//! ... one block per option, in canonical (authoring) order
18//! score: <|level|>\n{rubric}\n<|anchor|>\n (one per level; the level's
19//! ... NUMBER is never emitted)
20//! ```
21//!
22//! - The **anchor placeholder comes immediately AFTER each option's rubric text**
23//! (`<|anchor|>` stands in for the learned special token whose id the tokenizer assigns
24//! in P1.1/P1.2). Never a trailing anchor section: the shared scorer reads the hidden
25//! state at each anchor position, and option-adjacent placement keeps every option's
26//! local readout structurally identical through the backbone's causal linear-attention
27//! layers (S6b1).
28//! - **Noul has exactly ONE anchor**, not one per outcome: the noul statement has no
29//! per-option rubric blocks, so its single anchor follows the (optional) true/false
30//! criteria blocks and is the position where the sigmoid readout (P1.1) is taken —
31//! the same adjacent-after principle, applied to the statement's rubric blocks.
32//! - Score levels carry no index and no neighbour context — each level is judged
33//! independently against the state (upstream behavioral contract, S6a §2.2).
34//! - Question ids are caller-facing only and never appear in the serialization.
35//! - Non-string entries (objects/arrays) serialize as compact canonical JSON
36//! ([`crate::entry::Entry::canonical_text`]).
37//!
38//! ## Token budget (acceptance criterion)
39//!
40//! A serialized request must fit [`MAX_SERIALIZED_SPEC_TOKENS`] at the 2k-token serving
41//! context. When a choice question cannot fit its options within the budget — the
42//! canonical case being cardinality above the 255-option profile limit — the two-stage
43//! recipe (score candidates in parallel, then choose among the top scorers) is
44//! **mandatory**, not optional (S6a §2.4: both stages are client-side patterns, not wire
45//! features). Token counting is tokenizer-bound and lands with the P1.1 encoder; this
46//! module exposes [`serialized_bytes`] as the pre-tokenizer proxy.
47
48use crate::spec::{QuestionBody, QuestionSet, QuestionSpec};
49
50/// Placeholder for the learned special anchor token. The concrete token (and its id) is
51/// assigned when the tokenizer is extended in P1.1/P1.2; the serialization contract fixes
52/// only the *position*: immediately after each option's rubric text.
53pub const ANCHOR_PLACEHOLDER: &str = "<|anchor|>";
54
55/// The serialized-spec token budget at the 2k-token serving context. Exceeding it makes
56/// the two-stage >255 recipe mandatory. Enforced with the real tokenizer in P1.1; this
57/// constant is the contract.
58pub const MAX_SERIALIZED_SPEC_TOKENS: usize = 2048;
59
60/// Serialize one question to its canonical block (without the state header).
61pub fn serialize_question(question: &QuestionSpec) -> String {
62 let mut out = String::new();
63 out.push_str("<|question|>\n");
64 if let Some(instructions) = &question.instructions {
65 out.push_str(&instructions.canonical_text());
66 out.push('\n');
67 }
68 match &question.body {
69 QuestionBody::Noul { criteria } => {
70 if let Some(criteria) = criteria {
71 if let Some(on_true) = &criteria.on_true {
72 out.push_str("<|rubric|>\ntrue\n");
73 out.push_str(&on_true.canonical_text());
74 out.push('\n');
75 }
76 if let Some(on_false) = &criteria.on_false {
77 out.push_str("<|rubric|>\nfalse\n");
78 out.push_str(&on_false.canonical_text());
79 out.push('\n');
80 }
81 }
82 out.push_str(ANCHOR_PLACEHOLDER);
83 out.push('\n');
84 }
85 QuestionBody::Choice { options } => {
86 for option in options {
87 out.push_str("<|option|>\n");
88 out.push_str(&option.name);
89 out.push('\n');
90 if let Some(rubric) = &option.rubric {
91 out.push_str(&rubric.canonical_text());
92 out.push('\n');
93 }
94 out.push_str(ANCHOR_PLACEHOLDER);
95 out.push('\n');
96 }
97 }
98 QuestionBody::Score { levels } => {
99 for level in levels {
100 out.push_str("<|level|>\n");
101 if let Some(rubric) = level {
102 out.push_str(&rubric.canonical_text());
103 out.push('\n');
104 }
105 out.push_str(ANCHOR_PLACEHOLDER);
106 out.push('\n');
107 }
108 }
109 }
110 out
111}
112
113/// Serialize a full request: the state header followed by every question block in
114/// canonical (authoring) order.
115pub fn serialize_request(set: &QuestionSet) -> String {
116 let mut out = String::new();
117 out.push_str("<|state|>\n");
118 if let Some(state) = &set.state {
119 out.push_str(&state.canonical_text());
120 out.push('\n');
121 }
122 for question in &set.questions {
123 out.push_str(&serialize_question(question));
124 }
125 out
126}
127
128/// Byte length of the canonical serialization — the pre-tokenizer proxy for the
129/// [`MAX_SERIALIZED_SPEC_TOKENS`] budget (token counting is tokenizer-bound, P1.1).
130pub fn serialized_bytes(set: &QuestionSet) -> usize {
131 serialize_request(set).len()
132}
133
134#[cfg(test)]
135mod tests {
136 use super::*;
137 use crate::author::parse_yaml;
138
139 #[test]
140 fn anchor_follows_each_option_rubric_and_ids_never_appear() {
141 let set = parse_yaml(
142 r#"
143state: "The box was crushed."
144questions:
145 tone:
146 type: choice
147 instructions: "Classify the review."
148 criteria:
149 positive: "The review expresses approval."
150 negative: ~
151"#,
152 )
153 .unwrap_or_else(|error| panic!("fixture parses: {error}"));
154 let text = serialize_request(&set);
155 assert_eq!(
156 text,
157 "<|state|>\nThe box was crushed.\n\
158 <|question|>\nClassify the review.\n\
159 <|option|>\npositive\nThe review expresses approval.\n<|anchor|>\n\
160 <|option|>\nnegative\n<|anchor|>\n"
161 );
162 assert!(
163 !text.contains("tone"),
164 "question ids never reach the model: {text}"
165 );
166 }
167
168 #[test]
169 fn score_levels_carry_no_index() {
170 let set = parse_yaml(
171 r#"
172questions:
173 severity:
174 type: score
175 criteria: ["trivial", "critical"]
176"#,
177 )
178 .unwrap_or_else(|error| panic!("fixture parses: {error}"));
179 let text = serialize_request(&set);
180 assert_eq!(
181 text,
182 "<|state|>\n<|question|>\n\
183 <|level|>\ntrivial\n<|anchor|>\n\
184 <|level|>\ncritical\n<|anchor|>\n"
185 );
186 }
187
188 #[test]
189 fn noul_emits_one_anchor_after_criteria() {
190 let set = parse_yaml(
191 r#"
192questions:
193 is_refund:
194 type: noul
195 instructions: "The customer wants money back."
196 criteria:
197 "true": "Explicit refund request"
198"#,
199 )
200 .unwrap_or_else(|error| panic!("fixture parses: {error}"));
201 let text = serialize_request(&set);
202 assert_eq!(
203 text,
204 "<|state|>\n<|question|>\nThe customer wants money back.\n\
205 <|rubric|>\ntrue\nExplicit refund request\n<|anchor|>\n"
206 );
207 }
208}