hyprstream_decision_head/
encoder.rs1use hyprstream_decision::serialize;
6use hyprstream_decision::{QuestionBody, QuestionKind, QuestionSet};
7use tokenizers::{AddedToken, Tokenizer};
8
9use crate::Error;
10
11pub const ANCHOR_TOKEN: &str = serialize::ANCHOR_PLACEHOLDER;
16
17pub fn assign_anchor_token(tokenizer: &mut Tokenizer) -> Result<u32, Error> {
25 if let Some(id) = tokenizer.token_to_id(ANCHOR_TOKEN) {
26 return Ok(id);
27 }
28 tokenizer.add_special_tokens(&[AddedToken::from(ANCHOR_TOKEN, true)]);
29 tokenizer
30 .token_to_id(ANCHOR_TOKEN)
31 .ok_or(Error::AnchorTokenMissing {
32 token: ANCHOR_TOKEN,
33 })
34}
35
36#[derive(Debug, Clone, PartialEq, Eq)]
38pub struct QuestionAnchors {
39 pub question_id: String,
42 pub kind: QuestionKind,
44 pub anchor_positions: Vec<u32>,
47}
48
49#[derive(Debug, Clone, PartialEq, Eq)]
52pub struct EncodedRequest {
53 pub input_ids: Vec<u32>,
55 pub questions: Vec<QuestionAnchors>,
57}
58
59impl EncodedRequest {
60 pub fn token_count(&self) -> usize {
62 self.input_ids.len()
63 }
64}
65
66pub struct DecisionEncoder {
69 tokenizer: Tokenizer,
70 anchor_token_id: u32,
71 budget: usize,
72}
73
74impl DecisionEncoder {
75 pub fn new(mut tokenizer: Tokenizer) -> Result<Self, Error> {
80 let anchor_token_id = assign_anchor_token(&mut tokenizer)?;
81 Ok(Self {
82 tokenizer,
83 anchor_token_id,
84 budget: serialize::MAX_SERIALIZED_SPEC_TOKENS,
85 })
86 }
87
88 pub fn with_budget(mut self, budget: usize) -> Self {
91 self.budget = budget;
92 self
93 }
94
95 pub fn anchor_token_id(&self) -> u32 {
97 self.anchor_token_id
98 }
99
100 pub fn tokenizer(&self) -> &Tokenizer {
102 &self.tokenizer
103 }
104
105 pub fn encode(&self, set: &QuestionSet) -> Result<EncodedRequest, Error> {
112 let text = serialize::serialize_request(set);
113 let encoding = self
114 .tokenizer
115 .encode(text, false)
116 .map_err(|error| Error::Tokenizer(error.to_string()))?;
117 let ids = encoding.get_ids();
118 if ids.len() > self.budget {
119 return Err(Error::TokenBudgetExceeded {
120 tokens: ids.len(),
121 budget: self.budget,
122 });
123 }
124 let anchor_positions: Vec<u32> = ids
125 .iter()
126 .enumerate()
127 .filter(|(_, id)| **id == self.anchor_token_id)
128 .map(|(position, _)| position as u32)
129 .collect();
130 let expected: usize = set.questions.iter().map(anchor_arity).sum();
131 if anchor_positions.len() != expected {
132 return Err(Error::AnchorCountMismatch {
133 expected,
134 found: anchor_positions.len(),
135 });
136 }
137 let mut cursor = 0;
138 let questions = set
139 .questions
140 .iter()
141 .map(|question| {
142 let arity = anchor_arity(question);
143 let positions = anchor_positions[cursor..cursor + arity].to_vec();
144 cursor += arity;
145 QuestionAnchors {
146 question_id: question.id.clone(),
147 kind: question.kind,
148 anchor_positions: positions,
149 }
150 })
151 .collect();
152 Ok(EncodedRequest {
153 input_ids: ids.to_vec(),
154 questions,
155 })
156 }
157}
158
159fn anchor_arity(question: &hyprstream_decision::QuestionSpec) -> usize {
162 match &question.body {
163 QuestionBody::Noul { .. } => 1,
164 QuestionBody::Choice { options } => options.len(),
165 QuestionBody::Score { levels } => levels.len(),
166 }
167}
168
169#[cfg(test)]
170#[allow(clippy::unwrap_used, clippy::expect_used)]
171mod tests {
172 use super::*;
173 use hyprstream_decision::parse_yaml;
174
175 fn fixture_tokenizer() -> Tokenizer {
178 const TOK_JSON: &str = r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0},"unk_token":"[UNK]"}}"#;
179 Tokenizer::from_bytes(TOK_JSON.as_bytes()).expect("fixture tokenizer parses")
180 }
181
182 fn fixture_set() -> QuestionSet {
183 parse_yaml(
184 r#"
185state: "The box was crushed."
186questions:
187 tone:
188 type: choice
189 criteria:
190 positive: "The review expresses approval."
191 negative: "The review expresses disapproval."
192 severe:
193 type: score
194 criteria: ["trivial", "moderate", "critical"]
195 is_refund:
196 type: noul
197"#,
198 )
199 .expect("fixture spec parses")
200 }
201
202 #[test]
203 fn anchor_id_assignment_is_idempotent() {
204 let mut tokenizer = fixture_tokenizer();
205 let first = assign_anchor_token(&mut tokenizer).unwrap();
206 let second = assign_anchor_token(&mut tokenizer).unwrap();
207 assert_eq!(first, second, "re-assigning reuses the id");
208 assert_eq!(tokenizer.token_to_id(ANCHOR_TOKEN), Some(first));
209 }
210
211 #[test]
212 fn encode_maps_anchors_per_question_in_arity_order() {
213 let encoder = DecisionEncoder::new(fixture_tokenizer()).unwrap();
214 let encoded = encoder.encode(&fixture_set()).unwrap();
215 assert_eq!(encoded.questions.len(), 3);
216
217 let tone = &encoded.questions[0];
218 assert_eq!(tone.question_id, "tone");
219 assert_eq!(tone.kind, QuestionKind::Choice);
220 assert_eq!(tone.anchor_positions.len(), 2);
221
222 let severe = &encoded.questions[1];
223 assert_eq!(severe.kind, QuestionKind::Score);
224 assert_eq!(severe.anchor_positions.len(), 3);
225
226 let is_refund = &encoded.questions[2];
227 assert_eq!(is_refund.kind, QuestionKind::Noul);
228 assert_eq!(
229 is_refund.anchor_positions.len(),
230 1,
231 "noul has exactly one anchor"
232 );
233
234 let mut all: Vec<u32> = encoded
235 .questions
236 .iter()
237 .flat_map(|question| question.anchor_positions.iter().copied())
238 .collect();
239 let total = all.len();
240 all.dedup();
241 assert_eq!(all.len(), total, "anchor positions are distinct");
242 for window in all.windows(2) {
243 assert!(
244 window[0] < window[1],
245 "anchors appear in serialization order"
246 );
247 }
248 for position in &all {
249 assert_eq!(
250 encoded.input_ids[*position as usize],
251 encoder.anchor_token_id(),
252 "every mapped position holds the anchor token"
253 );
254 }
255 }
256
257 #[test]
258 fn budget_is_enforced_against_token_count() {
259 let encoder = DecisionEncoder::new(fixture_tokenizer())
260 .unwrap()
261 .with_budget(4);
262 let error = encoder.encode(&fixture_set()).unwrap_err();
263 match error {
264 Error::TokenBudgetExceeded { tokens, budget } => {
265 assert_eq!(budget, 4);
266 assert!(tokens > 4);
267 }
268 other => panic!("expected TokenBudgetExceeded, got {other}"),
269 }
270 }
271
272 #[test]
273 fn anchor_mismatch_fails_loudly() {
274 let encoder = DecisionEncoder::new(fixture_tokenizer()).unwrap();
278 let skewed = DecisionEncoder {
279 tokenizer: fixture_tokenizer(),
280 anchor_token_id: encoder.anchor_token_id(),
281 budget: serialize::MAX_SERIALIZED_SPEC_TOKENS,
282 };
283 let error = skewed.encode(&fixture_set()).unwrap_err();
284 assert!(
285 matches!(error, Error::AnchorCountMismatch { .. }),
286 "expected AnchorCountMismatch, got {error}"
287 );
288 }
289}