Skip to main content

hyprstream_decision_head/
encoder.rs

1//! Tokenizer-bound linearization: the canonical jev-1 serialization (P0.1a) turned
2//! into token ids plus a per-question anchor map, with the 2048-token spec budget
3//! enforced against the real tokenizer and the learned anchor token id assigned.
4
5use hyprstream_decision::serialize;
6use hyprstream_decision::{QuestionBody, QuestionKind, QuestionSet};
7use tokenizers::{AddedToken, Tokenizer};
8
9use crate::Error;
10
11/// The learned special anchor token string. Byte-identical to
12/// [`serialize::ANCHOR_PLACEHOLDER`]: the placeholder in the frozen serialization *is*
13/// the special token's text, so the tokenizer's added-token matcher maps every
14/// placeholder occurrence to the anchor id.
15pub const ANCHOR_TOKEN: &str = serialize::ANCHOR_PLACEHOLDER;
16
17/// Assign the learned anchor token id in `tokenizer`, adding it as a special token when
18/// absent. Idempotent: an existing entry (e.g. loaded from a converted bundle's
19/// tokenizer) is reused, so the id is stable across encode sites that share the bundle.
20///
21/// The assigned id is part of the model bundle recorded at conversion time (P1.2); the
22/// freshly extended backbone embedding row for this id is what "learned anchor" refers
23/// to — the token starts untrained and is learned during distillation (P1.4).
24pub fn assign_anchor_token(tokenizer: &mut Tokenizer) -> Result<u32, Error> {
25    if let Some(id) = tokenizer.token_to_id(ANCHOR_TOKEN) {
26        return Ok(id);
27    }
28    tokenizer.add_special_tokens(&[AddedToken::from(ANCHOR_TOKEN, true)]);
29    tokenizer
30        .token_to_id(ANCHOR_TOKEN)
31        .ok_or(Error::AnchorTokenMissing {
32            token: ANCHOR_TOKEN,
33        })
34}
35
36/// Per-question anchor readout positions in one encoded request.
37#[derive(Debug, Clone, PartialEq, Eq)]
38pub struct QuestionAnchors {
39    /// Caller-facing question id (never serialized to the model; used to align the
40    /// scorer's outputs with the request).
41    pub question_id: String,
42    /// The primitive type — selects the readout (sigmoid / softmax / binned softmax).
43    pub kind: QuestionKind,
44    /// Token positions of this question's anchor tokens, in canonical option/level
45    /// order. Exactly 1 for noul; `options.len()` for choice; `levels.len()` for score.
46    pub anchor_positions: Vec<u32>,
47}
48
49/// One tokenized decision request: the ids fed to the backbone and the anchor map the
50/// shared scorer reads at.
51#[derive(Debug, Clone, PartialEq, Eq)]
52pub struct EncodedRequest {
53    /// Token ids of the canonical serialization (state header + question blocks).
54    pub input_ids: Vec<u32>,
55    /// Per-question anchor maps, in canonical (authoring) question order.
56    pub questions: Vec<QuestionAnchors>,
57}
58
59impl EncodedRequest {
60    /// The encoded length in tokens — the quantity the spec budget constrains.
61    pub fn token_count(&self) -> usize {
62        self.input_ids.len()
63    }
64}
65
66/// The question/criteria encoder. Wraps the bundle tokenizer with the assigned anchor
67/// id and enforces the tokenizer-bound spec budget.
68pub struct DecisionEncoder {
69    tokenizer: Tokenizer,
70    anchor_token_id: u32,
71    budget: usize,
72}
73
74impl DecisionEncoder {
75    /// Build an encoder over `tokenizer`, assigning the anchor token id if the
76    /// tokenizer does not already carry it. The default budget is the contract
77    /// constant [`serialize::MAX_SERIALIZED_SPEC_TOKENS`] (2048 at the 2k serving
78    /// context).
79    pub fn new(mut tokenizer: Tokenizer) -> Result<Self, Error> {
80        let anchor_token_id = assign_anchor_token(&mut tokenizer)?;
81        Ok(Self {
82            tokenizer,
83            anchor_token_id,
84            budget: serialize::MAX_SERIALIZED_SPEC_TOKENS,
85        })
86    }
87
88    /// Override the token budget (tests and future context sizes). The production
89    /// contract is the default.
90    pub fn with_budget(mut self, budget: usize) -> Self {
91        self.budget = budget;
92        self
93    }
94
95    /// The assigned learned anchor token id.
96    pub fn anchor_token_id(&self) -> u32 {
97        self.anchor_token_id
98    }
99
100    /// The wrapped tokenizer (bundle tokenizer + anchor token).
101    pub fn tokenizer(&self) -> &Tokenizer {
102        &self.tokenizer
103    }
104
105    /// Encode a question set: canonical serialization → token ids → anchor map.
106    ///
107    /// Fails with [`Error::TokenBudgetExceeded`] over budget (the two-stage
108    /// score-then-choice recipe becomes mandatory there), and with
109    /// [`Error::AnchorCountMismatch`] if the encoding's anchor-token count disagrees
110    /// with the question arities — tokenizer/bundle skew must fail loudly.
111    pub fn encode(&self, set: &QuestionSet) -> Result<EncodedRequest, Error> {
112        let text = serialize::serialize_request(set);
113        let encoding = self
114            .tokenizer
115            .encode(text, false)
116            .map_err(|error| Error::Tokenizer(error.to_string()))?;
117        let ids = encoding.get_ids();
118        if ids.len() > self.budget {
119            return Err(Error::TokenBudgetExceeded {
120                tokens: ids.len(),
121                budget: self.budget,
122            });
123        }
124        let anchor_positions: Vec<u32> = ids
125            .iter()
126            .enumerate()
127            .filter(|(_, id)| **id == self.anchor_token_id)
128            .map(|(position, _)| position as u32)
129            .collect();
130        let expected: usize = set.questions.iter().map(anchor_arity).sum();
131        if anchor_positions.len() != expected {
132            return Err(Error::AnchorCountMismatch {
133                expected,
134                found: anchor_positions.len(),
135            });
136        }
137        let mut cursor = 0;
138        let questions = set
139            .questions
140            .iter()
141            .map(|question| {
142                let arity = anchor_arity(question);
143                let positions = anchor_positions[cursor..cursor + arity].to_vec();
144                cursor += arity;
145                QuestionAnchors {
146                    question_id: question.id.clone(),
147                    kind: question.kind,
148                    anchor_positions: positions,
149                }
150            })
151            .collect();
152        Ok(EncodedRequest {
153            input_ids: ids.to_vec(),
154            questions,
155        })
156    }
157}
158
159/// Anchors implied by one question's spec: 1 for noul (the single readout position
160/// after the optional criteria blocks), 1 per option for choice, 1 per level for score.
161fn anchor_arity(question: &hyprstream_decision::QuestionSpec) -> usize {
162    match &question.body {
163        QuestionBody::Noul { .. } => 1,
164        QuestionBody::Choice { options } => options.len(),
165        QuestionBody::Score { levels } => levels.len(),
166    }
167}
168
169#[cfg(test)]
170#[allow(clippy::unwrap_used, clippy::expect_used)]
171mod tests {
172    use super::*;
173    use hyprstream_decision::parse_yaml;
174
175    /// Minimal in-memory WordLevel tokenizer; unknown words map to `[UNK]`. The anchor
176    /// placeholder is not in the vocab — it arrives via `assign_anchor_token`.
177    fn fixture_tokenizer() -> Tokenizer {
178        const TOK_JSON: &str = r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0},"unk_token":"[UNK]"}}"#;
179        Tokenizer::from_bytes(TOK_JSON.as_bytes()).expect("fixture tokenizer parses")
180    }
181
182    fn fixture_set() -> QuestionSet {
183        parse_yaml(
184            r#"
185state: "The box was crushed."
186questions:
187  tone:
188    type: choice
189    criteria:
190      positive: "The review expresses approval."
191      negative: "The review expresses disapproval."
192  severe:
193    type: score
194    criteria: ["trivial", "moderate", "critical"]
195  is_refund:
196    type: noul
197"#,
198        )
199        .expect("fixture spec parses")
200    }
201
202    #[test]
203    fn anchor_id_assignment_is_idempotent() {
204        let mut tokenizer = fixture_tokenizer();
205        let first = assign_anchor_token(&mut tokenizer).unwrap();
206        let second = assign_anchor_token(&mut tokenizer).unwrap();
207        assert_eq!(first, second, "re-assigning reuses the id");
208        assert_eq!(tokenizer.token_to_id(ANCHOR_TOKEN), Some(first));
209    }
210
211    #[test]
212    fn encode_maps_anchors_per_question_in_arity_order() {
213        let encoder = DecisionEncoder::new(fixture_tokenizer()).unwrap();
214        let encoded = encoder.encode(&fixture_set()).unwrap();
215        assert_eq!(encoded.questions.len(), 3);
216
217        let tone = &encoded.questions[0];
218        assert_eq!(tone.question_id, "tone");
219        assert_eq!(tone.kind, QuestionKind::Choice);
220        assert_eq!(tone.anchor_positions.len(), 2);
221
222        let severe = &encoded.questions[1];
223        assert_eq!(severe.kind, QuestionKind::Score);
224        assert_eq!(severe.anchor_positions.len(), 3);
225
226        let is_refund = &encoded.questions[2];
227        assert_eq!(is_refund.kind, QuestionKind::Noul);
228        assert_eq!(
229            is_refund.anchor_positions.len(),
230            1,
231            "noul has exactly one anchor"
232        );
233
234        let mut all: Vec<u32> = encoded
235            .questions
236            .iter()
237            .flat_map(|question| question.anchor_positions.iter().copied())
238            .collect();
239        let total = all.len();
240        all.dedup();
241        assert_eq!(all.len(), total, "anchor positions are distinct");
242        for window in all.windows(2) {
243            assert!(
244                window[0] < window[1],
245                "anchors appear in serialization order"
246            );
247        }
248        for position in &all {
249            assert_eq!(
250                encoded.input_ids[*position as usize],
251                encoder.anchor_token_id(),
252                "every mapped position holds the anchor token"
253            );
254        }
255    }
256
257    #[test]
258    fn budget_is_enforced_against_token_count() {
259        let encoder = DecisionEncoder::new(fixture_tokenizer())
260            .unwrap()
261            .with_budget(4);
262        let error = encoder.encode(&fixture_set()).unwrap_err();
263        match error {
264            Error::TokenBudgetExceeded { tokens, budget } => {
265                assert_eq!(budget, 4);
266                assert!(tokens > 4);
267            }
268            other => panic!("expected TokenBudgetExceeded, got {other}"),
269        }
270    }
271
272    #[test]
273    fn anchor_mismatch_fails_loudly() {
274        // A tokenizer whose anchor id belongs to a *different* token than the
275        // serialization placeholder simulates bundle/tokenizer skew: the placeholders
276        // tokenize as ordinary text and no anchor tokens are found.
277        let encoder = DecisionEncoder::new(fixture_tokenizer()).unwrap();
278        let skewed = DecisionEncoder {
279            tokenizer: fixture_tokenizer(),
280            anchor_token_id: encoder.anchor_token_id(),
281            budget: serialize::MAX_SERIALIZED_SPEC_TOKENS,
282        };
283        let error = skewed.encode(&fixture_set()).unwrap_err();
284        assert!(
285            matches!(error, Error::AnchorCountMismatch { .. }),
286            "expected AnchorCountMismatch, got {error}"
287        );
288    }
289}