Skip to main content

hyprstream_inference/runtime/
inference_stage.rs

1//! Thread-local execution boundary for one contiguous decoder stage.
2//!
3//! A stage owns exactly one global layer range. The first stage receives token
4//! IDs and creates embeddings, middle stages receive an activation tensor, and
5//! the last stage turns its activation into logits. The stage object and its
6//! typed output are deliberately !Send; a future transport boundary must encode
7//! activations instead of moving this execution object between threads.
8
9use std::marker::PhantomData;
10use std::ops::Range;
11use std::path::Path;
12use std::rc::Rc;
13use std::sync::atomic::{AtomicU64, Ordering};
14use std::sync::Arc;
15
16use anyhow::{anyhow, bail, Result};
17use tch::{Device, Kind, Tensor};
18
19use crate::runtime::architectures::ModelOperations;
20use crate::runtime::model_factory::{ModelFactory, ModelStageRequest};
21use crate::runtime::KVQuantType;
22
23/// The range and endpoint responsibilities of one loaded decoder stage.
24#[derive(Clone, Debug, Eq, PartialEq)]
25pub struct StageContract {
26    layer_range: Range<usize>,
27    total_layers: usize,
28}
29
30impl StageContract {
31    /// Validate a contiguous global layer range against the model layer count.
32    pub fn new(layer_range: Range<usize>, total_layers: usize) -> Result<Self> {
33        if total_layers == 0 {
34            bail!("stage contract requires a model with at least one decoder layer");
35        }
36        if layer_range.start >= layer_range.end {
37            bail!(
38                "stage contract requires a non-empty layer range, got {:?}",
39                layer_range
40            );
41        }
42        if layer_range.end > total_layers {
43            bail!(
44                "stage contract range {:?} exceeds model layer count {}",
45                layer_range,
46                total_layers
47            );
48        }
49        Ok(Self {
50            layer_range,
51            total_layers,
52        })
53    }
54
55    /// Global decoder layers owned by this stage.
56    #[must_use]
57    pub fn layer_range(&self) -> Range<usize> {
58        self.layer_range.clone()
59    }
60
61    /// Whether this stage owns the embedding table and accepts token IDs.
62    #[must_use]
63    pub fn is_first(&self) -> bool {
64        self.layer_range.start == 0
65    }
66
67    /// Whether this stage owns final normalization and the language-model head.
68    #[must_use]
69    pub fn is_last(&self) -> bool {
70        self.layer_range.end == self.total_layers
71    }
72}
73
74/// Input admitted at a stage boundary.
75pub enum StageInput<'a> {
76    /// Token IDs, accepted only by the first stage.
77    TokenIds(&'a Tensor),
78    /// Hidden activation from the preceding stage, accepted only by a non-first stage.
79    Hidden(&'a Tensor),
80}
81
82/// Result emitted at a stage boundary.
83pub enum StageOutput {
84    /// Activation for the next stage.
85    Hidden(StageTensor),
86    /// Final vocabulary logits emitted by the last stage.
87    Logits(StageTensor),
88}
89
90/// A stage result that remains attached to the engine-owning thread.
91pub struct StageTensor {
92    tensor: Tensor,
93    _not_send: PhantomData<Rc<()>>,
94}
95
96impl StageTensor {
97    fn new(tensor: Tensor) -> Self {
98        Self {
99            tensor,
100            _not_send: PhantomData,
101        }
102    }
103
104    /// Borrow the tensor on the same engine-owning thread for encoding or the
105    /// next local stage call.
106    pub fn as_tensor(&self) -> &Tensor {
107        &self.tensor
108    }
109}
110
111/// The single cache timeline owned by one loaded stage.
112///
113/// A stage instance admits one sequence only. This prevents two request streams
114/// from accidentally sharing the model-local KV/SSM state while the lifecycle
115/// service has not yet installed per-request cache ownership.
116#[derive(Debug)]
117pub struct StageSequence {
118    id: u64,
119    next_start_pos: usize,
120    stage_identity: Option<Arc<()>>,
121}
122
123impl StageSequence {
124    /// Start a fresh stage-local cache timeline at position zero.
125    #[must_use]
126    pub fn new() -> Self {
127        static NEXT_STAGE_SEQUENCE_ID: AtomicU64 = AtomicU64::new(1);
128        Self {
129            id: NEXT_STAGE_SEQUENCE_ID.fetch_add(1, Ordering::Relaxed),
130            next_start_pos: 0,
131            stage_identity: None,
132        }
133    }
134
135    fn require_start_pos(&self, start_pos: usize) -> Result<()> {
136        if start_pos != self.next_start_pos {
137            bail!(
138                "stage sequence expected start_pos {}, got {}",
139                self.next_start_pos,
140                start_pos
141            );
142        }
143        Ok(())
144    }
145
146    fn advance(&mut self, tokens: usize) -> Result<()> {
147        self.next_start_pos = self
148            .next_start_pos
149            .checked_add(tokens)
150            .ok_or_else(|| anyhow!("stage sequence position overflow"))?;
151        Ok(())
152    }
153}
154
155impl Default for StageSequence {
156    fn default() -> Self {
157        Self::new()
158    }
159}
160
161/// A loaded, same-thread decoder stage.
162///
163/// Construction is the production consumer of [`ModelFactory::create_stage`].
164/// One instance is deliberately bound to one [`StageSequence`] and therefore
165/// one model-local cache timeline. The object intentionally has no transport
166/// implementation: callers encode activations at a process or host boundary.
167pub struct InferenceStage {
168    model: Box<dyn ModelOperations>,
169    contract: StageContract,
170    active_sequence: Option<u64>,
171    poisoned: bool,
172    identity: Arc<()>,
173    _not_send: PhantomData<Rc<()>>,
174}
175
176impl InferenceStage {
177    /// Load exactly the weights required for one contiguous stage.
178    /// `fp8_dequant_load` forwards the caller's explicit FP8 loading policy to
179    /// the model factory; stage loading does not change that policy.
180    pub async fn load(
181        model_path: &Path,
182        device: &Device,
183        dtype: Kind,
184        max_context: Option<usize>,
185        kv_quant_type: KVQuantType,
186        request: ModelStageRequest,
187        fp8_dequant_load: bool,
188    ) -> Result<Self> {
189        let layer_range = request.layer_range.clone();
190        let model = ModelFactory::create_stage(
191            model_path,
192            device,
193            dtype,
194            max_context,
195            kv_quant_type,
196            None,
197            request,
198            fp8_dequant_load,
199        )
200        .await?;
201        let contract = StageContract::new(layer_range, model.num_layers())?;
202        Ok(Self::from_loaded_model(model, contract))
203    }
204
205    fn from_loaded_model(model: Box<dyn ModelOperations>, contract: StageContract) -> Self {
206        Self {
207            model,
208            contract,
209            active_sequence: None,
210            poisoned: false,
211            identity: Arc::new(()),
212            _not_send: PhantomData,
213        }
214    }
215
216    /// Immutable stage role and range contract.
217    #[must_use]
218    pub fn contract(&self) -> &StageContract {
219        &self.contract
220    }
221
222    /// Execute one prefill or decode step for this stage.
223    ///
224    /// `sequence` is the sole cache timeline admitted by this stage instance.
225    /// `start_pos` must advance monotonically from zero for that sequence. The
226    /// loaded model keeps its per-layer KV (and architecture-specific recurrent)
227    /// state locally; the typed result stays on the engine-owning thread.
228    /// A failure after decoder execution starts can leave that cache partially
229    /// updated. Such a stage rejects further execution and must be reloaded
230    /// with a fresh sequence. A sequence cannot migrate to a different stage,
231    /// whose model-local cache would not contain that sequence's history.
232    pub fn execute(
233        &mut self,
234        sequence: &mut StageSequence,
235        input: StageInput<'_>,
236        start_pos: usize,
237    ) -> Result<StageOutput> {
238        if self.poisoned {
239            bail!("stage execution previously failed; the stage must be reloaded");
240        }
241        sequence.require_start_pos(start_pos)?;
242        if sequence
243            .stage_identity
244            .as_ref()
245            .is_some_and(|identity| !Arc::ptr_eq(identity, &self.identity))
246        {
247            bail!("a stage sequence is bound to another loaded stage");
248        }
249        if self
250            .active_sequence
251            .is_some_and(|active| active != sequence.id)
252        {
253            bail!("a loaded stage instance admits exactly one sequence");
254        }
255
256        let hidden = match input {
257            StageInput::TokenIds(token_ids) if self.contract.is_first() => {
258                self.model.embed_tokens(token_ids)?
259            }
260            StageInput::TokenIds(_) => {
261                bail!("token IDs are accepted only by the first decoder stage");
262            }
263            StageInput::Hidden(hidden) if !self.contract.is_first() => hidden.shallow_clone(),
264            StageInput::Hidden(_) => {
265                bail!("the first decoder stage requires token IDs, not a hidden activation");
266            }
267        };
268        self.active_sequence = Some(sequence.id);
269        sequence
270            .stage_identity
271            .get_or_insert_with(|| Arc::clone(&self.identity));
272        // ModelOperations has no rollback contract. Leave this set on every
273        // error after entering the decoder, including final projection errors.
274        self.poisoned = true;
275
276        let hidden =
277            self.model
278                .forward_layers(&hidden, self.contract.layer_range(), start_pos, None)?;
279        let token_count = hidden
280            .size()
281            .get(1)
282            .copied()
283            .ok_or_else(|| anyhow!("stage output must have a sequence dimension"))?;
284        let token_count = usize::try_from(token_count)
285            .map_err(|_| anyhow!("stage output sequence dimension must be non-negative"))?;
286        let output = if self.contract.is_last() {
287            let normalized = self.model.apply_final_norm(&hidden)?;
288            StageOutput::Logits(StageTensor::new(self.model.lm_head(&normalized)?))
289        } else {
290            StageOutput::Hidden(StageTensor::new(hidden))
291        };
292        sequence.advance(token_count)?;
293        self.poisoned = false;
294        Ok(output)
295    }
296}
297
298#[cfg(test)]
299mod tests {
300    use std::ops::Range;
301    use std::sync::Arc;
302
303    use anyhow::{anyhow, bail, Result};
304    use parking_lot::Mutex;
305    use tch::{Device, Kind, Tensor};
306
307    use super::{InferenceStage, StageContract, StageInput, StageOutput, StageSequence};
308    use crate::runtime::architectures::config::ArchitectureConfig;
309    use crate::runtime::architectures::{ModelArchitecture, ModelOperations};
310
311    fn valid_contract(range: std::ops::Range<usize>, total_layers: usize) -> StageContract {
312        match StageContract::new(range, total_layers) {
313            Ok(contract) => contract,
314            Err(error) => panic!("test contract must be valid: {error}"),
315        }
316    }
317
318    #[derive(Clone)]
319    struct TestConfig;
320
321    impl ArchitectureConfig for TestConfig {
322        fn num_attention_heads(&self) -> usize {
323            1
324        }
325
326        fn num_key_value_heads(&self) -> usize {
327            1
328        }
329
330        fn hidden_size(&self) -> usize {
331            4
332        }
333
334        fn intermediate_size(&self) -> usize {
335            8
336        }
337
338        fn vocab_size(&self) -> usize {
339            16
340        }
341
342        fn max_position_embeddings(&self) -> usize {
343            16
344        }
345
346        fn rope_theta(&self) -> Option<f32> {
347            None
348        }
349
350        fn rope_dim(&self) -> Option<usize> {
351            None
352        }
353
354        fn layer_norm_eps(&self) -> f32 {
355            1e-5
356        }
357
358        fn use_rms_norm(&self) -> bool {
359            true
360        }
361    }
362
363    struct TestModel {
364        config: TestConfig,
365        total_layers: usize,
366        observed_ranges: Arc<Mutex<Vec<Range<usize>>>>,
367        fail_at: Option<&'static str>,
368    }
369
370    impl ModelOperations for TestModel {
371        fn architecture(&self) -> ModelArchitecture {
372            ModelArchitecture::Llama { version: 3 }
373        }
374
375        fn config(&self) -> &dyn ArchitectureConfig {
376            &self.config
377        }
378
379        fn forward(&self, _input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
380            Err(anyhow!("whole-model forward is not used by a stage"))
381        }
382
383        fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor> {
384            Ok(Tensor::zeros(
385                [input_ids.size()[0], input_ids.size()[1], 4],
386                (Kind::Float, input_ids.device()),
387            ))
388        }
389
390        fn forward_layers(
391            &self,
392            hidden: &Tensor,
393            range: Range<usize>,
394            _start_pos: usize,
395            _delta: Option<&crate::training::TenantDelta>,
396        ) -> Result<Tensor> {
397            self.observed_ranges.lock().push(range);
398            if self.fail_at == Some("decoder") {
399                bail!("decoder failed after updating cache");
400            }
401            Ok(hidden.shallow_clone())
402        }
403
404        fn apply_final_norm(&self, hidden: &Tensor) -> Result<Tensor> {
405            if self.fail_at == Some("norm") {
406                bail!("final normalization failed");
407            }
408            Ok(hidden.shallow_clone())
409        }
410
411        fn lm_head(&self, hidden: &Tensor) -> Result<Tensor> {
412            if self.fail_at == Some("head") {
413                bail!("final projection failed");
414            }
415            Ok(hidden.shallow_clone())
416        }
417
418        fn num_layers(&self) -> usize {
419            self.total_layers
420        }
421
422        fn reshape_for_attention(&self, tensor: &Tensor, _is_key_value: bool) -> Result<Tensor> {
423            Ok(tensor.shallow_clone())
424        }
425
426        fn apply_rope(&self, tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor> {
427            Ok(tensor.shallow_clone())
428        }
429
430        fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
431            Ok(tensor.shallow_clone())
432        }
433
434        fn get_attention_mask(&self, _seq_len: usize, _past_kv_len: usize) -> Result<Tensor> {
435            Ok(Tensor::zeros([1], (Kind::Float, Device::Cpu)))
436        }
437
438        fn as_any(&self) -> &dyn std::any::Any {
439            self
440        }
441    }
442
443    fn test_stage(
444        range: Range<usize>,
445        total_layers: usize,
446    ) -> (InferenceStage, Arc<Mutex<Vec<Range<usize>>>>) {
447        let observed_ranges = Arc::new(Mutex::new(Vec::new()));
448        let model = TestModel {
449            config: TestConfig,
450            total_layers,
451            observed_ranges: Arc::clone(&observed_ranges),
452            fail_at: None,
453        };
454        let stage =
455            InferenceStage::from_loaded_model(Box::new(model), valid_contract(range, total_layers));
456        (stage, observed_ranges)
457    }
458
459    fn token_ids() -> Tensor {
460        Tensor::zeros([1, 2], (Kind::Int64, Device::Cpu))
461    }
462
463    fn hidden() -> Tensor {
464        Tensor::zeros([1, 2, 4], (Kind::Float, Device::Cpu))
465    }
466
467    #[test]
468    fn stage_contract_assigns_endpoint_roles_from_global_range() {
469        let first = valid_contract(0..2, 6);
470        assert!(first.is_first());
471        assert!(!first.is_last());
472
473        let middle = valid_contract(2..4, 6);
474        assert!(!middle.is_first());
475        assert!(!middle.is_last());
476
477        let last = valid_contract(4..6, 6);
478        assert!(!last.is_first());
479        assert!(last.is_last());
480    }
481
482    #[test]
483    fn stage_contract_rejects_empty_and_out_of_bounds_ranges() {
484        assert!(StageContract::new(2..2, 6).is_err());
485        assert!(StageContract::new(4..7, 6).is_err());
486        assert!(StageContract::new(0..1, 0).is_err());
487    }
488
489    #[test]
490    fn stage_execute_enforces_endpoint_inputs_and_forwards_its_global_range() {
491        let (mut first, first_ranges) = test_stage(0..2, 6);
492        let mut first_sequence = StageSequence::new();
493        let first_output =
494            match first.execute(&mut first_sequence, StageInput::TokenIds(&token_ids()), 0) {
495                Ok(output) => output,
496                Err(error) => panic!("first stage must execute: {error}"),
497            };
498        match first_output {
499            StageOutput::Hidden(hidden) => assert_eq!(hidden.as_tensor().size(), [1, 2, 4]),
500            StageOutput::Logits(_) => panic!("a non-final stage must emit hidden activation"),
501        }
502        assert_eq!(*first_ranges.lock(), vec![0..2]);
503
504        let (mut middle, _) = test_stage(2..4, 6);
505        let mut middle_sequence = StageSequence::new();
506        assert!(middle
507            .execute(&mut middle_sequence, StageInput::TokenIds(&token_ids()), 0)
508            .is_err());
509
510        let (mut final_stage, final_ranges) = test_stage(4..6, 6);
511        let mut final_sequence = StageSequence::new();
512        let final_output =
513            match final_stage.execute(&mut final_sequence, StageInput::Hidden(&hidden()), 0) {
514                Ok(output) => output,
515                Err(error) => panic!("final stage must execute: {error}"),
516            };
517        match final_output {
518            StageOutput::Logits(logits) => assert_eq!(logits.as_tensor().size(), [1, 2, 4]),
519            StageOutput::Hidden(_) => panic!("the final stage must emit logits"),
520        }
521        assert_eq!(*final_ranges.lock(), vec![4..6]);
522    }
523
524    #[test]
525    fn sequence_cannot_migrate_to_an_empty_stage_cache() {
526        let (mut first, _) = test_stage(0..2, 6);
527        let (mut fresh, fresh_ranges) = test_stage(0..2, 6);
528        let mut sequence = StageSequence::new();
529        assert!(first
530            .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
531            .is_ok());
532        assert!(fresh
533            .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 2)
534            .is_err());
535        assert!(
536            fresh_ranges.lock().is_empty(),
537            "foreign sequence must not reach decoder"
538        );
539        assert!(first
540            .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 2)
541            .is_ok());
542    }
543
544    #[test]
545    fn zero_position_sequence_stays_bound_after_stage_is_dropped() {
546        let (mut original, _) = test_stage(0..2, 6);
547        let mut sequence = StageSequence::new();
548        let empty_ids = Tensor::zeros([1, 0], (Kind::Int64, Device::Cpu));
549        assert!(original
550            .execute(&mut sequence, StageInput::TokenIds(&empty_ids), 0)
551            .is_ok());
552        assert_eq!(sequence.next_start_pos, 0);
553        drop(original);
554        let (mut fresh, fresh_ranges) = test_stage(0..2, 6);
555        assert!(fresh
556            .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
557            .is_err());
558        assert!(fresh_ranges.lock().is_empty());
559    }
560
561    #[test]
562    fn failed_stage_cannot_reuse_partially_updated_cache() {
563        for fail_at in ["decoder", "norm", "head"] {
564            let observed_ranges = Arc::new(Mutex::new(Vec::new()));
565            let model = TestModel {
566                config: TestConfig,
567                total_layers: 2,
568                observed_ranges: Arc::clone(&observed_ranges),
569                fail_at: Some(fail_at),
570            };
571            let mut stage =
572                InferenceStage::from_loaded_model(Box::new(model), valid_contract(0..2, 2));
573            let mut sequence = StageSequence::new();
574            assert!(stage
575                .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
576                .is_err());
577            assert_eq!(
578                sequence.next_start_pos, 0,
579                "failed output must not advance sequence"
580            );
581            assert!(stage
582                .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
583                .is_err());
584            let (mut fresh, fresh_ranges) = test_stage(0..2, 2);
585            assert!(fresh
586                .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
587                .is_err());
588            assert!(fresh_ranges.lock().is_empty());
589            assert_eq!(
590                observed_ranges.lock().len(),
591                1,
592                "poisoned cache must not be reused"
593            );
594        }
595    }
596
597    #[test]
598    fn stage_execute_rejects_wrong_input_and_a_second_cache_timeline() {
599        let (mut first, _) = test_stage(0..2, 6);
600        let mut sequence = StageSequence::new();
601        assert!(first
602            .execute(&mut sequence, StageInput::Hidden(&hidden()), 0)
603            .is_err());
604
605        let mut first_sequence = StageSequence::new();
606        if let Err(error) =
607            first.execute(&mut first_sequence, StageInput::TokenIds(&token_ids()), 0)
608        {
609            panic!("valid first-stage request must execute: {error}");
610        }
611        let mut second_sequence = StageSequence::new();
612        assert!(first
613            .execute(&mut second_sequence, StageInput::TokenIds(&token_ids()), 0)
614            .is_err());
615        assert!(first
616            .execute(&mut first_sequence, StageInput::TokenIds(&token_ids()), 0)
617            .is_err());
618    }
619}