1use std::marker::PhantomData;
10use std::ops::Range;
11use std::path::Path;
12use std::rc::Rc;
13use std::sync::atomic::{AtomicU64, Ordering};
14use std::sync::Arc;
15
16use anyhow::{anyhow, bail, Result};
17use tch::{Device, Kind, Tensor};
18
19use crate::runtime::architectures::ModelOperations;
20use crate::runtime::model_factory::{ModelFactory, ModelStageRequest};
21use crate::runtime::KVQuantType;
22
23#[derive(Clone, Debug, Eq, PartialEq)]
25pub struct StageContract {
26 layer_range: Range<usize>,
27 total_layers: usize,
28}
29
30impl StageContract {
31 pub fn new(layer_range: Range<usize>, total_layers: usize) -> Result<Self> {
33 if total_layers == 0 {
34 bail!("stage contract requires a model with at least one decoder layer");
35 }
36 if layer_range.start >= layer_range.end {
37 bail!(
38 "stage contract requires a non-empty layer range, got {:?}",
39 layer_range
40 );
41 }
42 if layer_range.end > total_layers {
43 bail!(
44 "stage contract range {:?} exceeds model layer count {}",
45 layer_range,
46 total_layers
47 );
48 }
49 Ok(Self {
50 layer_range,
51 total_layers,
52 })
53 }
54
55 #[must_use]
57 pub fn layer_range(&self) -> Range<usize> {
58 self.layer_range.clone()
59 }
60
61 #[must_use]
63 pub fn is_first(&self) -> bool {
64 self.layer_range.start == 0
65 }
66
67 #[must_use]
69 pub fn is_last(&self) -> bool {
70 self.layer_range.end == self.total_layers
71 }
72}
73
74pub enum StageInput<'a> {
76 TokenIds(&'a Tensor),
78 Hidden(&'a Tensor),
80}
81
82pub enum StageOutput {
84 Hidden(StageTensor),
86 Logits(StageTensor),
88}
89
90pub struct StageTensor {
92 tensor: Tensor,
93 _not_send: PhantomData<Rc<()>>,
94}
95
96impl StageTensor {
97 fn new(tensor: Tensor) -> Self {
98 Self {
99 tensor,
100 _not_send: PhantomData,
101 }
102 }
103
104 pub fn as_tensor(&self) -> &Tensor {
107 &self.tensor
108 }
109}
110
111#[derive(Debug)]
117pub struct StageSequence {
118 id: u64,
119 next_start_pos: usize,
120 stage_identity: Option<Arc<()>>,
121}
122
123impl StageSequence {
124 #[must_use]
126 pub fn new() -> Self {
127 static NEXT_STAGE_SEQUENCE_ID: AtomicU64 = AtomicU64::new(1);
128 Self {
129 id: NEXT_STAGE_SEQUENCE_ID.fetch_add(1, Ordering::Relaxed),
130 next_start_pos: 0,
131 stage_identity: None,
132 }
133 }
134
135 fn require_start_pos(&self, start_pos: usize) -> Result<()> {
136 if start_pos != self.next_start_pos {
137 bail!(
138 "stage sequence expected start_pos {}, got {}",
139 self.next_start_pos,
140 start_pos
141 );
142 }
143 Ok(())
144 }
145
146 fn advance(&mut self, tokens: usize) -> Result<()> {
147 self.next_start_pos = self
148 .next_start_pos
149 .checked_add(tokens)
150 .ok_or_else(|| anyhow!("stage sequence position overflow"))?;
151 Ok(())
152 }
153}
154
155impl Default for StageSequence {
156 fn default() -> Self {
157 Self::new()
158 }
159}
160
161pub struct InferenceStage {
168 model: Box<dyn ModelOperations>,
169 contract: StageContract,
170 active_sequence: Option<u64>,
171 poisoned: bool,
172 identity: Arc<()>,
173 _not_send: PhantomData<Rc<()>>,
174}
175
176impl InferenceStage {
177 pub async fn load(
181 model_path: &Path,
182 device: &Device,
183 dtype: Kind,
184 max_context: Option<usize>,
185 kv_quant_type: KVQuantType,
186 request: ModelStageRequest,
187 fp8_dequant_load: bool,
188 ) -> Result<Self> {
189 let layer_range = request.layer_range.clone();
190 let model = ModelFactory::create_stage(
191 model_path,
192 device,
193 dtype,
194 max_context,
195 kv_quant_type,
196 None,
197 request,
198 fp8_dequant_load,
199 )
200 .await?;
201 let contract = StageContract::new(layer_range, model.num_layers())?;
202 Ok(Self::from_loaded_model(model, contract))
203 }
204
205 fn from_loaded_model(model: Box<dyn ModelOperations>, contract: StageContract) -> Self {
206 Self {
207 model,
208 contract,
209 active_sequence: None,
210 poisoned: false,
211 identity: Arc::new(()),
212 _not_send: PhantomData,
213 }
214 }
215
216 #[must_use]
218 pub fn contract(&self) -> &StageContract {
219 &self.contract
220 }
221
222 pub fn execute(
233 &mut self,
234 sequence: &mut StageSequence,
235 input: StageInput<'_>,
236 start_pos: usize,
237 ) -> Result<StageOutput> {
238 if self.poisoned {
239 bail!("stage execution previously failed; the stage must be reloaded");
240 }
241 sequence.require_start_pos(start_pos)?;
242 if sequence
243 .stage_identity
244 .as_ref()
245 .is_some_and(|identity| !Arc::ptr_eq(identity, &self.identity))
246 {
247 bail!("a stage sequence is bound to another loaded stage");
248 }
249 if self
250 .active_sequence
251 .is_some_and(|active| active != sequence.id)
252 {
253 bail!("a loaded stage instance admits exactly one sequence");
254 }
255
256 let hidden = match input {
257 StageInput::TokenIds(token_ids) if self.contract.is_first() => {
258 self.model.embed_tokens(token_ids)?
259 }
260 StageInput::TokenIds(_) => {
261 bail!("token IDs are accepted only by the first decoder stage");
262 }
263 StageInput::Hidden(hidden) if !self.contract.is_first() => hidden.shallow_clone(),
264 StageInput::Hidden(_) => {
265 bail!("the first decoder stage requires token IDs, not a hidden activation");
266 }
267 };
268 self.active_sequence = Some(sequence.id);
269 sequence
270 .stage_identity
271 .get_or_insert_with(|| Arc::clone(&self.identity));
272 self.poisoned = true;
275
276 let hidden =
277 self.model
278 .forward_layers(&hidden, self.contract.layer_range(), start_pos, None)?;
279 let token_count = hidden
280 .size()
281 .get(1)
282 .copied()
283 .ok_or_else(|| anyhow!("stage output must have a sequence dimension"))?;
284 let token_count = usize::try_from(token_count)
285 .map_err(|_| anyhow!("stage output sequence dimension must be non-negative"))?;
286 let output = if self.contract.is_last() {
287 let normalized = self.model.apply_final_norm(&hidden)?;
288 StageOutput::Logits(StageTensor::new(self.model.lm_head(&normalized)?))
289 } else {
290 StageOutput::Hidden(StageTensor::new(hidden))
291 };
292 sequence.advance(token_count)?;
293 self.poisoned = false;
294 Ok(output)
295 }
296}
297
298#[cfg(test)]
299mod tests {
300 use std::ops::Range;
301 use std::sync::Arc;
302
303 use anyhow::{anyhow, bail, Result};
304 use parking_lot::Mutex;
305 use tch::{Device, Kind, Tensor};
306
307 use super::{InferenceStage, StageContract, StageInput, StageOutput, StageSequence};
308 use crate::runtime::architectures::config::ArchitectureConfig;
309 use crate::runtime::architectures::{ModelArchitecture, ModelOperations};
310
311 fn valid_contract(range: std::ops::Range<usize>, total_layers: usize) -> StageContract {
312 match StageContract::new(range, total_layers) {
313 Ok(contract) => contract,
314 Err(error) => panic!("test contract must be valid: {error}"),
315 }
316 }
317
318 #[derive(Clone)]
319 struct TestConfig;
320
321 impl ArchitectureConfig for TestConfig {
322 fn num_attention_heads(&self) -> usize {
323 1
324 }
325
326 fn num_key_value_heads(&self) -> usize {
327 1
328 }
329
330 fn hidden_size(&self) -> usize {
331 4
332 }
333
334 fn intermediate_size(&self) -> usize {
335 8
336 }
337
338 fn vocab_size(&self) -> usize {
339 16
340 }
341
342 fn max_position_embeddings(&self) -> usize {
343 16
344 }
345
346 fn rope_theta(&self) -> Option<f32> {
347 None
348 }
349
350 fn rope_dim(&self) -> Option<usize> {
351 None
352 }
353
354 fn layer_norm_eps(&self) -> f32 {
355 1e-5
356 }
357
358 fn use_rms_norm(&self) -> bool {
359 true
360 }
361 }
362
363 struct TestModel {
364 config: TestConfig,
365 total_layers: usize,
366 observed_ranges: Arc<Mutex<Vec<Range<usize>>>>,
367 fail_at: Option<&'static str>,
368 }
369
370 impl ModelOperations for TestModel {
371 fn architecture(&self) -> ModelArchitecture {
372 ModelArchitecture::Llama { version: 3 }
373 }
374
375 fn config(&self) -> &dyn ArchitectureConfig {
376 &self.config
377 }
378
379 fn forward(&self, _input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
380 Err(anyhow!("whole-model forward is not used by a stage"))
381 }
382
383 fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor> {
384 Ok(Tensor::zeros(
385 [input_ids.size()[0], input_ids.size()[1], 4],
386 (Kind::Float, input_ids.device()),
387 ))
388 }
389
390 fn forward_layers(
391 &self,
392 hidden: &Tensor,
393 range: Range<usize>,
394 _start_pos: usize,
395 _delta: Option<&crate::training::TenantDelta>,
396 ) -> Result<Tensor> {
397 self.observed_ranges.lock().push(range);
398 if self.fail_at == Some("decoder") {
399 bail!("decoder failed after updating cache");
400 }
401 Ok(hidden.shallow_clone())
402 }
403
404 fn apply_final_norm(&self, hidden: &Tensor) -> Result<Tensor> {
405 if self.fail_at == Some("norm") {
406 bail!("final normalization failed");
407 }
408 Ok(hidden.shallow_clone())
409 }
410
411 fn lm_head(&self, hidden: &Tensor) -> Result<Tensor> {
412 if self.fail_at == Some("head") {
413 bail!("final projection failed");
414 }
415 Ok(hidden.shallow_clone())
416 }
417
418 fn num_layers(&self) -> usize {
419 self.total_layers
420 }
421
422 fn reshape_for_attention(&self, tensor: &Tensor, _is_key_value: bool) -> Result<Tensor> {
423 Ok(tensor.shallow_clone())
424 }
425
426 fn apply_rope(&self, tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor> {
427 Ok(tensor.shallow_clone())
428 }
429
430 fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
431 Ok(tensor.shallow_clone())
432 }
433
434 fn get_attention_mask(&self, _seq_len: usize, _past_kv_len: usize) -> Result<Tensor> {
435 Ok(Tensor::zeros([1], (Kind::Float, Device::Cpu)))
436 }
437
438 fn as_any(&self) -> &dyn std::any::Any {
439 self
440 }
441 }
442
443 fn test_stage(
444 range: Range<usize>,
445 total_layers: usize,
446 ) -> (InferenceStage, Arc<Mutex<Vec<Range<usize>>>>) {
447 let observed_ranges = Arc::new(Mutex::new(Vec::new()));
448 let model = TestModel {
449 config: TestConfig,
450 total_layers,
451 observed_ranges: Arc::clone(&observed_ranges),
452 fail_at: None,
453 };
454 let stage =
455 InferenceStage::from_loaded_model(Box::new(model), valid_contract(range, total_layers));
456 (stage, observed_ranges)
457 }
458
459 fn token_ids() -> Tensor {
460 Tensor::zeros([1, 2], (Kind::Int64, Device::Cpu))
461 }
462
463 fn hidden() -> Tensor {
464 Tensor::zeros([1, 2, 4], (Kind::Float, Device::Cpu))
465 }
466
467 #[test]
468 fn stage_contract_assigns_endpoint_roles_from_global_range() {
469 let first = valid_contract(0..2, 6);
470 assert!(first.is_first());
471 assert!(!first.is_last());
472
473 let middle = valid_contract(2..4, 6);
474 assert!(!middle.is_first());
475 assert!(!middle.is_last());
476
477 let last = valid_contract(4..6, 6);
478 assert!(!last.is_first());
479 assert!(last.is_last());
480 }
481
482 #[test]
483 fn stage_contract_rejects_empty_and_out_of_bounds_ranges() {
484 assert!(StageContract::new(2..2, 6).is_err());
485 assert!(StageContract::new(4..7, 6).is_err());
486 assert!(StageContract::new(0..1, 0).is_err());
487 }
488
489 #[test]
490 fn stage_execute_enforces_endpoint_inputs_and_forwards_its_global_range() {
491 let (mut first, first_ranges) = test_stage(0..2, 6);
492 let mut first_sequence = StageSequence::new();
493 let first_output =
494 match first.execute(&mut first_sequence, StageInput::TokenIds(&token_ids()), 0) {
495 Ok(output) => output,
496 Err(error) => panic!("first stage must execute: {error}"),
497 };
498 match first_output {
499 StageOutput::Hidden(hidden) => assert_eq!(hidden.as_tensor().size(), [1, 2, 4]),
500 StageOutput::Logits(_) => panic!("a non-final stage must emit hidden activation"),
501 }
502 assert_eq!(*first_ranges.lock(), vec![0..2]);
503
504 let (mut middle, _) = test_stage(2..4, 6);
505 let mut middle_sequence = StageSequence::new();
506 assert!(middle
507 .execute(&mut middle_sequence, StageInput::TokenIds(&token_ids()), 0)
508 .is_err());
509
510 let (mut final_stage, final_ranges) = test_stage(4..6, 6);
511 let mut final_sequence = StageSequence::new();
512 let final_output =
513 match final_stage.execute(&mut final_sequence, StageInput::Hidden(&hidden()), 0) {
514 Ok(output) => output,
515 Err(error) => panic!("final stage must execute: {error}"),
516 };
517 match final_output {
518 StageOutput::Logits(logits) => assert_eq!(logits.as_tensor().size(), [1, 2, 4]),
519 StageOutput::Hidden(_) => panic!("the final stage must emit logits"),
520 }
521 assert_eq!(*final_ranges.lock(), vec![4..6]);
522 }
523
524 #[test]
525 fn sequence_cannot_migrate_to_an_empty_stage_cache() {
526 let (mut first, _) = test_stage(0..2, 6);
527 let (mut fresh, fresh_ranges) = test_stage(0..2, 6);
528 let mut sequence = StageSequence::new();
529 assert!(first
530 .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
531 .is_ok());
532 assert!(fresh
533 .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 2)
534 .is_err());
535 assert!(
536 fresh_ranges.lock().is_empty(),
537 "foreign sequence must not reach decoder"
538 );
539 assert!(first
540 .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 2)
541 .is_ok());
542 }
543
544 #[test]
545 fn zero_position_sequence_stays_bound_after_stage_is_dropped() {
546 let (mut original, _) = test_stage(0..2, 6);
547 let mut sequence = StageSequence::new();
548 let empty_ids = Tensor::zeros([1, 0], (Kind::Int64, Device::Cpu));
549 assert!(original
550 .execute(&mut sequence, StageInput::TokenIds(&empty_ids), 0)
551 .is_ok());
552 assert_eq!(sequence.next_start_pos, 0);
553 drop(original);
554 let (mut fresh, fresh_ranges) = test_stage(0..2, 6);
555 assert!(fresh
556 .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
557 .is_err());
558 assert!(fresh_ranges.lock().is_empty());
559 }
560
561 #[test]
562 fn failed_stage_cannot_reuse_partially_updated_cache() {
563 for fail_at in ["decoder", "norm", "head"] {
564 let observed_ranges = Arc::new(Mutex::new(Vec::new()));
565 let model = TestModel {
566 config: TestConfig,
567 total_layers: 2,
568 observed_ranges: Arc::clone(&observed_ranges),
569 fail_at: Some(fail_at),
570 };
571 let mut stage =
572 InferenceStage::from_loaded_model(Box::new(model), valid_contract(0..2, 2));
573 let mut sequence = StageSequence::new();
574 assert!(stage
575 .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
576 .is_err());
577 assert_eq!(
578 sequence.next_start_pos, 0,
579 "failed output must not advance sequence"
580 );
581 assert!(stage
582 .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
583 .is_err());
584 let (mut fresh, fresh_ranges) = test_stage(0..2, 2);
585 assert!(fresh
586 .execute(&mut sequence, StageInput::TokenIds(&token_ids()), 0)
587 .is_err());
588 assert!(fresh_ranges.lock().is_empty());
589 assert_eq!(
590 observed_ranges.lock().len(),
591 1,
592 "poisoned cache must not be reused"
593 );
594 }
595 }
596
597 #[test]
598 fn stage_execute_rejects_wrong_input_and_a_second_cache_timeline() {
599 let (mut first, _) = test_stage(0..2, 6);
600 let mut sequence = StageSequence::new();
601 assert!(first
602 .execute(&mut sequence, StageInput::Hidden(&hidden()), 0)
603 .is_err());
604
605 let mut first_sequence = StageSequence::new();
606 if let Err(error) =
607 first.execute(&mut first_sequence, StageInput::TokenIds(&token_ids()), 0)
608 {
609 panic!("valid first-stage request must execute: {error}");
610 }
611 let mut second_sequence = StageSequence::new();
612 assert!(first
613 .execute(&mut second_sequence, StageInput::TokenIds(&token_ids()), 0)
614 .is_err());
615 assert!(first
616 .execute(&mut first_sequence, StageInput::TokenIds(&token_ids()), 0)
617 .is_err());
618 }
619}