Skip to main content

hyprstream_core/storage/
pinned_model.rs

1//! Local admission and owned artifact acquisition for an exact commit model.
2
3use anyhow::{ensure, Context, Result};
4use git2::{Oid, StatusOptions};
5use git2db::pinned_disk::{DiskPinnedTreeProjection, ProjectionPolicy};
6use git2db::pinned_tree::{PinnedTree, SealedTreeProjection};
7use std::collections::HashMap;
8use std::path::{Path, PathBuf};
9use std::sync::{Arc, Weak};
10use tokio::sync::Mutex;
11
12/// Retain the selected commit's verified inputs through every loader read.
13/// Only the fixed staging profile needs the disk-backed, capacity-limited path.
14#[derive(Debug)]
15pub enum PinnedModelArtifact {
16    StagingDisk(DiskPinnedTreeProjection),
17    Sealed {
18        projection: SealedTreeProjection,
19        owner_pid: u32,
20    },
21}
22
23/// Reuse one immutable staging projection across tenant-specific Inference
24/// instances. The weak cache does not keep disk reservations alive by itself;
25/// the serving instances own strong references for as long as they need the
26/// verified file descriptors.
27#[derive(Default)]
28pub struct StagingPinnedArtifactCache {
29    entries:
30        Mutex<HashMap<(PathBuf, git2::Oid, ProjectionPolicy, PathBuf), Weak<PinnedModelArtifact>>>,
31}
32
33impl StagingPinnedArtifactCache {
34    pub async fn acquire(
35        &self,
36        worktree_path: &Path,
37        commit: git2::Oid,
38    ) -> Result<Arc<PinnedModelArtifact>> {
39        self.acquire_in(
40            worktree_path,
41            commit,
42            Path::new("/var/cache/hyprstream-pinned"),
43        )
44        .await
45    }
46
47    async fn acquire_in(
48        &self,
49        worktree_path: &Path,
50        commit: git2::Oid,
51        private_disk_parent: &Path,
52    ) -> Result<Arc<PinnedModelArtifact>> {
53        self.acquire_in_with_observer(
54            worktree_path,
55            commit,
56            private_disk_parent,
57            || Ok(()),
58            || Ok(()),
59        )
60        .await
61    }
62
63    async fn acquire_in_with_observer(
64        &self,
65        worktree_path: &Path,
66        commit: git2::Oid,
67        private_disk_parent: &Path,
68        after_admission: impl FnOnce() -> Result<()>,
69        after_capture: impl FnOnce() -> Result<()>,
70    ) -> Result<Arc<PinnedModelArtifact>> {
71        self.acquire_policy_with_observer(
72            worktree_path,
73            commit,
74            private_disk_parent,
75            ProjectionPolicy::default(),
76            after_admission,
77            after_capture,
78        )
79        .await
80    }
81
82    /// Server-owned per-pin policy; no request can supply expanded byte limits
83    /// or hydration paths. Distinct policies never share cache entries.
84    pub async fn acquire_with_policy(
85        &self,
86        worktree_path: &Path,
87        commit: git2::Oid,
88        policy: ProjectionPolicy,
89    ) -> Result<Arc<PinnedModelArtifact>> {
90        self.acquire_policy_with_observer(
91            worktree_path,
92            commit,
93            Path::new("/var/cache/hyprstream-pinned"),
94            policy,
95            || Ok(()),
96            || Ok(()),
97        )
98        .await
99    }
100
101    /// Isolated acceptance uses the SAME production bounded capture with a
102    /// task-owned private disk parent; no runtime env/RPC override exists.
103    #[cfg(test)]
104    pub(crate) async fn acquire_with_policy_in(
105        &self,
106        worktree_path: &Path,
107        commit: git2::Oid,
108        private_disk_parent: &Path,
109        policy: ProjectionPolicy,
110    ) -> Result<Arc<PinnedModelArtifact>> {
111        self.acquire_policy_with_observer(
112            worktree_path,
113            commit,
114            private_disk_parent,
115            policy,
116            || Ok(()),
117            || Ok(()),
118        )
119        .await
120    }
121
122    async fn acquire_policy_with_observer(
123        &self,
124        worktree_path: &Path,
125        commit: git2::Oid,
126        private_disk_parent: &Path,
127        policy: ProjectionPolicy,
128        after_admission: impl FnOnce() -> Result<()>,
129        after_capture: impl FnOnce() -> Result<()>,
130    ) -> Result<Arc<PinnedModelArtifact>> {
131        // A cache hit may share only the exact commit from the exact local
132        // checkout, and the checkout must still satisfy the current load's
133        // clean/HEAD admission rule.
134        verify_selected_checkout(worktree_path, commit)?;
135        after_admission()?;
136        let key = (
137            worktree_path.to_path_buf(),
138            commit,
139            policy.clone(),
140            private_disk_parent.to_path_buf(),
141        );
142        let mut entries = self.entries.lock().await;
143        // The mutex is held through a whole capture. A waiter's admission
144        // check above can predate that capture entirely, so re-observe the
145        // checkout after the wait before sharing or capturing anything.
146        verify_selected_checkout(worktree_path, commit)?;
147        entries.retain(|_, artifact| artifact.strong_count() > 0);
148        if let Some(artifact) = entries.get(&key).and_then(Weak::upgrade) {
149            return Ok(artifact);
150        }
151
152        // Hold the async cache lock while acquiring to make same-commit loads
153        // coalesce. The bounded acquisition itself runs on the projection I/O
154        // runtime, not the Model service reactor.
155        let projection = DiskPinnedTreeProjection::acquire_bounded_with_policy(
156            worktree_path,
157            commit,
158            private_disk_parent,
159            policy,
160        )
161        .await
162        .with_context(|| format!("capture model commit {commit}"))?;
163        after_capture()?;
164        // Capturing a large model can take long enough for a checkout update
165        // to race the admission. Do not publish an artifact unless the exact
166        // selected checkout is still clean and at the requested commit.
167        verify_selected_checkout(worktree_path, commit)?;
168        let artifact = Arc::new(PinnedModelArtifact::StagingDisk(projection));
169        entries.insert(key, Arc::downgrade(&artifact));
170        Ok(artifact)
171    }
172}
173
174impl PinnedModelArtifact {
175    pub fn commit(&self) -> Oid {
176        match self {
177            Self::StagingDisk(artifact) => artifact.commit(),
178            Self::Sealed { projection, .. } => projection.commit(),
179        }
180    }
181
182    pub fn root(&self) -> &Path {
183        match self {
184            Self::StagingDisk(artifact) => artifact.root(),
185            Self::Sealed { projection, .. } => projection.root(),
186        }
187    }
188
189    pub fn file_count(&self) -> usize {
190        match self {
191            Self::StagingDisk(artifact) => artifact.file_count(),
192            Self::Sealed { projection, .. } => projection.file_count(),
193        }
194    }
195
196    pub fn ensure_current_process(&self) -> Result<()> {
197        match self {
198            Self::StagingDisk(artifact) => artifact.ensure_current_process()?,
199            Self::Sealed { owner_pid, .. } => {
200                ensure!(
201                    *owner_pid == std::process::id(),
202                    "sealed pinned projection cannot cross a process boundary"
203                );
204            }
205        }
206        Ok(())
207    }
208}
209
210/// Admit only the selected, clean checkout of `commit`, then capture the
211/// requested Git tree independently of worktree bytes. The second checkout
212/// observation enforces the dirty policy during acquisition; the Git object
213/// capture and private disk projection, not these observations, prove loader bytes.
214pub async fn acquire_pinned_model(
215    worktree_path: &Path,
216    commit: Oid,
217) -> Result<Arc<PinnedModelArtifact>> {
218    verify_selected_checkout(worktree_path, commit)?;
219    let projection = PinnedTree::acquire(worktree_path, commit)
220        .await
221        .with_context(|| format!("capture model commit {commit}"))?
222        .into_sealed_projection()?;
223    verify_selected_checkout(worktree_path, commit)?;
224    ensure!(
225        projection.commit() == commit,
226        "pinned projection commit mismatch"
227    );
228    Ok(Arc::new(PinnedModelArtifact::Sealed {
229        projection,
230        owner_pid: std::process::id(),
231    }))
232}
233
234/// Bounded single-slot projection lease for the staging admission profile.
235pub async fn acquire_pinned_model_bounded(
236    worktree_path: &Path,
237    commit: Oid,
238) -> Result<Arc<PinnedModelArtifact>> {
239    StagingPinnedArtifactCache::default()
240        .acquire(worktree_path, commit)
241        .await
242}
243
244#[cfg(test)]
245pub(crate) async fn acquire_pinned_model_bounded_in(
246    cache: &StagingPinnedArtifactCache,
247    worktree_path: &Path,
248    commit: Oid,
249    private_disk_parent: &Path,
250) -> Result<Arc<PinnedModelArtifact>> {
251    cache
252        .acquire_in(worktree_path, commit, private_disk_parent)
253        .await
254}
255
256#[cfg(test)]
257pub(crate) async fn acquire_pinned_model_in(
258    worktree_path: &Path,
259    commit: Oid,
260    private_disk_parent: &Path,
261) -> Result<Arc<DiskPinnedTreeProjection>> {
262    Ok(Arc::new(
263        acquire_pinned_model_with_observer(
264            worktree_path,
265            commit,
266            private_disk_parent,
267            false,
268            || Ok(()),
269        )
270        .await?,
271    ))
272}
273
274#[cfg(test)]
275async fn acquire_pinned_model_with_observer(
276    worktree_path: &Path,
277    commit: Oid,
278    private_disk_parent: &Path,
279    bounded: bool,
280    after_capture: impl FnOnce() -> Result<()>,
281) -> Result<DiskPinnedTreeProjection> {
282    verify_selected_checkout(worktree_path, commit)?;
283    let projection = if bounded {
284        DiskPinnedTreeProjection::acquire_bounded_with_policy(
285            worktree_path,
286            commit,
287            private_disk_parent,
288            ProjectionPolicy::default(),
289        )
290        .await
291    } else {
292        DiskPinnedTreeProjection::acquire(worktree_path, commit, private_disk_parent).await
293    }
294    .with_context(|| format!("capture model commit {commit}"))?;
295    after_capture()?;
296    verify_selected_checkout(worktree_path, commit)?;
297    ensure!(
298        projection.commit() == commit,
299        "pinned projection commit mismatch"
300    );
301    Ok(projection)
302}
303
304pub fn verify_selected_checkout(worktree_path: &Path, commit: Oid) -> Result<()> {
305    let repo = git2db::GitManager::global()
306        .get_repository(worktree_path)
307        .context("open selected model worktree")?
308        .open()
309        .context("open selected model repository")?;
310    ensure!(!repo.is_bare(), "selected model checkout is bare");
311    let head = repo
312        .head()
313        .context("read selected model HEAD")?
314        .peel_to_commit()
315        .context("resolve selected model HEAD commit")?
316        .id();
317    ensure!(
318        head == commit,
319        "selected model checkout is at {head}, requested {commit}"
320    );
321    let mut options = StatusOptions::new();
322    options
323        .include_untracked(true)
324        .recurse_untracked_dirs(true)
325        .include_ignored(true)
326        .recurse_ignored_dirs(true);
327    let statuses = repo
328        .statuses(Some(&mut options))
329        .context("read selected model checkout status")?;
330    ensure!(statuses.is_empty(), "selected model checkout is dirty");
331    Ok(())
332}
333
334#[cfg(test)]
335mod tests {
336    use super::*;
337    use std::os::unix::fs::PermissionsExt;
338
339    // The bounded projection capacity is intentionally one process-global
340    // slot. Serialize tests that acquire it so parallel libtest scheduling
341    // cannot make the sharing assertion fail for unrelated test timing.
342    static BOUNDED_PROJECTION_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
343
344    #[tokio::test]
345    async fn projection_policy_cache_does_not_reuse_more_permissive_capture() -> Result<()> {
346        let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
347        let source = tempfile::tempdir()?;
348        let commit = commit_tracked_file(source.path(), b"12345678")?;
349        let parent = disk_backed_private_parent()?;
350        let cache = StagingPinnedArtifactCache::default();
351        let policy = ProjectionPolicy::from_trusted_contract(8, Default::default())?;
352        let a = cache
353            .acquire_policy_with_observer(
354                source.path(),
355                commit,
356                parent.path(),
357                policy.clone(),
358                || Ok(()),
359                || Ok(()),
360            )
361            .await?;
362        let b = cache
363            .acquire_policy_with_observer(
364                source.path(),
365                commit,
366                parent.path(),
367                policy,
368                || Ok(()),
369                || Ok(()),
370            )
371            .await?;
372        assert!(Arc::ptr_eq(&a, &b));
373        let lower = ProjectionPolicy::from_trusted_contract(7, Default::default())?;
374        assert!(cache
375            .acquire_policy_with_observer(
376                source.path(),
377                commit,
378                parent.path(),
379                lower.clone(),
380                || Ok(()),
381                || Ok(())
382            )
383            .await
384            .is_err());
385        drop(a);
386        drop(b);
387        let error = cache
388            .acquire_policy_with_observer(
389                source.path(),
390                commit,
391                parent.path(),
392                lower,
393                || Ok(()),
394                || Ok(()),
395            )
396            .await
397            .err()
398            .ok_or_else(|| anyhow::anyhow!("lower policy reused captured artifact"))?;
399        assert!(format!("{error:#}").contains("7-byte limit"));
400        Ok(())
401    }
402
403    fn commit_tracked_file(root: &Path, bytes: &[u8]) -> Result<Oid> {
404        let repo = git2db::Repository::init(root)?;
405        std::fs::write(root.join("config.json"), bytes)?;
406        let mut index = repo.index()?;
407        index.add_path(Path::new("config.json"))?;
408        index.write()?;
409        let tree = repo.find_tree(index.write_tree()?)?;
410        let sig = git2::Signature::now("test", "test@example.invalid")?;
411        let commit = repo.commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])?;
412        drop(tree);
413        drop(repo);
414        Ok(commit)
415    }
416
417    fn disk_backed_private_parent() -> Result<tempfile::TempDir> {
418        let parent = tempfile::tempdir_in(
419            std::env::current_exe()?
420                .parent()
421                .ok_or_else(|| anyhow::anyhow!("test binary has no parent directory"))?,
422        )?;
423        std::fs::set_permissions(parent.path(), std::fs::Permissions::from_mode(0o700))?;
424        Ok(parent)
425    }
426
427    #[tokio::test]
428    async fn generic_commit_uses_sealed_projection_without_private_disk_parent() -> Result<()> {
429        let dir = tempfile::tempdir()?;
430        let repo = git2db::Repository::init(dir.path())?;
431        std::fs::write(dir.path().join("config.json"), b"reviewed")?;
432        let mut index = repo.index()?;
433        index.add_path(Path::new("config.json"))?;
434        index.write()?;
435        let tree = repo.find_tree(index.write_tree()?)?;
436        let sig = git2::Signature::now("test", "test@example.invalid")?;
437        let commit = repo.commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])?;
438        drop(tree);
439        drop(repo);
440
441        let artifact = acquire_pinned_model(dir.path(), commit).await?;
442        assert!(matches!(&*artifact, PinnedModelArtifact::Sealed { .. }));
443        std::fs::write(dir.path().join("config.json"), b"changed")?;
444        assert_eq!(
445            std::fs::read(artifact.root().join("config.json"))?,
446            b"reviewed"
447        );
448        Ok(())
449    }
450
451    #[tokio::test]
452    async fn mismatched_or_dirty_checkout_fails_before_artifact_success() -> Result<()> {
453        let dir = tempfile::tempdir()?;
454        let parent = tempfile::tempdir()?;
455        std::fs::set_permissions(parent.path(), std::fs::Permissions::from_mode(0o700))?;
456        let repo = git2db::Repository::init(dir.path())?;
457        std::fs::write(dir.path().join("config.json"), b"reviewed")?;
458        let mut index = repo.index()?;
459        index.add_path(Path::new("config.json"))?;
460        index.write()?;
461        let tree = repo.find_tree(index.write_tree()?)?;
462        let sig = git2::Signature::now("test", "test@example.invalid")?;
463        let commit = repo.commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])?;
464        drop(tree);
465        drop(repo);
466        assert!(
467            acquire_pinned_model_in(dir.path(), Oid::zero(), parent.path())
468                .await
469                .is_err()
470        );
471        std::fs::write(dir.path().join("config.json"), b"changed")?;
472        assert!(acquire_pinned_model_in(dir.path(), commit, parent.path())
473            .await
474            .is_err());
475        Ok(())
476    }
477
478    #[tokio::test]
479    async fn mutation_during_acquisition_fails_closed() -> Result<()> {
480        let dir = tempfile::tempdir()?;
481        let parent = disk_backed_private_parent()?;
482        let commit = commit_tracked_file(dir.path(), b"reviewed")?;
483        let mut mutated = false;
484        let error = match acquire_pinned_model_with_observer(
485            dir.path(),
486            commit,
487            parent.path(),
488            false,
489            || {
490                mutated = true;
491                std::fs::write(dir.path().join("config.json"), b"changed")?;
492                Ok(())
493            },
494        )
495        .await
496        {
497            Ok(projection) => {
498                return Err(anyhow::anyhow!(
499                    "checkout mutation during acquisition must fail closed: {projection:?}"
500                ))
501            }
502            Err(error) => error,
503        };
504        assert!(
505            mutated,
506            "the after-capture witness must run before the failure"
507        );
508        assert!(
509            error
510                .to_string()
511                .contains("selected model checkout is dirty"),
512            "expected the dirty-checkout admission failure, got: {error:#}"
513        );
514        Ok(())
515    }
516
517    #[tokio::test]
518    async fn bounded_cache_rejects_checkout_mutation_during_capture() -> Result<()> {
519        let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
520        let dir = tempfile::tempdir()?;
521        let parent = disk_backed_private_parent()?;
522        let commit = commit_tracked_file(dir.path(), b"reviewed")?;
523
524        let cache = StagingPinnedArtifactCache::default();
525        let mut mutated = false;
526        let error = match cache
527            .acquire_in_with_observer(
528                dir.path(),
529                commit,
530                parent.path(),
531                || Ok(()),
532                || {
533                    mutated = true;
534                    std::fs::write(dir.path().join("config.json"), b"changed")?;
535                    Ok(())
536                },
537            )
538            .await
539        {
540            Ok(artifact) => {
541                return Err(anyhow::anyhow!(
542                    "checkout mutation during bounded capture must fail closed: {artifact:?}"
543                ))
544            }
545            Err(error) => error,
546        };
547        assert!(
548            mutated,
549            "the after-capture witness must run before the failure"
550        );
551        assert!(
552            error
553                .to_string()
554                .contains("selected model checkout is dirty"),
555            "expected the dirty-checkout admission failure, got: {error:#}"
556        );
557        assert!(cache.entries.lock().await.is_empty());
558        Ok(())
559    }
560
561    #[tokio::test]
562    async fn bounded_cache_waiter_revalidates_checkout_after_waiting_for_capture() -> Result<()> {
563        let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
564        let dir = tempfile::tempdir()?;
565        let parent = disk_backed_private_parent()?;
566        let commit = commit_tracked_file(dir.path(), b"reviewed")?;
567
568        let cache = Arc::new(StagingPinnedArtifactCache::default());
569        // Populate the cache and retain the strong artifact, as a serving
570        // Inference instance retains its projection.
571        let live =
572            acquire_pinned_model_bounded_in(&cache, dir.path(), commit, parent.path()).await?;
573
574        // Hold the entries mutex so the next acquire must wait past its own
575        // pre-lock admission check.
576        let entries_guard = cache.entries.lock().await;
577        let (admitted_tx, admitted_rx) = tokio::sync::oneshot::channel();
578        let waiting_cache = Arc::clone(&cache);
579        let waiting_path = dir.path().to_path_buf();
580        let waiting_parent = parent.path().to_path_buf();
581        let waiter = tokio::spawn(async move {
582            waiting_cache
583                .acquire_in_with_observer(
584                    &waiting_path,
585                    commit,
586                    &waiting_parent,
587                    || {
588                        let _ = admitted_tx.send(());
589                        Ok(())
590                    },
591                    || Ok(()),
592                )
593                .await
594        });
595        // The signal proves the waiter already passed its pre-lock admission
596        // check; with this test holding the mutex it can only be waiting.
597        tokio::time::timeout(std::time::Duration::from_secs(5), admitted_rx)
598            .await
599            .map_err(|_| anyhow::anyhow!("waiter did not reach the entries mutex"))?
600            .map_err(|_| anyhow::anyhow!("waiter observer dropped before signaling"))?;
601        let key = (
602            dir.path().to_path_buf(),
603            commit,
604            ProjectionPolicy::default(),
605            parent.path().to_path_buf(),
606        );
607        assert!(
608            entries_guard.get(&key).and_then(Weak::upgrade).is_some(),
609            "the live artifact must be shareable while the waiter is parked"
610        );
611
612        // Dirtied tracked bytes while the waiter is parked on the mutex.
613        std::fs::write(dir.path().join("config.json"), b"dirtied")?;
614        drop(entries_guard);
615
616        let outcome = waiter
617            .await
618            .map_err(|e| anyhow::anyhow!("waiter task failed: {e}"))?;
619        let error = match outcome {
620            Ok(artifact) => {
621                return Err(anyhow::anyhow!(
622                    "a waiter resuming after capture must revalidate admission: {artifact:?}"
623                ))
624            }
625            Err(error) => error,
626        };
627        assert!(
628            error
629                .to_string()
630                .contains("selected model checkout is dirty"),
631            "expected the dirty-checkout admission failure, got: {error:#}"
632        );
633        drop(live);
634        Ok(())
635    }
636
637    #[tokio::test]
638    async fn loader_inputs_remain_exact_after_checkout_replacement() -> Result<()> {
639        use crate::runtime::model_config::ModelConfig;
640        use tokenizers::{models::wordlevel::WordLevel, Tokenizer};
641
642        let dir = tempfile::tempdir()?;
643        let parent = tempfile::tempdir()?;
644        std::fs::set_permissions(parent.path(), std::fs::Permissions::from_mode(0o700))?;
645        let repo = git2db::Repository::init(dir.path())?;
646        let config = br#"{"model_type":"llama","hidden_size":8,"num_hidden_layers":1,"num_attention_heads":2}"#;
647        std::fs::write(dir.path().join("config.json"), config)?;
648        std::fs::write(dir.path().join("model.safetensors"), b"reviewed weights")?;
649        std::fs::write(
650            dir.path().join("generation_config.json"),
651            br#"{"temperature":0.25}"#,
652        )?;
653        Tokenizer::new(WordLevel::default())
654            .save(dir.path().join("tokenizer.json"), false)
655            .map_err(|e| anyhow::anyhow!(e.to_string()))?;
656        let mut index = repo.index()?;
657        for name in [
658            "config.json",
659            "model.safetensors",
660            "generation_config.json",
661            "tokenizer.json",
662        ] {
663            index.add_path(Path::new(name))?;
664        }
665        index.write()?;
666        let tree = repo.find_tree(index.write_tree()?)?;
667        let sig = git2::Signature::now("test", "test@example.invalid")?;
668        let commit = repo.commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])?;
669        drop(tree);
670        drop(repo);
671
672        let artifact = acquire_pinned_model_in(dir.path(), commit, parent.path()).await?;
673        std::fs::write(dir.path().join("config.json"), b"invalid replacement")?;
674        std::fs::write(dir.path().join("model.safetensors"), b"replacement weights")?;
675        std::fs::write(dir.path().join("tokenizer.json"), b"invalid replacement")?;
676        std::fs::write(
677            dir.path().join("generation_config.json"),
678            br#"{"temperature":0.9}"#,
679        )?;
680        std::fs::write(dir.path().join("untracked.safetensors"), b"unreviewed")?;
681
682        let loaded_config = ModelConfig::load(artifact.root(), &std::collections::HashMap::new())?;
683        assert_eq!(loaded_config.hidden_size, 8);
684        assert_eq!(
685            std::fs::read(artifact.root().join("model.safetensors"))?,
686            b"reviewed weights"
687        );
688        assert!(Tokenizer::from_file(artifact.root().join("tokenizer.json")).is_ok());
689        let sampling = crate::config::SamplingParams::from_model_path(artifact.root())
690            .await
691            .map_err(|e| anyhow::anyhow!(e.to_string()))?;
692        assert_eq!(sampling.temperature, Some(0.25));
693        assert!(!artifact.root().join("untracked.safetensors").exists());
694        Ok(())
695    }
696
697    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
698    async fn bounded_staging_loads_share_one_projection_for_the_same_commit() -> Result<()> {
699        let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
700        let repo_dir = tempfile::tempdir()?;
701        let private_parent = disk_backed_private_parent()?;
702        let commit = commit_tracked_file(repo_dir.path(), b"reviewed")?;
703
704        // A second load starts while the first still holds the entries mutex
705        // through its capture, then must coalesce onto the published artifact.
706        let cache = Arc::new(StagingPinnedArtifactCache::default());
707        let (captured_tx, captured_rx) = std::sync::mpsc::channel::<()>();
708        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
709        let capturing_cache = Arc::clone(&cache);
710        let capturing_path = repo_dir.path().to_path_buf();
711        let capturing_parent = private_parent.path().to_path_buf();
712        let capturing = tokio::spawn(async move {
713            capturing_cache
714                .acquire_in_with_observer(
715                    &capturing_path,
716                    commit,
717                    &capturing_parent,
718                    || Ok(()),
719                    move || {
720                        // The entries mutex is still held here: the capture
721                        // finished, but the artifact is not published yet.
722                        captured_tx
723                            .send(())
724                            .map_err(|e| anyhow::anyhow!("signal capture: {e}"))?;
725                        release_rx
726                            .recv_timeout(std::time::Duration::from_secs(10))
727                            .map_err(|e| anyhow::anyhow!("release capture: {e}"))?;
728                        Ok(())
729                    },
730                )
731                .await
732        });
733        captured_rx
734            .recv_timeout(std::time::Duration::from_secs(10))
735            .map_err(|e| anyhow::anyhow!("first acquire never captured: {e}"))?;
736
737        let (admitted_tx, admitted_rx) = tokio::sync::oneshot::channel();
738        let waiting_cache = Arc::clone(&cache);
739        let waiting_path = repo_dir.path().to_path_buf();
740        let waiting_parent = private_parent.path().to_path_buf();
741        let waiting = tokio::spawn(async move {
742            waiting_cache
743                .acquire_in_with_observer(
744                    &waiting_path,
745                    commit,
746                    &waiting_parent,
747                    || {
748                        let _ = admitted_tx.send(());
749                        Ok(())
750                    },
751                    || Ok(()),
752                )
753                .await
754        });
755        // The waiter's admission ran while the first acquire still holds the
756        // entries mutex, so it deterministically coalesces behind the capture.
757        tokio::time::timeout(std::time::Duration::from_secs(5), admitted_rx)
758            .await
759            .map_err(|_| anyhow::anyhow!("waiter did not reach the entries mutex"))?
760            .map_err(|_| anyhow::anyhow!("waiter observer dropped before signaling"))?;
761        release_tx
762            .send(())
763            .map_err(|e| anyhow::anyhow!("release the capture: {e}"))?;
764
765        let first = capturing
766            .await
767            .map_err(|e| anyhow::anyhow!("first acquire join: {e}"))??;
768        let second = waiting
769            .await
770            .map_err(|e| anyhow::anyhow!("second acquire join: {e}"))??;
771
772        assert!(Arc::ptr_eq(&first, &second));
773        assert_eq!(first.commit(), commit);
774        assert_eq!(
775            std::fs::read(first.root().join("config.json"))?,
776            b"reviewed"
777        );
778        Ok(())
779    }
780}