Skip to main content

git_xet_filter/
config.rs

1//! Configuration for XET filter
2
3use serde::{Deserialize, Serialize};
4use std::path::PathBuf;
5
6/// HuggingFace XET CAS endpoint
7pub const HUGGINGFACE_XET_ENDPOINT: &str = "https://transfer.xethub.hf.co";
8
9/// Read token from HuggingFace CLI config file
10///
11/// Token path resolution (matching huggingface_hub/constants.py):
12/// 1. `$HF_TOKEN_PATH` if set (explicit override)
13/// 2. `$HF_HOME/token` if `HF_HOME` is set
14/// 3. `~/.cache/huggingface/token` (default)
15fn read_huggingface_token_file() -> Option<String> {
16    let token_path = if let Ok(path) = std::env::var("HF_TOKEN_PATH") {
17        PathBuf::from(path)
18    } else if let Ok(hf_home) = std::env::var("HF_HOME") {
19        PathBuf::from(hf_home).join("token")
20    } else if let Ok(home) = std::env::var("HOME") {
21        PathBuf::from(home)
22            .join(".cache")
23            .join("huggingface")
24            .join("token")
25    } else {
26        return None;
27    };
28
29    std::fs::read_to_string(&token_path)
30        .ok()
31        .map(|s| s.trim().to_owned())
32        .filter(|s| !s.is_empty())
33}
34
35/// Configuration for XET large file storage
36#[derive(Clone, Serialize, Deserialize)]
37pub struct XetConfig {
38    /// CAS endpoint URL
39    ///
40    /// Can be set via:
41    /// - `GIT2DB_XET_ENDPOINT` environment variable
42    /// - `XETHUB_ENDPOINT` environment variable
43    /// - Programmatic configuration
44    ///
45    /// If empty, XET is disabled.
46    pub endpoint: String,
47
48    /// Authentication token
49    ///
50    /// Resolved in priority order:
51    /// 1. `XETHUB_TOKEN` environment variable
52    /// 2. `HF_TOKEN` environment variable
53    /// 3. HuggingFace CLI token file (`~/.cache/huggingface/token`)
54    pub token: Option<String>,
55
56    /// Compression scheme
57    #[cfg(feature = "xet-storage")]
58    #[serde(skip)]
59    pub compression: Option<cas_object::CompressionScheme>,
60
61    #[cfg(not(feature = "xet-storage"))]
62    #[serde(skip)]
63    pub compression: Option<()>,
64}
65
66impl std::fmt::Debug for XetConfig {
67    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
68        f.debug_struct("XetConfig")
69            .field("endpoint", &self.endpoint)
70            .field("token", &self.token.as_ref().map(|_| "[REDACTED]"))
71            .field("compression", &self.compression)
72            .finish()
73    }
74}
75
76impl Default for XetConfig {
77    fn default() -> Self {
78        // Check environment variables for endpoint (in priority order)
79        let endpoint = std::env::var("GIT2DB_XET_ENDPOINT")
80            .or_else(|_| std::env::var("XETHUB_ENDPOINT"))
81            .unwrap_or_default();
82
83        // Token priority (highest to lowest):
84        // 1. XETHUB_TOKEN env var (explicit XET token)
85        // 2. HF_TOKEN env var (explicit HuggingFace token)
86        // 3. HuggingFace CLI token file (~/.cache/huggingface/token)
87        let token = std::env::var("XETHUB_TOKEN")
88            .or_else(|_| std::env::var("HF_TOKEN"))
89            .ok()
90            .or_else(read_huggingface_token_file);
91
92        Self {
93            endpoint,
94            token,
95            compression: None,
96        }
97    }
98}
99
100impl XetConfig {
101    /// Create a new XET configuration with the given endpoint
102    pub fn new(endpoint: impl Into<String>) -> Self {
103        Self {
104            endpoint: endpoint.into(),
105            ..Default::default()
106        }
107    }
108
109    /// Create a configuration for HuggingFace repositories
110    ///
111    /// Uses the HuggingFace XET endpoint and automatically picks up
112    /// authentication from `HF_TOKEN` or `XETHUB_TOKEN` environment variables.
113    pub fn huggingface() -> Self {
114        Self {
115            endpoint: HUGGINGFACE_XET_ENDPOINT.to_owned(),
116            ..Default::default()
117        }
118    }
119
120    /// Check if this configuration is for HuggingFace
121    pub fn is_huggingface(&self) -> bool {
122        self.endpoint.contains("xethub.hf.co")
123    }
124
125    /// Check if XET is enabled (endpoint is not empty)
126    pub fn is_enabled(&self) -> bool {
127        !self.endpoint.is_empty()
128    }
129
130    /// Set the authentication token
131    pub fn with_token(mut self, token: impl Into<String>) -> Self {
132        self.token = Some(token.into());
133        self
134    }
135
136    /// Set the compression scheme
137    #[cfg(feature = "xet-storage")]
138    pub fn with_compression(mut self, compression: cas_object::CompressionScheme) -> Self {
139        self.compression = Some(compression);
140        self
141    }
142
143    /// Create XET config appropriate for a git URL
144    ///
145    /// Resolution order:
146    /// 1. Environment variable (XETHUB_ENDPOINT/GIT2DB_XET_ENDPOINT) - always takes priority
147    /// 2. URL pattern matching - convenience for known providers
148    /// 3. None - XET disabled for this URL
149    ///
150    /// # Examples
151    ///
152    /// ```ignore
153    /// // With env var set, always uses that endpoint
154    /// std::env::set_var("XETHUB_ENDPOINT", "https://my.xet.server");
155    /// let config = XetConfig::for_url("https://github.com/any/repo");
156    /// assert!(config.is_some());
157    ///
158    /// // Without env var, uses URL pattern matching
159    /// std::env::remove_var("XETHUB_ENDPOINT");
160    /// let config = XetConfig::for_url("https://hf.co/Qwen/Qwen3-0.6B");
161    /// assert!(config.unwrap().is_huggingface());
162    ///
163    /// // Unknown URL with no env var = None
164    /// let config = XetConfig::for_url("https://github.com/unknown/repo");
165    /// assert!(config.is_none());
166    /// ```
167    pub fn for_url(url: &str) -> Option<Self> {
168        // Priority 1: Environment variable always wins
169        let default = Self::default();
170        if default.is_enabled() {
171            return Some(default);
172        }
173
174        // Priority 2: Known provider patterns (convenience only)
175        if url.contains("huggingface.co") || url.contains("hf.co") {
176            return Some(Self::huggingface());
177        }
178        // Future providers can be added here as simple pattern matches
179
180        // No XET endpoint configured for this URL
181        None
182    }
183
184    /// Check if the endpoint uses SSH transport
185    pub fn is_ssh_transport(&self) -> bool {
186        self.endpoint.starts_with("ssh://")
187    }
188
189    /// Check if the endpoint uses HTTPS transport
190    pub fn is_https_transport(&self) -> bool {
191        self.endpoint.starts_with("https://") || self.endpoint.starts_with("http://")
192    }
193
194    /// Create the appropriate storage backend based on the endpoint URL.
195    ///
196    /// - `ssh://` endpoints use [`SshStorage`] (requires `ssh-transport` feature)
197    /// - `https://` endpoints use [`XetStorage`]
198    ///
199    /// # Example
200    ///
201    /// ```ignore
202    /// let config = XetConfig::new("ssh://user@host/path/to/cas");
203    /// let storage = config.create_storage().await?;
204    /// ```
205    #[cfg(feature = "xet-storage")]
206    pub async fn create_storage(
207        &self,
208    ) -> crate::error::Result<Box<dyn crate::storage::StorageBackend>> {
209        #[cfg(feature = "gittorrent-transport")]
210        if self.endpoint.starts_with("gittorrent://") {
211            // Initialize gittorrent service and create storage with HTTPS fallback
212            let gt_config = hyprstream_p2p::service::GitTorrentConfig::load().map_err(|e| {
213                crate::error::XetError::new(
214                    crate::error::XetErrorKind::RuntimeError,
215                    format!("Failed to load gittorrent config: {e}"),
216                )
217            })?;
218            let gt_service = std::sync::Arc::new(
219                hyprstream_p2p::service::GitTorrentService::new(gt_config)
220                    .await
221                    .map_err(|e| {
222                        crate::error::XetError::new(
223                            crate::error::XetErrorKind::RuntimeError,
224                            format!("Failed to start gittorrent: {e}"),
225                        )
226                    })?,
227            );
228            // Try to create HTTPS fallback storage
229            let https_endpoint = format!(
230                "https://{}",
231                self.endpoint
232                    .strip_prefix("gittorrent://")
233                    .unwrap_or(&self.endpoint)
234            );
235            let fallback_config = Self {
236                endpoint: https_endpoint,
237                ..self.clone()
238            };
239            let fallback = match crate::storage::XetStorage::new(&fallback_config).await {
240                Ok(s) => Some(Box::new(s) as Box<dyn crate::storage::StorageBackend>),
241                Err(e) => {
242                    tracing::warn!("Failed to initialize HTTPS fallback for gittorrent: {e}");
243                    None
244                }
245            };
246            return Ok(Box::new(crate::gittorrent_storage::GittorrentStorage::new(
247                gt_service, fallback,
248            )));
249        }
250
251        #[cfg(not(feature = "gittorrent-transport"))]
252        if self.endpoint.starts_with("gittorrent://") {
253            return Err(crate::error::XetError::new(
254                crate::error::XetErrorKind::InvalidConfig,
255                "Gittorrent transport requires the 'gittorrent-transport' feature",
256            ));
257        }
258
259        #[cfg(feature = "ssh-transport")]
260        if self.is_ssh_transport() {
261            let storage = crate::ssh_client::SshStorage::connect(&self.endpoint).await?;
262            return Ok(Box::new(storage));
263        }
264
265        #[cfg(not(feature = "ssh-transport"))]
266        if self.is_ssh_transport() {
267            return Err(crate::error::XetError::new(
268                crate::error::XetErrorKind::InvalidConfig,
269                "SSH transport requires the 'ssh-transport' feature",
270            ));
271        }
272
273        // Default to HTTPS transport
274        let storage = crate::storage::XetStorage::new(self).await?;
275        Ok(Box::new(storage))
276    }
277}
278
279// NOTE: Tests that modify environment variables use ENV_MUTEX to avoid race conditions.
280#[cfg(test)]
281mod tests {
282    use super::*;
283    use parking_lot::Mutex;
284
285    // Mutex to serialize tests that modify environment variables
286    static ENV_MUTEX: Mutex<()> = Mutex::new(());
287
288    /// Helper to clear XET-related env vars
289    fn clear_xet_env_vars() {
290        std::env::remove_var("GIT2DB_XET_ENDPOINT");
291        std::env::remove_var("XETHUB_ENDPOINT");
292    }
293
294    #[test]
295    fn test_default_config_disabled_without_env() {
296        let _guard = ENV_MUTEX.lock();
297        clear_xet_env_vars();
298
299        let config = XetConfig::default();
300        // XET is disabled by default when no env vars set
301        assert!(!config.is_enabled());
302    }
303
304    #[test]
305    fn test_config_builder() {
306        // No env var access, no lock needed
307        let config = XetConfig::new("https://custom.endpoint.dev").with_token("test_token");
308
309        assert_eq!(config.endpoint, "https://custom.endpoint.dev");
310        assert_eq!(config.token, Some("test_token".to_owned()));
311        assert!(config.is_enabled());
312    }
313
314    #[test]
315    fn test_huggingface_config() {
316        // No env var access, no lock needed
317        let config = XetConfig::huggingface();
318
319        assert_eq!(config.endpoint, HUGGINGFACE_XET_ENDPOINT);
320        assert!(config.is_huggingface());
321        assert!(config.is_enabled());
322    }
323
324    #[test]
325    fn test_for_url_huggingface() {
326        let _guard = ENV_MUTEX.lock();
327        clear_xet_env_vars();
328
329        // HuggingFace URLs should return huggingface config
330        let config = XetConfig::for_url("https://huggingface.co/Qwen/Qwen3-0.6B");
331        assert!(config.is_some());
332        if let Some(cfg) = config {
333            assert!(cfg.is_huggingface());
334        }
335
336        // Short hf.co URLs should also work
337        let config = XetConfig::for_url("https://hf.co/Qwen/Qwen3-0.6B");
338        assert!(config.is_some());
339        if let Some(cfg) = config {
340            assert!(cfg.is_huggingface());
341        }
342    }
343
344    #[test]
345    fn test_for_url_unknown() {
346        let _guard = ENV_MUTEX.lock();
347        clear_xet_env_vars();
348
349        // Unknown URLs should return None
350        let config = XetConfig::for_url("https://github.com/some/repo");
351        assert!(config.is_none());
352    }
353
354    #[test]
355    fn test_for_url_env_override() {
356        let _guard = ENV_MUTEX.lock();
357        clear_xet_env_vars();
358
359        // Set env var - should take priority over URL pattern
360        std::env::set_var("XETHUB_ENDPOINT", "https://custom.xet.server");
361
362        let config = XetConfig::for_url("https://github.com/any/repo");
363        assert!(config.is_some());
364        if let Some(cfg) = config {
365            assert_eq!(cfg.endpoint, "https://custom.xet.server");
366        }
367
368        // Clean up
369        std::env::remove_var("XETHUB_ENDPOINT");
370    }
371
372    #[test]
373    fn test_for_url_with_trailing_slash() {
374        let _guard = ENV_MUTEX.lock();
375        clear_xet_env_vars();
376
377        // URLs with trailing slash should still match
378        let config = XetConfig::for_url("https://huggingface.co/Qwen/Model/");
379        assert!(config.is_some());
380        if let Some(cfg) = config {
381            assert!(cfg.is_huggingface());
382        }
383    }
384
385    #[test]
386    fn test_debug_redacts_token() {
387        // No env var access, no lock needed
388        let config = XetConfig::new("https://example.com").with_token("secret_token_12345");
389
390        let debug_str = format!("{config:?}");
391
392        // Token should be redacted
393        assert!(debug_str.contains("[REDACTED]"));
394        assert!(!debug_str.contains("secret_token_12345"));
395        // Endpoint should still be visible
396        assert!(debug_str.contains("https://example.com"));
397    }
398
399    #[test]
400    fn test_debug_without_token() {
401        // No env var access, no lock needed
402        let config = XetConfig::new("https://example.com");
403
404        let debug_str = format!("{config:?}");
405
406        // Should show None for token (no redaction needed)
407        assert!(debug_str.contains("None"));
408        assert!(debug_str.contains("https://example.com"));
409    }
410}