cacheDir for env, config, and flag (#8947)

dimitropoulos · web-flow · commit 9ed775d1f019 · 2024-08-12T17:04:25.000-04:00
diff --git a/crates/turborepo-cache/src/async_cache.rs b/crates/turborepo-cache/src/async_cache.rs
@@ -218,6 +218,7 @@ mod tests {
     use std::{assert_matches::assert_matches, time::Duration};
 
     use anyhow::Result;
+    use camino::Utf8PathBuf;
     use futures::future::try_join_all;
     use tempfile::tempdir;
     use turbopath::AbsoluteSystemPathBuf;
@@ -253,7 +254,7 @@ mod tests {
         let hash = format!("{}-no-fs", test_case.hash);
 
         let opts = CacheOpts {
-            override_dir: None,
+            cache_dir: Utf8PathBuf::from(".turbo/cache"),
             remote_cache_read_only: false,
             skip_remote: false,
             skip_filesystem: true,
@@ -335,7 +336,7 @@ mod tests {
         let hash = format!("{}-no-remote", test_case.hash);
 
         let opts = CacheOpts {
-            override_dir: None,
+            cache_dir: Utf8PathBuf::from(".turbo/cache"),
             remote_cache_read_only: false,
             skip_remote: true,
             skip_filesystem: false,
@@ -427,7 +428,7 @@ mod tests {
         let hash = format!("{}-both", test_case.hash);
 
         let opts = CacheOpts {
-            override_dir: None,
+            cache_dir: Utf8PathBuf::from(".turbo/cache"),
             remote_cache_read_only: false,
             skip_remote: false,
             skip_filesystem: false,
diff --git a/crates/turborepo-cache/src/fs.rs b/crates/turborepo-cache/src/fs.rs
@@ -32,22 +32,18 @@ impl CacheMetadata {
 impl FSCache {
     fn resolve_cache_dir(
         repo_root: &AbsoluteSystemPath,
-        override_dir: Option<&Utf8Path>,
+        cache_dir: &Utf8Path,
     ) -> AbsoluteSystemPathBuf {
-        if let Some(override_dir) = override_dir {
-            AbsoluteSystemPathBuf::from_unknown(repo_root, override_dir)
-        } else {
-            repo_root.join_components(&[".turbo", "cache"])
-        }
+        AbsoluteSystemPathBuf::from_unknown(repo_root, cache_dir)
     }
 
     #[tracing::instrument(skip_all)]
     pub fn new(
-        override_dir: Option<&Utf8Path>,
+        cache_dir: &Utf8Path,
         repo_root: &AbsoluteSystemPath,
         analytics_recorder: Option<AnalyticsSender>,
     ) -> Result<Self, CacheError> {
-        let cache_directory = Self::resolve_cache_dir(repo_root, override_dir);
+        let cache_directory = Self::resolve_cache_dir(repo_root, cache_dir);
         cache_directory.create_dir_all()?;
 
         Ok(FSCache {
@@ -233,7 +229,11 @@ mod test {
         let (analytics_sender, analytics_handle) =
             start_analytics(api_auth.clone(), api_client.clone());
 
-        let cache = FSCache::new(None, repo_root_path, Some(analytics_sender.clone()))?;
+        let cache = FSCache::new(
+            Utf8Path::new(""),
+            repo_root_path,
+            Some(analytics_sender.clone()),
+        )?;
 
         let expected_miss = cache.fetch(repo_root_path, test_case.hash)?;
         assert!(expected_miss.is_none());
diff --git a/crates/turborepo-cache/src/lib.rs b/crates/turborepo-cache/src/lib.rs
@@ -105,7 +105,7 @@ pub struct CacheHitMetadata {
 
 #[derive(Debug, Default)]
 pub struct CacheOpts {
-    pub override_dir: Option<Utf8PathBuf>,
+    pub cache_dir: Utf8PathBuf,
     pub remote_cache_read_only: bool,
     pub skip_remote: bool,
     pub skip_filesystem: bool,
diff --git a/crates/turborepo-cache/src/multiplexer.rs b/crates/turborepo-cache/src/multiplexer.rs
@@ -48,13 +48,7 @@ impl CacheMultiplexer {
         }
 
         let fs_cache = use_fs_cache
-            .then(|| {
-                FSCache::new(
-                    opts.override_dir.as_deref(),
-                    repo_root,
-                    analytics_recorder.clone(),
-                )
-            })
+            .then(|| FSCache::new(&opts.cache_dir, repo_root, analytics_recorder.clone()))
             .transpose()?;
 
         let http_cache = use_http_cache
diff --git a/crates/turborepo-lib/src/commands/config.rs b/crates/turborepo-lib/src/commands/config.rs
@@ -1,3 +1,4 @@
+use camino::Utf8Path;
 use serde::Serialize;
 use turborepo_repository::{
     package_graph::PackageGraph, package_json::PackageJson, package_manager::PackageManager,
@@ -24,6 +25,7 @@ struct ConfigOutput<'a> {
     env_mode: EnvMode,
     scm_base: &'a str,
     scm_head: &'a str,
+    cache_dir: &'a Utf8Path,
 }
 
 pub async fn run(base: CommandBase) -> Result<(), cli::Error> {
@@ -55,6 +57,7 @@ pub async fn run(base: CommandBase) -> Result<(), cli::Error> {
             env_mode: config.env_mode(),
             scm_base: config.scm_base(),
             scm_head: config.scm_head(),
+            cache_dir: config.cache_dir(),
         })?
     );
     Ok(())
diff --git a/crates/turborepo-lib/src/commands/mod.rs b/crates/turborepo-lib/src/commands/mod.rs
@@ -103,6 +103,21 @@ impl CommandBase {
                             .and_then(|args| args.env_mode)
                     }),
             )
+            .with_cache_dir(
+                self.args
+                    .command
+                    .as_ref()
+                    .and_then(|c| match c {
+                        Command::Run { execution_args, .. } => execution_args.cache_dir.clone(),
+                        _ => None,
+                    })
+                    .or_else(|| {
+                        self.args
+                            .execution_args
+                            .as_ref()
+                            .and_then(|args| args.cache_dir.clone())
+                    }),
+            )
             .build()
     }
 
diff --git a/crates/turborepo-lib/src/config.rs b/crates/turborepo-lib/src/config.rs
@@ -4,12 +4,13 @@ use std::{
     io,
 };
 
+use camino::{Utf8Path, Utf8PathBuf};
 use convert_case::{Case, Casing};
 use miette::{Diagnostic, NamedSource, SourceSpan};
 use serde::Deserialize;
 use struct_iterable::Iterable;
 use thiserror::Error;
-use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPath};
+use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPath, RelativeUnixPath};
 use turborepo_auth::{TURBO_TOKEN_DIR, TURBO_TOKEN_FILE, VERCEL_TOKEN_DIR, VERCEL_TOKEN_FILE};
 use turborepo_dirs::{config_dir, vercel_config_dir};
 use turborepo_errors::TURBO_SITE;
@@ -161,6 +162,14 @@ pub enum Error {
     #[error(transparent)]
     #[diagnostic(transparent)]
     TurboJsonParseError(#[from] turbo_json::parser::Error),
+    #[error("found absolute path in `cacheDir`")]
+    #[diagnostic(help("if absolute paths are required, use `--cache-dir` or `TURBO_CACHE_DIR`"))]
+    AbsoluteCacheDir {
+        #[label("make `cacheDir` value a relative unix path")]
+        span: Option<SourceSpan>,
+        #[source_code]
+        text: NamedSource,
+    },
 }
 
 macro_rules! create_builder {
@@ -215,6 +224,8 @@ pub struct ConfigurationOptions {
     pub(crate) env_mode: Option<EnvMode>,
     pub(crate) scm_base: Option<String>,
     pub(crate) scm_head: Option<String>,
+    #[serde(rename = "cacheDir")]
+    pub(crate) cache_dir: Option<Utf8PathBuf>,
 }
 
 #[derive(Default)]
@@ -306,6 +317,16 @@ impl ConfigurationOptions {
     pub fn env_mode(&self) -> EnvMode {
         self.env_mode.unwrap_or_default()
     }
+
+    pub fn cache_dir(&self) -> &Utf8Path {
+        self.cache_dir.as_deref().unwrap_or_else(|| {
+            Utf8Path::new(if cfg!(windows) {
+                ".turbo\\cache"
+            } else {
+                ".turbo/cache"
+            })
+        })
+    }
 }
 
 // Maps Some("") to None to emulate how Go handles empty strings
@@ -332,6 +353,21 @@ impl ResolvedConfigurationOptions for RawTurboJson {
         } else {
             ConfigurationOptions::default()
         };
+
+        let cache_dir = if let Some(cache_dir) = self.cache_dir {
+            let cache_dir_str: &str = &cache_dir;
+            let cache_dir_unix = RelativeUnixPath::new(cache_dir_str).map_err(|_| {
+                let (span, text) = cache_dir.span_and_text("turbo.json");
+                Error::AbsoluteCacheDir { span, text }
+            })?;
+            // Convert the relative unix path to an anchored system path
+            // For unix/macos this is a no-op
+            let cache_dir_system = cache_dir_unix.to_anchored_system_path_buf();
+            Some(Utf8PathBuf::from(cache_dir_system.to_string()))
+        } else {
+            None
+        };
+
         // Don't allow token to be set for shared config.
         opts.token = None;
         opts.spaces_id = self
@@ -342,6 +378,7 @@ impl ResolvedConfigurationOptions for RawTurboJson {
         opts.allow_no_package_manager = self.allow_no_package_manager;
         opts.daemon = self.daemon.map(|daemon| *daemon.as_inner());
         opts.env_mode = self.env_mode;
+        opts.cache_dir = cache_dir;
         Ok(opts)
     }
 }
@@ -380,6 +417,7 @@ fn get_env_var_config(
     );
     turbo_mapping.insert(OsString::from("turbo_daemon"), "daemon");
     turbo_mapping.insert(OsString::from("turbo_env_mode"), "env_mode");
+    turbo_mapping.insert(OsString::from("turbo_cache_dir"), "cache_dir");
     turbo_mapping.insert(OsString::from("turbo_preflight"), "preflight");
     turbo_mapping.insert(OsString::from("turbo_scm_base"), "scm_base");
     turbo_mapping.insert(OsString::from("turbo_scm_head"), "scm_head");
@@ -490,6 +528,8 @@ fn get_env_var_config(
             _ => None,
         });
 
+    let cache_dir = output_map.get("cache_dir").map(|s| s.clone().into());
+
     // We currently don't pick up a Spaces ID via env var, we likely won't
     // continue using the Spaces name, we can add an env var when we have the
     // name we want to stick with.
@@ -517,6 +557,7 @@ fn get_env_var_config(
         upload_timeout,
         spaces_id,
         env_mode,
+        cache_dir,
     };
 
     Ok(output)
@@ -580,6 +621,7 @@ fn get_override_env_var_config(
         spaces_id: None,
         allow_no_package_manager: None,
         env_mode: None,
+        cache_dir: None,
     };
 
     Ok(output)
@@ -722,6 +764,7 @@ impl TurborepoConfigBuilder {
     );
     create_builder!(with_daemon, daemon, Option<bool>);
     create_builder!(with_env_mode, env_mode, Option<EnvMode>);
+    create_builder!(with_cache_dir, cache_dir, Option<Utf8PathBuf>);
 
     pub fn build(&self) -> Result<ConfigurationOptions, Error> {
         // Priority, from least significant to most significant:
@@ -816,6 +859,9 @@ impl TurborepoConfigBuilder {
                     if let Some(scm_head) = current_source_config.scm_head {
                         acc.scm_head = Some(scm_head);
                     }
+                    if let Some(cache_dir) = current_source_config.cache_dir {
+                        acc.cache_dir = Some(cache_dir);
+                    }
 
                     acc
                 })
@@ -828,6 +874,7 @@ impl TurborepoConfigBuilder {
 mod test {
     use std::{collections::HashMap, ffi::OsString};
 
+    use camino::Utf8PathBuf;
     use tempfile::TempDir;
     use turbopath::AbsoluteSystemPathBuf;
 
@@ -865,6 +912,7 @@ mod test {
         let turbo_team = "vercel";
         let turbo_teamid = "team_nLlpyC6REAqxydlFKbrMDlud";
         let turbo_token = "abcdef1234567890abcdef";
+        let cache_dir = Utf8PathBuf::from("nebulo9");
         let turbo_remote_cache_timeout = 200;
 
         env.insert("turbo_api".into(), turbo_api.into());
@@ -884,6 +932,7 @@ mod test {
         env.insert("turbo_daemon".into(), "true".into());
         env.insert("turbo_preflight".into(), "true".into());
         env.insert("turbo_env_mode".into(), "strict".into());
+        env.insert("turbo_cache_dir".into(), cache_dir.clone().into());
 
         let config = get_env_var_config(&env).unwrap();
         assert!(config.preflight());
@@ -897,6 +946,7 @@ mod test {
         assert_eq!(Some(true), config.allow_no_package_manager);
         assert_eq!(Some(true), config.daemon);
         assert_eq!(Some(EnvMode::Strict), config.env_mode);
+        assert_eq!(cache_dir, config.cache_dir.unwrap());
     }
 
     #[test]
diff --git a/crates/turborepo-lib/src/opts.rs b/crates/turborepo-lib/src/opts.rs
@@ -1,5 +1,6 @@
 use std::{backtrace, backtrace::Backtrace};
 
+use camino::Utf8PathBuf;
 use thiserror::Error;
 use turbopath::AnchoredSystemPathBuf;
 use turborepo_api_client::APIAuth;
@@ -149,6 +150,7 @@ pub struct RunOpts {
     pub(crate) concurrency: u32,
     pub(crate) parallel: bool,
     pub(crate) env_mode: EnvMode,
+    pub(crate) cache_dir: Utf8PathBuf,
     // Whether or not to infer the framework for each workspace.
     pub(crate) framework_inference: bool,
     pub profile: Option<String>,
@@ -263,6 +265,7 @@ impl<'a> TryFrom<OptsInputs<'a>> for RunOpts {
             graph,
             dry_run: inputs.run_args.dry_run,
             env_mode: inputs.config.env_mode(),
+            cache_dir: inputs.config.cache_dir().into(),
             is_github_actions,
         })
     }
@@ -358,7 +361,7 @@ impl<'a> From<OptsInputs<'a>> for CacheOpts {
         ));
 
         CacheOpts {
-            override_dir: inputs.execution_args.cache_dir.clone(),
+            cache_dir: inputs.config.cache_dir().into(),
             skip_filesystem: inputs.execution_args.remote_only,
             remote_cache_read_only: inputs.run_args.remote_cache_read_only,
             workers: inputs.run_args.cache_workers,
@@ -488,6 +491,7 @@ mod test {
             concurrency: 10,
             parallel: opts_input.parallel,
             env_mode: crate::cli::EnvMode::Loose,
+            cache_dir: camino::Utf8PathBuf::new(),
             framework_inference: true,
             profile: None,
             continue_on_error: opts_input.continue_on_error,
diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs
@@ -135,6 +135,8 @@ pub struct RawTurboJson {
     pub daemon: Option<Spanned<bool>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub env_mode: Option<EnvMode>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_dir: Option<Spanned<UnescapedString>>,
 
     #[deserializable(rename = "//")]
     #[serde(skip)]
diff --git a/crates/turborepo-lib/src/turbo_json/parser.rs b/crates/turborepo-lib/src/turbo_json/parser.rs
@@ -105,6 +105,7 @@ impl WithMetadata for RawTurboJson {
         self.global_env.add_text(text.clone());
         self.global_pass_through_env.add_text(text.clone());
         self.tasks.add_text(text.clone());
+        self.cache_dir.add_text(text.clone());
         self.pipeline.add_text(text);
     }
 
@@ -115,6 +116,7 @@ impl WithMetadata for RawTurboJson {
         self.global_env.add_path(path.clone());
         self.global_pass_through_env.add_path(path.clone());
         self.tasks.add_path(path.clone());
+        self.cache_dir.add_path(path.clone());
         self.pipeline.add_path(path);
     }
 }
diff --git a/turborepo-tests/integration/tests/config.t b/turborepo-tests/integration/tests/config.t

Original file line number	Diff line number	Diff line change
`@@ -105,6 +105,7 @@ impl WithMetadata for RawTurboJson {`
`105`	`105`	`self.global_env.add_text(text.clone());`
`106`	`106`	`self.global_pass_through_env.add_text(text.clone());`
`107`	`107`	`self.tasks.add_text(text.clone());`
	`108`	`+ self.cache_dir.add_text(text.clone());`
`108`	`109`	`self.pipeline.add_text(text);`
`109`	`110`	`}`
`110`	`111`
`@@ -115,6 +116,7 @@ impl WithMetadata for RawTurboJson {`
`115`	`116`	`self.global_env.add_path(path.clone());`
`116`	`117`	`self.global_pass_through_env.add_path(path.clone());`
`117`	`118`	`self.tasks.add_path(path.clone());`
	`119`	`+ self.cache_dir.add_path(path.clone());`
`118`	`120`	`self.pipeline.add_path(path);`
`119`	`121`	`}`
`120`	`122`	`}`