feat(clone): turbo clone (#9904)

### Description

Implements `turbo clone`, essentially a small wrapper around `git clone`
that does simple optimizations like blobless clones for local
development and treeless clones for CI.

### Testing Instructions

Wrote some tests but not convinced they're actually effective. See
comments below

Author: NicholasLYang

crates/turborepo-lib/src/cli/error.rs

@@ -26,6 +26,8 @@ pub enum Error {
     #[error(transparent)]
     Boundaries(#[from] crate::boundaries::Error),
     #[error(transparent)]
+    Clone(#[from] crate::commands::clone::Error),
+    #[error(transparent)]
     Path(#[from] turbopath::PathError),
     #[error(transparent)]
     #[diagnostic(transparent)]

crates/turborepo-lib/src/cli/mod.rs

@@ -28,8 +28,8 @@ use turborepo_ui::{ColorConfig, GREY};
 use crate::{
     cli::error::print_potential_tasks,
     commands::{
-        bin, boundaries, config, daemon, generate, info, link, login, logout, ls, prune, query,
-        run, scan, telemetry, unlink, CommandBase,
+        bin, boundaries, clone, config, daemon, generate, info, link, login, logout, ls, prune,
+        query, run, scan, telemetry, unlink, CommandBase,
     },
     get_version,
     run::watch::WatchClient,
@@ -588,6 +588,17 @@ pub enum Command {
         #[clap(short = 'F', long, group = "scope-filter-group")]
         filter: Vec<String>,
     },
+    #[clap(hide = true)]
+    Clone {
+        url: String,
+        dir: Option<String>,
+        #[clap(long, conflicts_with = "local")]
+        ci: bool,
+        #[clap(long, conflicts_with = "ci")]
+        local: bool,
+        #[clap(long)]
+        depth: Option<usize>,
+    },
     /// Generate the autocompletion script for the specified shell
     Completion { shell: Shell },
     /// Runs the Turborepo background daemon
@@ -1360,7 +1371,6 @@ pub async fn run(
     };
 
     cli_args.command = Some(command);
-    cli_args.cwd = Some(repo_root.as_path().to_owned());
 
     let root_telemetry = GenericEventBuilder::new();
     root_telemetry.track_start();
@@ -1389,6 +1399,18 @@ pub async fn run(
 
             Ok(boundaries::run(base, event).await?)
         }
+        Command::Clone {
+            url,
+            dir,
+            ci,
+            local,
+            depth,
+        } => {
+            let event = CommandEventBuilder::new("clone").with_parent(&root_telemetry);
+            event.track_call();
+
+            Ok(clone::run(cwd, url, dir.as_deref(), *ci, *local, *depth)?)
+        }
         #[allow(unused_variables)]
         Command::Daemon { command, idle_time } => {
             let event = CommandEventBuilder::new("daemon").with_parent(&root_telemetry);

crates/turborepo-lib/src/commands/clone.rs

@@ -0,0 +1,53 @@
+use std::env::current_dir;
+
+use camino::Utf8Path;
+use thiserror::Error;
+use turbopath::AbsoluteSystemPathBuf;
+use turborepo_ci::is_ci;
+use turborepo_scm::clone::{CloneMode, Git};
+
+#[derive(Debug, Error)]
+pub enum Error {
+    #[error("path is not valid UTF-8")]
+    Path(#[from] camino::FromPathBufError),
+
+    #[error(transparent)]
+    Turbopath(#[from] turbopath::PathError),
+
+    #[error("failed to clone repository")]
+    Scm(#[from] turborepo_scm::Error),
+}
+
+pub fn run(
+    cwd: Option<&Utf8Path>,
+    url: &str,
+    dir: Option<&str>,
+    ci: bool,
+    local: bool,
+    depth: Option<usize>,
+) -> Result<i32, Error> {
+    // We do *not* want to use the repo root but the actual, literal cwd for clone
+    let cwd = if let Some(cwd) = cwd {
+        cwd.to_owned()
+    } else {
+        current_dir()
+            .expect("could not get current directory")
+            .try_into()?
+    };
+    let abs_cwd = AbsoluteSystemPathBuf::from_cwd(cwd)?;
+
+    let clone_mode = if ci {
+        CloneMode::CI
+    } else if local {
+        CloneMode::Local
+    } else if is_ci() {
+        CloneMode::CI
+    } else {
+        CloneMode::Local
+    };
+
+    let git = Git::find()?;
+    git.clone(url, abs_cwd, dir, None, clone_mode, depth)?;
+
+    Ok(0)
+}

crates/turborepo-lib/src/commands/mod.rs

@@ -16,6 +16,7 @@ use crate::{
 
 pub(crate) mod bin;
 pub(crate) mod boundaries;
+pub(crate) mod clone;
 pub(crate) mod config;
 pub(crate) mod daemon;
 pub(crate) mod generate;

crates/turborepo-lib/src/diagnostics.rs

@@ -11,7 +11,7 @@ use tokio::{
 };
 use turbo_updater::check_for_updates;
 use turbopath::AbsoluteSystemPathBuf;
-use turborepo_scm::Git;
+use turborepo_scm::GitRepo;
 
 use crate::{
     commands::{
@@ -157,7 +157,7 @@ impl Diagnostic for GitDaemonDiagnostic {
                     // get the current setting
                     let stdout = Stdio::piped();
 
-                    let Ok(git_path) = Git::find_bin() else {
+                    let Ok(git_path) = GitRepo::find_bin() else {
                         return Err("git not found");
                     };
 

crates/turborepo-scm/src/clone.rs

@@ -0,0 +1,88 @@
+use std::{backtrace::Backtrace, process::Command};
+
+use turbopath::AbsoluteSystemPathBuf;
+
+use crate::{Error, GitRepo};
+
+pub enum CloneMode {
+    /// Cloning locally, do a blobless clone (good UX and reasonably fast)
+    Local,
+    /// Cloning on CI, do a treeless clone (worse UX but fastest)
+    CI,
+}
+
+// Provide a sane maximum depth for cloning. If a user needs more history than
+// this, they can override or fetch it themselves.
+const MAX_CLONE_DEPTH: usize = 64;
+
+/// A wrapper around the git binary that is not tied to a specific repo.
+pub struct Git {
+    bin: AbsoluteSystemPathBuf,
+}
+
+impl Git {
+    pub fn find() -> Result<Self, Error> {
+        Ok(Self {
+            bin: GitRepo::find_bin()?,
+        })
+    }
+
+    pub fn spawn_git_command(
+        &self,
+        cwd: &AbsoluteSystemPathBuf,
+        args: &[&str],
+        pathspec: &str,
+    ) -> Result<(), Error> {
+        let mut command = Command::new(self.bin.as_std_path());
+        command
+            .args(args)
+            .current_dir(cwd)
+            .env("GIT_OPTIONAL_LOCKS", "0");
+
+        if !pathspec.is_empty() {
+            command.arg("--").arg(pathspec);
+        }
+
+        let output = command.spawn()?.wait_with_output()?;
+
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr).to_string();
+            Err(Error::Git(stderr, Backtrace::capture()))
+        } else {
+            Ok(())
+        }
+    }
+
+    pub fn clone(
+        &self,
+        url: &str,
+        cwd: AbsoluteSystemPathBuf,
+        dir: Option<&str>,
+        branch: Option<&str>,
+        mode: CloneMode,
+        depth: Option<usize>,
+    ) -> Result<(), Error> {
+        let depth = depth.unwrap_or(MAX_CLONE_DEPTH).to_string();
+        let mut args = vec!["clone", "--depth", &depth];
+        if let Some(branch) = branch {
+            args.push("--branch");
+            args.push(branch);
+        }
+        match mode {
+            CloneMode::Local => {
+                args.push("--filter=blob:none");
+            }
+            CloneMode::CI => {
+                args.push("--filter=tree:0");
+            }
+        }
+        args.push(url);
+        if let Some(dir) = dir {
+            args.push(dir);
+        }
+
+        self.spawn_git_command(&cwd, &args, "")?;
+
+        Ok(())
+    }
+}

crates/turborepo-scm/src/git.rs

@@ -14,7 +14,7 @@ use turbopath::{
 };
 use turborepo_ci::Vendor;
 
-use crate::{Error, Git, SCM};
+use crate::{Error, GitRepo, SCM};
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct InvalidRange {
@@ -171,7 +171,7 @@ impl CIEnv {
     }
 }
 
-impl Git {
+impl GitRepo {
     fn get_current_branch(&self) -> Result<String, Error> {
         let output = self.execute_git_command(&["branch", "--show-current"], "")?;
         let output = String::from_utf8(output)?;
@@ -324,7 +324,7 @@ impl Git {
         Ok(files)
     }
 
-    fn execute_git_command(&self, args: &[&str], pathspec: &str) -> Result<Vec<u8>, Error> {
+    pub fn execute_git_command(&self, args: &[&str], pathspec: &str) -> Result<Vec<u8>, Error> {
         let mut command = Command::new(self.bin.as_std_path());
         command
             .args(args)
@@ -432,7 +432,7 @@ mod tests {
     use super::{previous_content, CIEnv, InvalidRange};
     use crate::{
         git::{GitHubCommit, GitHubEvent},
-        Error, Git, SCM,
+        Error, GitRepo, SCM,
     };
 
     fn setup_repository(
@@ -1045,7 +1045,7 @@ mod tests {
             repo.branch(branch, &commit, true).unwrap();
         });
 
-        let thing = Git::find(&root).unwrap();
+        let thing = GitRepo::find(&root).unwrap();
         let actual = thing.resolve_base(target_branch, CIEnv::none()).ok();
 
         assert_eq!(actual.as_deref(), expected);
@@ -1281,7 +1281,7 @@ mod tests {
             Err(VarError::NotPresent)
         };
 
-        let actual = Git::get_github_base_ref(CIEnv {
+        let actual = GitRepo::get_github_base_ref(CIEnv {
             is_github_actions: test_case.env.is_github_actions,
             github_base_ref: test_case.env.github_base_ref,
             github_event_path: temp_file

crates/turborepo-scm/src/lib.rs

@@ -19,6 +19,7 @@ use thiserror::Error;
 use tracing::debug;
 use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError, RelativeUnixPathBuf};
 
+pub mod clone;
 pub mod git;
 mod hash_object;
 mod ls_tree;
@@ -171,7 +172,7 @@ pub(crate) fn wait_for_success<R: Read, T>(
 }
 
 #[derive(Debug, Clone)]
-pub struct Git {
+pub struct GitRepo {
     root: AbsoluteSystemPathBuf,
     bin: AbsoluteSystemPathBuf,
 }
@@ -184,7 +185,7 @@ enum GitError {
     Root(AbsoluteSystemPathBuf, Error),
 }
 
-impl Git {
+impl GitRepo {
     fn find(path_in_repo: &AbsoluteSystemPath) -> Result<Self, GitError> {
         // If which produces an invalid absolute path, it's not an execution error, it's
         // a programming error. We expect it to always give us an absolute path
@@ -236,17 +237,19 @@ fn find_git_root(turbo_root: &AbsoluteSystemPath) -> Result<AbsoluteSystemPathBu
 
 #[derive(Debug, Clone)]
 pub enum SCM {
-    Git(Git),
+    Git(GitRepo),
     Manual,
 }
 
 impl SCM {
     #[tracing::instrument]
     pub fn new(path_in_repo: &AbsoluteSystemPath) -> SCM {
-        Git::find(path_in_repo).map(SCM::Git).unwrap_or_else(|e| {
-            debug!("{}, continuing with manual hashing", e);
-            SCM::Manual
-        })
+        GitRepo::find(path_in_repo)
+            .map(SCM::Git)
+            .unwrap_or_else(|e| {
+                debug!("{}, continuing with manual hashing", e);
+                SCM::Manual
+            })
     }
 
     pub fn is_manual(&self) -> bool {

crates/turborepo-scm/src/ls_tree.rs

@@ -6,9 +6,9 @@ use std::{
 use nom::Finish;
 use turbopath::{AbsoluteSystemPathBuf, RelativeUnixPathBuf};
 
-use crate::{wait_for_success, Error, Git, GitHashes};
+use crate::{wait_for_success, Error, GitHashes, GitRepo};
 
-impl Git {
+impl GitRepo {
     #[tracing::instrument(skip(self))]
     pub fn git_ls_tree(&self, root_path: &AbsoluteSystemPathBuf) -> Result<GitHashes, Error> {
         let mut hashes = GitHashes::new();

crates/turborepo-scm/src/package_deps.rs

@@ -8,7 +8,7 @@ use turborepo_telemetry::events::task::{FileHashMethod, PackageTaskEventBuilder}
 
 #[cfg(feature = "git2")]
 use crate::hash_object::hash_objects;
-use crate::{Error, Git, GitHashes, SCM};
+use crate::{Error, GitHashes, GitRepo, SCM};
 
 pub const INPUT_INCLUDE_DEFAULT_FILES: &str = "$TURBO_DEFAULT$";
 
@@ -112,7 +112,7 @@ impl SCM {
     }
 }
 
-impl Git {
+impl GitRepo {
     fn get_package_file_hashes<S: AsRef<str>>(
         &self,
         turbo_root: &AbsoluteSystemPath,

crates/turborepo-scm/src/status.rs

@@ -9,9 +9,9 @@ use std::{
 use nom::Finish;
 use turbopath::{AbsoluteSystemPath, RelativeUnixPathBuf};
 
-use crate::{wait_for_success, Error, Git, GitHashes};
+use crate::{wait_for_success, Error, GitHashes, GitRepo};
 
-impl Git {
+impl GitRepo {
     #[tracing::instrument(skip(self, root_path, hashes))]
     pub(crate) fn append_git_status(
         &self,