Merge branch 'main' into shew-e2ab2

CONTRIBUTING.md

@@ -188,6 +188,11 @@ See [the publishing guide](./release.md).
 
 Contributing to examples helps the Turborepo community by showcasing how to use Turborepo in real-world scenarios with other tools and frameworks. They can be found in [the examples directory](https://github.com/vercel/turborepo/tree/main/examples) of this repository.
 
+> [!IMPORTANT]
+> As Turborepo usage has grown, the community has contributed more and more examples to the repository. While this is exciting for us on the core team, we're unable to maintain the full surface area of every example, given the constant updates across the breadth of tooling that Turborepo works with in the examples.
+>
+> Because of this, a handful of the examples are explictly marked as maintained by the core team. For the rest, we work with the community to keep them as up to date and correct as possible. If you find a problem with a community-supported template, we ask that you do not open a GitHub Issue for it. Instead, please open a pull request with the needed fixes.
+
 The `basic` example is the default used by `create-turbo`.
 
 For simplicity, each example is treated as a standalone "repository", separate from the rest of the repository, with its own dependencies, lockfile, `turbo` version, etc. You are able to run code and make code updates in an example without needing to install the dependencies of the rest of the repository.

cli/Makefile

@@ -5,7 +5,7 @@ CLI_DIR = $(shell pwd)
 
 # This only builds JS packages
 build:
-	cd $(CLI_DIR)/../ && turbo build \
+	cd $(CLI_DIR)/../ && turbo build copy-schema \
 		--filter=create-turbo \
 		--filter=@turbo/codemod \
 		--filter=turbo-ignore \

crates/turborepo-cache/src/config.rs

@@ -213,6 +213,10 @@ mod test {
     )]
     #[test_case("local:r", Ok(CacheConfig { local: CacheActions { read: true, write: false }, remote: CacheActions { read: false, write: false } }) ; "local:r"
     )]
+    #[test_case("remote:r", Ok(CacheConfig { local: CacheActions { read: false, write: false }, remote: CacheActions { read: true, write: false } }) ; "remote:r"
+    )]
+    #[test_case("local:rw,remote:r", Ok(CacheConfig { local: CacheActions { read: true, write: true }, remote: CacheActions { read: true, write: false } }) ; "local:rw,remote:r"
+    )]
     #[test_case("local:", Ok(CacheConfig { local: CacheActions { read: false, write: false }, remote: CacheActions { read: false, write: false } }) ; "empty action"
     )]
     #[test_case("local:,remote:", Ok(CacheConfig { local: CacheActions { read: false, write: false }, remote: CacheActions { read: false, write: false } }) ; "multiple empty actions"

crates/turborepo-env/src/lib.rs

@@ -13,7 +13,7 @@ use thiserror::Error;
 
 pub mod platform;
 
-const DEFAULT_ENV_VARS: &[&str] = ["VERCEL_ANALYTICS_ID", "VERCEL_ENV"].as_slice();
+const DEFAULT_ENV_VARS: &[&str] = ["VERCEL_ANALYTICS_ID", "VERCEL_TARGET_ENV"].as_slice();
 
 #[derive(Clone, Debug, Error)]
 pub enum Error {
@@ -218,7 +218,7 @@ impl EnvironmentVariableMap {
     // that user exclusions have primacy over inferred inclusions.
     pub fn wildcard_map_from_wildcards_unresolved(
         &self,
-        wildcard_patterns: &[String],
+        wildcard_patterns: &[impl AsRef<str>],
     ) -> Result<WildcardMaps, Error> {
         if wildcard_patterns.is_empty() {
             return Ok(WildcardMaps {
@@ -229,6 +229,60 @@ impl EnvironmentVariableMap {
 
         self.wildcard_map_from_wildcards(wildcard_patterns)
     }
+
+    /// Return a detailed map for which environment variables are factored into
+    /// the task's hash
+    pub fn hashable_task_env(
+        &self,
+        computed_wildcards: &[String],
+        task_env: &[String],
+    ) -> Result<DetailedMap, Error> {
+        let mut explicit_env_var_map = EnvironmentVariableMap::default();
+        let mut all_env_var_map = EnvironmentVariableMap::default();
+        let mut matching_env_var_map = EnvironmentVariableMap::default();
+        let inference_env_var_map = self.from_wildcards(computed_wildcards)?;
+
+        let user_env_var_set = self.wildcard_map_from_wildcards_unresolved(task_env)?;
+
+        all_env_var_map.union(&user_env_var_set.inclusions);
+        all_env_var_map.union(&inference_env_var_map);
+        all_env_var_map.difference(&user_env_var_set.exclusions);
+
+        explicit_env_var_map.union(&user_env_var_set.inclusions);
+        explicit_env_var_map.difference(&user_env_var_set.exclusions);
+
+        matching_env_var_map.union(&inference_env_var_map);
+        matching_env_var_map.difference(&user_env_var_set.exclusions);
+
+        Ok(DetailedMap {
+            all: all_env_var_map,
+            by_source: BySource {
+                explicit: explicit_env_var_map,
+                matching: matching_env_var_map,
+            },
+        })
+    }
+
+    /// Constructs an environment map that contains pass through environment
+    /// variables
+    pub fn pass_through_env(
+        &self,
+        builtins: &[&str],
+        global_env: &Self,
+        task_pass_through: &[impl AsRef<str>],
+    ) -> Result<Self, Error> {
+        let mut pass_through_env = EnvironmentVariableMap::default();
+        let default_env_var_pass_through_map = self.from_wildcards(builtins)?;
+        let task_pass_through_env =
+            self.wildcard_map_from_wildcards_unresolved(task_pass_through)?;
+
+        pass_through_env.union(&default_env_var_pass_through_map);
+        pass_through_env.union(global_env);
+        pass_through_env.union(&task_pass_through_env.inclusions);
+        pass_through_env.difference(&task_pass_through_env.exclusions);
+
+        Ok(pass_through_env)
+    }
 }
 
 const WILDCARD: char = '*';
@@ -336,13 +390,13 @@ mod tests {
         }
     }
 
-    #[test_case(&[], &["VERCEL_ANALYTICS_ID", "VERCEL_ENV"] ; "defaults")]
+    #[test_case(&[], &["VERCEL_ANALYTICS_ID", "VERCEL_TARGET_ENV"] ; "defaults")]
     #[test_case(&["!VERCEL*"], &[] ; "removing defaults")]
-    #[test_case(&["FOO*", "!FOOD"], &["FOO", "FOOBAR", "VERCEL_ANALYTICS_ID", "VERCEL_ENV"] ; "intersecting globs")]
+    #[test_case(&["FOO*", "!FOOD"], &["FOO", "FOOBAR", "VERCEL_ANALYTICS_ID", "VERCEL_TARGET_ENV"] ; "intersecting globs")]
     fn test_global_env(inputs: &[&str], expected: &[&str]) {
         let env_at_start = EnvironmentVariableMap(
             vec![
-                ("VERCEL_ENV", "prod"),
+                ("VERCEL_TARGET_ENV", "prod"),
                 ("VERCEL_ANALYTICS_ID", "1"),
                 ("FOO", "bar"),
                 ("FOOBAR", "baz"),
@@ -358,4 +412,58 @@ mod tests {
         actual.sort();
         assert_eq!(actual, expected);
     }
+
+    #[test_case(&["FOO*"], &["BAR"], &["BAR", "FOO", "FOOBAR", "FOOD"] ; "wildcard")]
+    #[test_case(&["FOO*", "!FOOBAR"], &["BAR"], &["BAR", "FOO", "FOOD"] ; "omit wild")]
+    #[test_case(&["FOO*"], &["!FOOBAR"], &["FOO", "FOOD"] ; "omit task")]
+    fn test_hashable_env(wildcards: &[&str], task: &[&str], expected: &[&str]) {
+        let env_at_start = EnvironmentVariableMap(
+            vec![
+                ("FOO", "bar"),
+                ("FOOBAR", "baz"),
+                ("FOOD", "cheese"),
+                ("BAR", "nuts"),
+            ]
+            .into_iter()
+            .map(|(k, v)| (k.to_owned(), v.to_owned()))
+            .collect(),
+        );
+        let wildcards: Vec<_> = wildcards.iter().map(|s| s.to_string()).collect();
+        let task: Vec<_> = task.iter().map(|s| s.to_string()).collect();
+        let output = env_at_start.hashable_task_env(&wildcards, &task).unwrap();
+        let mut actual: Vec<_> = output.all.keys().map(|s| s.as_str()).collect();
+        actual.sort();
+        assert_eq!(actual, expected);
+    }
+
+    #[test_case(&["FOO*"], &["FOO", "FOOBAR", "FOOD", "PATH"] ; "folds 3 sources")]
+    #[test_case(&["!FOO"], &["PATH"] ; "remove global")]
+    #[test_case(&["!PATH"], &["FOO"] ; "remove builtin")]
+    #[test_case(&["FOO*", "!FOOD"], &["FOO", "FOOBAR", "PATH"] ; "mixing negations")]
+    fn test_pass_through_env(task: &[&str], expected: &[&str]) {
+        let env_at_start = EnvironmentVariableMap(
+            vec![
+                ("PATH", "of"),
+                ("FOO", "bar"),
+                ("FOOBAR", "baz"),
+                ("FOOD", "cheese"),
+                ("BAR", "nuts"),
+            ]
+            .into_iter()
+            .map(|(k, v)| (k.to_owned(), v.to_owned()))
+            .collect(),
+        );
+        let global_env = EnvironmentVariableMap(
+            vec![("FOO", "bar")]
+                .into_iter()
+                .map(|(k, v)| (k.to_owned(), v.to_owned()))
+                .collect(),
+        );
+        let output = env_at_start
+            .pass_through_env(&["PATH"], &global_env, task)
+            .unwrap();
+        let mut actual: Vec<_> = output.keys().map(|s| s.as_str()).collect();
+        actual.sort();
+        assert_eq!(actual, expected);
+    }
 }

crates/turborepo-filewatch/src/optional_watch.rs

@@ -48,7 +48,7 @@ impl<T> OptionalWatch<T> {
 
 pub struct SomeRef<'a, T>(pub(crate) Ref<'a, Option<T>>);
 
-impl<'a, T> std::ops::Deref for SomeRef<'a, T> {
+impl<T> std::ops::Deref for SomeRef<'_, T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {

crates/turborepo-filewatch/src/scm_resource.rs

@@ -41,7 +41,7 @@ impl SCMResource {
     }
 }
 
-impl<'a> Deref for SCMPermit<'a> {
+impl Deref for SCMPermit<'_> {
     type Target = SCM;
 
     fn deref(&self) -> &Self::Target {

crates/turborepo-fs/Cargo.toml

@@ -11,9 +11,9 @@ workspace = true
 
 [dependencies]
 fs-err = "2.9.0"
+ignore = "0.4.22"
 thiserror = "1.0.38"
 turbopath = { workspace = true }
-walkdir = "2.3.3"
 
 [dev-dependencies]
 tempfile = { workspace = true }

crates/turborepo-fs/src/lib.rs

@@ -6,8 +6,8 @@ use std::{
 };
 
 use fs_err as fs;
+use ignore::WalkBuilder;
 use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf};
-use walkdir::WalkDir;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
@@ -16,7 +16,7 @@ pub enum Error {
     #[error(transparent)]
     Io(#[from] io::Error),
     #[error("Error walking directory during recursive copy: {0}")]
-    Walk(#[from] walkdir::Error),
+    Walk(#[from] ignore::Error),
 }
 
 pub fn recursive_copy(
@@ -28,8 +28,14 @@ pub fn recursive_copy(
     let src_metadata = src.symlink_metadata()?;
 
     if src_metadata.is_dir() {
-        let walker = WalkDir::new(src.as_path()).follow_links(false);
-        for entry in walker.into_iter() {
+        let walker = WalkBuilder::new(src.as_path())
+            .hidden(false)
+            .git_ignore(true)
+            .git_global(false)
+            .git_exclude(true)
+            .build();
+
+        for entry in walker {
             match entry {
                 Err(e) => {
                     if e.io_error().is_some() {
@@ -43,7 +49,9 @@ pub fn recursive_copy(
                 Ok(entry) => {
                     let path = entry.path();
                     let path = AbsoluteSystemPath::from_std_path(path)?;
-                    let file_type = entry.file_type();
+                    let file_type = entry
+                        .file_type()
+                        .expect("all dir entries aside from stdin should have a file type");
 
                     // Note that we also don't currently copy broken symlinks
                     if file_type.is_symlink() && path.stat().is_err() {
@@ -310,6 +318,51 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_recursive_copy_gitignore() -> Result<(), Error> {
+        // Directory layout:
+        //
+        // <src>/
+        //   .gitignore
+        //   invisible.txt <- ignored
+        //   dist/ <- ignored
+        //     output.txt
+        //   child/
+        //     seen.txt
+        //     .hidden
+        let (_src_tmp, src_dir) = tmp_dir()?;
+        // Need to create this for `.gitignore` to be respected
+        src_dir.join_component(".git").create_dir_all()?;
+        src_dir
+            .join_component(".gitignore")
+            .create_with_contents("invisible.txt\ndist/\n")?;
+        src_dir
+            .join_component("invisible.txt")
+            .create_with_contents("not here")?;
+        let output = src_dir.join_components(&["dist", "output.txt"]);
+        output.ensure_dir()?;
+        output.create_with_contents("hi!")?;
+
+        let child = src_dir.join_component("child");
+        let seen = child.join_component("seen.txt");
+        seen.ensure_dir()?;
+        seen.create_with_contents("here")?;
+        let hidden = child.join_component(".hidden");
+        hidden.create_with_contents("polo")?;
+
+        let (_dst_tmp, dst_dir) = tmp_dir()?;
+        recursive_copy(&src_dir, &dst_dir)?;
+
+        assert!(dst_dir.join_component(".gitignore").exists());
+        assert!(!dst_dir.join_component("invisible.txt").exists());
+        assert!(!dst_dir.join_component("dist").exists());
+        assert!(dst_dir.join_component("child").exists());
+        assert!(dst_dir.join_components(&["child", "seen.txt"]).exists());
+        assert!(dst_dir.join_components(&["child", ".hidden"]).exists());
+
+        Ok(())
+    }
+
     fn assert_file_matches(a: impl AsRef<AbsoluteSystemPath>, b: impl AsRef<AbsoluteSystemPath>) {
         let a = a.as_ref();
         let b = b.as_ref();