diff --git a/packages/zpm-config/schema.json b/packages/zpm-config/schema.json index 3a019028..95050edb 100644 --- a/packages/zpm-config/schema.json +++ b/packages/zpm-config/schema.json @@ -267,6 +267,16 @@ "description": "The linker to use for node_modules", "default": "pnp" }, + "nodeExperimentalPackageMap": { + "type": "boolean", + "description": "Whether to inject the generated Node.js package map into NODE_OPTIONS for node-modules and pnpm installs", + "default": false + }, + "nodePackageMapType": { + "type": "crate::NodePackageMapType", + "description": "Whether generated package maps should reflect declared dependencies (standard) or the installed node_modules layout (loose)", + "default": "standard" + }, "nmHoistingLimits": { "type": "crate::NmHoistingLimits", "description": "How far the node-modules linker is allowed to hoist a workspace's dependencies (none / workspaces / dependencies)", diff --git a/packages/zpm-config/src/lib.rs b/packages/zpm-config/src/lib.rs index b2518b0f..5c1acef9 100644 --- a/packages/zpm-config/src/lib.rs +++ b/packages/zpm-config/src/lib.rs @@ -1398,6 +1398,7 @@ merge_optional_settings!(zpm_utils::Os); merge_optional_settings!(zpm_utils::Secret); merge_settings!(crate::types::NodeLinker, |s: &str| FromFileString::from_file_string(s).unwrap()); +merge_settings!(crate::types::NodePackageMapType, |s: &str| FromFileString::from_file_string(s).unwrap()); merge_settings!(crate::types::IslandLinker, |s: &str| FromFileString::from_file_string(s).unwrap()); merge_settings!(crate::types::PnpFallbackMode, |s: &str| FromFileString::from_file_string(s).unwrap()); merge_settings!(crate::types::NmHoistingLimits, |s: &str| FromFileString::from_file_string(s).unwrap()); @@ -1407,6 +1408,7 @@ merge_settings!(crate::types::LogLevel, |s: &str| FromFileString::from_file_stri merge_settings!(crate::types::NpmPublishAccess, |s: &str| FromFileString::from_file_string(s).unwrap()); merge_settings!(crate::types::EcosystemFilter, |s: &str| FromFileString::from_file_string(s).unwrap()); merge_optional_settings!(crate::types::NodeLinker); +merge_optional_settings!(crate::types::NodePackageMapType); merge_optional_settings!(crate::types::IslandLinker); merge_optional_settings!(crate::types::PnpFallbackMode); merge_optional_settings!(crate::types::NmHoistingLimits); diff --git a/packages/zpm-config/src/types.rs b/packages/zpm-config/src/types.rs index 5676c4c8..eb147296 100644 --- a/packages/zpm-config/src/types.rs +++ b/packages/zpm-config/src/types.rs @@ -15,6 +15,16 @@ pub enum NodeLinker { NodeModules, } +#[zpm_enum(error = ConfigurationError, or_else = |s| Err(ConfigurationError::EnumError(s.to_string())))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NodePackageMapType { + #[literal("standard")] + Standard, + + #[literal("loose")] + Loose, +} + #[zpm_enum(error = ConfigurationError, or_else = |s| Err(ConfigurationError::EnumError(s.to_string())))] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IslandLinker { diff --git a/packages/zpm-utils/src/path.rs b/packages/zpm-utils/src/path.rs index 7d9d9d01..36b42f6d 100644 --- a/packages/zpm-utils/src/path.rs +++ b/packages/zpm-utils/src/path.rs @@ -341,6 +341,22 @@ impl Path { } } + pub fn without_trailing_separators(&self) -> Path { + if self.is_root() { + return self.clone(); + } + + let trimmed = self.path.trim_end_matches('/'); + + if trimmed.len() == self.path.len() { + self.clone() + } else { + Path { + path: trimmed.to_string(), + } + } + } + pub fn extname<'a>(&'a self) -> Option<&'a str> { self.basename().and_then(|basename| { if let Some(mut last_dot) = basename.rfind('.') { @@ -1154,5 +1170,27 @@ impl ToHumanString for Path { } } +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::Path; + + #[test] + fn normalizes_repeated_trailing_separators() { + assert_eq!(Path::from_str("foo//").unwrap().as_str(), "foo/"); + assert_eq!(Path::from_str("/foo///").unwrap().as_str(), "/foo/"); + assert_eq!(Path::from_str("///").unwrap().as_str(), "/"); + } + + #[test] + fn removes_trailing_separators() { + assert_eq!(Path::from_str("foo/").unwrap().without_trailing_separators().as_str(), "foo"); + assert_eq!(Path::from_str("/foo/").unwrap().without_trailing_separators().as_str(), "/foo"); + assert_eq!(Path::from_str("/").unwrap().without_trailing_separators().as_str(), "/"); + assert_eq!(Path::new().without_trailing_separators().as_str(), ""); + } +} + impl_file_string_from_str!(Path); impl_file_string_serialization!(Path); diff --git a/packages/zpm/src/error.rs b/packages/zpm/src/error.rs index 712ece13..0464cc3a 100644 --- a/packages/zpm/src/error.rs +++ b/packages/zpm/src/error.rs @@ -295,6 +295,9 @@ pub enum Error { #[error("Lockfile generation error: {0}")] LockfileGenerationError(zpm_parsers::Error), + #[error("Package map generation error: {0}")] + PackageMapGenerationError(String), + #[error("Incompatible options: {}", .0.join(", "))] IncompatibleOptions(Vec), diff --git a/packages/zpm/src/linker/mod.rs b/packages/zpm/src/linker/mod.rs index f992fe55..6684fcec 100644 --- a/packages/zpm/src/linker/mod.rs +++ b/packages/zpm/src/linker/mod.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use zpm_config::{IslandLinker, NodeLinker}; use zpm_primitives::Locator; -use zpm_utils::Path; +use zpm_utils::{IoResultExt, Path}; use crate::{ build::BuildRequests, @@ -13,6 +13,7 @@ use crate::{ pub mod helpers; pub mod nm; +pub mod package_map; pub mod pnpm; pub mod pnp; pub mod venv; @@ -73,5 +74,11 @@ fn cleanup_inactive_linker_artifacts(project: &Project) -> Result<(), Error> { } } + if active == NodeLinker::Pnp { + project.package_map_path(None) + .fs_rm() + .ok_missing()?; + } + Ok(()) } diff --git a/packages/zpm/src/linker/nm/mod.rs b/packages/zpm/src/linker/nm/mod.rs index a0c6841b..6deac142 100644 --- a/packages/zpm/src/linker/nm/mod.rs +++ b/packages/zpm/src/linker/nm/mod.rs @@ -5,7 +5,7 @@ use zpm_sync::{SyncItem, SyncTemplate, SyncTree}; use zpm_utils::{FromFileString, IoResultExt, Path, ToHumanString}; use crate::{ - build::{self, BuildRequest, BuildRequests}, content_flags, error::Error, fetchers::PackageData, install::Install, linker::{self, LinkResult, helpers::PackageMeta, nm::hoist::{Hoister, WorkTree}}, project::Project + build::{self, BuildRequest, BuildRequests}, content_flags, error::Error, fetchers::PackageData, install::Install, linker::{self, LinkResult, helpers::PackageMeta, nm::hoist::{Hoister, WorkTree}, package_map::{NodeModulesPackageMapBuilder, persist_package_map, persist_package_map_at}}, project::Project }; pub mod hoist; @@ -86,6 +86,7 @@ fn register_workspace_symlinks_at( project: &Project, install: &Install, workspace_nm_tree: &mut SyncTree, + mut package_map_builder: Option<&mut NodeModulesPackageMapBuilder>, host_node: &hoist::WorkNode, host_abs_path: &Path, candidate_workspaces: impl IntoIterator, @@ -147,9 +148,20 @@ fn register_workspace_symlinks_at( = workspace_dir .relative_to(&host_abs_path.with_join(&symlink_path).dirname().unwrap_or_default()); + let symlink_location + = host_abs_path.with_join(&symlink_path); + workspace_nm_tree.register_entry(symlink_path, SyncItem::Symlink { target_path, })?; + + if let Some(package_map_builder) = package_map_builder.as_deref_mut() { + package_map_builder.register_package( + symlink_location, + workspace_dir, + &target_locator, + ); + } } Ok(()) @@ -187,11 +199,15 @@ fn generate_workspace_node_modules( install: &Install, work_tree: &WorkTree, workspace_node_idx: usize, + package_map_builder: Option<&mut NodeModulesPackageMapBuilder>, packages_by_location: &mut BTreeMap, canonical_build_locations: &mut BTreeMap, force_rebuild_locators: &mut BTreeSet, cas_extractions: &mut Vec<(Path, Locator)>, ) -> Result<(), Error> { + let mut package_map_builder + = package_map_builder; + let hardlinks_mode = matches!( project.config.settings.nm_mode.value, zpm_config::NmMode::HardlinksLocal | zpm_config::NmMode::HardlinksGlobal, @@ -213,6 +229,14 @@ fn generate_workspace_node_modules( = project.project_cwd .with_join(&workspace.rel_path); + if let Some(package_map_builder) = package_map_builder.as_deref_mut() { + package_map_builder.register_package( + workspace_dir.clone(), + workspace_dir.clone(), + &workspace_node.locator, + ); + } + let workspace_abs_path = workspace_dir .with_join_str("node_modules"); @@ -255,6 +279,7 @@ fn generate_workspace_node_modules( project, install, &mut workspace_nm_tree, + package_map_builder.as_deref_mut(), &work_tree.nodes[workspace_node_idx], &workspace_abs_path, candidate_workspaces, @@ -315,6 +340,14 @@ fn generate_workspace_node_modules( let child_abs_path = workspace_abs_path.with_join(&child_rel_path); + if let Some(package_map_builder) = package_map_builder.as_deref_mut() { + package_map_builder.register_package( + child_abs_path.clone(), + package_directory.clone(), + &child_node.locator, + ); + } + let target_path = package_directory.relative_to(&child_abs_path.dirname().unwrap()); @@ -328,6 +361,14 @@ fn generate_workspace_node_modules( }, Some(PackageData::Zip {archive_path, package_directory, ..}) => { + if let Some(package_map_builder) = package_map_builder.as_deref_mut() { + package_map_builder.register_package( + abs_path.clone(), + abs_path.clone(), + &child_node.locator, + ); + } + // SyncTree re-extracts user-deleted destinations // automatically; we just need to flag for rebuild // so the build cache doesn't short-circuit. @@ -373,6 +414,14 @@ fn generate_workspace_node_modules( let target_path = Path::from_file_string(¶ms.path)?; + if let Some(package_map_builder) = package_map_builder.as_deref_mut() { + package_map_builder.register_package( + abs_path.clone(), + target_path.clone(), + &child_node.locator, + ); + } + workspace_nm_tree.register_entry(child_rel_path, SyncItem::Symlink { target_path, })?; @@ -492,10 +541,16 @@ pub async fn link_island_nm( = BTreeSet::new(); let mut cas_extractions = Vec::new(); + let mut package_maps + = Vec::new(); for workspace_ident in &island.workspace_idents { let workspace = project.workspace_by_ident(workspace_ident)?; let workspace_locator = workspace.locator(); + let package_map_base_path + = workspace.path.with_join_str("node_modules"); + let mut package_map_builder + = NodeModulesPackageMapBuilder::new_at(project, install, package_map_base_path.clone()); let mut work_tree = WorkTree::new_for_island_workspace( project, @@ -513,15 +568,25 @@ pub async fn link_island_nm( install, &work_tree, 0, + Some(&mut package_map_builder), &mut packages_by_location, &mut canonical_build_locations, &mut force_rebuild_locators, &mut cas_extractions, )?; + + package_maps.push(( + project.package_map_path(Some(workspace)), + package_map_builder.build()?, + )); } run_cas_extractions(project, install, &cas_extractions)?; + for (package_map_path, package_map) in package_maps { + persist_package_map_at(&package_map_path, &package_map)?; + } + let dependencies_meta = linker::helpers::TopLevelConfiguration::from_project(project); @@ -772,6 +837,9 @@ pub async fn link_project_nm(project: &Project, install: &Install) -> Result Result Result, +} + +#[derive(Debug, Serialize)] +struct PackageMapPackage { + url: String, + dependencies: BTreeMap, +} + +#[derive(Debug)] +struct PackageMapNode { + id: String, + package_path: Path, + dependency_names: Option>, +} + +pub struct NodeModulesPackageMapBuilder<'a> { + project: &'a Project, + install: &'a Install, + base_path: Path, + package_map_nodes: BTreeMap, + package_locations_by_node_modules_path: BTreeMap>, +} + +pub struct PnpmPackageMapBuilder { + base_path: Path, + top_level_locator: Locator, + package_map_type: NodePackageMapType, + package_map_nodes_by_locator: BTreeMap, +} + +#[derive(Debug)] +struct PnpmPackageMapNode { + package_location: Path, + dependencies: BTreeMap, +} + +impl<'a> NodeModulesPackageMapBuilder<'a> { + pub fn new(project: &'a Project, install: &'a Install) -> Self { + Self::new_at(project, install, project.nm_path()) + } + + pub fn new_at(project: &'a Project, install: &'a Install, base_path: Path) -> Self { + Self { + project, + install, + base_path, + package_map_nodes: BTreeMap::new(), + package_locations_by_node_modules_path: BTreeMap::new(), + } + } + + pub fn register_package(&mut self, location: Path, package_path: Path, locator: &Locator) { + let normalized_location + = location.without_trailing_separators(); + let normalized_package_path + = package_path.without_trailing_separators(); + + let package_map_node = PackageMapNode { + id: get_package_id(&self.base_path, &normalized_location), + package_path: normalized_package_path.clone(), + dependency_names: Some(get_package_dependency_names(self.project, self.install, locator, &normalized_package_path)), + }; + + self.package_map_nodes.insert(normalized_location.clone(), package_map_node); + + if let Some((node_modules_path, package_name)) = get_package_name(&normalized_location) { + self.package_locations_by_node_modules_path + .entry(node_modules_path) + .or_default() + .insert(package_name, normalized_location); + } + } + + pub fn build(&self) -> Result { + let package_map_type + = self.project.config.settings.node_package_map_type.value; + + let mut packages + = BTreeMap::new(); + + for package_map_node in self.package_map_nodes.values() { + packages.insert(package_map_node.id.clone(), PackageMapPackage { + url: get_relative_url(&self.base_path, &package_map_node.package_path), + dependencies: self.get_package_dependencies( + &package_map_node.package_path, + match package_map_type { + NodePackageMapType::Standard => package_map_node.dependency_names.as_ref(), + NodePackageMapType::Loose => None, + }, + )?, + }); + } + + Ok(PackageMap { + packages, + }) + } + + fn get_package_dependencies(&self, package_path: &Path, dependency_names: Option<&BTreeSet>) -> Result, Error> { + let mut dependencies + = BTreeMap::new(); + + let mut current_path + = package_path.clone(); + + loop { + let node_modules_path + = current_path.with_join_str("node_modules"); + + if let Some(package_locations) = self.package_locations_by_node_modules_path.get(&node_modules_path) { + for (dependency_name, dependency_location) in package_locations { + if let Some(dependency_names) = dependency_names { + if !dependency_names.contains(dependency_name) { + continue; + } + } + + if dependencies.contains_key(dependency_name) { + continue; + } + + let dependency + = self.package_map_nodes + .get(dependency_location) + .ok_or_else(|| package_map_error(format!("expected {dependency_location:?} to have been registered")))?; + + dependencies.insert(dependency_name.clone(), dependency.id.clone()); + } + } + + let Some(parent_path) = current_path.dirname() else { + break; + }; + + if parent_path == current_path { + break; + } + + current_path = parent_path; + } + + Ok(dependencies) + } +} + +impl PnpmPackageMapBuilder { + pub fn new(project: &Project) -> Self { + Self { + base_path: project.nm_path(), + top_level_locator: project.root_workspace().locator(), + package_map_type: project.config.settings.node_package_map_type.value, + package_map_nodes_by_locator: BTreeMap::new(), + } + } + + pub fn register_package(&mut self, locator: &Locator, package_location: Path) { + self.package_map_nodes_by_locator + .entry(locator.clone()) + .or_insert_with(|| PnpmPackageMapNode { + package_location: package_location.without_trailing_separators(), + dependencies: BTreeMap::new(), + }); + } + + pub fn register_dependency(&mut self, locator: &Locator, dependency_name: &Ident, dependency_locator: &Locator) -> Result<(), Error> { + if !self.package_map_nodes_by_locator.contains_key(dependency_locator) { + return Err(package_map_error(format!("expected dependency {} to have been registered", dependency_locator.to_print_string()))); + } + + let package_map_node + = self.package_map_nodes_by_locator + .get_mut(locator) + .ok_or_else(|| package_map_error(format!("expected {} to have been registered", locator.to_print_string())))?; + + package_map_node.dependencies.insert(dependency_name.as_str().to_string(), dependency_locator.clone()); + + Ok(()) + } + + pub fn build(&self) -> Result { + let top_level_package_map_node + = self.package_map_nodes_by_locator + .get(&self.top_level_locator) + .ok_or_else(|| package_map_error("expected the top-level package to have been registered"))?; + + let package_ids_by_locator: BTreeMap + = self.package_map_nodes_by_locator + .iter() + .map(|(locator, package_map_node)| { + (locator.clone(), get_package_id(&self.base_path, &package_map_node.package_location)) + }) + .collect(); + + let mut packages + = BTreeMap::new(); + + let mut package_map_nodes + = self.package_map_nodes_by_locator.values().collect::>(); + + package_map_nodes.sort_by_key(|package_map_node| { + get_package_id(&self.base_path, &package_map_node.package_location) + }); + + for package_map_node in package_map_nodes { + let dependencies = match self.package_map_type { + NodePackageMapType::Standard => package_map_node.dependencies.clone(), + NodePackageMapType::Loose => { + let mut dependencies + = top_level_package_map_node.dependencies.clone(); + + dependencies.extend(package_map_node.dependencies.clone()); + dependencies + }, + }; + + packages.insert(get_package_id(&self.base_path, &package_map_node.package_location), PackageMapPackage { + url: get_relative_url(&self.base_path, &package_map_node.package_location), + dependencies: serialize_pnpm_dependencies(&dependencies, &package_ids_by_locator)?, + }); + } + + Ok(PackageMap { + packages, + }) + } +} + +pub fn persist_package_map(project: &Project, package_map: &PackageMap) -> Result<(), Error> { + persist_package_map_at(&project.package_map_path(None), package_map) +} + +pub fn persist_package_map_at(package_map_path: &Path, package_map: &PackageMap) -> Result<(), Error> { + if let Some(parent) = package_map_path.dirname() { + parent.fs_create_dir_all()?; + } + + package_map_path.fs_change(format!("{}\n", JsonDocument::to_string_pretty(package_map)?), false)?; + + Ok(()) +} + +fn serialize_pnpm_dependencies(dependencies: &BTreeMap, package_ids_by_locator: &BTreeMap) -> Result, Error> { + dependencies + .iter() + .map(|(dependency_name, dependency_locator)| { + let package_id + = package_ids_by_locator + .get(dependency_locator) + .ok_or_else(|| package_map_error(format!("expected dependency {} to have a package id", dependency_locator.to_print_string())))?; + + Ok((dependency_name.clone(), package_id.clone())) + }) + .collect() +} + +fn package_map_error(message: impl Into) -> Error { + Error::PackageMapGenerationError(message.into()) +} + +fn get_package_dependency_names(project: &Project, install: &Install, locator: &Locator, package_path: &Path) -> BTreeSet { + let tree + = &install.install_state.resolution_tree; + + let mut dependency_names = resolution_dependency_names(tree, locator) + .or_else(|| workspace_package_dependency_names(project, tree, package_path)) + .unwrap_or_default(); + + // Add implicit self-dependency for non-workspace packages when there's no explicit self-dependency + if !locator.reference.is_workspace_reference() && !dependency_names.contains(locator.ident.as_str()) { + dependency_names.insert(locator.ident.as_str().to_string()); + } + + dependency_names +} + +fn resolution_dependency_names(tree: &ResolutionTree, locator: &Locator) -> Option> { + tree.locator_resolutions + .get(locator) + .map(|resolution| { + resolution.dependencies + .keys() + .map(|ident| ident.as_str().to_string()) + .collect() + }) +} + +fn workspace_package_dependency_names(project: &Project, tree: &ResolutionTree, package_path: &Path) -> Option> { + let package_rel_path = package_path.forward_relative_to(&project.project_cwd)?; + let workspace = project.try_closest_workspace_by_rel_path(&package_rel_path)?; + + resolution_dependency_names(tree, &workspace.locator()) +} + +fn get_package_name(location: &Path) -> Option<(Path, String)> { + let segments + = location.components().collect::>(); + + let node_modules_index + = segments.iter().rposition(|segment| *segment == "node_modules")?; + + let scope_or_name + = segments.get(node_modules_index + 1)?; + + let node_modules_path + = Path::from_str(&segments[..=node_modules_index].join("/")).ok()?; + + if !scope_or_name.starts_with('@') { + return Some((node_modules_path, (*scope_or_name).to_string())); + } + + let name + = segments.get(node_modules_index + 2)?; + + Some((node_modules_path, format!("{scope_or_name}/{name}"))) +} + +fn get_relative_url(from: &Path, to: &Path) -> String { + let relative_path + = to.relative_to(from); + + let relative_path + = if relative_path.is_empty() { + ".".to_string() + } else { + relative_path.to_file_string() + }; + + if relative_path.starts_with('.') { + relative_path + } else { + format!("./{relative_path}") + } +} + +fn get_package_id(base_path: &Path, location: &Path) -> String { + let relative_path + = location.relative_to(base_path); + + let relative_path + = if relative_path.is_empty() { + ".".to_string() + } else { + relative_path.to_file_string() + }; + + if relative_path == ".." { + ".".to_string() + } else { + relative_path + } +} diff --git a/packages/zpm/src/linker/pnpm.rs b/packages/zpm/src/linker/pnpm.rs index 49c462c2..c9a2f9e1 100644 --- a/packages/zpm/src/linker/pnpm.rs +++ b/packages/zpm/src/linker/pnpm.rs @@ -8,7 +8,7 @@ use crate::{ error::Error, fetchers::PackageData, install::Install, - linker::{self, LinkResult}, + linker::{self, LinkResult, package_map::{PnpmPackageMapBuilder, persist_package_map}}, project::Project, tree_resolver::ResolutionTree, }; @@ -64,6 +64,8 @@ pub async fn link_project_pnpm<'a>(project: &'a Project, install: &'a Install) - = BTreeMap::new(); let mut locations_by_package = BTreeMap::new(); + let mut package_map_builder + = PnpmPackageMapBuilder::new(project); let mut all_build_entries = Vec::new(); @@ -115,6 +117,8 @@ pub async fn link_project_pnpm<'a>(project: &'a Project, install: &'a Install) - package_location_rel.clone(), ); + package_map_builder.register_package(locator, package_location_abs.clone()); + // We don't create node_modules directories and we don't build // local packages that are not fully contained within the project if matches!(physical_package_data, PackageData::Local {package_directory, ..} if !project.project_cwd.contains(package_directory)) { @@ -258,12 +262,21 @@ pub async fn link_project_pnpm<'a>(project: &'a Project, install: &'a Install) - let is_local = matches!(physical_package_data, PackageData::Local {..}); + let mut has_explicit_self_dependency + = false; + for (dep_name, descriptor) in &resolution.dependencies { + if dep_name == &locator.ident { + has_explicit_self_dependency = true; + } + let dep_locator = tree.descriptor_to_locator .get(descriptor) .expect("Failed to find dependency resolution"); + package_map_builder.register_dependency(locator, dep_name, dep_locator)?; + if !is_local && !locator.reference.is_workspace_reference() { if let Some(hoisted_locator) = hoisted_packages.get(dep_name) { // If the exact same version is hoisted, skip creating the symlink @@ -310,8 +323,14 @@ pub async fn link_project_pnpm<'a>(project: &'a Project, install: &'a Install) - .fs_create_parent()? .fs_symlink(&symlink_target)?; } + + if !has_explicit_self_dependency && !locator.reference.is_workspace_reference() { + package_map_builder.register_dependency(locator, &locator.ident, locator)?; + } } + persist_package_map(project, &package_map_builder.build()?)?; + let package_build_dependencies = linker::helpers::populate_build_entry_dependencies( &package_build_entries, &tree.locator_resolutions, diff --git a/packages/zpm/src/project.rs b/packages/zpm/src/project.rs index cabd8c18..f5a1c6dd 100644 --- a/packages/zpm/src/project.rs +++ b/packages/zpm/src/project.rs @@ -2,7 +2,7 @@ use std::{collections::{BTreeMap, BTreeSet, HashSet}, io::ErrorKind, sync::{Arc, use colored::Colorize; use globset::{GlobBuilder, GlobSetBuilder}; -use zpm_config::{Configuration, ConfigurationContext, Source}; +use zpm_config::{Configuration, ConfigurationContext, IslandLinker, Source}; use zpm_macro_enum::zpm_enum; use zpm_parsers::JsonDocument; use zpm_primitives::{Descriptor, Ident, Locator, Range, Reference, WorkspaceIdentReference, WorkspaceMagicRange, WorkspacePathReference}; @@ -33,6 +33,7 @@ pub const MANIFEST_NAME: &str = "package.json"; pub const PNP_CJS_NAME: &str = ".pnp.cjs"; pub const PNP_ESM_NAME: &str = ".pnp.loader.mjs"; pub const PNP_DATA_NAME: &str = ".pnp.data.json"; +pub const PACKAGE_MAP_NAME: &str = ".package-map.json"; const LOCKFILE_DIFF_LINE_LIMIT: usize = 100; const LOCKFILE_DIFF_TIMEOUT: Duration = Duration::from_secs(3); @@ -287,6 +288,13 @@ impl Project { self.project_cwd.with_join_str("node_modules") } + pub fn package_map_path(&self, workspace: Option<&Workspace>) -> Path { + workspace + .map(|workspace| workspace.path.with_join_str("node_modules")) + .unwrap_or_else(|| self.nm_path()) + .with_join_str(PACKAGE_MAP_NAME) + } + pub fn ignore_path(&self) -> Path { self.project_cwd.with_join_str(".yarn/ignore") } @@ -723,6 +731,29 @@ impl Project { Ok(workspace) } + pub fn try_closest_workspace_by_rel_path(&self, rel_path: &Path) -> Option<&Workspace> { + for candidate_path in rel_path.iter_path().rev() { + if let Some(workspace) = self.try_workspace_by_rel_path(&candidate_path).ok().flatten() { + return Some(workspace); + } + } + + None + } + + pub fn try_island_by_rel_path(&self, rel_path: &Path, linker: IslandLinker) -> Option<&Workspace> { + let workspace + = self.try_closest_workspace_by_rel_path(rel_path)?; + + self.config.settings.unstable_islands + .values() + .any(|island| { + island.linker.value == linker + && island.workspaces.iter().any(|glob| glob.value.check(&workspace.name)) + }) + .then_some(workspace) + } + pub fn package_location(&self, locator: &Locator) -> Result<&Path, Error> { if let Some(locator_workspace) = self.try_workspace_by_locator(locator)? { return Ok(&locator_workspace.path); diff --git a/packages/zpm/src/script.rs b/packages/zpm/src/script.rs index 7febba0d..a8347e83 100644 --- a/packages/zpm/src/script.rs +++ b/packages/zpm/src/script.rs @@ -16,6 +16,7 @@ use crate::{ static CJS_LOADER_MATCHER: LazyLock = LazyLock::new(|| regex::Regex::new(r"\s*--require\s+\S*\.pnp\.c?js\s*").unwrap()); static ESM_LOADER_MATCHER: LazyLock = LazyLock::new(|| regex::Regex::new(r"\s*--experimental-loader\s+\S*\.pnp\.loader\.mjs\s*").unwrap()); +static PACKAGE_MAP_MATCHER: LazyLock = LazyLock::new(|| regex::Regex::new(r#"\s*--experimental-package-map(?:=|\s+)(?:"[^"]*"|'[^']*'|\S+)\s*"#).unwrap()); static JS_EXTENSION: LazyLock = LazyLock::new(|| regex::Regex::new(r"\.[cm]?[jt]sx?$").unwrap()); type TrustPromptResultCache = BTreeMap; @@ -44,6 +45,14 @@ fn make_python_entry_point_snippet(binary_name: &str, package_path: &Path, modul ) } +fn quote_path_if_needed(path: &str) -> String { + if path.chars().any(char::is_whitespace) { + serde_json::to_string(path).expect("expected valid path") + } else { + path.to_string() + } +} + fn make_executable_wrapper(bin_dir: &Path, name: &str, argv0: &str, args: &[String]) -> Result<(), Error> { if cfg!(windows) { let escaped_args = args @@ -547,6 +556,8 @@ impl ScriptEnvironment { self.append_env("NODE_OPTIONS", ' ', &format!("--experimental-loader {}", pnp_loader_path.to_file_string())); } + self.refresh_package_map(project); + self.env.insert("PROJECT_CWD".to_string(), Some(project.project_cwd.to_file_string())); self.env.insert("INIT_CWD".to_string(), Some(project.project_cwd.with_join(&project.shell_cwd).to_file_string())); self.env.insert("CACHE_CWD".to_string(), Some(project.preferred_cache_path().to_file_string())); @@ -566,6 +577,7 @@ impl ScriptEnvironment { let updated = CJS_LOADER_MATCHER.replace_all(¤t, " "); let updated = ESM_LOADER_MATCHER.replace_all(&updated, " "); + let updated = PACKAGE_MAP_MATCHER.replace_all(&updated, " "); let updated = updated.trim(); if current != updated { @@ -579,6 +591,55 @@ impl ScriptEnvironment { } } + fn refresh_package_map(&mut self, project: &Project) { + self.remove_package_map(); + + if !project.config.settings.node_experimental_package_map.value { + return; + } + + let cwd_rel_path = if self.cwd.is_absolute() { + self.cwd.forward_relative_to(&project.project_cwd) + } else { + Some(self.cwd.clone()) + }; + + let package_map_workspace = cwd_rel_path + .as_ref() + .and_then(|cwd_rel_path| project.try_island_by_rel_path(cwd_rel_path, zpm_config::IslandLinker::NodeModules)); + + if package_map_workspace.is_none() + && project.config.settings.node_linker.value == zpm_config::NodeLinker::Pnp + { + return; + } + + if let Some(package_map_path) = project.package_map_path(package_map_workspace).if_exists() { + self.append_env("NODE_OPTIONS", ' ', &format!("--experimental-package-map={}", quote_path_if_needed(&package_map_path.to_file_string()))); + } + } + + fn remove_package_map(&mut self) { + let current = self.env.get("NODE_OPTIONS") + .and_then(|opt| opt.clone()) + .or_else(|| std::env::var("NODE_OPTIONS").ok()); + + let Some(current) = current else { + return; + }; + + let updated = PACKAGE_MAP_MATCHER.replace_all(¤t, " "); + let updated = updated.trim(); + + if current != updated { + if updated.is_empty() { + self.env.insert("NODE_OPTIONS".to_string(), None); + } else { + self.env.insert("NODE_OPTIONS".to_string(), Some(updated.to_string())); + } + } + } + pub fn without_pnp_loader(mut self) -> Self { self.remove_pnp_loader(); self @@ -724,6 +785,7 @@ impl ScriptEnvironment { .with_join(package_cwd_rel); self.attach_package_variables(project, locator)?; + self.refresh_package_map(project); let binaries = project.package_visible_binaries(locator)?; diff --git a/tests/acceptance-tests/pkg-tests-specs/sources/features/islands.test.ts b/tests/acceptance-tests/pkg-tests-specs/sources/features/islands.test.ts index c495b291..8376f4e0 100644 --- a/tests/acceptance-tests/pkg-tests-specs/sources/features/islands.test.ts +++ b/tests/acceptance-tests/pkg-tests-specs/sources/features/islands.test.ts @@ -1576,6 +1576,7 @@ describe(`Features`, () => { // workspace-a should have a node_modules directory const nmPath = `${path}/packages/workspace-a/node_modules` as PortablePath; expect(await xfs.existsPromise(nmPath)).toBe(true); + expect(await xfs.existsPromise(`${nmPath}/.package-map.json` as PortablePath)).toBe(true); // no-deps should be resolvable via node_modules await expect( @@ -1588,6 +1589,54 @@ describe(`Features`, () => { ), ); + test( + `it should inject island package maps into script environments`, + makeTemporaryMonorepoEnv( + { + workspaces: [`packages/*`], + }, + { + [`packages/workspace-a`]: { + name: `workspace-a`, + version: `1.0.0`, + dependencies: { + [`no-deps`]: `1.0.0`, + }, + }, + }, + { + nodeLinker: `pnp`, + nodeExperimentalPackageMap: true, + }, + async ({path, run}) => { + await yarn.writeConfiguration(path, { + unstableIslands: { + main: { + workspaces: [`workspace-a`], + linker: `node-modules`, + }, + }, + }); + + await run(`install`); + + const islandPackageMapPath = `${path}/packages/workspace-a/node_modules/.package-map.json` as PortablePath; + const {stdout: islandNodeOptions} = await run(`exec`, `bash`, `-c`, `printf %s "$NODE_OPTIONS"`, { + cwd: `${path}/packages/workspace-a` as PortablePath, + }); + + expect(islandNodeOptions).toContain(`--experimental-package-map=`); + expect(islandNodeOptions).toContain(npath.fromPortablePath(islandPackageMapPath)); + + const {stdout: rootNodeOptions} = await run(`exec`, `bash`, `-c`, `printf %s "$NODE_OPTIONS"`, { + cwd: path, + }); + + expect(rootNodeOptions).not.toContain(`--experimental-package-map=`); + }, + ), + ); + test( `it should allow mixed PnP and node-modules island resolution`, makeTemporaryMonorepoEnv( diff --git a/tests/acceptance-tests/pkg-tests-specs/sources/features/packageMaps.test.ts b/tests/acceptance-tests/pkg-tests-specs/sources/features/packageMaps.test.ts new file mode 100644 index 00000000..0dfd9541 --- /dev/null +++ b/tests/acceptance-tests/pkg-tests-specs/sources/features/packageMaps.test.ts @@ -0,0 +1,460 @@ +import {Filename, PortablePath, npath, ppath, xfs} from '@yarnpkg/fslib'; +import {yarn} from 'pkg-tests-core'; + +const { + fs: {writeFile, writeJson}, +} = require(`pkg-tests-core`); + +const PACKAGE_MAP = `.package-map.json` as Filename; + +const getPackageMapPath = (path: PortablePath) => { + return ppath.join(path, Filename.nodeModules, PACKAGE_MAP); +}; + +const requireFromPackage = (packageName: string, request: string) => { + return `require('module').createRequire(require('path').join(process.cwd(), 'node_modules', ${JSON.stringify(packageName)}, 'index.js'))(${JSON.stringify(request)})`; +}; + +const supportsPackageMaps = Number(process.versions.node.split(`.`)[0]) >= 27; +const describePackageMaps = supportsPackageMaps ? describe : describe.skip; + +describePackageMaps(`Package maps`, () => { + it(`should allow packages to require their declared dependencies`, + makeTemporaryEnv( + { + dependencies: { + [`one-fixed-dep`]: `1.0.0`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + }, + async ({run, source}) => { + await run(`install`); + + await expect(source(requireFromPackage(`one-fixed-dep`, `.`))).resolves.toMatchObject({ + name: `one-fixed-dep`, + version: `1.0.0`, + dependencies: { + [`no-deps`]: { + name: `no-deps`, + version: `1.0.0`, + }, + }, + }); + }, + ), + ); + + it(`should reject undeclared dependencies even when they are hoisted`, + makeTemporaryEnv( + { + dependencies: { + [`no-deps`]: `1.0.0`, + [`various-requires`]: `1.0.0`, + }, + }, + { + nodeLinker: `node-modules`, + }, + async ({run, source}) => { + await run(`install`); + + await expect(source(requireFromPackage(`various-requires`, `./invalid-require`))).resolves.toMatchObject({ + name: `no-deps`, + version: `1.0.0`, + }); + + await run(`config`, `set`, `nodeExperimentalPackageMap`, `true`); + + await expect(source(requireFromPackage(`various-requires`, `./invalid-require`))).rejects.toMatchObject({ + externalException: { + code: `MODULE_NOT_FOUND`, + }, + }); + }, + ), + ); + + it(`should allow node-modules installs to use a loose package map`, + makeTemporaryEnv( + { + dependencies: { + [`no-deps`]: `1.0.0`, + [`various-requires`]: `1.0.0`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + nodePackageMapType: `loose`, + }, + async ({run, source}) => { + await run(`install`); + + await expect(source(requireFromPackage(`various-requires`, `./invalid-require`))).resolves.toMatchObject({ + name: `no-deps`, + version: `1.0.0`, + }); + }, + ), + ); + + it(`should allow scoped packages to be required in node-modules installs`, + makeTemporaryEnv( + { + dependencies: { + [`@scoped/release-date`]: `1.0.0`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + }, + async ({run, source}) => { + await run(`install`); + + await expect(source(requireFromPackage(`@scoped/release-date`, `./package.json`))).resolves.toMatchObject({ + name: `@scoped/release-date`, + version: `1.0.0`, + }); + }, + ), + ); + + it(`should install scoped workspaces in node-modules installs`, + makeTemporaryEnv( + { + private: true, + workspaces: [`workspace`], + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + }, + async ({path, run}) => { + await writeJson(ppath.join(path, `workspace/package.json` as PortablePath), { + name: `@scope/workspace`, + version: `1.0.0`, + dependencies: { + [`no-deps`]: `1.0.0`, + }, + }); + await writeFile(ppath.join(path, `workspace/index.js` as PortablePath), ``); + + await run(`install`); + + await expect(xfs.existsPromise(getPackageMapPath(path))).resolves.toEqual(true); + await expect(xfs.existsPromise(ppath.join(path, `node_modules/@scope/workspace` as PortablePath))).resolves.toEqual(true); + }, + ), + ); + + it(`should enforce declared dependencies for pnpm standard package maps`, + makeTemporaryEnv( + { + dependencies: { + [`no-deps`]: `1.0.0`, + [`one-fixed-dep`]: `1.0.0`, + [`various-requires`]: `1.0.0`, + }, + }, + { + nodeLinker: `pnpm`, + nodeExperimentalPackageMap: true, + }, + async ({path, run, source}) => { + await run(`install`); + + await expect(xfs.existsPromise(getPackageMapPath(path))).resolves.toEqual(true); + await expect(source(requireFromPackage(`one-fixed-dep`, `.`))).resolves.toMatchObject({ + name: `one-fixed-dep`, + version: `1.0.0`, + dependencies: { + [`no-deps`]: { + name: `no-deps`, + version: `1.0.0`, + }, + }, + }); + await expect(source(requireFromPackage(`various-requires`, `./invalid-require`))).rejects.toMatchObject({ + externalException: { + code: `MODULE_NOT_FOUND`, + }, + }); + }, + ), + ); + + it(`should allow pnpm installs to use a loose package map`, + makeTemporaryEnv( + { + dependencies: { + [`no-deps`]: `1.0.0`, + [`various-requires`]: `1.0.0`, + }, + }, + { + nodeLinker: `pnpm`, + nodeExperimentalPackageMap: true, + nodePackageMapType: `loose`, + }, + async ({path, run, source}) => { + await run(`install`); + + await expect(xfs.existsPromise(getPackageMapPath(path))).resolves.toEqual(true); + await expect(source(requireFromPackage(`various-requires`, `./invalid-require`))).resolves.toMatchObject({ + name: `no-deps`, + version: `1.0.0`, + }); + }, + ), + ); + + it(`should allow package extensions to declare additional dependencies`, + makeTemporaryEnv( + { + dependencies: { + [`various-requires`]: `1.0.0`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + }, + async ({path, run, source}) => { + await yarn.writeConfiguration(path, { + packageExtensions: { + [`various-requires@*`]: { + dependencies: { + [`no-deps`]: `1.0.0`, + }, + }, + }, + }); + + await run(`install`); + + await expect(source(requireFromPackage(`various-requires`, `./invalid-require`))).resolves.toMatchObject({ + name: `no-deps`, + version: `1.0.0`, + }); + }, + ), + ); + + it(`should resolve aliases through their alias name`, + makeTemporaryEnv( + { + dependencies: { + [`requester`]: `file:./requester`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + }, + async ({path, run, source}) => { + await writeJson(ppath.join(path, `requester/package.json` as PortablePath), { + name: `requester`, + version: `1.0.0`, + dependencies: { + [`no-deps2`]: `npm:no-deps@2.0.0`, + }, + }); + await writeFile(ppath.join(path, `requester/index.js` as PortablePath), ``); + + await run(`install`); + + await expect(source(requireFromPackage(`requester`, `no-deps2`))).resolves.toMatchObject({ + name: `no-deps`, + version: `2.0.0`, + }); + + await expect(source(requireFromPackage(`requester`, `no-deps`))).rejects.toMatchObject({ + externalException: { + code: `MODULE_NOT_FOUND`, + }, + }); + }, + ), + ); + + it(`should resolve dependencies from the issuer package instance`, + makeTemporaryEnv( + { + private: true, + workspaces: [`workspace`], + dependencies: { + [`one-fixed-dep`]: `1.0.0`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + nmHoistingLimits: `workspaces`, + }, + async ({path, run, source}) => { + await writeJson(ppath.join(path, `workspace/package.json` as PortablePath), { + name: `workspace`, + version: `1.0.0`, + dependencies: { + [`one-fixed-dep`]: `1.0.0`, + }, + }); + await writeFile(ppath.join(path, `workspace/index.js` as PortablePath), ``); + + await run(`install`); + + await expect(source(`{ + const path = require('path'); + const rootOneFixedDepRequire = require('module').createRequire(path.join(process.cwd(), 'node_modules/one-fixed-dep/index.js')); + const workspaceOneFixedDepPath = path.join(process.cwd(), 'workspace/node_modules/one-fixed-dep'); + const workspaceOneFixedDepRequire = require('module').createRequire(path.join(workspaceOneFixedDepPath, 'index.js')); + + return { + rootDependencyLocation: rootOneFixedDepRequire.resolve('.'), + rootDependency: rootOneFixedDepRequire('.'), + workspaceDependencyLocation: workspaceOneFixedDepRequire.resolve(workspaceOneFixedDepPath), + workspaceDependency: workspaceOneFixedDepRequire(workspaceOneFixedDepPath), + }; + }`)).resolves.toMatchObject({ + rootDependencyLocation: expect.stringContaining(`${npath.sep}node_modules${npath.sep}one-fixed-dep${npath.sep}index.js`), + rootDependency: { + dependencies: { + [`no-deps`]: { + name: `no-deps`, + version: `1.0.0`, + }, + }, + }, + workspaceDependencyLocation: expect.stringContaining(`${npath.sep}workspace${npath.sep}node_modules${npath.sep}one-fixed-dep${npath.sep}index.js`), + workspaceDependency: { + dependencies: { + [`no-deps`]: { + name: `no-deps`, + version: `1.0.0`, + }, + }, + }, + }); + }, + ), + ); + + it(`should refresh dependency access after package extensions are removed`, + makeTemporaryEnv( + { + dependencies: { + [`various-requires`]: `1.0.0`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + }, + async ({path, run, source}) => { + await yarn.writeConfiguration(path, { + packageExtensions: { + [`various-requires@*`]: { + dependencies: { + [`no-deps`]: `1.0.0`, + }, + }, + }, + }); + + await run(`install`); + + await expect(source(requireFromPackage(`various-requires`, `./invalid-require`))).resolves.toMatchObject({ + name: `no-deps`, + version: `1.0.0`, + }); + + await xfs.removePromise(ppath.join(path, `.yarnrc.yml` as Filename)); + await run(`install`); + + await expect(source(requireFromPackage(`various-requires`, `./invalid-require`))).rejects.toMatchObject({ + externalException: { + code: `MODULE_NOT_FOUND`, + }, + }); + }, + ), + ); +}); + +describe(`Package map generation`, () => { + it(`should use workspace dependency names for workspace link locators in standard node-modules maps`, + makeTemporaryEnv( + { + private: true, + workspaces: [`workspace`], + dependencies: { + workspace: `workspace:*`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + }, + async ({path, run}) => { + await writeJson(ppath.join(path, `workspace/package.json` as PortablePath), { + name: `workspace`, + version: `1.0.0`, + dependencies: { + [`no-deps`]: `1.0.0`, + }, + }); + await writeFile(ppath.join(path, `workspace/index.js` as PortablePath), ``); + + await run(`install`); + + const packageMap = await xfs.readJsonPromise(getPackageMapPath(path)); + + expect(packageMap.packages.workspace.dependencies).toMatchObject({ + [`no-deps`]: `no-deps`, + }); + }, + ), + ); + + it(`should include workspace self-reference symlinks in loose node-modules maps`, + makeTemporaryEnv( + { + private: true, + workspaces: [`workspace`], + dependencies: { + [`various-requires`]: `1.0.0`, + }, + }, + { + nodeLinker: `node-modules`, + nodeExperimentalPackageMap: true, + nodePackageMapType: `loose`, + }, + async ({path, run}) => { + await writeJson(ppath.join(path, `workspace/package.json` as PortablePath), { + name: `workspace`, + version: `1.0.0`, + }); + await writeFile(ppath.join(path, `workspace/index.js` as PortablePath), ``); + + await run(`install`); + + await expect(xfs.existsPromise(ppath.join(path, `node_modules/workspace` as PortablePath))).resolves.toEqual(true); + + const packageMap = await xfs.readJsonPromise(getPackageMapPath(path)); + + expect(packageMap.packages).toHaveProperty(`workspace`); + expect(packageMap.packages.workspace).toMatchObject({ + url: `../workspace`, + }); + expect(packageMap.packages[`various-requires`].dependencies).toMatchObject({ + workspace: `workspace`, + }); + }, + ), + ); +}); diff --git a/tests/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts b/tests/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts index 5ed94955..001602f3 100644 --- a/tests/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts +++ b/tests/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts @@ -2134,6 +2134,7 @@ describe(`Node Modules`, () => { await run(`install`); await expect(xfs.readdirPromise(ppath.join(path, Filename.nodeModules))).resolves.toEqual([ + `.package-map.json`, `native`, `native-foo-x64`, `native-foo-x86`, diff --git a/website/config/yarnrc.json b/website/config/yarnrc.json index 76324058..769998c1 100644 --- a/website/config/yarnrc.json +++ b/website/config/yarnrc.json @@ -420,6 +420,21 @@ } ] }, + "nodeExperimentalPackageMap": { + "_package": "@yarnpkg/plugin-pnp", + "title": "Define whether Yarn should enable Node.js package maps for installs that support them.", + "description": "If true, Yarn will inject Node.js' experimental `--experimental-package-map` option into `NODE_OPTIONS` when using `nodeLinker: node-modules` or `nodeLinker: pnpm` and a package map has been generated.", + "type": "boolean", + "default": false + }, + "nodePackageMapType": { + "_package": "@yarnpkg/plugin-pnp", + "title": "Define how Yarn should generate Node.js package maps.", + "description": "Possible values are:\n\n- If `standard` (the default), package maps will reflect the dependency graph and reject undeclared dependencies.\n- If `loose`, package maps will reflect the hoisted node_modules layout and mimic Node.js resolution more closely.\n\nThis setting applies to both `nodeLinker: node-modules` and `nodeLinker: pnpm`.", + "type": "string", + "enum": ["standard", "loose"], + "default": "standard" + }, "pnpmStoreFolder": { "_package": "@yarnpkg/plugin-pnpm", "title": "Path where the pnpm store will be stored",