diff --git a/Cargo.lock b/Cargo.lock index 58768895..b0c300f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,7 +401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -410,7 +410,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -592,6 +592,20 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -1026,6 +1040,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.3" @@ -3607,7 +3627,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -4101,6 +4121,7 @@ dependencies = [ "clipanion", "colored 3.0.0", "convert_case 0.8.0", + "dashmap", "dialoguer", "env_logger", "fancy-regex", diff --git a/packages/zpm/Cargo.toml b/packages/zpm/Cargo.toml index 723b5a5f..ca61d1ee 100644 --- a/packages/zpm/Cargo.toml +++ b/packages/zpm/Cargo.toml @@ -48,3 +48,4 @@ hickory-resolver = "0.25.2" indexmap = {version = "2.11.0", features = ["serde"]} sha2 = "0.10.8" hex = "0.4.3" +dashmap = "6.1.0" diff --git a/packages/zpm/src/commands/debug/print_hoisting.rs b/packages/zpm/src/commands/debug/print_hoisting.rs index 224dc7f0..ab0a1efd 100644 --- a/packages/zpm/src/commands/debug/print_hoisting.rs +++ b/packages/zpm/src/commands/debug/print_hoisting.rs @@ -35,7 +35,9 @@ impl PrintHoisting { = Hoister::new(&mut work_tree); hoister.set_print_logs(self.verbose); + println!("Hoisting..."); hoister.hoist(); + println!("Hoisted!"); let root_node = hoist::TreeRenderer::new(&work_tree).convert(); diff --git a/packages/zpm/src/install.rs b/packages/zpm/src/install.rs index 6d47b36b..f7f4ecf9 100644 --- a/packages/zpm/src/install.rs +++ b/packages/zpm/src/install.rs @@ -1,5 +1,6 @@ use std::{collections::{BTreeMap, BTreeSet}, hash::Hash, marker::PhantomData, sync::LazyLock}; +use dashmap::DashMap; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use zpm_config::PackageExtension; use zpm_primitives::{Descriptor, Ident, Locator, PatchRange, PeerRange, Range, RegistrySemverRange, RegistryTagRange, SemverDescriptor, SemverPeerRange}; @@ -9,7 +10,7 @@ use serde::{Deserialize, Serialize}; use zpm_utils::{FromFileString, ToFileString}; use crate::{ - build, cache::CompositeCache, content_flags::ContentFlags, error::Error, fetchers::{fetch_locator, patch::has_builtin_patch, try_fetch_locator_sync, PackageData, SyncFetchAttempt}, graph::{GraphCache, GraphIn, GraphOut, GraphTasks}, linker, lockfile::{Lockfile, LockfileEntry, LockfileMetadata}, primitives_exts::RangeExt, project::{InstallMode, Project}, report::{async_section, with_context_result, ReportContext}, resolvers::{resolve_descriptor, resolve_locator, try_resolve_descriptor_sync, validate_resolution, Resolution, SyncResolutionAttempt}, system, tree_resolver::{ResolutionTree, TreeResolver} + build, cache::CompositeCache, content_flags::ContentFlags, error::Error, fetchers::{fetch_locator, patch::has_builtin_patch, try_fetch_locator_sync, PackageData, SyncFetchAttempt}, graph::{GraphCache, GraphIn, GraphOut, GraphTasks}, linker, lockfile::{Lockfile, LockfileEntry, LockfileMetadata}, primitives_exts::RangeExt, project::{InstallMode, Project}, report::{async_section, with_context_result, ReportContext}, resolvers::{npm::NpmPayload, resolve_descriptor, resolve_locator, try_resolve_descriptor_sync, validate_resolution, Resolution, SyncResolutionAttempt}, system, tree_resolver::{ResolutionTree, TreeResolver} }; @@ -21,6 +22,7 @@ pub struct InstallContext<'a> { pub check_checksums: bool, pub check_resolutions: bool, pub enforced_resolutions: BTreeMap, + pub npm_metadata_cache: Option<&'a DashMap>, pub refresh_lockfile: bool, pub mode: Option, } @@ -34,6 +36,7 @@ impl<'a> Default for InstallContext<'a> { check_checksums: false, check_resolutions: false, enforced_resolutions: BTreeMap::new(), + npm_metadata_cache: None, refresh_lockfile: false, mode: None, } @@ -41,6 +44,11 @@ impl<'a> Default for InstallContext<'a> { } impl<'a> InstallContext<'a> { + pub fn with_npm_metadata_cache(mut self, npm_metadata_cache: Option<&'a DashMap>) -> Self { + self.npm_metadata_cache = npm_metadata_cache; + self + } + pub fn with_package_cache(mut self, package_cache: Option<&'a CompositeCache>) -> Self { self.package_cache = package_cache; self diff --git a/packages/zpm/src/project.rs b/packages/zpm/src/project.rs index bab02b89..ce2a9a61 100644 --- a/packages/zpm/src/project.rs +++ b/packages/zpm/src/project.rs @@ -1,5 +1,6 @@ use std::{collections::{BTreeMap, HashSet}, io::ErrorKind, sync::Arc, time::UNIX_EPOCH}; +use dashmap::DashMap; use globset::{GlobBuilder, GlobSetBuilder}; use zpm_config::{Configuration, ConfigurationContext}; use zpm_macro_enum::zpm_enum; @@ -625,7 +626,11 @@ impl Project { } } + let npm_metadata_cache + = DashMap::new(); + let install_context = InstallContext::default() + .with_npm_metadata_cache(Some(&npm_metadata_cache)) .with_package_cache(Some(&package_cache)) .with_project(Some(self)) .set_check_checksums(options.check_checksums) diff --git a/packages/zpm/src/resolvers/npm.rs b/packages/zpm/src/resolvers/npm.rs index 57a316f8..dea827d9 100644 --- a/packages/zpm/src/resolvers/npm.rs +++ b/packages/zpm/src/resolvers/npm.rs @@ -1,10 +1,9 @@ -use std::{collections::BTreeMap, fmt, marker::PhantomData, str::FromStr, sync::{Arc, LazyLock}}; +use std::{collections::BTreeMap, str::FromStr, sync::{Arc, LazyLock}}; use regex::Regex; -use serde::{de::{self, DeserializeOwned, DeserializeSeed, IgnoredAny, Visitor}, Deserialize, Deserializer}; +use serde::Deserialize; use zpm_config::ConfigExt; use zpm_primitives::{Descriptor, Ident, Locator, RegistryReference, RegistrySemverRange, RegistryTagRange, UrlReference}; -use zpm_utils::ToFileString; use crate::{ error::Error, @@ -17,136 +16,30 @@ use crate::{ static NODE_GYP_IDENT: LazyLock = LazyLock::new(|| Ident::from_str("node-gyp").unwrap()); static NODE_GYP_MATCH: LazyLock = LazyLock::new(|| Regex::new(r"\b(node-gyp|prebuild-install)\b").unwrap()); -/** - * Deserializer that only deserializes the requested field and skips all others. - */ -pub struct FindFieldNested<'a, T> { - field: &'a str, - nested: T, -} - -impl<'de, T> Visitor<'de> for FindFieldNested<'_, T> where T: DeserializeSeed<'de> + Clone { - type Value = T::Value; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - write!(formatter, "a map with a matching field") - } - - fn visit_map(self, mut map: A) -> Result where A: de::MapAccess<'de> { - let mut selected = None; - - while let Some(key) = map.next_key::()? { - if key == self.field { - selected = Some(map.next_value_seed(self.nested.clone())?); - } else { - let _ = map.next_value::(); - } - } - - selected - .ok_or(de::Error::missing_field("")) - } -} - -/** - * Deserializer that only deserializes the value for the highest key matching the provided semver range. - */ -#[derive(Clone)] -pub struct FindHighestCompatibleVersion { - range: zpm_semver::Range, - phantom: PhantomData, -} - -impl<'de, T> DeserializeSeed<'de> for FindHighestCompatibleVersion where T: DeserializeOwned { - type Value = Option<(zpm_semver::Version, T)>; - - fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de> { - deserializer.deserialize_map(self) - } -} - -impl<'de, T> Visitor<'de> for FindHighestCompatibleVersion where T: DeserializeOwned { - type Value = Option<(zpm_semver::Version, T)>; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - write!(formatter, "a map with a matching version") - } - - fn visit_map(self, mut map: A) -> Result where A: de::MapAccess<'de> { - let mut selected = None; - - while let Some(key) = map.next_key::()? { - let version - = zpm_semver::Version::from_str(key.as_str()).unwrap(); - - if self.range.check(&version) && selected.as_ref().map(|(current_version, _)| *current_version < version).unwrap_or(true) { - selected = Some((version, map.next_value::()?)); - } else { - map.next_value::()?; - } - } - - let Some((version, version_payload)) = selected else { - return Ok(None); - }; - - let deserialized_payload - = T::deserialize(&version_payload).unwrap(); - - Ok(Some((version, deserialized_payload))) - } -} - -/** - * Deserializer that only deserializes the value for the highest key matching the provided semver range. - */ -#[derive(Clone)] -pub struct FindField<'a, TVal> { - value: &'a str, - phantom: PhantomData, -} - -impl<'de, TVal> DeserializeSeed<'de> for FindField<'de, TVal> where TVal: Deserialize<'de> + std::fmt::Debug { - type Value = Option; - - fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de> { - deserializer.deserialize_map(self) - } -} - -impl<'de, TVal> Visitor<'de> for FindField<'de, TVal> where TVal: Deserialize<'de> + std::fmt::Debug { - type Value = Option; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - write!(formatter, "a map with a matching version") - } - - fn visit_map(self, mut map: A) -> Result where A: de::MapAccess<'de> { - let mut res = None; - - while let Some(key) = map.next_key::()? { - if self.value == key { - res = Some(map.next_value::()?); - } else { - map.next_value::()?; - } - } - - Ok(res) - } +#[derive(Clone, Deserialize, Debug)] +pub struct InterestingScriptsOnly { + preinstall: Option, + install: Option, + postinstall: Option, } /** * We need to read the scripts to figure out whether the package has an implicit node-gyp dependency. */ #[derive(Clone, Deserialize, Debug)] -struct RemoteManifestWithScripts { +pub struct RemoteManifestWithScripts { #[serde(flatten)] remote: RemoteManifest, #[serde(default)] - #[serde(skip_serializing_if = "BTreeMap::is_empty")] - scripts: BTreeMap, + scripts: Option, +} + +#[derive(Clone, Deserialize, Debug)] +pub struct NpmPayload { + #[serde(rename = "dist-tags")] + dist_tags: BTreeMap, + versions: BTreeMap, } fn fix_manifest(manifest: &mut RemoteManifestWithScripts) { @@ -156,10 +49,14 @@ fn fix_manifest(manifest: &mut RemoteManifestWithScripts) { // Also, node-gyp is not always set as a dependency in packages, so it will also be added if used in scripts. // if !manifest.remote.dependencies.contains_key(&NODE_GYP_IDENT) && !manifest.remote.peer_dependencies.contains_key(&NODE_GYP_IDENT) { - for script in manifest.scripts.values() { - if NODE_GYP_MATCH.is_match(script.as_str()) { - manifest.remote.dependencies.insert(NODE_GYP_IDENT.clone(), Descriptor::new_semver(NODE_GYP_IDENT.clone(), "*").unwrap()); - break; + if let Some(scripts) = &manifest.scripts { + for maybe_script in [&scripts.preinstall, &scripts.install, &scripts.postinstall] { + if let Some(script) = maybe_script { + if NODE_GYP_MATCH.is_match(script.as_str()) { + manifest.remote.dependencies.insert(NODE_GYP_IDENT.clone(), Descriptor::new_semver(NODE_GYP_IDENT.clone(), "*").unwrap()); + break; + } + } } } } @@ -207,13 +104,21 @@ pub async fn resolve_semver_or_workspace_descriptor(context: &InstallContext<'_> resolve_semver_descriptor(context, descriptor, params).await } -pub async fn resolve_semver_descriptor(context: &InstallContext<'_>, descriptor: &Descriptor, params: &RegistrySemverRange) -> Result { +async fn process_registry_data(context: &InstallContext<'_>, package_ident: &Ident, f: impl FnOnce(&NpmPayload) -> Result) -> Result { + let npm_metadata_cache + = context.npm_metadata_cache + .expect("The npm metadata cache is required for resolving a semver descriptor"); + + let cached_registry_data + = npm_metadata_cache.get(package_ident); + + if let Some(cached_registry_data) = cached_registry_data { + return Ok(f(&cached_registry_data.value())?); + } + let project = context.project .expect("The project is required for resolving a workspace package"); - let package_ident = params.ident.as_ref() - .unwrap_or(&descriptor.ident); - let registry_url = npm::registry_url_for_all_versions(&project.config.registry_base_for(package_ident), package_ident); @@ -222,21 +127,28 @@ pub async fn resolve_semver_descriptor(context: &InstallContext<'_>, descriptor: let registry_text = response.text().await .map_err(|err| Error::RemoteRegistryError(Arc::new(err)))?; + let registry_data: NpmPayload + = sonic_rs::from_str(registry_text.as_str())?; - let mut deserializer - = sonic_rs::Deserializer::from_str(registry_text.as_str()); + npm_metadata_cache.insert(package_ident.clone(), registry_data.clone()); - let (version, manifest) = deserializer.deserialize_map(FindFieldNested { - field: "versions", - nested: FindHighestCompatibleVersion { - range: params.range.clone(), - phantom: PhantomData::, - }, - })?.ok_or_else(|| { - Error::NoCandidatesFound(descriptor.range.clone()) - })?; + Ok(f(®istry_data)?) +} + +pub async fn resolve_semver_descriptor(context: &InstallContext<'_>, descriptor: &Descriptor, params: &RegistrySemverRange) -> Result { + let package_ident = params.ident.as_ref() + .unwrap_or(&descriptor.ident); + + let (version, manifest) = process_registry_data(context, package_ident, |registry_data| { + registry_data.versions.iter() + .filter(|(version, _)| params.range.check(version)) + .max_by(|(version, _), (other_version, _)| version.cmp(other_version)) + .ok_or_else(|| Error::NoCandidatesFound(descriptor.range.clone())) + .map(|(version, manifest)| (version.clone(), manifest.clone())) + }).await?; - Ok(build_resolution_result(context, descriptor, package_ident, version, manifest)) + + Ok(build_resolution_result(context, descriptor, package_ident, version.clone(), manifest.clone())) } pub async fn resolve_tag_or_workspace_descriptor(context: &InstallContext<'_>, descriptor: &Descriptor, params: &RegistryTagRange) -> Result { @@ -253,46 +165,22 @@ pub async fn resolve_tag_or_workspace_descriptor(context: &InstallContext<'_>, d } pub async fn resolve_tag_descriptor(context: &InstallContext<'_>, descriptor: &Descriptor, params: &RegistryTagRange) -> Result { - let project = context.project - .expect("The project is required for resolving a workspace package"); - let package_ident = params.ident.as_ref() .unwrap_or(&descriptor.ident); - let registry_url - = npm::registry_url_for_all_versions(&project.config.registry_base_for(package_ident), package_ident); - - let response - = project.http_client.get(®istry_url)?.send().await?; - - let registry_text = response.text().await - .map_err(|err| Error::RemoteRegistryError(Arc::new(err)))?; - - #[derive(Deserialize)] - struct RegistryMetadata { - #[serde(rename(deserialize = "dist-tags"))] - dist_tags: sonic_rs::Value, - versions: sonic_rs::Value, - } - - let registry_data: RegistryMetadata = sonic_rs::from_str(registry_text.as_str()) - .map_err(Arc::new)?; + let (version, manifest) = process_registry_data(context, package_ident, |registry_data| { + let version = registry_data.dist_tags + .get(params.tag.as_str()) + .ok_or_else(|| Error::TagNotFound(params.tag.clone()))?; - let version = registry_data.dist_tags.deserialize_map(FindField { - value: params.tag.as_str(), - phantom: PhantomData::, - })?.ok_or_else(|| { - Error::TagNotFound(params.tag.clone()) - })?; + let manifest = registry_data.versions + .get(version) + .ok_or_else(|| Error::TagNotFound(params.tag.clone()))?; - let manifest = registry_data.versions.deserialize_map(FindField { - value: &version.to_file_string(), - phantom: PhantomData::, - })?.ok_or_else(|| { - Error::NoCandidatesFound(descriptor.range.clone()) - })?; + Ok((version.clone(), manifest.clone())) + }).await?; - Ok(build_resolution_result(context, descriptor, package_ident, version, manifest)) + Ok(build_resolution_result(context, descriptor, package_ident, version.clone(), manifest.clone())) } pub async fn resolve_locator(context: &InstallContext<'_>, locator: &Locator, params: &RegistryReference) -> Result {