diff --git a/packages/zpm/src/commands/add.rs b/packages/zpm/src/commands/add.rs index e5e75aa8..541cf205 100644 --- a/packages/zpm/src/commands/add.rs +++ b/packages/zpm/src/commands/add.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::{collections::{HashMap, HashSet}, time::UNIX_EPOCH}; use clipanion::cli; use zpm_parsers::{Document, JsonDocument, Value}; @@ -11,6 +11,7 @@ use crate::{ descriptor_loose::{self, LooseDescriptor}, error::Error, install::InstallContext, + manifest::helpers::parse_manifest_from_bytes, project::{self, InstallMode} }; @@ -194,7 +195,7 @@ pub struct Add { impl Add { pub async fn execute(&self) -> Result<(), Error> { - let project + let mut project = project::Project::new(None).await?; let range_kind = if self.fixed { @@ -320,16 +321,30 @@ impl Add { manifest_path .fs_change(&document.input, false)?; - let mut project - = project::Project::new(None).await?; + let manifest_meta = manifest_path + .fs_metadata()?; + let last_changed_at = manifest_meta.modified()? + .duration_since(UNIX_EPOCH).unwrap() + .as_nanos(); + + let active_workspace = project.active_workspace_mut()?; + active_workspace.manifest = parse_manifest_from_bytes(&document.input)?; + active_workspace.last_changed_at = last_changed_at; + project.last_modified_at.update(last_changed_at); let enforced_resolutions + = resolutions.iter() + .filter_map(|resolution| resolution.locator.clone().map(|locator| (resolution.descriptor.clone(), Some(locator)))) + .collect(); + + let enforced_resolution_results = resolutions.into_iter() - .filter_map(|resolution| resolution.locator.map(|locator| (resolution.descriptor, Some(locator)))) + .filter_map(|resolution| resolution.resolution.map(|result| (resolution.descriptor, result))) .collect(); project.run_install(project::RunInstallOptions { mode: self.mode, + enforced_resolution_results, enforced_resolutions, silent_or_error: self.silent, ..Default::default() diff --git a/packages/zpm/src/descriptor_loose.rs b/packages/zpm/src/descriptor_loose.rs index 028686b8..201f7cf8 100644 --- a/packages/zpm/src/descriptor_loose.rs +++ b/packages/zpm/src/descriptor_loose.rs @@ -1,14 +1,18 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::{collections::{BTreeMap, BTreeSet}, io::ErrorKind, time::{Duration, SystemTime}}; +use chrono::{DateTime, Utc}; use futures::{future::BoxFuture, FutureExt}; +use serde::Deserialize; +use serde_with::{MapSkipError, serde_as}; use wax::{Glob, Program}; use zpm_formats::{iter_ext::IterExt, tar, tar_iter}; use zpm_macro_enum::zpm_enum; +use zpm_parsers::{JsonDocument, RawJsonValue}; use zpm_primitives::{AnonymousSemverRange, AnonymousTagRange, Descriptor, FolderRange, Ident, Locator, Range, RegistrySemverRange, RegistryTagRange, TarballRange, WorkspaceMagicRange}; use zpm_semver::RangeKind; -use zpm_utils::Path; +use zpm_utils::{Hash64, Path}; -use crate::{error::Error, install::InstallContext, manifest::helpers::{parse_manifest_from_bytes, read_manifest}, project::Project, report::{with_report_result, StreamReport, StreamReportConfig}, resolvers}; +use crate::{error::Error, http_npm, install::{InstallContext, ResolutionResult}, manifest::helpers::{parse_manifest_from_bytes, read_manifest}, npm, project::Project, report::{with_report_result, StreamReport, StreamReportConfig}, resolvers}; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct ResolveOptions { @@ -18,10 +22,141 @@ pub struct ResolveOptions { pub allow_reuse: bool, } -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Debug)] pub struct LooseResolution { pub descriptor: Descriptor, pub locator: Option, + pub resolution: Option, +} + +const ADD_PACKUMENT_CACHE_TTL: Duration = Duration::from_secs(30); + +#[serde_as] +#[derive(Deserialize)] +struct RegistryMetadata<'a> { + #[serde(default)] + #[serde(rename(deserialize = "dist-tags"))] + dist_tags: BTreeMap, + + #[serde_as(as = "Option>")] + time: Option>>, + + #[serde(borrow)] + versions: BTreeMap>, +} + +fn add_packument_cache_path(project: &Project, registry_base: &str, registry_path: &str) -> Path { + let cache_key + = Hash64::from_data(format!("{registry_base}{registry_path}")); + + project.ignore_path() + .with_join_str("npm-metadata") + .with_join_str(format!("{}.json", cache_key.short())) +} + +fn read_cached_packument(cache_path: &Path, allow_stale: bool) -> Result>, Error> { + let metadata = match cache_path.fs_metadata() { + Ok(metadata) => metadata, + + Err(error) if error.io_kind() == Some(ErrorKind::NotFound) => { + return Ok(None); + }, + + Err(error) => { + return Err(error.into()); + }, + }; + + if !allow_stale { + let age = SystemTime::now() + .duration_since(metadata.modified()?) + .unwrap_or_default(); + + if age > ADD_PACKUMENT_CACHE_TTL { + return Ok(None); + } + } + + Ok(Some(cache_path.fs_read_prealloc()?)) +} + +async fn fetch_registry_metadata(context: &InstallContext<'_>, package_ident: &Ident) -> Result, Error> { + let project = context.project.as_ref() + .expect("Project is required for resolving registry metadata"); + + let registry_base + = http_npm::get_registry(&project.config, package_ident.scope(), false)?; + let registry_path + = npm::registry_url_for_all_versions(package_ident); + + let authorization + = http_npm::get_authorization(&http_npm::GetAuthorizationOptions { + configuration: &project.config, + http_client: &project.http_client, + registry: ®istry_base, + ident: Some(package_ident), + auth_mode: http_npm::AuthorizationMode::RespectConfiguration, + allow_oidc: false, + }).await?; + + let cache_path = authorization.is_none() + .then(|| add_packument_cache_path(project, ®istry_base, ®istry_path)); + + if let Some(cache_path) = cache_path.as_ref() { + if let Some(bytes) = read_cached_packument(cache_path, false)? { + return Ok(bytes); + } + } + + let bytes = match http_npm::get(&http_npm::NpmHttpParams { + http_client: &project.http_client, + registry: ®istry_base, + path: ®istry_path, + authorization: authorization.as_deref(), + otp: None, + }).await { + Ok(bytes) => bytes.to_vec(), + + Err(error @ Error::NetworkDisabledError(_)) => { + if let Some(cache_path) = cache_path.as_ref() { + if let Some(bytes) = read_cached_packument(cache_path, true)? { + return Ok(bytes); + } + } + + return Err(error); + }, + + Err(error) => { + return Err(error); + }, + }; + + if let Some(cache_path) = cache_path.as_ref() { + let _ = cache_path.fs_create_parent() + .and_then(|_| cache_path.fs_write(&bytes)); + } + + Ok(bytes) +} + +fn find_semver_candidate<'a>(context: &InstallContext<'_>, package_ident: &Ident, range: &zpm_semver::Range, registry_data: &'a RegistryMetadata<'a>) -> Option<(&'a zpm_semver::Version, &'a RawJsonValue<'a>)> { + let project = context.project.as_ref() + .expect("Project is required for resolving semver candidates"); + + registry_data.versions.iter().rev() + .filter(|(version, _)| range.check(version)) + .find(|(version, _)| { + let release_time = project.config.settings.npm_minimal_age_gate.value + .and_then(|_| registry_data.time.as_ref()) + .and_then(|times| times.get(*version)); + + resolvers::npm::is_package_approved(context, package_ident, version, release_time) + }) +} + +fn build_registry_resolution_result(context: &InstallContext<'_>, descriptor: &Descriptor, package_ident: &Ident, version: &zpm_semver::Version, manifest: resolvers::npm::RemoteManifestWithScripts) -> Result { + resolvers::npm::build_resolution_result(context, descriptor, package_ident, version.clone(), manifest) } #[zpm_enum(or_else = |s| Err(Error::InvalidRange(s.to_string())))] @@ -161,6 +296,7 @@ impl LooseDescriptor { Ok(LooseResolution { descriptor, locator: None, + resolution: None, }) } @@ -182,6 +318,7 @@ impl LooseDescriptor { Ok(LooseResolution { descriptor, locator: None, + resolution: None, }) }, @@ -209,6 +346,7 @@ impl LooseDescriptor { Ok(LooseResolution { descriptor: descriptor.clone(), locator: None, + resolution: None, }) }, @@ -223,6 +361,7 @@ impl LooseDescriptor { return Ok(LooseResolution { descriptor, locator: None, + resolution: None, }); } @@ -231,6 +370,7 @@ impl LooseDescriptor { return Ok(LooseResolution { descriptor: descriptor.clone(), locator: None, + resolution: None, }); } } @@ -256,22 +396,37 @@ impl LooseDescriptor { return Ok(LooseResolution { descriptor, locator: None, + resolution: None, }); }; - // Otherwise we resolve them - let resolution_result - = resolvers::npm::resolve_semver_descriptor(context, &descriptor, &range_params).await?; + let package_ident = range_ident + .unwrap_or(ident); - let range = resolution_result.resolution.version + let bytes + = fetch_registry_metadata(context, package_ident).await?; + let registry_data: RegistryMetadata<'_> + = JsonDocument::hydrate_from_slice(&bytes[..])?; + + let (resolved_version, manifest_value) = find_semver_candidate(context, package_ident, &range_params.range, ®istry_data) + .ok_or_else(|| Error::NoCandidatesFound(descriptor.range.clone()))?; + + let manifest: resolvers::npm::RemoteManifestWithScripts + = JsonDocument::hydrate_from_value(manifest_value)?; + + let range = resolved_version .to_range(range_kind); let descriptor = Descriptor::new(ident.clone(), RegistrySemverRange {ident: range_ident.cloned(), range: range.clone()}.into()); + let resolution + = build_registry_resolution_result(context, &descriptor, package_ident, resolved_version, manifest)?; + let locator = resolution.resolution.locator.clone(); Ok(LooseResolution { descriptor, - locator: Some(resolution_result.resolution.locator), + locator: Some(locator), + resolution: Some(resolution), }) } @@ -283,60 +438,59 @@ impl LooseDescriptor { return Ok(LooseResolution { descriptor, locator: None, + resolution: None, }); } - let descriptor - = Descriptor::new(ident.clone(), RegistryTagRange {ident: range_ident.cloned(), tag: tag.into()}.into()); + let package_ident = range_ident + .unwrap_or(ident); - let Range::RegistryTag(range_params) = &descriptor.range else { - panic!("Invalid range"); - }; - - let resolution_result - = resolvers::npm::resolve_tag_descriptor(context, &descriptor, &range_params).await?; - - let range = resolution_result.resolution.version - .to_range(options.range_kind); + let bytes + = fetch_registry_metadata(context, package_ident).await?; + let registry_data: RegistryMetadata<'_> + = JsonDocument::hydrate_from_slice(&bytes[..])?; - let descriptor - = Descriptor::new(ident.clone(), RegistrySemverRange {ident: range_ident.cloned(), range: range.clone()}.into()); + let latest_version = registry_data.dist_tags + .get(tag) + .ok_or_else(|| Error::TagNotFound(tag.to_string()))?; - let Range::RegistrySemver(range_params) = &descriptor.range else { - panic!("Invalid range"); - }; + let (resolved_version, manifest_value) = registry_data.versions.iter().rev() + .filter(|(version, _)| *version <= latest_version) + .filter(|(version, _)| !version.rc.is_some() || latest_version.rc.is_some()) + .find(|(version, _)| { + let release_time = registry_data.time.as_ref() + .and_then(|times| times.get(*version)); - // We must check whether resolving the range would yield a - // different version than the one we just resolved (this can - // happen if, say, we have `rc: 1.0.0-rc.1`, and there's a - // release version `1.1.0`). - // - // In that case we force the descriptor to use a fixed version - // rather than the requested range_kind. + resolvers::npm::is_package_approved(context, package_ident, version, release_time) + }) + .ok_or_else(|| Error::NoCandidatesFound(AnonymousSemverRange {range: zpm_semver::Range::lte(latest_version.clone())}.into()))?; - let resolution_check_result - = resolvers::npm::resolve_semver_descriptor(context, &descriptor, &range_params).await?; + let manifest: resolvers::npm::RemoteManifestWithScripts + = JsonDocument::hydrate_from_value(manifest_value)?; - if resolution_check_result.resolution.version == resolution_result.resolution.version { - let descriptor - = Descriptor::new(ident.clone(), RegistrySemverRange {ident: range_ident.cloned(), range: range.clone()}.into()); + let derived_range = resolved_version + .to_range(options.range_kind); + let range_matches = find_semver_candidate(context, package_ident, &derived_range, ®istry_data) + .map(|(version, _)| version == resolved_version) + .unwrap_or(false); - Ok(LooseResolution { - descriptor, - locator: Some(resolution_check_result.resolution.locator), - }) + let final_range = if range_matches { + derived_range } else { - let fixed_range = resolution_result.resolution.version - .to_range(RangeKind::Exact); + resolved_version.to_range(RangeKind::Exact) + }; - let descriptor - = Descriptor::new(ident.clone(), RegistrySemverRange {ident: range_ident.cloned(), range: fixed_range.clone()}.into()); + let descriptor + = Descriptor::new(ident.clone(), RegistrySemverRange {ident: range_ident.cloned(), range: final_range}.into()); + let resolution + = build_registry_resolution_result(context, &descriptor, package_ident, resolved_version, manifest)?; + let locator = resolution.resolution.locator.clone(); - Ok(LooseResolution { - descriptor, - locator: Some(resolution_result.resolution.locator), - }) - } + Ok(LooseResolution { + descriptor, + locator: Some(locator), + resolution: Some(resolution), + }) } } diff --git a/packages/zpm/src/install.rs b/packages/zpm/src/install.rs index 436cd5b0..6a5ce81e 100644 --- a/packages/zpm/src/install.rs +++ b/packages/zpm/src/install.rs @@ -35,6 +35,7 @@ pub struct InstallContext<'a> { pub systems: Option<&'a Vec>, pub check_checksums: bool, pub check_resolutions: bool, + pub enforced_resolution_results: BTreeMap, pub prune_dev_dependencies: bool, pub enforced_resolutions: BTreeMap>, pub refresh_lockfile: bool, @@ -50,6 +51,7 @@ impl<'a> Default for InstallContext<'a> { systems: None, check_checksums: false, check_resolutions: false, + enforced_resolution_results: BTreeMap::new(), prune_dev_dependencies: false, enforced_resolutions: BTreeMap::new(), refresh_lockfile: false, @@ -80,6 +82,11 @@ impl<'a> InstallContext<'a> { self } + pub fn set_enforced_resolution_results(mut self, enforced_resolution_results: BTreeMap) -> Self { + self.enforced_resolution_results = enforced_resolution_results; + self + } + pub fn set_enforced_resolutions(mut self, enforced_resolutions: BTreeMap>) -> Self { self.enforced_resolutions = enforced_resolutions; self @@ -255,6 +262,7 @@ fn resolve_descriptor_impl<'a>( match cached { CacheHit::Full(result) => { start_fetch(&result, ctx, maps).await; + return Ok(result); }, @@ -275,6 +283,7 @@ fn resolve_descriptor_impl<'a>( }).await.map_err(Arc::new)?; start_fetch(&result, ctx, maps).await; + return Ok(result); }, } @@ -526,6 +535,10 @@ enum CacheHit { /// Check if a descriptor can be resolved from the lockfile cache. fn check_resolution_cache(ctx: &InstallContext<'_>, lockfile: &Lockfile, descriptor: &Descriptor) -> Result, Error> { + if let Some(result) = ctx.enforced_resolution_results.get(descriptor) { + return Ok(Some(CacheHit::Full(result.clone()))); + } + let range_details = descriptor.range.details(); diff --git a/packages/zpm/src/main.rs b/packages/zpm/src/main.rs index 7f502b42..a20fadab 100644 --- a/packages/zpm/src/main.rs +++ b/packages/zpm/src/main.rs @@ -4,7 +4,9 @@ use std::process::ExitCode; #[tokio::main] async fn main() -> ExitCode { - env_logger::init(); + if std::env::var_os("RUST_LOG").is_some() { + let _ = env_logger::try_init(); + } zpm::commands::run_default(None).await } diff --git a/packages/zpm/src/project.rs b/packages/zpm/src/project.rs index cfe4c4bd..5380cbb6 100644 --- a/packages/zpm/src/project.rs +++ b/packages/zpm/src/project.rs @@ -17,7 +17,7 @@ use crate::{ error::Error, git::{GitOperation, detect_git_operation}, http::HttpClient, - install::{InstallContext, InstallManager, InstallResult, InstallState}, + install::{InstallContext, InstallManager, InstallResult, InstallState, ResolutionResult}, lockfile::{Lockfile, from_legacy_berry_lockfile, from_pnpm_node_modules}, manifest::{Manifest, helpers::read_manifest_with_size}, manifest_finder::CachedManifestFinder, @@ -53,6 +53,7 @@ pub enum InstallMode { pub struct RunInstallOptions { pub check_checksums: bool, pub check_resolutions: bool, + pub enforced_resolution_results: BTreeMap, pub enforced_resolutions: BTreeMap>, pub prune_dev_dependencies: bool, pub mode: Option, @@ -882,6 +883,7 @@ impl Project { .with_project(Some(self)) .set_check_checksums(options.check_checksums) .set_enforced_resolutions(options.enforced_resolutions) + .set_enforced_resolution_results(options.enforced_resolution_results) .set_prune_dev_dependencies(options.prune_dev_dependencies) .set_refresh_lockfile(options.refresh_lockfile) .set_mode(options.mode) diff --git a/packages/zpm/src/resolvers/npm.rs b/packages/zpm/src/resolvers/npm.rs index a24229fe..d095199d 100644 --- a/packages/zpm/src/resolvers/npm.rs +++ b/packages/zpm/src/resolvers/npm.rs @@ -24,7 +24,7 @@ static NODE_GYP_MATCH: LazyLock = LazyLock::new(|| Regex::new(r"\b(node-g * We need to read the scripts to figure out whether the package has an implicit node-gyp dependency. */ #[derive(Clone, Deserialize, Debug)] -struct RemoteManifestWithScripts { +pub(crate) struct RemoteManifestWithScripts { #[serde(flatten)] remote: RemoteManifest, @@ -49,7 +49,7 @@ fn fix_manifest(manifest: &mut RemoteManifestWithScripts) { } } -fn build_resolution_result(context: &InstallContext, descriptor: &Descriptor, package_ident: &Ident, version: zpm_semver::Version, mut manifest: RemoteManifestWithScripts) -> Result { +pub(crate) fn build_resolution_result(context: &InstallContext, descriptor: &Descriptor, package_ident: &Ident, version: zpm_semver::Version, mut manifest: RemoteManifestWithScripts) -> Result { let project = context.project .expect("The project is required for resolving a workspace package"); @@ -97,7 +97,7 @@ pub async fn resolve_semver_or_workspace_descriptor(context: &InstallContext<'_> resolve_semver_descriptor(context, descriptor, params).await } -fn is_package_approved(context: &InstallContext<'_>, ident: &Ident, version: &zpm_semver::Version, release_time: Option<&DateTime>) -> bool { +pub(crate) fn is_package_approved(context: &InstallContext<'_>, ident: &Ident, version: &zpm_semver::Version, release_time: Option<&DateTime>) -> bool { let project = context.project .expect("The project is required for resolving a workspace package");