diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/NpmCliExtractor.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/NpmCliExtractor.java index 012d0f1b7f..33fe07158d 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/NpmCliExtractor.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/NpmCliExtractor.java @@ -26,6 +26,7 @@ import com.blackduck.integration.detectable.extraction.Extraction; import com.blackduck.integration.detectable.util.ToolVersionLogger; import com.blackduck.integration.executable.ExecutableOutput; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; public class NpmCliExtractor { public static final String OUTPUT_FILE = "detect_npm_proj_dependencies.json"; @@ -36,12 +37,20 @@ public class NpmCliExtractor { private final NpmCliParser npmCliParser; private final Gson gson; private final ToolVersionLogger toolVersionLogger; + @Nullable private final ExcludedIncludedWildcardFilter workspaceFilter; public NpmCliExtractor(DetectableExecutableRunner executableRunner, NpmCliParser npmCliParser, Gson gson, ToolVersionLogger toolVersionLogger) { + this(executableRunner, npmCliParser, gson, toolVersionLogger, null); + } + + public NpmCliExtractor(DetectableExecutableRunner executableRunner, NpmCliParser npmCliParser, + Gson gson, ToolVersionLogger toolVersionLogger, + @Nullable ExcludedIncludedWildcardFilter workspaceFilter) { this.executableRunner = executableRunner; this.npmCliParser = npmCliParser; this.gson = gson; this.toolVersionLogger = toolVersionLogger; + this.workspaceFilter = workspaceFilter; } public Extraction extract(File directory, ExecutableTarget npmExe, @Nullable String npmArguments, File packageJsonFile) { @@ -77,7 +86,7 @@ public Extraction extract(File directory, ExecutableTarget npmExe, @Nullable Str } else if (StringUtils.isNotBlank(standardOutput)) { logger.debug("Parsing npm ls file."); logger.debug(standardOutput); - NpmPackagerResult result = npmCliParser.generateCodeLocation(standardOutput, combinedPackageJson); + NpmPackagerResult result = npmCliParser.generateCodeLocation(standardOutput, combinedPackageJson, workspaceFilter); String projectName = result.getProjectName() != null ? result.getProjectName() : combinedPackageJson.getName(); String projectVersion = result.getProjectVersion() != null ? result.getProjectVersion() : combinedPackageJson.getVersion(); return new Extraction.Builder().success(result.getCodeLocation()).projectName(projectName).projectVersion(projectVersion).build(); @@ -91,7 +100,7 @@ private CombinedPackageJson parsePackageJson(File packageJson) throws IOExceptio String packageJsonText = FileUtils.readFileToString(packageJson, StandardCharsets.UTF_8); CombinedPackageJsonExtractor extractor = new CombinedPackageJsonExtractor(gson); - CombinedPackageJson combinedPackageJson = extractor.constructCombinedPackageJson(packageJson.getPath(), packageJsonText); + CombinedPackageJson combinedPackageJson = extractor.constructCombinedPackageJson(packageJson.getPath(), packageJsonText, workspaceFilter); return combinedPackageJson; } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/NpmCliExtractorOptions.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/NpmCliExtractorOptions.java index 7c5562c94b..41e82ad3db 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/NpmCliExtractorOptions.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/NpmCliExtractorOptions.java @@ -1,5 +1,7 @@ package com.blackduck.integration.detectable.detectables.npm.cli; +import java.util.Collections; +import java.util.List; import java.util.Optional; import com.blackduck.integration.detectable.detectable.util.EnumListFilter; @@ -8,10 +10,33 @@ public class NpmCliExtractorOptions { private final EnumListFilter npmDependencyTypeFilter; private final String npmArguments; + private final List excludedWorkspaceNames; + private final List includedWorkspaceNames; + private final boolean ignoreAllWorkspaces; - public NpmCliExtractorOptions(EnumListFilter npmDependencyTypeFilter, String npmArguments) { + public NpmCliExtractorOptions(EnumListFilter npmDependencyTypeFilter, + String npmArguments) { + this(npmDependencyTypeFilter, npmArguments, + Collections.emptyList(), Collections.emptyList(), false); + } + + public NpmCliExtractorOptions(EnumListFilter npmDependencyTypeFilter, + String npmArguments, + List excludedWorkspaceNames, + List includedWorkspaceNames) { + this(npmDependencyTypeFilter, npmArguments, excludedWorkspaceNames, includedWorkspaceNames, false); + } + + public NpmCliExtractorOptions(EnumListFilter npmDependencyTypeFilter, + String npmArguments, + List excludedWorkspaceNames, + List includedWorkspaceNames, + boolean ignoreAllWorkspaces) { this.npmDependencyTypeFilter = npmDependencyTypeFilter; this.npmArguments = npmArguments; + this.excludedWorkspaceNames = excludedWorkspaceNames; + this.includedWorkspaceNames = includedWorkspaceNames; + this.ignoreAllWorkspaces = ignoreAllWorkspaces; } public EnumListFilter getDependencyTypeFilter() { @@ -21,4 +46,16 @@ public EnumListFilter getDependencyTypeFilter() { public Optional getNpmArguments() { return Optional.ofNullable(npmArguments); } + + public List getExcludedWorkspaceNames() { + return excludedWorkspaceNames; + } + + public List getIncludedWorkspaceNames() { + return includedWorkspaceNames; + } + + public boolean isIgnoreAllWorkspaces() { + return ignoreAllWorkspaces; + } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/parse/NpmCliParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/parse/NpmCliParser.java index fe62ac26be..f3bf82b681 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/parse/NpmCliParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/cli/parse/NpmCliParser.java @@ -9,6 +9,7 @@ import org.apache.commons.collections4.MultiValuedMap; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +25,7 @@ import com.blackduck.integration.detectable.detectables.npm.NpmDependencyType; import com.blackduck.integration.detectable.detectables.npm.lockfile.result.NpmPackagerResult; import com.blackduck.integration.detectable.detectables.npm.packagejson.CombinedPackageJson; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -45,41 +47,45 @@ public NpmCliParser(ExternalIdFactory externalIdFactory, EnumListFilter aliasMapping = buildAliasMapping(combinedPackageJson); - populateChildren(graph, null, npmJson.getAsJsonObject(JSON_DEPENDENCIES), true, combinedPackageJson, aliasMapping); + populateChildren(graph, null, npmJson.getAsJsonObject(JSON_DEPENDENCIES), true, + combinedPackageJson, aliasMapping, workspaceFilter); ExternalId externalId = externalIdFactory.createNameVersionExternalId(Forge.NPMJS, projectName, projectVersion); CodeLocation codeLocation = new CodeLocation(graph, externalId); return new NpmPackagerResult(projectName, projectVersion, codeLocation); - } /** @@ -99,7 +105,9 @@ private Map buildAliasMapping(CombinedPackageJson combinedPackag ); } - private void populateChildren(DependencyGraph graph, Dependency parentDependency, JsonObject parentNodeChildren, boolean isRootDependency, CombinedPackageJson combinedPackageJson, Map aliasMapping) { + private void populateChildren(DependencyGraph graph, Dependency parentDependency, JsonObject parentNodeChildren, + boolean isRootDependency, CombinedPackageJson combinedPackageJson, Map aliasMapping, + @Nullable ExcludedIncludedWildcardFilter workspaceFilter) { if (parentNodeChildren == null) { return; } @@ -113,15 +121,16 @@ private void populateChildren(DependencyGraph graph, Dependency parentDependency // Transitives can be both application and dev/peer/optional dependency graphs, but Detect shouldn't be walking a dev, peer, or optional dependency tree unless it passed the filter already. return true; } - boolean excludingBecauseDev = (npmDependencyTypeFilter.shouldExclude(NpmDependencyType.DEV, combinedPackageJson.getDevDependencies()) && combinedPackageJson.getDevDependencies().containsKey( - elementEntry.getKey())); + boolean excludingBecauseDev = (npmDependencyTypeFilter.shouldExclude(NpmDependencyType.DEV, combinedPackageJson.getDevDependencies()) + && combinedPackageJson.getDevDependencies().containsKey(elementEntry.getKey())); boolean excludingBecausePeer = (npmDependencyTypeFilter.shouldExclude(NpmDependencyType.PEER, combinedPackageJson.getPeerDependencies()) && combinedPackageJson.getPeerDependencies().containsKey(elementEntry.getKey())); boolean excludingBecauseOptional = (npmDependencyTypeFilter.shouldExclude(NpmDependencyType.OPTIONAL, combinedPackageJson.getOptionalDependencies()) && combinedPackageJson.getOptionalDependencies().containsKey(elementEntry.getKey())); return !excludingBecauseDev && !excludingBecausePeer && !excludingBecauseOptional; }) - .forEach(elementEntry -> processChild(elementEntry, graph, parentDependency, isRootDependency, combinedPackageJson, aliasMapping)); + .forEach(elementEntry -> processChild(elementEntry, graph, parentDependency, isRootDependency, + combinedPackageJson, aliasMapping, workspaceFilter)); } private void processChild( @@ -130,7 +139,8 @@ private void processChild( Dependency parentDependency, boolean isRootDependency, CombinedPackageJson combinedPackageJson, - Map aliasMapping + Map aliasMapping, + @Nullable ExcludedIncludedWildcardFilter workspaceFilter ) { JsonObject element = elementEntry.getValue().getAsJsonObject(); String name = elementEntry.getKey(); @@ -150,29 +160,44 @@ private void processChild( } ExternalId externalId = externalIdFactory.createNameVersionExternalId(Forge.NPMJS, actualName, version); Dependency child = new Dependency(actualName, version, externalId); - - // Any workspace dependency is considered a direct dependency + + // Any workspace dependency is considered a direct dependency (unless filtered out) boolean directWorkspaceDependency = false; String possibleWorkspaceDependency = Optional.ofNullable(element.getAsJsonPrimitive("resolved")) - .filter(JsonPrimitive::isString) - .map(JsonPrimitive::getAsString) - .orElse(null); - + .filter(JsonPrimitive::isString) + .map(JsonPrimitive::getAsString) + .orElse(null); + + // Whether this workspace was detected but explicitly excluded by the filter + boolean workspaceExcludedByFilter = false; + if (combinedPackageJson.getRelativeWorkspaces() != null && possibleWorkspaceDependency != null) { - // workspaces under the root resolve as file../ - // remove that and see if any absolute workspace paths have this subpath - String convertedPossibleWorkspaceDependency = - possibleWorkspaceDependency.replace("file:../", ""); - - directWorkspaceDependency = - combinedPackageJson.getRelativeWorkspaces().stream().anyMatch(workspace -> workspace.equals(convertedPossibleWorkspaceDependency)); + // Strip "file:" prefix and any leading "../" or "./" segments; the remaining + // path is relative to the project root, matching entries in relativeWorkspaces. + String convertedPath = possibleWorkspaceDependency.replaceFirst("^file:(\\.\\./|\\./)*", ""); + boolean isWorkspace = combinedPackageJson.getRelativeWorkspaces().stream() + .anyMatch(workspace -> workspace.equals(convertedPath)); + + if (isWorkspace) { + if (workspaceFilter != null) { + directWorkspaceDependency = workspaceFilter.shouldInclude(convertedPath); + workspaceExcludedByFilter = !directWorkspaceDependency; + } else { + directWorkspaceDependency = true; + } + } } - populateChildren(graph, child, children, directWorkspaceDependency, combinedPackageJson, aliasMapping); + // When a workspace is excluded by the filter, do not traverse its children — + // treat it as a leaf (regular dependency with no transitive graph) + if (!workspaceExcludedByFilter) { + populateChildren(graph, child, children, directWorkspaceDependency, + combinedPackageJson, aliasMapping, workspaceFilter); + } - if (isRootDependency || directWorkspaceDependency) { + if ((isRootDependency && !workspaceExcludedByFilter) || directWorkspaceDependency) { graph.addChildToRoot(child); - } else { + } else if (!workspaceExcludedByFilter) { graph.addParentWithChild(parentDependency, child); } } else { diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/lockfile/NpmLockfileOptions.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/lockfile/NpmLockfileOptions.java index ab3ae5e331..28e50c287a 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/lockfile/NpmLockfileOptions.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/lockfile/NpmLockfileOptions.java @@ -1,17 +1,52 @@ package com.blackduck.integration.detectable.detectables.npm.lockfile; +import java.util.List; + import com.blackduck.integration.detectable.detectable.util.EnumListFilter; import com.blackduck.integration.detectable.detectables.npm.NpmDependencyType; // TODO: Identical to NpmPackageJsonParseDetectableOptions. Similar to NpmCliExtractorOptions. Common base Options class? JM-01/2022 public class NpmLockfileOptions { private final EnumListFilter npmDependencyTypeFilter; + private final List excludedWorkspaceNames; + private final List includedWorkspaceNames; + private final boolean ignoreAllWorkspaces; public NpmLockfileOptions(EnumListFilter npmDependencyTypeFilter) { + this(npmDependencyTypeFilter, java.util.Collections.emptyList(), java.util.Collections.emptyList(), false); + } + + public NpmLockfileOptions( + EnumListFilter npmDependencyTypeFilter, + List excludedWorkspaceNames, + List includedWorkspaceNames) { + this(npmDependencyTypeFilter, excludedWorkspaceNames, includedWorkspaceNames, false); + } + + public NpmLockfileOptions( + EnumListFilter npmDependencyTypeFilter, + List excludedWorkspaceNames, + List includedWorkspaceNames, + boolean ignoreAllWorkspaces) { this.npmDependencyTypeFilter = npmDependencyTypeFilter; + this.excludedWorkspaceNames = excludedWorkspaceNames; + this.includedWorkspaceNames = includedWorkspaceNames; + this.ignoreAllWorkspaces = ignoreAllWorkspaces; } public EnumListFilter getNpmDependencyTypeFilter() { return npmDependencyTypeFilter; } + + public List getExcludedWorkspaceNames() { + return excludedWorkspaceNames; + } + + public List getIncludedWorkspaceNames() { + return includedWorkspaceNames; + } + + public boolean isIgnoreAllWorkspaces() { + return ignoreAllWorkspaces; + } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/lockfile/parse/NpmLockfilePackager.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/lockfile/parse/NpmLockfilePackager.java index af160093f9..cd45145db1 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/lockfile/parse/NpmLockfilePackager.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/lockfile/parse/NpmLockfilePackager.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.Nullable; @@ -22,6 +23,7 @@ import com.blackduck.integration.detectable.detectables.npm.lockfile.result.NpmPackagerResult; import com.blackduck.integration.detectable.detectables.npm.packagejson.CombinedPackageJson; import com.blackduck.integration.detectable.detectables.npm.packagejson.CombinedPackageJsonExtractor; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; import com.blackduck.integration.util.NameVersion; public class NpmLockfilePackager { @@ -29,12 +31,21 @@ public class NpmLockfilePackager { private final ExternalIdFactory externalIdFactory; private final NpmLockFileProjectIdTransformer projectIdTransformer; private final NpmLockfileGraphTransformer graphTransformer; + @Nullable private final ExcludedIncludedWildcardFilter workspaceFilter; public NpmLockfilePackager(Gson gson, ExternalIdFactory externalIdFactory, NpmLockFileProjectIdTransformer projectIdTransformer, NpmLockfileGraphTransformer graphTransformer) { + this(gson, externalIdFactory, projectIdTransformer, graphTransformer, null); + } + + public NpmLockfilePackager(Gson gson, ExternalIdFactory externalIdFactory, + NpmLockFileProjectIdTransformer projectIdTransformer, + NpmLockfileGraphTransformer graphTransformer, + @Nullable ExcludedIncludedWildcardFilter workspaceFilter) { this.gson = gson; this.externalIdFactory = externalIdFactory; this.projectIdTransformer = projectIdTransformer; this.graphTransformer = graphTransformer; + this.workspaceFilter = workspaceFilter; } public NpmPackagerResult parseAndTransform(@Nullable String rootJsonPath, @Nullable String packageJsonText, String lockFileText) throws IOException { @@ -43,7 +54,7 @@ public NpmPackagerResult parseAndTransform(@Nullable String rootJsonPath, @Nulla public NpmPackagerResult parseAndTransform(@Nullable String rootJsonPath, @Nullable String packageJsonText, String lockFileText, List externalDependencies) throws IOException { CombinedPackageJsonExtractor extractor = new CombinedPackageJsonExtractor(gson); - CombinedPackageJson combinedPackageJson = extractor.constructCombinedPackageJson(rootJsonPath, packageJsonText); + CombinedPackageJson combinedPackageJson = extractor.constructCombinedPackageJson(rootJsonPath, packageJsonText, workspaceFilter); lockFileText = removePathInfoFromPackageName(lockFileText); @@ -68,12 +79,25 @@ public NpmPackagerResult parseAndTransform(@Nullable String rootJsonPath, @Nulla : new HashMap<>(); DependencyGraph dependencyGraph = graphTransformer.transform(packageLock, project, externalDependencies, - combinedPackageJson == null ? null : combinedPackageJson.getRelativeWorkspaces(), aliasMapping); + getFilteredWorkspaces(combinedPackageJson), aliasMapping); ExternalId projectId = projectIdTransformer.transform(combinedPackageJson, packageLock); CodeLocation codeLocation = new CodeLocation(dependencyGraph, projectId); return new NpmPackagerResult(projectId.getName(), projectId.getVersion(), codeLocation); } + private List getFilteredWorkspaces(@Nullable CombinedPackageJson combinedPackageJson) { + if (combinedPackageJson == null) { + return null; + } + List workspaces = combinedPackageJson.getRelativeWorkspaces(); + if (workspaceFilter == null || workspaces.isEmpty()) { + return workspaces; + } + return workspaces.stream() + .filter(path -> workspaceFilter.shouldInclude(path)) + .collect(Collectors.toList()); + } + public String removePathInfoFromPackageName(String lockFileText) { List searchList = new ArrayList<>(Arrays.asList("/node_modules/", "node_modules/")); List replaceList = new ArrayList<>(Arrays.asList("*", "")); diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/CombinedPackageJson.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/CombinedPackageJson.java index eeafee1017..f50580690b 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/CombinedPackageJson.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/CombinedPackageJson.java @@ -11,7 +11,7 @@ public class CombinedPackageJson { private String name; private String version; private List relativeWorkspaces = new ArrayList<>(); - + private MultiValuedMap dependencies; private MultiValuedMap devDependencies; private MultiValuedMap peerDependencies; @@ -40,7 +40,7 @@ public MultiValuedMap getOptionalDependencies() { return optionalDependencies; } - public List getRelativeWorkspaces() { + public List getRelativeWorkspaces() { return relativeWorkspaces; } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/CombinedPackageJsonExtractor.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/CombinedPackageJsonExtractor.java index f17f0363cc..d6cc397d2d 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/CombinedPackageJsonExtractor.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/CombinedPackageJsonExtractor.java @@ -11,6 +11,7 @@ import java.util.Optional; import com.blackduck.integration.detectable.util.JsonSanitizer; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; import com.google.gson.Gson; import org.apache.commons.io.FileUtils; @@ -28,56 +29,72 @@ public CombinedPackageJsonExtractor(Gson gson) { * Merge the root package.json with any potential workspace package.json files. */ public CombinedPackageJson constructCombinedPackageJson(String rootJsonPath, String packageJsonText) throws IOException { + return constructCombinedPackageJson(rootJsonPath, packageJsonText, null); + } + + public CombinedPackageJson constructCombinedPackageJson(String rootJsonPath, String packageJsonText, + ExcludedIncludedWildcardFilter workspaceFilter) throws IOException { if (packageJsonText == null) { return null; } - + PackageJson packageJson = Optional.ofNullable(JsonSanitizer.sanitize(packageJsonText)) .map(content -> gson.fromJson(content, PackageJson.class)) .orElse(null); - + if (packageJson == null) { return null; } - + CombinedPackageJson combinedPackageJson = new CombinedPackageJson(); - + // Take fields that will be related to BD projects from the root project.json combinedPackageJson.setName(packageJson.name); combinedPackageJson.setVersion(packageJson.version); - + // Add dependencies from the root of the project combinedPackageJson.getDependencies().putAll(packageJson.dependencies); combinedPackageJson.getDevDependencies().putAll(packageJson.devDependencies); combinedPackageJson.getPeerDependencies().putAll(packageJson.peerDependencies); combinedPackageJson.getOptionalDependencies().putAll(packageJson.optionalDependencies); - + if (packageJson.workspaces != null && rootJsonPath != null) { // If there are workspaces there are additional package.json's we need to parse String projectRoot = rootJsonPath.substring(0, rootJsonPath.lastIndexOf(File.separator) + 1); - - List convertedWorkspaces = + + List convertedWorkspaces = convertWorkspaceWildcards(projectRoot, packageJson.workspaces); - + for(String convertedWorkspace : convertedWorkspaces) { Path workspaceJsonPath = Paths.get(convertedWorkspace + "/package.json").normalize(); - + // We are looking for a package.json but they aren't always where we expect them. // Don't try to read a file that doesn't exist. if (!Files.exists(workspaceJsonPath)) { continue; - } else { - addRelativeWorkspace(combinedPackageJson, projectRoot, convertedWorkspace); } - - String workspaceJsonString - = FileUtils.readFileToString(new File(workspaceJsonPath.toString()), StandardCharsets.UTF_8); - + + // Always register the workspace path so downstream code (CLI parser, lockfile packager) + // can identify workspace packages — even ones that will be excluded. Without this, + // the CLI parser can't set workspaceExcludedByFilter and the package slips through as + // a regular root dependency. + addRelativeWorkspace(combinedPackageJson, projectRoot, convertedWorkspace); + + // Skip merging deps from excluded workspaces into the combined package.json. + // This prevents excluded workspace deps from appearing as declared root dependencies. + String relativePath = getRelativePath(projectRoot, convertedWorkspace); + if (workspaceFilter != null && relativePath != null && !workspaceFilter.shouldInclude(relativePath)) { + continue; + } + + String workspaceJsonString = + FileUtils.readFileToString(new File(workspaceJsonPath.toString()), StandardCharsets.UTF_8); + PackageJson workspacePackageJson = Optional.ofNullable(workspaceJsonString) .map(content -> gson.fromJson(content, PackageJson.class)) .orElse(null); - + if (workspacePackageJson != null) { combinedPackageJson.getDependencies().putAll(workspacePackageJson.dependencies); combinedPackageJson.getDevDependencies().putAll(workspacePackageJson.devDependencies); @@ -86,7 +103,7 @@ public CombinedPackageJson constructCombinedPackageJson(String rootJsonPath, Str } } } - + return combinedPackageJson; } @@ -101,16 +118,23 @@ public CombinedPackageJson constructCombinedPackageJson(String rootJsonPath, Str */ private void addRelativeWorkspace(CombinedPackageJson combinedPackageJson, String projectRoot, String convertedWorkspace) { + String relativePath = getRelativePath(projectRoot, convertedWorkspace); + if (relativePath != null) { + combinedPackageJson.getRelativeWorkspaces().add(relativePath); + } + } + + private String getRelativePath(String projectRoot, String convertedWorkspace) { int rootIndex = convertedWorkspace.indexOf(projectRoot); if (rootIndex != -1) { int packageStartIndex = rootIndex + projectRoot.length(); if (packageStartIndex < convertedWorkspace.length()) { // Replace any \'s with /'s, so we can properly compare workspace names with what is in // the package-lock.json file. - String relativeWorkspace = convertedWorkspace.substring(packageStartIndex).replace("\\", "/"); - combinedPackageJson.getRelativeWorkspaces().add(relativeWorkspace); + return convertedWorkspace.substring(packageStartIndex).replace("\\", "/"); } } + return null; } /** diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/NpmPackageJsonParseDetectableOptions.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/NpmPackageJsonParseDetectableOptions.java index a4be77e7fc..088306dfb4 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/NpmPackageJsonParseDetectableOptions.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/NpmPackageJsonParseDetectableOptions.java @@ -1,16 +1,45 @@ package com.blackduck.integration.detectable.detectables.npm.packagejson; +import java.util.Collections; +import java.util.List; + import com.blackduck.integration.detectable.detectable.util.EnumListFilter; import com.blackduck.integration.detectable.detectables.npm.NpmDependencyType; public class NpmPackageJsonParseDetectableOptions { private final EnumListFilter npmDependencyTypeFilter; + private final List excludedWorkspaceNames; + private final List includedWorkspaceNames; + private final boolean ignoreAllWorkspaces; public NpmPackageJsonParseDetectableOptions(EnumListFilter npmDependencyTypeFilter) { + this(npmDependencyTypeFilter, Collections.emptyList(), Collections.emptyList(), false); + } + + public NpmPackageJsonParseDetectableOptions( + EnumListFilter npmDependencyTypeFilter, + List excludedWorkspaceNames, + List includedWorkspaceNames, + boolean ignoreAllWorkspaces) { this.npmDependencyTypeFilter = npmDependencyTypeFilter; + this.excludedWorkspaceNames = excludedWorkspaceNames; + this.includedWorkspaceNames = includedWorkspaceNames; + this.ignoreAllWorkspaces = ignoreAllWorkspaces; } public EnumListFilter getNpmDependencyTypeFilter() { return npmDependencyTypeFilter; } + + public List getExcludedWorkspaceNames() { + return excludedWorkspaceNames; + } + + public List getIncludedWorkspaceNames() { + return includedWorkspaceNames; + } + + public boolean isIgnoreAllWorkspaces() { + return ignoreAllWorkspaces; + } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/PackageJsonExtractor.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/PackageJsonExtractor.java index 4276fa3f57..d6700a9be5 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/PackageJsonExtractor.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/npm/packagejson/PackageJsonExtractor.java @@ -19,22 +19,33 @@ import com.blackduck.integration.bdio.model.externalid.ExternalId; import com.blackduck.integration.bdio.model.externalid.ExternalIdFactory; import com.blackduck.integration.detectable.detectable.codelocation.CodeLocation; +import org.jetbrains.annotations.Nullable; + import com.blackduck.integration.detectable.detectable.util.EnumListFilter; import com.blackduck.integration.detectable.detectable.util.SemVerComparator; import com.blackduck.integration.detectable.detectables.npm.NpmAliasParser; import com.blackduck.integration.detectable.detectables.npm.NpmDependencyType; import com.blackduck.integration.detectable.extraction.Extraction; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; import com.google.gson.Gson; public class PackageJsonExtractor { private final Gson gson; private final ExternalIdFactory externalIdFactory; private final EnumListFilter npmDependencyTypeFilter; + @Nullable private final ExcludedIncludedWildcardFilter workspaceFilter; public PackageJsonExtractor(Gson gson, ExternalIdFactory externalIdFactory, EnumListFilter npmDependencyTypeFilter) { + this(gson, externalIdFactory, npmDependencyTypeFilter, null); + } + + public PackageJsonExtractor(Gson gson, ExternalIdFactory externalIdFactory, + EnumListFilter npmDependencyTypeFilter, + @Nullable ExcludedIncludedWildcardFilter workspaceFilter) { this.gson = gson; this.externalIdFactory = externalIdFactory; this.npmDependencyTypeFilter = npmDependencyTypeFilter; + this.workspaceFilter = workspaceFilter; } public Extraction extract(File packageJsonFile) throws IOException { @@ -46,7 +57,7 @@ public Extraction extract(File packageJsonFile) throws IOException { } CombinedPackageJsonExtractor extractor = new CombinedPackageJsonExtractor(gson); - CombinedPackageJson combinedPackageJson = extractor.constructCombinedPackageJson(packagePath, packageText); + CombinedPackageJson combinedPackageJson = extractor.constructCombinedPackageJson(packagePath, packageText, workspaceFilter); return extract(combinedPackageJson); } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/factory/DetectableFactory.java b/detectable/src/main/java/com/blackduck/integration/detectable/factory/DetectableFactory.java index f27fc36ae6..c210c9402e 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/factory/DetectableFactory.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/factory/DetectableFactory.java @@ -1,6 +1,8 @@ package com.blackduck.integration.detectable.factory; import java.io.File; +import java.util.Collections; +import java.util.List; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; @@ -307,6 +309,7 @@ import com.blackduck.integration.detectable.detectables.opam.lockfile.OpamLockFileExtractor; import com.blackduck.integration.detectable.detectables.opam.transform.OpamGraphTransformer; import com.blackduck.integration.detectable.util.ToolVersionLogger; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; /* Entry point for creating detectables using most @@ -574,7 +577,11 @@ public Conan2CliDetectable createConan2CliDetectable(DetectableEnvironment envir public NpmCliDetectable createNpmCliDetectable(DetectableEnvironment environment, NpmResolver npmResolver, NpmCliExtractorOptions npmCliExtractorOptions) { NpmCliParser npmCliParser = new NpmCliParser(externalIdFactory, npmCliExtractorOptions.getDependencyTypeFilter()); - NpmCliExtractor npmCliExtractor = new NpmCliExtractor(executableRunner, npmCliParser, gson, toolVersionLogger); + ExcludedIncludedWildcardFilter workspaceFilter = buildNpmWorkspaceFilter( + npmCliExtractorOptions.isIgnoreAllWorkspaces(), + npmCliExtractorOptions.getExcludedWorkspaceNames(), + npmCliExtractorOptions.getIncludedWorkspaceNames()); + NpmCliExtractor npmCliExtractor = new NpmCliExtractor(executableRunner, npmCliParser, gson, toolVersionLogger, workspaceFilter); return new NpmCliDetectable(environment, fileFinder, npmResolver, npmCliExtractor, npmCliExtractorOptions); } @@ -595,7 +602,11 @@ public NpmShrinkwrapDetectable createNpmShrinkwrapDetectable(DetectableEnvironme } public NpmPackageJsonParseDetectable createNpmPackageJsonParseDetectable(DetectableEnvironment environment, NpmPackageJsonParseDetectableOptions npmPackageJsonOptions) { - PackageJsonExtractor packageJsonExtractor = new PackageJsonExtractor(gson, externalIdFactory, npmPackageJsonOptions.getNpmDependencyTypeFilter()); + ExcludedIncludedWildcardFilter workspaceFilter = buildNpmWorkspaceFilter( + npmPackageJsonOptions.isIgnoreAllWorkspaces(), + npmPackageJsonOptions.getExcludedWorkspaceNames(), + npmPackageJsonOptions.getIncludedWorkspaceNames()); + PackageJsonExtractor packageJsonExtractor = new PackageJsonExtractor(gson, externalIdFactory, npmPackageJsonOptions.getNpmDependencyTypeFilter(), workspaceFilter); return new NpmPackageJsonParseDetectable(environment, fileFinder, packageJsonExtractor); } @@ -1027,7 +1038,18 @@ private ConanCliExtractor conanCliExtractor(ConanCliOptions options) { private NpmLockfilePackager npmLockfilePackager(NpmLockfileOptions npmLockfileOptions) { NpmLockfileGraphTransformer npmLockfileGraphTransformer = new NpmLockfileGraphTransformer(npmLockfileOptions.getNpmDependencyTypeFilter()); - return new NpmLockfilePackager(gson, externalIdFactory, npmLockFileProjectIdTransformer(), npmLockfileGraphTransformer); + ExcludedIncludedWildcardFilter workspaceFilter = buildNpmWorkspaceFilter( + npmLockfileOptions.isIgnoreAllWorkspaces(), + npmLockfileOptions.getExcludedWorkspaceNames(), + npmLockfileOptions.getIncludedWorkspaceNames()); + return new NpmLockfilePackager(gson, externalIdFactory, npmLockFileProjectIdTransformer(), npmLockfileGraphTransformer, workspaceFilter); + } + + private ExcludedIncludedWildcardFilter buildNpmWorkspaceFilter(boolean ignoreAllWorkspaces, List excluded, List included) { + if (ignoreAllWorkspaces) { + return ExcludedIncludedWildcardFilter.fromCollections(Collections.singletonList("*"), Collections.emptyList()); + } + return ExcludedIncludedWildcardFilter.fromCollections(excluded, included); } private NpmLockFileProjectIdTransformer npmLockFileProjectIdTransformer() { diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/cli/parse/NpmCliParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/cli/parse/NpmCliParserTest.java index 08abd52e63..4cc87d8bd7 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/cli/parse/NpmCliParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/cli/parse/NpmCliParserTest.java @@ -4,18 +4,22 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import com.blackduck.integration.bdio.graph.DependencyGraph; +import com.blackduck.integration.bdio.model.Forge; import com.blackduck.integration.bdio.model.dependency.Dependency; import com.blackduck.integration.bdio.model.externalid.ExternalIdFactory; import com.blackduck.integration.detectable.detectable.util.EnumListFilter; import com.blackduck.integration.detectable.detectables.npm.NpmDependencyType; import com.blackduck.integration.detectable.detectables.npm.lockfile.result.NpmPackagerResult; import com.blackduck.integration.detectable.detectables.npm.packagejson.CombinedPackageJson; +import com.blackduck.integration.detectable.util.graph.GraphAssert; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; class NpmCliParserTest { @@ -180,4 +184,50 @@ void testParseNpmLsWithDevDependencyAlias() { assertEquals("devpackage", dependency.getName()); assertEquals("1.0.0", dependency.getVersion()); } + + @Test + public void excludedWorkspaceIsNotTreatedAsWorkspace() { + // npm ls -json output: root project has "my-ui" workspace and "express" regular dep. + // my-ui has lodash as its own dep. + // When my-ui is excluded, lodash should NOT become a root dependency. + String npmLsOutput = "{" + + "\"name\":\"my-project\"," + + "\"version\":\"1.0.0\"," + + "\"dependencies\":{" + + "\"express\":{\"version\":\"4.18.0\"}," + + "\"my-ui\":{" + + "\"version\":\"1.0.0\"," + + "\"resolved\":\"file:../packages/ui\"," + + "\"dependencies\":{" + + "\"lodash\":{\"version\":\"4.17.21\"}" + + "}" + + "}" + + "}" + + "}"; + + CombinedPackageJson combinedPackageJson = new CombinedPackageJson(); + combinedPackageJson.setName("my-project"); + combinedPackageJson.setVersion("1.0.0"); + combinedPackageJson.getRelativeWorkspaces().add("packages/ui"); + + EnumListFilter depFilter = EnumListFilter.excludeNone(); + NpmCliParser parser = new NpmCliParser(externalIdFactory, depFilter); + + // Exclude by relative path + ExcludedIncludedWildcardFilter workspaceFilter = + ExcludedIncludedWildcardFilter.fromCollections( + Collections.singletonList("packages/ui"), + Collections.emptyList()); + + NpmPackagerResult result = parser.generateCodeLocation(npmLsOutput, combinedPackageJson, workspaceFilter); + + DependencyGraph graph = result.getCodeLocation().getDependencyGraph(); + GraphAssert graphAssert = new GraphAssert(Forge.NPMJS, graph); + graphAssert.hasRootDependency( + externalIdFactory.createNameVersionExternalId(Forge.NPMJS, "express", "4.18.0")); + graphAssert.hasNoDependency( + externalIdFactory.createNameVersionExternalId(Forge.NPMJS, "lodash", "4.17.21")); + graphAssert.hasNoDependency( + externalIdFactory.createNameVersionExternalId(Forge.NPMJS, "my-ui", "1.0.0")); + } } diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/lockfile/unit/NpmWorkspaceFilterTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/lockfile/unit/NpmWorkspaceFilterTest.java new file mode 100644 index 0000000000..0ea8d03910 --- /dev/null +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/lockfile/unit/NpmWorkspaceFilterTest.java @@ -0,0 +1,126 @@ +package com.blackduck.integration.detectable.detectables.npm.lockfile.unit; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import org.junit.jupiter.api.Test; + +import com.google.gson.Gson; +import com.blackduck.integration.bdio.graph.DependencyGraph; +import com.blackduck.integration.bdio.model.Forge; +import com.blackduck.integration.bdio.model.dependency.Dependency; +import com.blackduck.integration.bdio.model.externalid.ExternalId; +import com.blackduck.integration.bdio.model.externalid.ExternalIdFactory; +import com.blackduck.integration.detectable.detectables.npm.lockfile.NpmLockfileOptions; +import com.blackduck.integration.detectable.detectables.npm.lockfile.parse.NpmLockfileGraphTransformer; +import com.blackduck.integration.detectable.detectables.npm.lockfile.parse.NpmLockfilePackager; +import com.blackduck.integration.detectable.detectables.npm.lockfile.parse.NpmLockFileProjectIdTransformer; +import com.blackduck.integration.detectable.detectables.npm.lockfile.result.NpmPackagerResult; +import com.blackduck.integration.detectable.detectable.util.EnumListFilter; +import com.blackduck.integration.detectable.detectables.npm.NpmDependencyType; +import com.blackduck.integration.detectable.util.FunctionalTestFiles; +import com.blackduck.integration.detectable.util.graph.GraphAssert; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; + +public class NpmWorkspaceFilterTest { + + // Fixture files live under src/test/resources/detectables/functional/npm/workspace-filter-test/ + // The fixture has two workspaces: + // packages/api (package name "my-api") — has express as a dependency + // packages/ui (package name "my-ui") — has lodash as a dependency + // The root package.json declares express as a direct dependency. + private static final String FIXTURE_PATH = "/npm/workspace-filter-test/package.json"; + private static final String PACKAGE_JSON = FunctionalTestFiles.asString(FIXTURE_PATH); + private static final String LOCKFILE = FunctionalTestFiles.asString("/npm/workspace-filter-test/package-lock.json"); + + @Test + public void excludedWorkspaceIsNotTreatedAsWorkspace() throws IOException { + NpmLockfileOptions options = new NpmLockfileOptions( + EnumListFilter.excludeNone(), + Arrays.asList("packages/ui"), // exclude by relative path + Collections.emptyList() + ); + NpmPackagerResult result = buildResult(options); + // lodash is only brought in as a workspace dep of my-ui. + // When my-ui is excluded as a workspace, lodash should not be promoted to root. + // express (from my-api workspace) should still appear as a root dep. + DependencyGraph graph = result.getCodeLocation().getDependencyGraph(); + ExternalId expressId = new ExternalIdFactory().createNameVersionExternalId(Forge.NPMJS, "express", "4.18.0"); + ExternalId lodashId = new ExternalIdFactory().createNameVersionExternalId(Forge.NPMJS, "lodash", "4.17.21"); + + boolean expressAtRoot = graph.getRootDependencies().stream() + .map(Dependency::getExternalId) + .anyMatch(expressId::equals); + boolean lodashAtRoot = graph.getRootDependencies().stream() + .map(Dependency::getExternalId) + .anyMatch(lodashId::equals); + + assertTrue(expressAtRoot, "express should be a root dependency (promoted from my-api workspace)"); + assertFalse(lodashAtRoot, "lodash should NOT be a root dependency when my-ui workspace is excluded"); + } + + @Test + public void includedWorkspaceOnlyIncludesThatWorkspace() throws IOException { + NpmLockfileOptions options = new NpmLockfileOptions( + EnumListFilter.excludeNone(), + Collections.emptyList(), + Arrays.asList("packages/api") // only include by relative path + ); + NpmPackagerResult result = buildResult(options); + DependencyGraph graph = result.getCodeLocation().getDependencyGraph(); + ExternalId expressId = new ExternalIdFactory().createNameVersionExternalId(Forge.NPMJS, "express", "4.18.0"); + ExternalId lodashId = new ExternalIdFactory().createNameVersionExternalId(Forge.NPMJS, "lodash", "4.17.21"); + + boolean expressAtRoot = graph.getRootDependencies().stream() + .map(Dependency::getExternalId) + .anyMatch(expressId::equals); + boolean lodashAtRoot = graph.getRootDependencies().stream() + .map(Dependency::getExternalId) + .anyMatch(lodashId::equals); + + assertTrue(expressAtRoot, "express should be a root dependency (promoted from my-api workspace)"); + assertFalse(lodashAtRoot, "lodash should NOT be a root dependency when only my-api workspace is included"); + } + + @Test + public void ignoreAllWorkspacesExcludesAllWorkspaceDeps() throws IOException { + NpmLockfileOptions options = new NpmLockfileOptions( + EnumListFilter.excludeNone(), + Collections.emptyList(), + Collections.emptyList(), + true + ); + NpmPackagerResult result = buildResult(options); + DependencyGraph graph = result.getCodeLocation().getDependencyGraph(); + ExternalId expressId = new ExternalIdFactory().createNameVersionExternalId(Forge.NPMJS, "express", "4.18.0"); + ExternalId lodashId = new ExternalIdFactory().createNameVersionExternalId(Forge.NPMJS, "lodash", "4.17.21"); + + boolean expressAtRoot = graph.getRootDependencies().stream() + .map(Dependency::getExternalId) + .anyMatch(expressId::equals); + boolean lodashAtRoot = graph.getRootDependencies().stream() + .map(Dependency::getExternalId) + .anyMatch(lodashId::equals); + + assertTrue(expressAtRoot, "express should be a root dependency (declared in root package.json)"); + assertFalse(lodashAtRoot, "lodash should NOT be a root dependency when all workspaces are ignored"); + } + + private NpmPackagerResult buildResult(NpmLockfileOptions options) throws IOException { + String rootJsonPath = FunctionalTestFiles.resolvePath(FIXTURE_PATH); + Gson gson = new Gson(); + ExternalIdFactory externalIdFactory = new ExternalIdFactory(); + NpmLockfileGraphTransformer transformer = + new NpmLockfileGraphTransformer(options.getNpmDependencyTypeFilter()); + ExcludedIncludedWildcardFilter workspaceFilter = options.isIgnoreAllWorkspaces() + ? ExcludedIncludedWildcardFilter.fromCollections(Collections.singletonList("*"), Collections.emptyList()) + : ExcludedIncludedWildcardFilter.fromCollections(options.getExcludedWorkspaceNames(), options.getIncludedWorkspaceNames()); + NpmLockfilePackager packager = + new NpmLockfilePackager(gson, externalIdFactory, + new NpmLockFileProjectIdTransformer(gson, externalIdFactory), transformer, workspaceFilter); + return packager.parseAndTransform(rootJsonPath, PACKAGE_JSON, LOCKFILE); + } +} diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/packagejson/unit/CombinedPackageJsonExtractorTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/packagejson/unit/CombinedPackageJsonExtractorTest.java index 4c95e96568..d84f619f96 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/packagejson/unit/CombinedPackageJsonExtractorTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/packagejson/unit/CombinedPackageJsonExtractorTest.java @@ -1,7 +1,6 @@ package com.blackduck.integration.detectable.detectables.npm.packagejson.unit; import java.io.IOException; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collection; import java.util.Map.Entry; diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/packagejson/unit/PackageJsonExtractorTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/packagejson/unit/PackageJsonExtractorTest.java index 6d396827a7..b7903ed788 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/packagejson/unit/PackageJsonExtractorTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/npm/packagejson/unit/PackageJsonExtractorTest.java @@ -1,6 +1,13 @@ package com.blackduck.integration.detectable.detectables.npm.packagejson.unit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -9,6 +16,7 @@ import com.google.gson.GsonBuilder; import com.blackduck.integration.bdio.graph.DependencyGraph; import com.blackduck.integration.bdio.model.Forge; +import com.blackduck.integration.bdio.model.dependency.Dependency; import com.blackduck.integration.bdio.model.externalid.ExternalId; import com.blackduck.integration.bdio.model.externalid.ExternalIdFactory; import com.blackduck.integration.detectable.annotations.UnitTest; @@ -18,7 +26,9 @@ import com.blackduck.integration.detectable.detectables.npm.packagejson.CombinedPackageJson; import com.blackduck.integration.detectable.detectables.npm.packagejson.PackageJsonExtractor; import com.blackduck.integration.detectable.extraction.Extraction; +import com.blackduck.integration.detectable.util.FunctionalTestFiles; import com.blackduck.integration.detectable.util.graph.GraphAssert; +import com.blackduck.integration.util.ExcludedIncludedWildcardFilter; @UnitTest class PackageJsonExtractorTest { @@ -36,12 +46,79 @@ void setUp() { testDevDep2 = externalIdFactory.createNameVersionExternalId(Forge.NPMJS, "nameDev2", "versionDev2"); } + // Workspace filtering tests use the workspace-filter-test fixture: + // root package.json: { "dependencies": { "express": "4.18.0" }, "workspaces": ["packages/api", "packages/ui"] } + // packages/api/package.json: { "dependencies": { "express": "4.18.0" } } + // packages/ui/package.json: { "dependencies": { "lodash": "4.17.21" } } + + @Test + void excludedWorkspaceDepDoesNotAppearInOutput() throws IOException { + ExcludedIncludedWildcardFilter filter = ExcludedIncludedWildcardFilter.fromCollections( + Arrays.asList("packages/ui"), Collections.emptyList()); + PackageJsonExtractor extractor = createExtractorWithFilter(filter); + DependencyGraph graph = extractFromFixture(extractor); + + ExternalIdFactory factory = new ExternalIdFactory(); + ExternalId expressId = factory.createNameVersionExternalId(Forge.NPMJS, "express", "4.18.0"); + ExternalId lodashId = factory.createNameVersionExternalId(Forge.NPMJS, "lodash", "4.17.21"); + + assertTrue(graph.getRootDependencies().stream().map(Dependency::getExternalId).anyMatch(expressId::equals), + "express should appear (declared in root package.json)"); + assertFalse(graph.getRootDependencies().stream().map(Dependency::getExternalId).anyMatch(lodashId::equals), + "lodash should NOT appear when packages/ui is excluded"); + } + + @Test + void includedWorkspaceOnlyMergesThatWorkspacesDeps() throws IOException { + ExcludedIncludedWildcardFilter filter = ExcludedIncludedWildcardFilter.fromCollections( + Collections.emptyList(), Arrays.asList("packages/api")); + PackageJsonExtractor extractor = createExtractorWithFilter(filter); + DependencyGraph graph = extractFromFixture(extractor); + + ExternalIdFactory factory = new ExternalIdFactory(); + ExternalId expressId = factory.createNameVersionExternalId(Forge.NPMJS, "express", "4.18.0"); + ExternalId lodashId = factory.createNameVersionExternalId(Forge.NPMJS, "lodash", "4.17.21"); + + assertTrue(graph.getRootDependencies().stream().map(Dependency::getExternalId).anyMatch(expressId::equals), + "express should appear (declared in root package.json and packages/api)"); + assertFalse(graph.getRootDependencies().stream().map(Dependency::getExternalId).anyMatch(lodashId::equals), + "lodash should NOT appear when only packages/api is included"); + } + + @Test + void ignoreAllWorkspacesExcludesAllWorkspaceDeps() throws IOException { + ExcludedIncludedWildcardFilter filter = ExcludedIncludedWildcardFilter.fromCollections( + Collections.singletonList("*"), Collections.emptyList()); + PackageJsonExtractor extractor = createExtractorWithFilter(filter); + DependencyGraph graph = extractFromFixture(extractor); + + ExternalIdFactory factory = new ExternalIdFactory(); + ExternalId expressId = factory.createNameVersionExternalId(Forge.NPMJS, "express", "4.18.0"); + ExternalId lodashId = factory.createNameVersionExternalId(Forge.NPMJS, "lodash", "4.17.21"); + + assertTrue(graph.getRootDependencies().stream().map(Dependency::getExternalId).anyMatch(expressId::equals), + "express should appear (declared in root package.json)"); + assertFalse(graph.getRootDependencies().stream().map(Dependency::getExternalId).anyMatch(lodashId::equals), + "lodash should NOT appear when all workspaces are ignored"); + } + + private DependencyGraph extractFromFixture(PackageJsonExtractor extractor) throws IOException { + File packageJsonFile = FunctionalTestFiles.asFile("/npm/workspace-filter-test/package.json"); + Extraction extraction = extractor.extract(packageJsonFile); + return extraction.getCodeLocations().get(0).getDependencyGraph(); + } + private PackageJsonExtractor createExtractor(NpmDependencyType... excludedTypes) { Gson gson = new GsonBuilder().setPrettyPrinting().create(); EnumListFilter npmDependencyTypeFilter = EnumListFilter.fromExcluded(excludedTypes); return new PackageJsonExtractor(gson, new ExternalIdFactory(), npmDependencyTypeFilter); } + private PackageJsonExtractor createExtractorWithFilter(ExcludedIncludedWildcardFilter workspaceFilter) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + return new PackageJsonExtractor(gson, new ExternalIdFactory(), EnumListFilter.excludeNone(), workspaceFilter); + } + @Test void extractWithNoDevOrPeerDependencies() { CombinedPackageJson packageJson = createPackageJson(); diff --git a/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/package-lock.json b/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/package-lock.json new file mode 100644 index 0000000000..a79dba4614 --- /dev/null +++ b/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/package-lock.json @@ -0,0 +1,45 @@ +{ + "name": "my-project", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "my-project", + "version": "1.0.0", + "workspaces": [ + "packages/api", + "packages/ui" + ], + "dependencies": { + "express": "4.18.0" + } + }, + "node_modules/express": { + "version": "4.18.0" + }, + "node_modules/lodash": { + "version": "4.17.21" + }, + "node_modules/my-api": { + "resolved": "packages/api", + "link": true + }, + "node_modules/my-ui": { + "resolved": "packages/ui", + "link": true + }, + "packages/api": { + "version": "1.0.0", + "dependencies": { + "express": "4.18.0" + } + }, + "packages/ui": { + "version": "1.0.0", + "dependencies": { + "lodash": "4.17.21" + } + } + } +} diff --git a/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/package.json b/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/package.json new file mode 100644 index 0000000000..3c0fcede65 --- /dev/null +++ b/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/package.json @@ -0,0 +1,11 @@ +{ + "name": "my-project", + "version": "1.0.0", + "workspaces": [ + "packages/api", + "packages/ui" + ], + "dependencies": { + "express": "4.18.0" + } +} diff --git a/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/packages/api/package.json b/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/packages/api/package.json new file mode 100644 index 0000000000..7a481b81bb --- /dev/null +++ b/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/packages/api/package.json @@ -0,0 +1,7 @@ +{ + "name": "my-api", + "version": "1.0.0", + "dependencies": { + "express": "4.18.0" + } +} diff --git a/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/packages/ui/package.json b/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/packages/ui/package.json new file mode 100644 index 0000000000..a0c8321acb --- /dev/null +++ b/detectable/src/test/resources/detectables/functional/npm/workspace-filter-test/packages/ui/package.json @@ -0,0 +1,4 @@ +{ + "name": "my-ui", + "version": "1.0.0" +} diff --git a/src/main/java/com/blackduck/integration/detect/configuration/DetectProperties.java b/src/main/java/com/blackduck/integration/detect/configuration/DetectProperties.java index e9f5fed3d6..cee570df05 100644 --- a/src/main/java/com/blackduck/integration/detect/configuration/DetectProperties.java +++ b/src/main/java/com/blackduck/integration/detect/configuration/DetectProperties.java @@ -1362,6 +1362,37 @@ private DetectProperties() { .setGroups(DetectGroup.NPM, DetectGroup.GLOBAL, DetectGroup.SOURCE_SCAN) .build(); + public static final CaseSensitiveStringListProperty DETECT_NPM_EXCLUDED_WORKSPACES = + CaseSensitiveStringListProperty.newBuilder("detect.npm.excluded.workspaces") + .setInfo("NPM Exclude Workspaces", DetectPropertyFromVersion.VERSION_12_0_0) + .setHelp( + "A comma-separated list of npm workspace relative paths to exclude.", + "By default, Detect includes all workspaces. Workspaces are identified by their path relative to the project root (e.g. packages/react-components). This property accepts filename globbing-style wildcards. For more information, refer to the Property wildcard support page." + ) + .setGroups(DetectGroup.NPM, DetectGroup.SOURCE_SCAN) + .setCategory(DetectCategory.Advanced) + .setExample("packages/test-harness,packages/internal-*") + .build(); + + public static final CaseSensitiveStringListProperty DETECT_NPM_INCLUDED_WORKSPACES = + CaseSensitiveStringListProperty.newBuilder("detect.npm.included.workspaces") + .setInfo("NPM Include Workspaces", DetectPropertyFromVersion.VERSION_12_0_0) + .setHelp( + "A comma-separated list of npm workspace relative paths to include.", + "By default, Detect includes all workspaces. If workspaces are excluded or included, Detect will include any workspace included by this property that is not excluded. Exclusion rules always win. Workspaces are identified by their path relative to the project root (e.g. packages/react-components). This property accepts filename globbing-style wildcards. For more information, refer to the Property wildcard support page." + ) + .setGroups(DetectGroup.NPM, DetectGroup.SOURCE_SCAN) + .setCategory(DetectCategory.Advanced) + .setExample("packages/frontend,packages/api") + .build(); + + public static final BooleanProperty DETECT_NPM_IGNORE_ALL_WORKSPACES_MODE = + BooleanProperty.newBuilder("detect.npm.ignore.all.workspaces", false) + .setInfo("Ignore All Workspaces", DetectPropertyFromVersion.VERSION_12_0_0) + .setHelp("All workspaces are ignored by the NPM detector for increased performance and precision to scan a massive codebase.") + .setGroups(DetectGroup.NPM, DetectGroup.SOURCE_SCAN) + .build(); + public static final NullablePathProperty DETECT_NPM_PATH = NullablePathProperty.newBuilder("detect.npm.path") .setInfo("NPM Executable", DetectPropertyFromVersion.VERSION_3_0_0) diff --git a/src/main/java/com/blackduck/integration/detect/configuration/DetectPropertyFromVersion.java b/src/main/java/com/blackduck/integration/detect/configuration/DetectPropertyFromVersion.java index e447c92b8d..7142d113ee 100644 --- a/src/main/java/com/blackduck/integration/detect/configuration/DetectPropertyFromVersion.java +++ b/src/main/java/com/blackduck/integration/detect/configuration/DetectPropertyFromVersion.java @@ -60,7 +60,8 @@ public enum DetectPropertyFromVersion implements PropertyVersion { VERSION_11_0_0("11.0.0"), VERSION_11_2_0("11.2.0"), VERSION_11_3_0("11.3.0"), - VERSION_11_4_0("11.4.0"); + VERSION_11_4_0("11.4.0"), + VERSION_12_0_0("12.0.0"); private final String version; diff --git a/src/main/java/com/blackduck/integration/detect/configuration/DetectableOptionFactory.java b/src/main/java/com/blackduck/integration/detect/configuration/DetectableOptionFactory.java index 92ace8c329..4009939e6e 100644 --- a/src/main/java/com/blackduck/integration/detect/configuration/DetectableOptionFactory.java +++ b/src/main/java/com/blackduck/integration/detect/configuration/DetectableOptionFactory.java @@ -246,17 +246,26 @@ public ConanLockfileExtractorOptions createConanLockfileOptions() { public NpmCliExtractorOptions createNpmCliExtractorOptions() { EnumListFilter npmDependencyTypeFilter = createNpmDependencyTypeFilter(); String npmArguments = detectConfiguration.getNullableValue(DetectProperties.DETECT_NPM_ARGUMENTS); - return new NpmCliExtractorOptions(npmDependencyTypeFilter, npmArguments); + List excludedWorkspaceNames = detectConfiguration.getValue(DetectProperties.DETECT_NPM_EXCLUDED_WORKSPACES); + List includedWorkspaceNames = detectConfiguration.getValue(DetectProperties.DETECT_NPM_INCLUDED_WORKSPACES); + boolean ignoreAllWorkspaces = detectConfiguration.getValue(DetectProperties.DETECT_NPM_IGNORE_ALL_WORKSPACES_MODE); + return new NpmCliExtractorOptions(npmDependencyTypeFilter, npmArguments, excludedWorkspaceNames, includedWorkspaceNames, ignoreAllWorkspaces); } public NpmLockfileOptions createNpmLockfileOptions() { EnumListFilter npmDependencyTypeFilter = createNpmDependencyTypeFilter(); - return new NpmLockfileOptions(npmDependencyTypeFilter); + List excludedWorkspaceNames = detectConfiguration.getValue(DetectProperties.DETECT_NPM_EXCLUDED_WORKSPACES); + List includedWorkspaceNames = detectConfiguration.getValue(DetectProperties.DETECT_NPM_INCLUDED_WORKSPACES); + boolean ignoreAllWorkspaces = detectConfiguration.getValue(DetectProperties.DETECT_NPM_IGNORE_ALL_WORKSPACES_MODE); + return new NpmLockfileOptions(npmDependencyTypeFilter, excludedWorkspaceNames, includedWorkspaceNames, ignoreAllWorkspaces); } public NpmPackageJsonParseDetectableOptions createNpmPackageJsonParseDetectableOptions() { EnumListFilter npmDependencyTypeFilter = createNpmDependencyTypeFilter(); - return new NpmPackageJsonParseDetectableOptions(npmDependencyTypeFilter); + List excludedWorkspaceNames = detectConfiguration.getValue(DetectProperties.DETECT_NPM_EXCLUDED_WORKSPACES); + List includedWorkspaceNames = detectConfiguration.getValue(DetectProperties.DETECT_NPM_INCLUDED_WORKSPACES); + boolean ignoreAllWorkspaces = detectConfiguration.getValue(DetectProperties.DETECT_NPM_IGNORE_ALL_WORKSPACES_MODE); + return new NpmPackageJsonParseDetectableOptions(npmDependencyTypeFilter, excludedWorkspaceNames, includedWorkspaceNames, ignoreAllWorkspaces); } private EnumListFilter createNpmDependencyTypeFilter() {