perf(jvm): stop merging a jvm_artifact's transitive jars into its entry

docs/notes/2.34.x.md

@@ -80,6 +80,10 @@ Fixed `java` processes run against a Java 9+ JDK without nailgun (e.g. JUnit/Sca
 
 Fixed a bug where a single, unparametrized `protobuf_sources` target could not be depended on by both a `java_sources` and a `scala_sources` target at the same time: Pants would raise an ambiguity error when resolving the classpath entry and when hydrating the generated sources, since it couldn't tell which language's codegen a given dependent wanted. Both are now resolved per dependency edge, based on which language is actually asking, with no BUILD file changes required.
 
+A `jvm_artifact`'s classpath entry no longer merges in its transitive jars, so a jar that several `jvm_artifact`s depend on, such as `scala-library`, appears once on a compile's classpath instead of once per artifact that pulled it in. This lowers `scalac`'s heap use on large classpaths. Some JVM compiles, `scalafix` runs, tests and `deploy_jar` builds miss the cache once after upgrading.
+
+`deploy_jar` no longer receives the same jar twice, which changes some `duplicate_policy` outcomes: `concat` and `concat_text` no longer repeat a jar's content, `replace` keeps the copy from the last distinct jar, and `throw` no longer fails on a jar duplicated only against itself.
+
 #### Python
 
 With `[mypy].config_discovery` enabled (the default), each Python source file now uses the mypy config in its nearest ancestor directory, rather than only one at the build root, following [mypy's filename precedence](https://mypy.readthedocs.io/en/stable/config_file.html). Files with different configs are type-checked in separate mypy runs, and previously ignored nested configs may now change results. Behavior is unchanged when `[mypy].config` is set or discovery is disabled.
@@ -161,6 +165,8 @@ Fixed a sandbox leak where Go module downloads left their extracted module cache
 
 `Field` subclasses now resolve `default`, `required` and `none_is_valid_value`, and whether `removal_version` is set, once per field type and cache the result. A field type that computes any of these dynamically, for example through a descriptor returning a different value on each access, will now only ever see the first value.
 
+`ClasspathEntry` has a new `exports` field: a `jvm_artifact`'s entry now holds only its own jar and exports one entry per transitive jar, and entries that used to merge other entries into their `digest` export them instead. Code that reads `digest` or `filenames` directly misses exported jars, and `merge` lists the merged entries' exports after their own jars.
+
 ## Full Changelog
 
 For the full changelog, see the individual GitHub Releases for this series: <https://github.com/pantsbuild/pants/releases>

src/python/pants/backend/java/compile/javac.py

@@ -127,10 +127,7 @@ async def compile_java_source(
     ]
     if not component_members_and_java_source_files:
         # Is a generator, and so exports all of its direct deps.
-        exported_digest = await merge_digests(
-            MergeDigests(cpe.digest for cpe in direct_dependency_classpath_entries)
-        )
-        classpath_entry = ClasspathEntry.merge(exported_digest, direct_dependency_classpath_entries)
+        classpath_entry = ClasspathEntry(EMPTY_DIGEST, exports=direct_dependency_classpath_entries)
         return FallibleClasspathEntry(
             description=str(request.component),
             result=CompileResult.SUCCEEDED,
@@ -233,18 +230,12 @@ async def compile_java_source(
             **implicitly(StripJarRequest(digest=jar_output_digest, filenames=output_files))
         )
     output_classpath = ClasspathEntry(
-        jar_output_digest, output_files, direct_dependency_classpath_entries
+        jar_output_digest,
+        output_files,
+        direct_dependency_classpath_entries,
+        exports=export_classpath_entries,
     )
 
-    if export_classpath_entries:
-        merged_export_digest = await merge_digests(
-            MergeDigests((output_classpath.digest, *(i.digest for i in export_classpath_entries)))
-        )
-        merged_classpath = ClasspathEntry.merge(
-            merged_export_digest, (output_classpath, *export_classpath_entries)
-        )
-        output_classpath = merged_classpath
-
     return FallibleClasspathEntry.from_fallible_process_result(
         str(request.component),
         compile_result,

src/python/pants/backend/kotlin/compile/kotlinc.py

@@ -110,10 +110,7 @@ async def compile_kotlin_source(
 
     if not component_members_and_kotlin_source_files:
         # Is a generator, and so exports all of its direct deps.
-        exported_digest = await merge_digests(
-            MergeDigests(cpe.digest for cpe in direct_dependency_classpath_entries)
-        )
-        classpath_entry = ClasspathEntry.merge(exported_digest, direct_dependency_classpath_entries)
+        classpath_entry = ClasspathEntry(EMPTY_DIGEST, exports=direct_dependency_classpath_entries)
         return FallibleClasspathEntry(
             description=str(request.component),
             result=CompileResult.SUCCEEDED,

src/python/pants/backend/kotlin/compile/kotlinc_plugins.py

@@ -184,7 +184,7 @@ async def fetch_kotlinc_plugins(request: KotlincPluginsRequest) -> KotlincPlugin
 
     entries = list(ClasspathEntry.closure(artifacts))
     merged_classpath_digest = await merge_digests(MergeDigests(entry.digest for entry in entries))
-    merged = ClasspathEntry.merge(merged_classpath_digest, entries)
+    merged = ClasspathEntry(merged_classpath_digest, ClasspathEntry.args(entries))
 
     ids = tuple(_plugin_id(target) for target in request.plugins)
 

src/python/pants/backend/kotlin/compile/kotlinc_test.py

@@ -13,6 +13,7 @@
 from pants.backend.kotlin.compile import kotlinc_plugins
 from pants.backend.kotlin.compile.kotlinc import CompileKotlinSourceRequest
 from pants.backend.kotlin.compile.kotlinc import rules as kotlinc_rules
+from pants.backend.kotlin.compile.kotlinc_plugins import KotlincPlugins, KotlincPluginsRequest
 from pants.backend.kotlin.compile.testutil import _KOTLIN_VERSION, KOTLIN_STDLIB_REQUIREMENTS
 from pants.backend.kotlin.dependency_inference.rules import rules as kotlin_dep_inf_rules
 from pants.backend.kotlin.goals.check import KotlincCheckRequest
@@ -23,7 +24,8 @@
 from pants.core.goals.check import CheckResults
 from pants.core.util_rules import config_files, source_files, system_binaries
 from pants.engine.addresses import Addresses
-from pants.engine.target import CoarsenedTargets
+from pants.engine.fs import Digest, Snapshot
+from pants.engine.target import CoarsenedTargets, Targets
 from pants.jvm import jdk_rules, testutil
 from pants.jvm.compile import ClasspathEntry, CompileResult, FallibleClasspathEntry
 from pants.jvm.resolve import jvm_tool
@@ -63,6 +65,8 @@ def rule_runner() -> RuleRunner:
             QueryRule(FallibleClasspathEntry, (CompileKotlinSourceRequest,)),
             QueryRule(RenderedClasspath, (CompileKotlinSourceRequest,)),
             QueryRule(ClasspathEntry, (CompileKotlinSourceRequest,)),
+            QueryRule(KotlincPlugins, (KotlincPluginsRequest,)),
+            QueryRule(Snapshot, (Digest,)),
         ],
         target_types=[JvmArtifactTarget, KotlinSourcesGeneratorTarget, KotlincPluginTarget],
     )
@@ -455,3 +459,51 @@ class B: A() {
     fallible_result = rule_runner.request(FallibleClasspathEntry, [request])
     print(f"stdout:\n{fallible_result.stdout}\nstderr:\n{fallible_result.stderr}")
     assert fallible_result.result == CompileResult.SUCCEEDED
+
+
+@maybe_skip_jdk_test
+def test_plugin_classpath_lists_a_jar_shared_by_two_plugins_once(
+    rule_runner: RuleRunner, kotlinc_allopen_lockfile: JVMLockfileFixture
+) -> None:
+    rule_runner.write_files(
+        {
+            "lib/BUILD": dedent(
+                """\
+                kotlinc_plugin(
+                    name="reflect",
+                    plugin_id="reflect",
+                    artifact="3rdparty/jvm:org.jetbrains.kotlin_kotlin-reflect",
+                )
+
+                kotlinc_plugin(
+                    name="stdlib",
+                    plugin_id="stdlib",
+                    artifact="3rdparty/jvm:org.jetbrains.kotlin_kotlin-stdlib",
+                )
+                """
+            ),
+            "3rdparty/jvm/BUILD": kotlinc_allopen_lockfile.requirements_as_jvm_artifact_targets(),
+            "3rdparty/jvm/default.lock": kotlinc_allopen_lockfile.serialized_lockfile,
+        }
+    )
+    plugins = [
+        rule_runner.get_target(Address("lib", target_name=name)) for name in ("reflect", "stdlib")
+    ]
+    artifacts = [
+        rule_runner.get_target(Address("3rdparty/jvm", target_name=name))
+        for name in ("org.jetbrains.kotlin_kotlin-reflect", "org.jetbrains.kotlin_kotlin-stdlib")
+    ]
+
+    plugins_classpath = rule_runner.request(
+        KotlincPlugins,
+        [KotlincPluginsRequest(Targets(plugins), Targets(artifacts), make_resolve(rule_runner))],
+    ).classpath
+
+    assert sorted(plugins_classpath.filenames) == [
+        "org.jetbrains.kotlin_kotlin-reflect_1.6.20.jar",
+        "org.jetbrains.kotlin_kotlin-stdlib-common_1.6.20.jar",
+        "org.jetbrains.kotlin_kotlin-stdlib_1.6.20.jar",
+        "org.jetbrains_annotations_13.0.jar",
+    ]
+    snapshot = rule_runner.request(Snapshot, [plugins_classpath.digest])
+    assert set(snapshot.files) == set(plugins_classpath.filenames)

src/python/pants/backend/scala/bsp/rules.py

@@ -78,7 +78,7 @@
 from pants.jvm.bsp.resources import _jvm_bsp_resources
 from pants.jvm.bsp.resources import rules as jvm_resources_rules
 from pants.jvm.bsp.spec import JvmBuildTarget
-from pants.jvm.compile import ClasspathEntryRequestFactory
+from pants.jvm.compile import ClasspathEntry, ClasspathEntryRequestFactory
 from pants.jvm.jdk_rules import DefaultJdk, JdkRequest, prepare_jdk_environment
 from pants.jvm.resolve.common import ArtifactRequirements
 from pants.jvm.resolve.coursier_fetch import (
@@ -338,8 +338,7 @@ async def handle_bsp_scalac_options_request(
         build_root.pathlib_path.joinpath(
             f".pants.d/bsp/{thirdparty_modules_prefix}/{filename}"
         ).as_uri()
-        for cp_entry in thirdparty_modules.entries.values()
-        for filename in cp_entry.filenames
+        for filename in ClasspathEntry.args(thirdparty_modules.entries.values())
     )
 
     return HandleScalacOptionsResult(

src/python/pants/backend/scala/compile/scalac.py

@@ -138,10 +138,7 @@ async def compile_scala_source(
 
     if not component_members_and_scala_source_files:
         # Is a generator, and so exports all of its direct deps.
-        exported_digest = await merge_digests(
-            MergeDigests(cpe.digest for cpe in direct_dependency_classpath_entries)
-        )
-        classpath_entry = ClasspathEntry.merge(exported_digest, direct_dependency_classpath_entries)
+        classpath_entry = ClasspathEntry(EMPTY_DIGEST, exports=direct_dependency_classpath_entries)
         return FallibleClasspathEntry(
             description=str(request.component),
             result=CompileResult.SUCCEEDED,

src/python/pants/backend/scala/compile/scalac_plugins.py

@@ -239,8 +239,9 @@ async def fetch_plugins(request: ScalaPluginsRequest) -> ScalaPlugins:
         failed = [i for i in fallible_artifacts if i.exit_code != 0]
         raise Exception(f"Fetching local scala plugins failed: {failed}")
 
-    merged_classpath_digest = await merge_digests(MergeDigests(i.digest for i in artifacts))
-    merged = ClasspathEntry.merge(merged_classpath_digest, artifacts)
+    entries = list(ClasspathEntry.closure(artifacts))
+    merged_classpath_digest = await merge_digests(MergeDigests(entry.digest for entry in entries))
+    merged = ClasspathEntry(merged_classpath_digest, ClasspathEntry.args(entries))
 
     names = tuple(_plugin_name(target) for target in request.plugins)
 

src/python/pants/backend/scala/compile/scalac_test.py

@@ -14,6 +14,7 @@
 )
 from pants.backend.scala.compile.scalac import CompileScalaSourceRequest
 from pants.backend.scala.compile.scalac import rules as scalac_rules
+from pants.backend.scala.compile.scalac_plugins import ScalaPlugins, ScalaPluginsRequest
 from pants.backend.scala.dependency_inference.rules import rules as scala_dep_inf_rules
 from pants.backend.scala.goals.check import ScalacCheckRequest
 from pants.backend.scala.goals.check import rules as scalac_check_rules
@@ -28,9 +29,10 @@
 from pants.core.goals.check import CheckResults
 from pants.core.util_rules import source_files, stripped_source_files, system_binaries
 from pants.engine.addresses import Addresses
+from pants.engine.fs import Digest, Snapshot
 from pants.engine.internals.parametrize import Parametrize
 from pants.engine.internals.scheduler import ExecutionError
-from pants.engine.target import CoarsenedTargets
+from pants.engine.target import CoarsenedTargets, Targets
 from pants.jvm import jdk_rules, testutil
 from pants.jvm.compile import ClasspathEntry, CompileResult, FallibleClasspathEntry
 from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
@@ -68,6 +70,8 @@ def rule_runner() -> RuleRunner:
             QueryRule(FallibleClasspathEntry, (CompileScalaSourceRequest,)),
             QueryRule(RenderedClasspath, (CompileScalaSourceRequest,)),
             QueryRule(ClasspathEntry, (CompileScalaSourceRequest,)),
+            QueryRule(ScalaPlugins, (ScalaPluginsRequest,)),
+            QueryRule(Snapshot, (Digest,)),
         ],
         target_types=[
             JvmArtifactTarget,
@@ -730,6 +734,60 @@ def example: Option[String] = {
     rule_runner.request(RenderedClasspath, [request])
 
 
+@maybe_skip_jdk_test
+def test_plugin_classpath_includes_each_transitive_jar_once(
+    rule_runner: RuleRunner, multiple_scala_plugins_jvm_lockfile: JVMLockfileFixture
+) -> None:
+    rule_runner.write_files(
+        {
+            "lib/BUILD": dedent(
+                """\
+                scalac_plugin(
+                    name="kind-projector",
+                    plugin_name="kind-projector",
+                    artifact="3rdparty/jvm:org.typelevel_kind-projector_2.13.8",
+                )
+
+                scalac_plugin(
+                    name="better-monadic-for",
+                    plugin_name="bm4",
+                    artifact="3rdparty/jvm:com.olegpy_better-monadic-for_2.13",
+                )
+                """
+            ),
+            "3rdparty/jvm/BUILD": multiple_scala_plugins_jvm_lockfile.requirements_as_jvm_artifact_targets(),
+            "3rdparty/jvm/default.lock": multiple_scala_plugins_jvm_lockfile.serialized_lockfile,
+        }
+    )
+    plugins = [
+        rule_runner.get_target(Address("lib", target_name=name))
+        for name in ("kind-projector", "better-monadic-for")
+    ]
+    artifacts = [
+        rule_runner.get_target(Address("3rdparty/jvm", target_name=name))
+        for name in ("org.typelevel_kind-projector_2.13.8", "com.olegpy_better-monadic-for_2.13")
+    ]
+
+    scala_plugins = rule_runner.request(
+        ScalaPlugins,
+        [ScalaPluginsRequest(Targets(plugins), Targets(artifacts), make_resolve(rule_runner))],
+    )
+
+    filenames = scala_plugins.classpath.filenames
+    assert len(filenames) == len(set(filenames))
+    assert set(filenames) == {
+        "org.typelevel_kind-projector_2.13.8_0.13.2.jar",
+        "com.olegpy_better-monadic-for_2.13_0.3.1.jar",
+        "net.java.dev.jna_jna_5.9.0.jar",
+        "org.jline_jline_3.21.0.jar",
+        "org.scala-lang_scala-compiler_2.13.8.jar",
+        "org.scala-lang_scala-library_2.13.8.jar",
+        "org.scala-lang_scala-reflect_2.13.8.jar",
+    }
+    snapshot = rule_runner.request(Snapshot, [scala_plugins.classpath.digest])
+    assert set(snapshot.files) == set(filenames)
+
+
 @pytest.fixture
 def scala_2_12_lockfile_def() -> JVMLockfileFixtureDefinition:
     return JVMLockfileFixtureDefinition(

src/python/pants/bsp/protocol_test.py

@@ -33,6 +33,7 @@
 )
 from pants.bsp.spec.resources import ResourcesParams
 from pants.bsp.spec.targets import (
+    DependencyModulesParams,
     DependencySourcesParams,
     SourcesParams,
     SourcesResult,
@@ -262,10 +263,28 @@ class ExampleLibSpec extends AnyFunSpec {
                 "buildTarget/scalacOptions", ScalacOptionsParams(target_ids).to_json_dict()
             ).result(timeout=timeout)
         )
-        assert scalac_options.items[0].classpath
+        classpath = [Path(urlparse(uri).path) for uri in scalac_options.items[0].classpath]
+        assert len(classpath) == len(set(classpath))
+        assert all(jar.is_file() for jar in classpath)
+        assert any(jar.name.startswith("org.scalactic_scalactic_2.13") for jar in classpath)
         class_directory = Path(urlparse(scalac_options.items[0].class_directory).path)
         assert not class_directory.exists()
 
+        # buildTarget/dependencyModules
+        dependency_modules = endpoint.request(
+            "buildTarget/dependencyModules",
+            DependencyModulesParams(target_ids).to_json_dict(),
+        ).result(timeout=timeout)
+        (dependency_modules_item,) = dependency_modules["items"]
+        modules = {module["name"]: module for module in dependency_modules_item["modules"]}
+        scalatest_jars = [
+            Path(urlparse(artifact["uri"]).path)
+            for artifact in modules["org.scalatest:scalatest_2.13"]["data"]["artifacts"]
+        ]
+        assert scalatest_jars[0].name == "org.scalatest_scalatest_2.13_3.2.10.jar"
+        assert len(scalatest_jars) == len(set(scalatest_jars))
+        assert set(scalatest_jars) <= set(classpath)
+
         # buildTarget/compile
         compile_result = CompileResult.from_json_dict(
             endpoint.request(

src/python/pants/jvm/bsp/dependencies.py

@@ -152,7 +152,9 @@ async def collect_thirdparty_modules(
         required_classfiles(fce) for fce in fallible_classpath_entries
     )
 
-    resolve_digest = await merge_digests(MergeDigests(cpe.digest for cpe in classpath_entries))
+    resolve_digest = await merge_digests(
+        MergeDigests(cpe.digest for cpe in ClasspathEntry.export_closure(classpath_entries))
+    )
 
     return ThirdpartyModules(
         resolve,
@@ -407,7 +409,7 @@ async def _jvm_bsp_dependency_modules(
                             f".pants.d/bsp/jvm/resolves/{resolve.name}/lib/{filename}"
                         ).as_uri()
                     )
-                    for filename in cp_entry.filenames
+                    for filename in ClasspathEntry.args([cp_entry])
                 ),
             ),
         )