go: golangci-lint sees Go generated from other targets

docs/notes/2.34.x.md

@@ -204,6 +204,8 @@ Go test coverage now follows `go test` when the race detector is enabled: the co
 `--go-test-cover-mode=atomic` now works for packages which do not already import `sync/atomic`. Atomic instrumentation rewrites the sources to call `sync/atomic.AddUint32`, but nothing added that package to the covered package's `importcfg`, so those packages failed to compile with `could not import sync/atomic`.
 Fixed a sandbox leak where Go module downloads left their extracted module cache read-only, so sandbox cleanup could not remove it. Download processes now pass `-modcacherw` so the sandboxes can be torn down.
 
+`golangci-lint` now sees Go generated from other targets, such as `protobuf_sources`. It used to copy only the plain sources of the linted packages' dependencies, so a package importing generated Go failed `typecheck`, and with it every other linter on that package. Each generated package is placed in the module at the directory its import path names, which is where `go` looks for it; generated packages whose import path is outside the module are still left out. See [#23707](https://github.com/pantsbuild/pants/issues/23707).
+
 ### Plugin API changes
 
 `Target`, `TargetAdaptor`, `SourceBlock`, `SourceBlocks`, `TextBlock` and `Hunk` are now backed by native Rust implementations. They are still importable from their previous locations and their public constructors, attributes and methods are unchanged, but they are no longer Python dataclasses and they are built in `__new__` rather than `__init__`, so `dataclasses.is_dataclass`, `dataclasses.fields` and `dataclasses.replace` no longer apply to them. Defining subclasses in Python, and setting attributes on those subclasses, continues to work as before.

src/python/pants/backend/go/lint/golangci_lint/rules.py

@@ -9,6 +9,10 @@
 from pants.backend.go.lint.golangci_lint.skip_field import SkipGolangciLintField
 from pants.backend.go.lint.golangci_lint.subsystem import GolangciLint
 from pants.backend.go.subsystems.golang import GolangSubsystem
+from pants.backend.go.target_type_rules import (
+    GoImportPathMappingRequest,
+    map_import_paths_to_packages,
+)
 from pants.backend.go.target_types import GoPackageSourcesField
 from pants.backend.go.util_rules.build_opts import (
     GoBuildOptionsFromTargetRequest,
@@ -29,14 +33,26 @@
 from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files
 from pants.core.util_rules.system_binaries import BashBinary
 from pants.engine.addresses import Address
-from pants.engine.fs import CreateDigest, FileContent, MergeDigests
+from pants.engine.fs import CreateDigest, Digest, FileContent, FileEntry, MergeDigests
+from pants.engine.internals.graph import hydrate_sources
 from pants.engine.internals.graph import transitive_targets as transitive_targets_get
 from pants.engine.internals.selectors import concurrently
-from pants.engine.intrinsics import create_digest, execute_process, merge_digests
+from pants.engine.intrinsics import (
+    create_digest,
+    execute_process,
+    get_digest_entries,
+    merge_digests,
+)
 from pants.engine.platform import Platform
 from pants.engine.process import Process
 from pants.engine.rules import collect_rules, implicitly, rule
-from pants.engine.target import FieldSet, SourcesField, Target, TransitiveTargetsRequest
+from pants.engine.target import (
+    FieldSet,
+    HydrateSourcesRequest,
+    SourcesField,
+    Target,
+    TransitiveTargetsRequest,
+)
 from pants.engine.unions import UnionRule
 from pants.util.logging import LogLevel
 
@@ -94,6 +110,73 @@ async def partition_golangci_lint(
     )
 
 
+@dataclass(frozen=True)
+class GeneratedGoSourcesRequest:
+    go_mod_address: Address
+    targets: tuple[Target, ...]
+
+
+@dataclass(frozen=True)
+class GeneratedGoSources:
+    digest: Digest
+
+
+@rule(desc="Lay out generated Go for golangci-lint", level=LogLevel.DEBUG)
+async def generated_go_sources(request: GeneratedGoSourcesRequest) -> GeneratedGoSources:
+    """The Go that `targets` generate, at the directory `go` resolves each package's import path to.
+
+    Codegen writes its output beside its own sources (a `.pb.go` beside its `.proto`), but `go`
+    finds a package of the main module at the `go.mod` directory joined with the import path
+    relative to the module path. Generated packages outside the module path are left out: no
+    directory in the module provides them.
+    """
+    go_mod_info, import_paths = await concurrently(
+        determine_go_mod_info(GoModInfoRequest(request.go_mod_address)),
+        map_import_paths_to_packages(
+            GoImportPathMappingRequest(request.go_mod_address), **implicitly()
+        ),
+    )
+    module_path = go_mod_info.import_path
+    package_dirs: dict[Address, str] = {}
+    for tgt in request.targets:
+        if tgt.has_field(GoPackageSourcesField) or not tgt.has_field(SourcesField):
+            continue
+        import_path = import_paths.address_to_import_path.get(tgt.address)
+        if import_path == module_path:
+            package_dirs[tgt.address] = ""
+        elif import_path and import_path.startswith(f"{module_path}/"):
+            package_dirs[tgt.address] = import_path[len(module_path) + 1 :]
+    codegen_targets = [tgt for tgt in request.targets if tgt.address in package_dirs]
+
+    generated = await concurrently(
+        hydrate_sources(
+            HydrateSourcesRequest(
+                tgt[SourcesField],
+                for_sources_types=(GoPackageSourcesField,),
+                enable_codegen=True,
+            ),
+            **implicitly(),
+        )
+        for tgt in codegen_targets
+    )
+    entries = await concurrently(
+        get_digest_entries(sources.snapshot.digest) for sources in generated
+    )
+
+    go_mod_dir = request.go_mod_address.spec_path
+    relocated = [
+        FileEntry(
+            os.path.join(go_mod_dir, package_dirs[tgt.address], os.path.basename(entry.path)),
+            entry.file_digest,
+            entry.is_executable,
+        )
+        for tgt, target_entries in zip(codegen_targets, entries)
+        for entry in target_entries
+        if isinstance(entry, FileEntry)
+    ]
+    return GeneratedGoSources(await create_digest(CreateDigest(relocated)))
+
+
 @rule(desc="Lint with golangci-lint", level=LogLevel.DEBUG)
 async def run_golangci_lint(
     request: GolangciLintRequest.Batch[GolangciLintFieldSet, GolangciLintPartitionMetadata],
@@ -126,6 +209,9 @@ async def run_golangci_lint(
     go_build_opts_request = go_extract_build_options_from_target(
         GoBuildOptionsFromTargetRequest(go_mod_address), **implicitly()
     )
+    generated_go_sources_request = generated_go_sources(
+        GeneratedGoSourcesRequest(go_mod_address, tuple(transitive_targets.closure))
+    )
 
     (
         target_source_files,
@@ -134,13 +220,15 @@ async def run_golangci_lint(
         config_files,
         go_mod_info,
         go_build_opts,
+        generated_go,
     ) = await concurrently(
         target_source_files_request,
         all_source_files_request,
         downloaded_golangci_lint_request,
         config_files_request,
         go_mod_info_request,
         go_build_opts_request,
+        generated_go_sources_request,
     )
 
     cgo_enabled = go_build_opts.cgo_enabled
@@ -195,6 +283,7 @@ async def run_golangci_lint(
                 config_files.snapshot.digest,
                 target_source_files.snapshot.digest,
                 all_source_files.snapshot.digest,
+                generated_go.digest,
                 go_mod_info.digest,
             ]
         )

src/python/pants/backend/go/lint/golangci_lint/rules_integration_test.py

@@ -7,6 +7,13 @@
 
 import pytest
 
+from pants.backend.codegen.protobuf import protobuf_dependency_inference
+from pants.backend.codegen.protobuf import target_types as protobuf_target_types
+from pants.backend.codegen.protobuf.go.rules import rules as go_protobuf_rules
+from pants.backend.codegen.protobuf.target_types import (
+    ProtobufSourcesGeneratorTarget,
+    ProtobufSourceTarget,
+)
 from pants.backend.go import target_type_rules
 from pants.backend.go.lint.golangci_lint import skip_field
 from pants.backend.go.lint.golangci_lint.rules import (
@@ -20,6 +27,7 @@
 from pants.backend.go.util_rules import (
     assembly,
     build_pkg,
+    build_pkg_target,
     first_party_pkg,
     go_mod,
     import_analysis,
@@ -28,7 +36,13 @@
     third_party_pkg,
 )
 from pants.core.goals.lint import LintResult, Partitions
-from pants.core.util_rules import config_files, external_tool, source_files, system_binaries
+from pants.core.util_rules import (
+    config_files,
+    external_tool,
+    source_files,
+    stripped_source_files,
+    system_binaries,
+)
 from pants.engine.addresses import Address
 from pants.engine.target import Target
 from pants.testutil.rule_runner import QueryRule, RuleRunner
@@ -37,20 +51,30 @@
 @pytest.fixture()
 def rule_runner() -> RuleRunner:
     rule_runner = RuleRunner(
-        target_types=[GoModTarget, GoPackageTarget],
+        target_types=[
+            GoModTarget,
+            GoPackageTarget,
+            ProtobufSourceTarget,
+            ProtobufSourcesGeneratorTarget,
+        ],
         rules=[
             *assembly.rules(),
             *build_pkg.rules(),
+            *build_pkg_target.rules(),
             *config_files.rules(),
             *external_tool.rules(),
             *first_party_pkg.rules(),
             *go_mod.rules(),
+            *go_protobuf_rules(),
             *golangci_lint_rules(),
             *import_analysis.rules(),
             *link.rules(),
+            *protobuf_dependency_inference.rules(),
+            *protobuf_target_types.rules(),
             *sdk.rules(),
             *skip_field.rules(),
             *source_files.rules(),
+            *stripped_source_files.rules(),
             *system_binaries.rules(),
             *target_type_rules.rules(),
             *third_party_pkg.rules(),
@@ -249,3 +273,70 @@ def test_multiple_go_mods(rule_runner: RuleRunner) -> None:
     # Verify the failure is in mod_b
     failing_result = results_by_exit_code[1]
     assert "bad.go" in failing_result.stdout or "bad.go" in failing_result.stderr
+
+
+@pytest.mark.platform_specific_behavior
+def test_generated_go(rule_runner: RuleRunner) -> None:
+    """A package importing Go generated from a `protobuf_sources` target type-checks.
+
+    The proto's `go_package` is not the directory it sits in, so this also checks that the
+    generated files are laid out where `go` resolves the import path, not where codegen wrote
+    them.
+    """
+    rule_runner.write_files(
+        {
+            "go.mod": dedent(
+                """\
+                module example.com/lint
+                go 1.17
+                require google.golang.org/protobuf v1.27.1
+                """
+            ),
+            "go.sum": dedent(
+                """\
+                github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
+                github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+                github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+                github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+                golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+                golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+                google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+                google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+                google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+                """
+            ),
+            "BUILD": "go_mod(name='mod')\n",
+            "protos/greeting.proto": dedent(
+                """\
+                syntax = "proto3";
+
+                option go_package = "example.com/lint/gen/greeting";
+
+                package greeting;
+
+                message Greeting {
+                  string text = 1;
+                }
+                """
+            ),
+            "protos/BUILD": "protobuf_sources()\n",
+            "app/main.go": dedent(
+                """\
+                package main
+                import (
+                \t"fmt"
+                \t"example.com/lint/gen/greeting"
+                )
+                func main() {
+                \tg := &greeting.Greeting{Text: "Hello World"}
+                \tfmt.Printf("%s", g.GetText())
+                }
+                """
+            ),
+            "app/BUILD": "go_package()\n",
+        }
+    )
+    tgt = rule_runner.get_target(Address("app"))
+    lint_results = run_golangci_lint(rule_runner, [tgt])
+    assert len(lint_results) == 1
+    assert lint_results[0].exit_code == 0, lint_results[0].stdout