Terraform: upgrade python-hcl2 to 8.1.4

docs/notes/2.34.x.md

@@ -170,6 +170,12 @@ Fixed S3 downloads failing with HTTP 403 `SignatureDoesNotMatch` when the object
 
 #### Shell
 
+#### Terraform
+
+The default version of `python-hcl2`, which the `terraform-hcl2-parser` subsystem uses to infer dependencies between Terraform modules, has been upgraded from `4.3.0` to [`8.1.4`](https://github.com/amplify-education/python-hcl2/releases/tag/v8.1.4). This fixes parse errors on valid Terraform, such as `for` used as an attribute name ([#21845](https://github.com/pantsbuild/pants/issues/21845)), `in` used as a block name ([#22401](https://github.com/pantsbuild/pants/issues/22401)), and conditional expressions split over multiple lines.
+
+If you install the parser from your own resolve with `[terraform-hcl2-parser].install_from_resolve`, support for `python-hcl2` versions older than 8.0 in that resolve is deprecated and will be removed in Pants 2.35. Pants logs a deprecation warning when it parses Terraform with one of them. Upgrade that resolve to `python-hcl2` 8.1.4 or newer.
+
 #### Javascript
 
 #### TypeScript

src/python/pants/backend/terraform/dependency_inference.py

@@ -15,6 +15,7 @@
     setup_venv_pex_process,
 )
 from pants.backend.python.util_rules.pex import rules as pex_rules
+from pants.backend.terraform.hcl2_parser import LEGACY_HCL2_MARKER
 from pants.backend.terraform.target_types import (
     TerraformBackendTarget,
     TerraformDependenciesField,
@@ -23,6 +24,7 @@
     TerraformModuleSourcesField,
     TerraformVarFileTarget,
 )
+from pants.base.deprecated import warn_or_error
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
 from pants.base.specs import DirGlobSpec, DirLiteralSpec, RawSpecs
 from pants.core.target_types import LockfileTarget
@@ -61,8 +63,7 @@ class TerraformHcl2Parser(PythonToolRequirementsBase):
     options_scope = "terraform-hcl2-parser"
     help_short = "Used to parse Terraform modules to infer their dependencies."
 
-    # versions 4.3.2+ have parsing issues; bump once resolved
-    default_requirements = ["python-hcl2>=3.0.5,<=4.3.0"]
+    default_requirements = ["python-hcl2>=8.1.4,<9"]
 
     register_interpreter_constraints = True
 
@@ -120,6 +121,23 @@ async def setup_process_for_parse_terraform_module_sources(
     return process
 
 
+def warn_if_legacy_hcl2(parser_stderr: bytes) -> None:
+    """Warn if the parser reported running with a python-hcl2 release older than 8.0."""
+    if LEGACY_HCL2_MARKER not in parser_stderr:
+        return
+    scope = TerraformHcl2Parser.options_scope
+    warn_or_error(
+        removal_version="2.35.0.dev0",
+        entity=f"support for `python-hcl2` versions older than 8.0 in `[{scope}]`",
+        hint=softwrap(
+            f"""
+            Pants now uses `python-hcl2` 8.1.4 by default. Upgrade `python-hcl2` to 8.1.4 or
+            newer in the resolve named by `[{scope}].install_from_resolve`.
+            """
+        ),
+    )
+
+
 @dataclass(frozen=True)
 class TerraformModuleDependenciesInferenceFieldSet(FieldSet):
     required_fields = (TerraformModuleSourcesField, TerraformDependenciesField)
@@ -257,6 +275,7 @@ async def _infer_dependencies_from_sources(
             )
         )
     )
+    warn_if_legacy_hcl2(result.stderr)
     candidate_spec_paths = [line for line in result.stdout.decode("utf-8").split("\n") if line]
     # For each path, see if there is a `terraform_module` target at the specified spec_path.
     candidate_targets = await resolve_targets(

src/python/pants/backend/terraform/dependency_inference_test.py

@@ -12,8 +12,10 @@
     TerraformDeploymentDependenciesInferenceFieldSet,
     TerraformHcl2Parser,
     TerraformModuleDependenciesInferenceFieldSet,
+    warn_if_legacy_hcl2,
 )
 from pants.backend.terraform.goals.lockfiles import rules as terraform_lockfile_rules
+from pants.backend.terraform.hcl2_parser import LEGACY_HCL2_MARKER
 from pants.backend.terraform.target_types import (
     TerraformBackendTarget,
     TerraformDeploymentTarget,
@@ -242,6 +244,71 @@ def test_hcl_parser_wrapper_runs(rule_runner: RuleRunner, major_minor_interprete
 
     lines = {line for line in result.stdout.decode("utf-8").splitlines() if line}
     assert lines == {"grok", "foo/hello/world"}
+    assert LEGACY_HCL2_MARKER not in result.stderr
+
+
+def test_hcl_parser_handles_keywords_and_multiline_conditionals(rule_runner: RuleRunner) -> None:
+    """python-hcl2 4.3.0 and earlier failed to parse each of these constructs."""
+    rule_runner.write_files(
+        {
+            "foo/BUILD": "terraform_module(name='t')\n",
+            "foo/main.tf": textwrap.dedent(
+                """\
+                # `for` as an attribute name: https://github.com/pantsbuild/pants/issues/21845
+                resource "grafana_rule_group" "example" {
+                  rule {
+                    name = "example"
+                    for  = "5m"
+                  }
+                }
+
+                # `in` as a block name: https://github.com/pantsbuild/pants/issues/22401
+                data "snowflake_schemas" "in" {
+                  in {
+                    database = "example"
+                  }
+                }
+
+                # A conditional expression split over multiple lines.
+                locals {
+                  flavor = (
+                    var.enabled
+                    ? "on"
+                    : "off"
+                  )
+                }
+
+                module "foo" {
+                  source = "../grok"
+                }
+                """
+            ),
+        }
+    )
+    target = rule_runner.get_target(Address("foo", target_name="t"))
+    sources = rule_runner.request(HydratedSources, [HydrateSourcesRequest(target[SourcesField])])
+    result = rule_runner.request(
+        ProcessResult,
+        [
+            ParseTerraformModuleSources(
+                sources_digest=sources.snapshot.digest, paths=("foo/main.tf",)
+            )
+        ],
+    )
+
+    lines = {line for line in result.stdout.decode("utf-8").splitlines() if line}
+    assert lines == {"grok"}
+    assert LEGACY_HCL2_MARKER not in result.stderr
+
+
+def test_warn_if_legacy_hcl2(caplog: pytest.LogCaptureFixture) -> None:
+    warn_if_legacy_hcl2(b"")
+    assert not caplog.records
+
+    warn_if_legacy_hcl2(LEGACY_HCL2_MARKER + b"\n")
+    assert len(caplog.records) == 1
+    assert "DEPRECATED: support for `python-hcl2` versions older than 8.0" in caplog.text
+    assert "[terraform-hcl2-parser].install_from_resolve" in caplog.text
 
 
 def test_generate_lockfile_without_python_backend() -> None: