diff --git a/e2e/cases/py-cc-toolchain-1095/BUILD.bazel b/e2e/cases/py-cc-toolchain-1095/BUILD.bazel new file mode 100644 index 000000000..6a1c1303c --- /dev/null +++ b/e2e/cases/py-cc-toolchain-1095/BUILD.bazel @@ -0,0 +1,72 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_shell//shell:sh_test.bzl", "sh_test") + +# Reproduces https://github.com/aspect-build/rules_py/issues/1095 +# +# The `python_interpreters` module extension is positioned as a drop-in +# replacement for rules_python's `python.toolchain()`: it sources hermetic PBS +# interpreters and registers Python toolchains, so a project can drop the +# `rules_python` dependency. But the hub repo it generates +# (`@python_interpreters//:BUILD.bazel`) registers only two toolchain types per +# platform: +# +# - @bazel_tools//tools/python:toolchain_type (runtime) +# - @aspect_rules_py//py/private/toolchain:exec_tools_toolchain_type (exec tools) +# +# It never registers a Python C/C++ headers toolchain +# (`@rules_python//python/cc:toolchain_type`), which `python_register_toolchains` +# in rules_python *does* register. nanobind-bazel and pybind11 resolve Python +# headers through `@rules_python//python/cc:current_py_cc_headers`, which consumes +# `@rules_python//python/cc:toolchain_type`. With only `python_interpreters` in +# play, that toolchain type has no registered toolchain and native extension +# builds fail at analysis time with: +# +# ERROR: .../external/rules_python+/python/cc/BUILD.bazel:15:22: While resolving +# toolchains for target @@rules_python+//python/cc:current_py_cc_headers: No +# matching toolchains found for types: +# @@rules_python+//python/cc:toolchain_type +# +# The PBS interpreters already ship the `include/` headers, so (per the issue) +# "the missing piece is purely the registration". +# +# This test captures the contents of the generated hub BUILD file and asserts it +# registers a `@rules_python//python/cc:toolchain_type` toolchain. It FAILS today +# (reproducing the bug) and should PASS once `python_interpreters` registers a +# py_cc toolchain per platform. +# +# NOTE: We assert on the registration rather than building a real native +# extension because, in a graph that still has `rules_python` present, the gap is +# masked: rules_python's own MODULE.bazel auto-registers py_cc toolchains for its +# default interpreters, so `current_py_cc_headers` resolves to *those* instead of +# failing. The registration is the thing `python_interpreters` is missing and the +# thing it must add to be a true replacement. + +genrule( + name = "hub_build", + srcs = ["@python_interpreters//:BUILD.bazel"], + outs = ["hub_build.txt"], + cmd = "cp $< $@", +) + +sh_test( + name = "test", + srcs = ["test.sh"], + data = [":hub_build"], +) + +# End-to-end guard: compiling against through +# @rules_python//python/cc:current_py_cc_headers (exactly how nanobind/pybind11 +# resolve Python headers) requires the py_cc toolchain registered by +# python_interpreters. This is the concrete native-extension build the issue is +# about. +cc_library( + name = "python_h_smoke", + srcs = ["python_h_smoke.cc"], + deps = ["@rules_python//python/cc:current_py_cc_headers"], +) + +build_test( + name = "python_h_smoke_test", + targets = [":python_h_smoke"], +) diff --git a/e2e/cases/py-cc-toolchain-1095/python_h_smoke.cc b/e2e/cases/py-cc-toolchain-1095/python_h_smoke.cc new file mode 100644 index 000000000..904fb9b49 --- /dev/null +++ b/e2e/cases/py-cc-toolchain-1095/python_h_smoke.cc @@ -0,0 +1,12 @@ +// Smoke source for https://github.com/aspect-build/rules_py/issues/1095. +// +// Mirrors what a native Python extension (nanobind/pybind11) needs: resolve and +// compile against the active Python toolchain's headers via +// @rules_python//python/cc:current_py_cc_headers. If python_interpreters does +// not register a @rules_python//python/cc:toolchain_type toolchain, analysis of +// this target fails with "No matching toolchains found". +#include + +int python_abi_version() { + return PY_MAJOR_VERSION * 100 + PY_MINOR_VERSION; +} diff --git a/e2e/cases/py-cc-toolchain-1095/test.sh b/e2e/cases/py-cc-toolchain-1095/test.sh new file mode 100755 index 000000000..e3eed51fa --- /dev/null +++ b/e2e/cases/py-cc-toolchain-1095/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Reproduces https://github.com/aspect-build/rules_py/issues/1095 +# +# Asserts that the @python_interpreters hub registers a Python C/C++ headers +# toolchain (@rules_python//python/cc:toolchain_type). Without it, native Python +# extensions (nanobind/pybind11) cannot resolve Python.h through +# @rules_python//python/cc:current_py_cc_headers when only python_interpreters is +# providing toolchains. + +set -euo pipefail + +hub_build="${TEST_SRCDIR}/_main/cases/py-cc-toolchain-1095/hub_build.txt" + +if [[ ! -f "$hub_build" ]]; then + echo "FAIL: could not find generated hub BUILD at $hub_build" + exit 1 +fi + +echo "Toolchain types registered by @python_interpreters//:all:" +grep -o 'toolchain_type = "[^"]*"' "$hub_build" | sort -u | sed 's/^/ /' + +if grep -q 'python/cc:toolchain_type' "$hub_build"; then + echo "PASS: python_interpreters registers a @rules_python//python/cc:toolchain_type (py_cc) toolchain" + exit 0 +fi + +echo +echo "FAIL: python_interpreters does NOT register a @rules_python//python/cc:toolchain_type (py_cc) toolchain." +echo " Native Python extensions resolving @rules_python//python/cc:current_py_cc_headers" +echo " will fail with: No matching toolchains found for types:" +echo " @@rules_python+//python/cc:toolchain_type" +echo " See https://github.com/aspect-build/rules_py/issues/1095" +exit 1 diff --git a/py/private/interpreter/extension.bzl b/py/private/interpreter/extension.bzl index 7f2765574..dda24f93f 100644 --- a/py/private/interpreter/extension.bzl +++ b/py/private/interpreter/extension.bzl @@ -384,6 +384,10 @@ def _python_interpreters_impl(module_ctx): "config_settings": tag.config_settings, "target_compatible_with": tag.target_compatible_with, "exec_compatible_with": tag.exec_compatible_with, + # PBS interpreters ship a Bazel-controlled include/ layout, so + # the interpreter repo defines a py_cc_toolchain. Local/system + # interpreters do not (their headers aren't Bazel-managed). + "py_cc": True, })) if not version_found: diff --git a/py/private/interpreter/repository.bzl b/py/private/interpreter/repository.bzl index 8204ccb7c..a85d4b70d 100644 --- a/py/private/interpreter/repository.bzl +++ b/py/private/interpreter/repository.bzl @@ -43,6 +43,8 @@ def _python_interpreter_impl(rctx): micro = micro, python_bin = python_bin, is_windows = is_windows, + is_macos = "darwin" in platform or "apple" in platform, + freethreaded = rctx.attr.freethreaded, )) if not features.external_deps.extension_metadata_has_reproducible: @@ -84,11 +86,48 @@ filegroup( return "\n".join(lines), all_excludes -def _build_file_content(major, minor, micro, python_bin, is_windows): +def _build_file_content(major, minor, micro, python_bin, is_windows, is_macos = False, freethreaded = False): """Generate the full BUILD.bazel content for an interpreter repo.""" feature_targets, feature_excludes = _feature_filegroups(major, minor, is_windows) + # Freethreaded builds suffix the version with "t" (the ABI flag) in both the + # include dir and the shared-library names, e.g. python3.13t / libpython3.13t. + abi_suffix = "t" if freethreaded else "" + + # Directory holding the C headers (Python.h, etc.). PBS lays these out as + # include/python{major}.{minor} (and include/python{major}.{minor}t for + # freethreaded builds) on POSIX, and include/ on Windows. + if is_windows: + headers_include_dir = "include" + else: + headers_include_dir = "include/python{major}.{minor}{suffix}".format( + major = major, + minor = minor, + suffix = abi_suffix, + ) + + # Shared library / import library for linking against the interpreter. Used + # by @rules_python//python/cc:current_py_cc_libs and required to link native + # extensions on Windows. PBS lays these out per-OS; each interpreter repo is + # single-platform, so we emit the exact files (no select()). + if is_windows: + libpython_srcs = [ + "python3{abi}.dll".format(abi = abi_suffix), + "python{major}{minor}{abi}.dll".format(major = major, minor = minor, abi = abi_suffix), + "libs/python{major}{minor}{abi}.lib".format(major = major, minor = minor, abi = abi_suffix), + "libs/python3{abi}.lib".format(abi = abi_suffix), + ] + elif is_macos: + libpython_srcs = [ + "lib/libpython{major}.{minor}{abi}.dylib".format(major = major, minor = minor, abi = abi_suffix), + ] + else: # linux + libpython_srcs = [ + "lib/libpython{major}.{minor}{abi}.so".format(major = major, minor = minor, abi = abi_suffix), + "lib/libpython{major}.{minor}{abi}.so.1.0".format(major = major, minor = minor, abi = abi_suffix), + ] + if is_windows: core_include = '["**/*.py", "**/*.pyd", "**/*.dll", "**/*.exe", "include/**", "Lib/**"]' core_exclude = '["Lib/**/test/**", "Lib/**/tests/**", "**/__pycache__/*.pyc*"]' @@ -116,8 +155,10 @@ def _build_file_content(major, minor, micro, python_bin, is_windows): """.format(feature = feature_name) return """\ +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python:py_runtime.bzl", "py_runtime") load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") +load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") load("@aspect_rules_py//py/private/exec_tools:defs.bzl", "py_exec_tools_toolchain") package(default_visibility = ["//visibility:public"]) @@ -163,6 +204,34 @@ py_runtime_pair( py_exec_tools_toolchain( name = "exec_tools_toolchain", ) + +# Python C/C++ headers, consumed by native extensions (nanobind, pybind11, ...) +# via @rules_python//python/cc:current_py_cc_headers. The PBS interpreter already +# ships these under {headers_include_dir}. +cc_library( + name = "python_headers", + hdrs = glob(["include/**/*.h"], allow_empty = True), + includes = ["{headers_include_dir}"], +) + +# Python shared/import library, consumed via +# @rules_python//python/cc:current_py_cc_libs. +cc_library( + name = "libpython", + srcs = {libpython_srcs}, +) + +# py_cc toolchain so that @rules_python//python/cc:toolchain_type resolves when +# python_interpreters is the only registered toolchain provider. Without this, +# building any native Python extension fails at analysis time with +# "No matching toolchains found for types: .../python/cc:toolchain_type". +py_cc_toolchain( + name = "py_cc_toolchain", + headers = ":python_headers", + headers_abi3 = ":python_headers", + libs = ":libpython", + python_version = "{major}.{minor}", +) """.format( python_bin = python_bin, major = major, @@ -172,6 +241,8 @@ py_exec_tools_toolchain( feature_selects = feature_selects, core_include = core_include, core_exclude = core_exclude, + headers_include_dir = headers_include_dir, + libpython_srcs = repr(libpython_srcs), ) python_interpreter = repository_rule( @@ -314,6 +385,31 @@ config_setting( target_compatible_with = info["compatible_with"] + extra_target_compatible exec_compatible_with = info["compatible_with"] + extra_exec_compatible + # py_cc toolchain registration, only for interpreters whose repo defines + # a py_cc_toolchain target (PBS interpreters). Local/system interpreters + # don't ship a Bazel-managed include/ layout, so we skip it for them. + py_cc_toolchain_block = "" + if info.get("py_cc", False): + py_cc_toolchain_block = """ +# Python C/C++ headers toolchain (@rules_python//python/cc:toolchain_type). +# Selected by the TARGET platform, like the runtime toolchain: native extensions +# are compiled against the headers of the interpreter they run on. Registering it +# here lets python_interpreters fully replace rules_python's python.toolchain(), +# which registers this same toolchain type via python_register_toolchains. +toolchain( + name = "{name}_py_cc", + target_compatible_with = {target_compatible_with}, + target_settings = {target_settings}, + toolchain = "@{repo}//:py_cc_toolchain", + toolchain_type = "@rules_python//python/cc:toolchain_type", +) +""".format( + name = info["name"], + repo = info["repo"], + target_compatible_with = target_compatible_with, + target_settings = target_settings, + ) + content.append(""" # The Python interpreter toolchain has no exec_compatible_with: the interpreter # runs on the TARGET platform (inside the virtualenv), not on the exec host. @@ -329,7 +425,7 @@ toolchain( toolchain = "@{repo}//:runtime_pair", toolchain_type = "@bazel_tools//tools/python:toolchain_type", ) - +{py_cc_toolchain_block} # Exec tools toolchain: selected by exec platform (not target platform) so # that build actions using the interpreter (e.g. compileall) get a runnable # binary on the build host regardless of the target platform being built for. @@ -345,6 +441,7 @@ toolchain( exec_compatible_with = exec_compatible_with, target_compatible_with = target_compatible_with, target_settings = target_settings, + py_cc_toolchain_block = py_cc_toolchain_block, )) content.append("""