diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index 0cfe49d5d8..a34623ea08 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "7vRndkQ5a5Q2gcPIP8Jd/AkNRuB4n7SofpNFmFvodG8=", + "bzlTransitiveDigest": "3vqCp6yUy32HDgpZG9L+zqedcsEnHZu0Y7hfoRk3owY=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "DQe4hZM+myEcJ/pVW54jl5vWJOw+oZNBZfE0WOX/S9g=", + "bzlTransitiveDigest": "DdkE6u15ketmEdOGiZ1UcHX7sZV/xpVbFQLWBKjOAYM=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl new file mode 100644 index 0000000000..4b5a3c6810 --- /dev/null +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -0,0 +1,161 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Setup a python-build-standalone based toolchain.""" + +load("@rules_cc//cc:defs.bzl", "cc_import", "cc_library") +load("//python:py_runtime.bzl", "py_runtime") +load("//python:py_runtime_pair.bzl", "py_runtime_pair") +load("//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") +load(":py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") +load(":semver.bzl", "semver") + +def define_hermetic_runtime_toolchain_impl( + *, + name, + extra_files_glob_include, + extra_files_glob_exclude, + python_version, + python_bin, + coverage_tool): + """Define a toolchain implementation for a python-build-standalone repo. + + It expected this macro is called in the top-level package of an extracted + python-build-standalone repository. See + python/private/python_repositories.bzl for how it is invoked. + + Args: + name: {type}`str` name used for tools to identify the invocation. + extra_files_glob_include: {type}`list[str]` additional glob include + patterns for the target runtime files (the one included in + binaries). + extra_files_glob_exclude: {type}`list[str]` additional glob exclude + patterns for the target runtime files. + python_version: {type}`str` The Python version, in `major.minor.micro` + format. + python_bin: {type}`str` The path to the Python binary within the + repositoroy. + coverage_tool: {type}`str` optional target to the coverage tool to + use. + """ + _ = name # @unused + version_info = semver(python_version) + version_dict = version_info.to_dict() + native.filegroup( + name = "files", + srcs = native.glob( + include = [ + "bin/**", + "extensions/**", + "include/**", + "libs/**", + "share/**", + ] + extra_files_glob_include, + # Platform-agnostic filegroup can't match on all patterns. + allow_empty = True, + exclude = [ + "**/* *", # Bazel does not support spaces in file names. + # Unused shared libraries. `python` executable and the `:libpython` target + # depend on `libpython{python_version}.so.1.0`. + "lib/libpython{major}.{minor}.so".format(**version_dict), + # static libraries + "lib/**/*.a", + # tests for the standard libraries. + "lib/python{major}.{minor}/**/test/**".format(**version_dict), + "lib/python{major}.{minor}/**/tests/**".format(**version_dict), + "**/__pycache__/*.pyc.*", # During pyc creation, temp files named *.pyc.NNN are created + ] + extra_files_glob_exclude, + ), + ) + cc_import( + name = "interface", + interface_library = "libs/python{major}{minor}.lib".format(**version_dict), + system_provided = True, + ) + + native.filegroup( + name = "includes", + srcs = native.glob(["include/**/*.h"]), + ) + cc_library( + name = "python_headers", + deps = select({ + "@bazel_tools//src/conditions:windows": [":interface"], + "//conditions:default": None, + }), + hdrs = [":includes"], + includes = [ + "include", + "include/python{major}.{minor}".format(**version_dict), + "include/python{major}.{minor}m".format(**version_dict), + ], + ) + cc_library( + name = "libpython", + hdrs = [":includes"], + srcs = select({ + "@platforms//os:linux": [ + "lib/libpython{major}.{minor}.so".format(**version_dict), + "lib/libpython{major}.{minor}.so.1.0".format(**version_dict), + ], + "@platforms//os:macos": ["lib/libpython{major}.{minor}.dylib".format(**version_dict)], + "@platforms//os:windows": ["python3.dll", "libs/python{major}{minor}.lib".format(**version_dict)], + }), + ) + + native.exports_files(["python", python_bin]) + + # Used to only download coverage toolchain when the coverage is collected by + # bazel. + native.config_setting( + name = "coverage_enabled", + values = {"collect_code_coverage": "true"}, + visibility = ["//visibility:private"], + ) + + py_runtime( + name = "py3_runtime", + files = [":files"], + interpreter = python_bin, + interpreter_version_info = { + "major": str(version_info.major), + "micro": str(version_info.patch), + "minor": str(version_info.minor), + }, + # Convert empty string to None + coverage_tool = coverage_tool or None, + python_version = "PY3", + implementation_name = "cpython", + # See https://peps.python.org/pep-3147/ for pyc tag infix format + pyc_tag = "cpython-{major}{minor}".format(**version_dict), + ) + + py_runtime_pair( + name = "python_runtimes", + py2_runtime = None, + py3_runtime = ":py3_runtime", + ) + + py_cc_toolchain( + name = "py_cc_toolchain", + headers = ":python_headers", + libs = ":libpython", + python_version = python_version, + ) + + py_exec_tools_toolchain( + name = "py_exec_tools_toolchain", + # This macro is called in another repo: use Label() to ensure it + # resolves in the rules_python context. + precompiler = Label("//tools/precompiler:precompiler"), + ) diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index c4988ee691..0286160b52 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -34,6 +34,7 @@ load(":coverage_deps.bzl", "coverage_dep") load(":full_version.bzl", "full_version") load(":internal_config_repo.bzl", "internal_config_repo") load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load(":text_util.bzl", "render") load( ":toolchains_repo.bzl", "host_toolchain", @@ -246,20 +247,6 @@ def _python_repository_impl(rctx): python_bin = "python.exe" if ("windows" in platform) else "bin/python3" - glob_include = [] - glob_exclude = [ - "**/* *", # Bazel does not support spaces in file names. - # Unused shared libraries. `python` executable and the `:libpython` target - # depend on `libpython{python_version}.so.1.0`. - "lib/libpython{python_version}.so".format(python_version = python_short_version), - # static libraries - "lib/**/*.a", - # tests for the standard libraries. - "lib/python{python_version}/**/test/**".format(python_version = python_short_version), - "lib/python{python_version}/**/tests/**".format(python_version = python_short_version), - "**/__pycache__/*.pyc.*", # During pyc creation, temp files named *.pyc.NNN are created - ] - if "linux" in platform: # Workaround around https://github.com/indygreg/python-build-standalone/issues/231 for url in urls: @@ -281,6 +268,8 @@ def _python_repository_impl(rctx): rctx.delete("share/terminfo") break + glob_include = [] + glob_exclude = [] if rctx.attr.ignore_root_user_error or "windows" in platform: glob_exclude += [ # These pycache files are created on first use of the associated python files. @@ -295,148 +284,42 @@ def _python_repository_impl(rctx): glob_include += [ "*.exe", "*.dll", - "bin/**", "DLLs/**", - "extensions/**", - "include/**", "Lib/**", - "libs/**", "Scripts/**", - "share/**", "tcl/**", ] else: - glob_include += [ - "bin/**", - "extensions/**", - "include/**", + glob_include.append( "lib/**", - "libs/**", - "share/**", - ] + ) - if rctx.attr.coverage_tool: - if "windows" in platform: - coverage_tool = None - else: - coverage_tool = '"{}"'.format(rctx.attr.coverage_tool) - - coverage_attr_text = """\ - coverage_tool = select({{ - ":coverage_enabled": {coverage_tool}, - "//conditions:default": None - }}), -""".format(coverage_tool = coverage_tool) + if "windows" in platform: + coverage_tool = None else: - coverage_attr_text = " # coverage_tool attribute not supported by this Bazel version" + coverage_tool = rctx.attr.coverage_tool build_content = """\ -# Generated by python/repositories.bzl +# Generated by python/private/python_repositories.bzl -load("@rules_python//python:py_runtime.bzl", "py_runtime") -load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") -load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") -load("@rules_python//python/private:py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") +load("@rules_python//python/private:hermetic_runtime_repo_setup.bzl", "define_hermetic_runtime_toolchain_impl") package(default_visibility = ["//visibility:public"]) -filegroup( - name = "files", - srcs = glob( - include = {glob_include}, - # Platform-agnostic filegroup can't match on all patterns. - allow_empty = True, - exclude = {glob_exclude}, - ), -) - -cc_import( - name = "interface", - interface_library = "libs/python{python_version_nodot}.lib", - system_provided = True, -) - -filegroup( - name = "includes", - srcs = glob(["include/**/*.h"]), -) - -cc_library( - name = "python_headers", - deps = select({{ - "@bazel_tools//src/conditions:windows": [":interface"], - "//conditions:default": None, - }}), - hdrs = [":includes"], - includes = [ - "include", - "include/python{python_version}", - "include/python{python_version}m", - ], -) - -cc_library( - name = "libpython", - hdrs = [":includes"], - srcs = select({{ - "@platforms//os:windows": ["python3.dll", "libs/python{python_version_nodot}.lib"], - "@platforms//os:macos": ["lib/libpython{python_version}.dylib"], - "@platforms//os:linux": ["lib/libpython{python_version}.so", "lib/libpython{python_version}.so.1.0"], - }}), -) - -exports_files(["python", "{python_path}"]) - -# Used to only download coverage toolchain when the coverage is collected by -# bazel. -config_setting( - name = "coverage_enabled", - values = {{"collect_code_coverage": "true"}}, - visibility = ["//visibility:private"], -) - -py_runtime( - name = "py3_runtime", - files = [":files"], -{coverage_attr} - interpreter = "{python_path}", - interpreter_version_info = {{ - "major": "{interpreter_version_info_major}", - "minor": "{interpreter_version_info_minor}", - "micro": "{interpreter_version_info_micro}", - }}, - python_version = "PY3", - implementation_name = 'cpython', - pyc_tag = "cpython-{interpreter_version_info_major}{interpreter_version_info_minor}", -) - -py_runtime_pair( - name = "python_runtimes", - py2_runtime = None, - py3_runtime = ":py3_runtime", -) - -py_cc_toolchain( - name = "py_cc_toolchain", - headers = ":python_headers", - libs = ":libpython", - python_version = "{python_version}", -) - -py_exec_tools_toolchain( - name = "py_exec_tools_toolchain", - precompiler = "@rules_python//tools/precompiler:precompiler", +define_hermetic_runtime_toolchain_impl( + name = "define_runtime", + extra_files_glob_include = {extra_files_glob_include}, + extra_files_glob_exclude = {extra_files_glob_exclude}, + python_version = {python_version}, + python_bin = {python_bin}, + coverage_tool = {coverage_tool}, ) """.format( - glob_exclude = repr(glob_exclude), - glob_include = repr(glob_include), - python_path = python_bin, - python_version = python_short_version, - python_version_nodot = python_short_version.replace(".", ""), - coverage_attr = coverage_attr_text, - interpreter_version_info_major = python_version_info[0], - interpreter_version_info_minor = python_version_info[1], - interpreter_version_info_micro = python_version_info[2], + extra_files_glob_exclude = render.list(glob_exclude), + extra_files_glob_include = render.list(glob_include), + python_bin = render.str(python_bin), + python_version = render.str(rctx.attr.python_version), + coverage_tool = render.str(coverage_tool), ) rctx.delete("python") rctx.symlink(python_bin, "python") diff --git a/python/private/semver.bzl b/python/private/semver.bzl index 9a240d46b7..d77249ff9f 100644 --- a/python/private/semver.bzl +++ b/python/private/semver.bzl @@ -34,6 +34,15 @@ def _key(version): version.build, ) +def _to_dict(self): + return { + "build": self.build, + "major": self.major, + "minor": self.minor, + "patch": self.patch, + "pre_release": self.pre_release, + } + def semver(version): """Parse the semver version and return the values as a struct. @@ -50,7 +59,8 @@ def semver(version): patch, _, build = tail.partition("+") patch, _, pre_release = patch.partition("-") - public = struct( + # buildifier: disable=uninitialized + self = struct( major = int(major), minor = int(minor or "0"), # NOTE: this is called `micro` in the Python interpreter versioning scheme @@ -58,8 +68,8 @@ def semver(version): pre_release = pre_release, build = build, # buildifier: disable=uninitialized - key = lambda: _key(self.actual), + key = lambda: _key(self), str = lambda: version, + to_dict = lambda: _to_dict(self), ) - self = struct(actual = public) - return public + return self