diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 0000000..b70f6a3 --- /dev/null +++ b/.bazelignore @@ -0,0 +1 @@ +bazel diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 0000000..776e61c --- /dev/null +++ b/.bazelrc @@ -0,0 +1,6 @@ +common --enable_bzlmod +test --test_output=errors + +# The C++ toolchain (clang from the LLVM tree) and all ABI-critical flags are +# centralized in the cppjit_bazel module, which registers @llvm//:cc_toolchain. +# No per-repo compiler wiring needed. diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000..56b6be4 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +8.3.1 diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml new file mode 100644 index 0000000..1c70b82 --- /dev/null +++ b/.github/workflows/bazel.yml @@ -0,0 +1,51 @@ +# Best-effort, NON-GATING Bazel build. The supported build is setup.py/CMake; +# this job never blocks a PR (continue-on-error). The build and test steps run +# for real and surface a regression as a warning, not a false green. +# +# The build is standalone: CppInterOp comes from the pinned commit in +# MODULE.bazel, so no sibling clone is needed. Only LLVM comes from the host. +name: bazel (best-effort) + +on: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +jobs: + bazel: + name: bazel (best-effort) llvm${{ matrix.llvm }} + runs-on: ubuntu-24.04 + continue-on-error: true + strategy: + fail-fast: false + matrix: + llvm: ["22"] + steps: + - name: Checkout cppjit + uses: actions/checkout@v7 + + - name: Setup LLVM ${{ matrix.llvm }} + uses: compiler-research/ci-workflows/actions/setup-llvm@main + with: + version: ${{ matrix.llvm }} + os: ubuntu-24.04 + + - name: Point LLVM_DIR at the tree root for the @llvm Bazel extension + run: | + # setup-llvm sets LLVM_DIR=/lib/cmake/llvm (CMake convention); + # bazel/llvm.bzl needs the tree root holding bin/llvm-config. + echo "LLVM_DIR=${GITHUB_WORKSPACE}/install" >> "$GITHUB_ENV" + + - name: bazel build //... + run: | + bazelisk build //... \ + || echo "::warning::bazel build //... failed (best-effort, non-gating)" + + - name: bazel test //... + run: | + bazelisk test //... \ + || echo "::warning::bazel test //... failed (best-effort, non-gating)" diff --git a/.gitignore b/.gitignore index 2013091..1b2ccc7 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,13 @@ venv/ .envrc .python-version +# Bazel convenience symlinks and module lock +/bazel-bin +/bazel-out +/bazel-testlogs +/bazel-cppjit +MODULE.bazel.lock + # Editors / OS .idea/ .vscode/ diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 0000000..cbb64ae --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,444 @@ +load("@cppjit_bazel//:defs.bzl", "BASE_COPTS", "LLVM_MAJOR", "jit_cxx_interp_args", "repo_loc") +load("@cppjit_bazel//:rules.bzl", "cppjit_py_test", "cppjit_selflocation_py_test", "cppjit_test_dict_sos") +load("@cppjit_bazel//:staging.bzl", "stage_files") +load("@@rules_cc+//cc:defs.bzl", "cc_library", "cc_shared_library") +load("@rules_python//python:defs.bzl", "py_library") + +# cppjit is a sanitizer boundary: libcppjit.so is dlopen'd by a (possibly +# non-instrumented) host interpreter and works directly with the non-sanitized +# LLVM/clang runtime, so an asan/ubsan-instrumented build would die on undefined +# __asan_* symbols at load. Opt the package out; harmless when the consumer +# build isn't sanitized. +package( + default_visibility = ["//visibility:public"], + features = [ + "-asan", + "-ubsan", + ], +) + +# The pytest the py_tests run under. Defaults to the standalone build's hermetic +# hub; a consumer whose Python toolchain already provides pytest (e.g. a conda +# interpreter) overrides this to :ambient_pytest so no hub wheel is needed: +# --@cppjit//:pytest=@cppjit//:ambient_pytest +label_flag( + name = "pytest", + build_setting_default = "@cppjit_test_deps//pytest", +) + +# Empty stand-in: pytest comes from the interpreter, not a Bazel dep. +py_library(name = "ambient_pytest") + +# The JIT C++ toolchain the py_tests need at RUN time (clang-repl compiles +# in-process). Default empty -> standalone uses the host. A consumer without one +# stages headers/clang into runfiles here, paired with :jit_cxx_interp_args: +# --@cppjit//:jit_cxx_data=//my/pkg:cppjit_jit_cxx_data +label_flag( + name = "jit_cxx_data", + build_setting_default = ":jit_cxx_data_default", +) + +filegroup( + name = "jit_cxx_data_default", + srcs = [], +) + +# The matching --gcc-toolchain / -stdlib++-isystem args (CPPINTEROP_JIT_CXX_ARGS +# make-var, appended to the py_tests' interpreter env). Default empty; a consumer +# staging :jit_cxx_data overrides with its own jit_cxx_interp_args() target: +# --@cppjit//:jit_cxx_interp_args=//my/pkg:cppjit_jit_cxx_interp_args +label_flag( + name = "jit_cxx_interp_args", + build_setting_default = ":jit_cxx_interp_args_default", +) + +jit_cxx_interp_args( + name = "jit_cxx_interp_args_default", + args = "", +) + +# The public C-API headers, installed as cppjit/interop/include/cpyrt/*.h. +filegroup( + name = "headers", + srcs = [ + "src/cpyrt/API.h", + "src/cpyrt/CommonDefs.h", + "src/cpyrt/DispatchPtr.h", + "src/cpyrt/PyException.h", + "src/cpyrt/Reflex.h", + ], +) + +# Public headers as a cc_library carrying the right include root, so a +# cross-Python consumer (see :srcs) gets resolvable by depending on +# this instead of re-deriving the include path in its own package. +cc_library( + name = "headers_lib", + hdrs = [":headers"], + includes = ["src"], +) + +# The extension module's source + private headers, exposed so a consumer can +# build the extension against a different Python (e.g. a consumer building per-conda- +# env variants); the :solib target here is single-version. +filegroup( + name = "srcs", + srcs = glob([ + "src/cpyrt/*.cxx", + "src/cpyrt/*.h", + "src/cpyrt/*.inc", + ]), +) + +# The interop wrapper's own headers. The CppInterOp API headers stay in +# @cppinterop; only the wrapper-local ones live here. +filegroup( + name = "interop_headers", + srcs = glob(["src/interop/*.h"]), +) + +# The two interop translation units the merged library compiles (the CMake +# INTEROP_SOURCES list), for the same cross-Python case as :srcs. +filegroup( + name = "interop_srcs", + srcs = [ + "src/interop/cppinterop_dispatch.cxx", + "src/interop/interop_wrapper.cxx", + ], +) + +# One merged library, mirroring CMake's single `add_library(cppjit SHARED ...)`: +# every src/cpyrt/*.cxx plus the two interop translation units. +cc_library( + name = "solib_lib", + srcs = glob(["src/cpyrt/*.cxx"]) + [":interop_srcs"], + hdrs = glob([ + "src/cpyrt/*.h", + "src/cpyrt/*.inc", + "src/interop/*.h", + ]) + ["@cppinterop//:headers"], + copts = BASE_COPTS + [ + # BASE_COPTS sets -fno-exceptions/-fno-rtti; both halves need them + # (throw std::runtime_error / PyException, typeid for AutoCastRTTI). + "-fexceptions", + "-frtti", + # CppInterOp headers are ; include root is the source + # tree plus the genfiles tree where the tblgen'd .inc headers land. + "-I" + repo_loc("@cppinterop") + "/include", + "-I$(GENDIR)/" + repo_loc("@cppinterop") + "/include", + # CMAKE: the relative layout cppinterop_paths() anchors at libcppjit.so's + # own directory. The fallback prefix is the runfiles path of this repo's + # python/ dir, which is where the solib and the staged tree land. + "-DCPPINTEROP_INSTALL_PREFIX='\"python/cppjit\"'", + "-DCPPINTEROP_LIBRARY='\"interop/lib/libclangCppInterOp.so\"'", + "-DCPPINTEROP_INCLUDE_DIR='\"interop/include\"'", + "-DCPPJIT_CLANG_MAJOR='\"" + LLVM_MAJOR + "\"'", + "-DCPPJIT_CLANG_INCLUDE_DIR='\"interop/lib/clang/" + LLVM_MAJOR + "\"'", + # CMAKE: -Wall -Wno-strict-aliasing -Wno-register. + "-Wall", + "-Wno-register", # C++17 vs. Python headers + "-Wno-strict-aliasing", # not all Pythons provide this + "-Wno-cast-function-type", # g++: CPyFunction casts + # Intentional non-override overrides, an unused var and a non-canonical + # dtor name; demote so a consumer toolchain with -Werror does not make + # them fatal. + "-Wno-error=inconsistent-missing-override", + "-Wno-inconsistent-missing-override", + "-Wno-error=unused-variable", + "-Wno-unused-variable", + "-Wno-error=dtor-name", + "-Wno-dtor-name", + "-Wno-error=unused-parameter", + "-Wno-unused-parameter", + ], + data = ["@cppinterop//:headers"], + # CMAKE: target_include_directories PRIVATE src, src/cpyrt, src/interop. + includes = [ + "src", + "src/cpyrt", + "src/interop", + ], + deps = [ + "@rules_python//python/cc:current_py_cc_headers", + "@rules_python//python/cc:current_py_cc_libs", + ], +) + +# Output name MUST be libcppjit.so inside the cppjit package: +# _preload_backend_library imports it as `cppjit.libcppjit`, and +# cppinterop_paths() anchors the staged interop/ tree at this directory. No link +# against CppInterOp: cppinterop_dispatch.cxx defines every Cpp:: entry point +# and dlopen's the real library at runtime (mirrors CMake, which only +# add_dependencies). +cc_shared_library( + name = "solib", + shared_lib_name = "python/cppjit/libcppjit.so", + user_link_flags = [ + "-Wl,-Bsymbolic-functions", + # Bazel omits the soname when shared_lib_name carries a directory; + # CMake always sets one, so set it back for artifact parity. + "-Wl,-soname,libcppjit.so", + ], + deps = [":solib_lib"], +) + +# Wheel-layout staging: the wheel bundles CppInterOp, the cpyrt API headers and +# clang's builtin headers under cppjit/interop, and libcppjit.so self-locates +# them beside itself. Recreate that shape in the Bazel tree so the same lookup +# works here. +stage_files( + name = "staged_cppinterop_solib", + srcs = ["@cppinterop//:solib"], + out_dir = "python/cppjit/interop/lib", + path_anchor = "lib", +) + +stage_files( + name = "staged_cppinterop_headers", + srcs = ["@cppinterop//:headers"], + out_dir = "python/cppjit/interop/include", + path_anchor = "include", +) + +stage_files( + name = "staged_cpyrt_headers", + srcs = [":headers"], + out_dir = "python/cppjit/interop/include/cpyrt", + path_anchor = "cpyrt", +) + +# The build clang's builtin headers, laid out as a headers-only resource dir. +# Their presence is what makes acquireOrCreateInterpreter pin -resource-dir +# instead of probing the host for a same-major clang. +stage_files( + name = "staged_clang_resource_headers", + srcs = ["@llvm//:clang_resource_files"], + out_dir = "python/cppjit/interop/lib/clang", + path_anchor = "clang", +) + +# The wheel payload as one tree: everything under python/ re-rooted at +# site_packages/, which is what `pip install .` writes into site-packages. +# Same inputs as :data plus the pure-Python sources, so the layout the tests +# exercise and the layout an installer ships stay one thing. +stage_files( + name = "site_packages", + srcs = [ + ":solib", + ":staged_clang_resource_headers", + ":staged_cppinterop_headers", + ":staged_cppinterop_solib", + ":staged_cpyrt_headers", + ] + glob(["python/cppjit/**/*.py"]), + out_dir = "site_packages", + path_anchor = "python", +) + +# Convenience aggregate of the runtime artifacts the tests load. +filegroup( + name = "data", + srcs = [ + ":solib", + ":staged_clang_resource_headers", + ":staged_cppinterop_headers", + ":staged_cppinterop_solib", + ":staged_cpyrt_headers", + "@cppinterop//:solib", + ], +) + +# The cppjit package: pure Python plus the runtime libs staged at their +# wheel-layout spots, so anything depending on it gets libcppjit.so + +# libclangCppInterOp.so + the headers in its runfiles. +py_library( + name = "lib", + srcs = glob(["python/cppjit/**/*.py"]), + data = [ + ":headers", + ":solib", + ":staged_clang_resource_headers", + ":staged_cppinterop_headers", + ":staged_cppinterop_solib", + ":staged_cpyrt_headers", + "@cppinterop//:headers", + "@cppinterop//:solib", + ], + imports = ["python"], +) + +# test/cpp/*.cxx + test/cpp/*.h consumed by the dict solibs and as py_test data. +filegroup( + name = "cxxh", + srcs = glob([ + "test/cpp/*.cxx", + "test/cpp/*.h", + ]), +) + +# Per-key .a + Dict.so, mirroring test/Makefile's dict list. +cppjit_test_dict_sos([ + "advancedcpp", + "advancedcpp2", + "conversions", + "cpp11features", + "crossinheritance", + "datatypes", + "doc_helper", + "example01", + "fragile", + "operators", + "overloads", + "pythonizables", + "std_streams", + "stltypes", + "templates", +]) + +# One py_test per test/test_*.py; sokeys are the dicts each loads via setup_make(). +cppjit_py_test( + name = "test_aclassloader", + sokeys = ["example01"], +) + +cppjit_py_test( + name = "test_advancedcpp", + sokeys = [ + "advancedcpp", + "advancedcpp2", + ], +) + +cppjit_py_test(name = "test_api") + +cppjit_py_test( + name = "test_basic_api", + sokeys = ["example01"], +) + +cppjit_py_test(name = "test_boost") + +cppjit_py_test(name = "test_concurrent") + +cppjit_py_test( + name = "test_conversions", + sokeys = ["conversions"], +) + +cppjit_py_test( + name = "test_cpp11features", + sokeys = ["cpp11features"], +) + +# The C++23 cases skip unless the interpreter itself runs at -std=c++23, so this +# cell raises the standard for its own process. +cppjit_py_test( + name = "test_cpp23features", + extra_env = {"CPPINTEROP_EXTRA_INTERPRETER_ARGS": "-std=c++23"}, +) + +cppjit_py_test( + name = "test_crossinheritance", + sokeys = ["crossinheritance"], +) + +cppjit_py_test( + name = "test_datatypes", + sokeys = ["datatypes"], +) + +cppjit_py_test( + name = "test_doc_features", + sokeys = ["doc_helper"], +) + +cppjit_py_test(name = "test_eigen") + +cppjit_py_test( + name = "test_fragile", + sokeys = ["fragile"], +) + +cppjit_py_test( + name = "test_lowlevel", + sokeys = ["datatypes"], +) + +cppjit_py_test( + name = "test_operators", + sokeys = ["operators"], +) + +cppjit_py_test( + name = "test_overloads", + sokeys = ["overloads"], +) + +cppjit_py_test( + name = "test_pythonify", + sokeys = ["example01"], +) + +cppjit_py_test( + name = "test_pythonization", + sokeys = ["pythonizables"], +) + +cppjit_py_test(name = "test_regression") + +cppjit_py_test( + name = "test_stltypes", + sokeys = ["stltypes"], +) + +cppjit_py_test( + name = "test_streams", + sokeys = ["std_streams"], +) + +cppjit_py_test( + name = "test_templates", + sokeys = ["templates"], +) + +# The self-location property gate: no path env vars, foreign cwd. See the +# macro's docstring in cppjit_bazel/rules.bzl. +cppjit_selflocation_py_test(name = "test_selflocation") + +# leakcheck and numba are flaky/heavy in upstream CI. The frozen cppjit_py_test +# macro takes no tags param, so they are declared as normal targets but kept +# OUT of the :tests suite below. +cppjit_py_test(name = "test_leakcheck") + +cppjit_py_test(name = "test_numba") + +# Cross-repo test_suite cannot use wildcards; list every gated target. +test_suite( + name = "tests", + tests = [ + ":test_aclassloader", + ":test_advancedcpp", + ":test_api", + ":test_basic_api", + ":test_boost", + ":test_concurrent", + ":test_conversions", + ":test_cpp11features", + ":test_cpp23features", + ":test_crossinheritance", + ":test_datatypes", + ":test_doc_features", + ":test_eigen", + ":test_fragile", + ":test_lowlevel", + ":test_operators", + ":test_overloads", + ":test_pythonify", + ":test_pythonization", + ":test_regression", + ":test_selflocation", + ":test_stltypes", + ":test_streams", + ":test_templates", + # test_leakcheck, test_numba intentionally excluded (flaky/heavy). + ], +) diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000..36d25aa --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,62 @@ +module( + name = "cppjit", + version = "0.1.0", +) + +bazel_dep(name = "cppjit_bazel", version = "0.1.0") +local_path_override( + module_name = "cppjit_bazel", + path = "bazel", +) + +bazel_dep(name = "cppinterop", version = "0.1.0") + +# Fetch the same CppInterOp commit the CMake build pins (CPPINTEROP_GIT_TAG in +# CMakeLists.txt), so both builds compile the same backend sources and no +# sibling checkout is needed. Keep the two pins in step. +# The patch only renames the shared Bazel module (cppyy_bazel -> cppjit_bazel) +# in the fetched tree; drop it once the pinned commit carries the rename. +# To build a fork or a newer commit, swap urls/integrity/strip_prefix together. +archive_override( + module_name = "cppinterop", + integrity = "sha256-Oo15vfnyOyZMFWZ/lzm20v/UVc5lWm5HSMcELmOXUrs=", + patch_strip = 1, + patches = ["//:bazel-support/cppinterop-cppjit-bazel-module.patch"], + strip_prefix = "CppInterOp-9802d61921ad5688ae42e4e628d754fc1192244d", + urls = ["https://github.com/compiler-research/CppInterOp/archive/9802d61921ad5688ae42e4e628d754fc1192244d.tar.gz"], +) + +# Dev loop: build a sibling CppInterOp checkout instead of the pinned archive +# (the Bazel equivalent of CMake's CPPINTEROP_SOURCE_DIR). Bazel allows one +# override per module, so comment out the archive_override above when you +# enable this. Keep it commented out in a commit: an enabled override makes +# the build depend on a checkout outside the repo, which nobody else has. +# local_path_override( +# module_name = "cppinterop", +# path = "../CppInterOp", +# ) + +bazel_dep(name = "rules_python", version = "1.8.5") + +# @llvm comes from the local LLVM/Clang tree selected by LLVM_DIR. +llvm = use_extension("@cppjit_bazel//:llvm.bzl", "llvm") + +# Match the CMake build's CMAKE_CXX_STANDARD. A mixed standard across the +# libcppjit / libclangCppInterOp boundary silently changes inline bodies. +llvm.config(cxx_std = "c++20") +use_repo(llvm, llvm = "cppjit_llvm") + +# Opt in to the centralized clang toolchain from cppjit_bazel (standalone build). +register_toolchains("@llvm//:cc_toolchain") + +# Match the primary CI matrix cell in ci.yml. +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.14") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "cppjit_test_deps", + python_version = "3.14", + requirements_lock = "//:requirements_bazel.txt", +) +use_repo(pip, "cppjit_test_deps") diff --git a/bazel-support/cppinterop-cppjit-bazel-module.patch b/bazel-support/cppinterop-cppjit-bazel-module.patch new file mode 100644 index 0000000..36d281b --- /dev/null +++ b/bazel-support/cppinterop-cppjit-bazel-module.patch @@ -0,0 +1,56 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +index dd26bb5..f64dd0b 100644 +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -1,18 +1,18 @@ + """Experimental Bazel build for CppInterOp. CMake is the supported build; + this translates the CMake recipe to build libclangCppInterOp against a local +-LLVM tree (selected via the LLVM_DIR env var, see @cppyy_bazel//:llvm.bzl).""" ++LLVM tree (selected via the LLVM_DIR env var, see @cppjit_bazel//:llvm.bzl).""" + + load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag") +-load("@cppyy_bazel//:defs.bzl", "BASE_COPTS", "CPPINTEROP_COPTS", "DEFAULT_LTO_OPT_LEVEL", "LTO_OPT_LEVELS", "is_main_repo", "jit_cxx_interp_args", "llvm_linkopts", "llvm_lto_opt_linkopts", "llvm_system_libs", "llvm_tblgen_linkopts", "repo_rloc") ++load("@cppjit_bazel//:defs.bzl", "BASE_COPTS", "CPPINTEROP_COPTS", "DEFAULT_LTO_OPT_LEVEL", "LTO_OPT_LEVELS", "is_main_repo", "jit_cxx_interp_args", "llvm_linkopts", "llvm_lto_opt_linkopts", "llvm_system_libs", "llvm_tblgen_linkopts", "repo_rloc") + load( +- "@cppyy_bazel//:rules.bzl", ++ "@cppjit_bazel//:rules.bzl", + "cppinterop_buildinfo_inc", + "cppinterop_cc_test", + "cppinterop_cxx_shim", + "cppinterop_dispatch_cc_test", + "cppinterop_tblgen_inc_files", + ) + + # libclangCppInterOp is a sanitizer boundary: it's dlopen'd by a (possibly + # non-instrumented) host interpreter and works directly with the non-sanitized + # LLVM/clang libraries, so an asan/ubsan-instrumented build would die on +diff --git a/MODULE.bazel b/MODULE.bazel +index 07e401d..938a05b 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -1,20 +1,20 @@ + module( + name = "cppinterop", + version = "0.1.0", + ) + +-bazel_dep(name = "cppyy_bazel", version = "0.1.0") ++bazel_dep(name = "cppjit_bazel", version = "0.1.0") + local_path_override( +- module_name = "cppyy_bazel", +- path = "../cppyy/bazel", ++ module_name = "cppjit_bazel", ++ path = "../cppjit/bazel", + ) + + bazel_dep(name = "bazel_skylib", version = "1.7.1") + bazel_dep(name = "googletest", version = "1.15.2") + bazel_dep(name = "google_benchmark", version = "1.9.1") + +-llvm = use_extension("@cppyy_bazel//:llvm.bzl", "llvm") ++llvm = use_extension("@cppjit_bazel//:llvm.bzl", "llvm") + use_repo(llvm, llvm = "cppjit_llvm") + +-# Opt in to the centralized clang toolchain from cppyy_bazel (standalone build). ++# Opt in to the centralized clang toolchain from cppjit_bazel (standalone build). + register_toolchains("@llvm//:cc_toolchain") diff --git a/bazel/.bazelversion b/bazel/.bazelversion new file mode 100644 index 0000000..56b6be4 --- /dev/null +++ b/bazel/.bazelversion @@ -0,0 +1 @@ +8.3.1 diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel new file mode 100644 index 0000000..954de1f --- /dev/null +++ b/bazel/BUILD.bazel @@ -0,0 +1,19 @@ +# Explicit load instead of the native symbol: consumers may build with +# Starlark rule autoloads disabled. +load("@rules_python//python:defs.bzl", "py_library") + +exports_files([ + "llvm.bzl", + "defs.bzl", + "rules.bzl", +]) + +# Shim that fixes the hermetic interpreter's stale sysconfig INCLUDEPY; +# py_tests depend on it. imports=["."] puts this dir on sys.path, so +# test_main.py can import it before it imports cppjit. +py_library( + name = "sitecustomize", + srcs = ["sitecustomize.py"], + imports = ["."], + visibility = ["//visibility:public"], +) diff --git a/bazel/MODULE.bazel b/bazel/MODULE.bazel new file mode 100644 index 0000000..5797f31 --- /dev/null +++ b/bazel/MODULE.bazel @@ -0,0 +1,27 @@ +# cppjit_bazel: shared Bazel machinery (copts, linkopts, rules, the @llvm repo) +# for the cppjit stack (cppinterop, cppjit). +# +# It lives in cppjit/bazel/ but is its OWN module so the consumer repos can +# depend on it without forming a bzlmod cycle: cppjit depends on cppjit_bazel, so +# cppjit_bazel cannot depend back on cppjit. Keeping it module-local (rather than a +# registry artifact) keeps the monorepo self-contained. +module( + name = "cppjit_bazel", + version = "0.1.0", +) + +bazel_dep(name = "rules_python", version = "1.0.0") +bazel_dep(name = "googletest", version = "1.15.2") +bazel_dep(name = "platforms", version = "0.0.10") + +# cppjit_bazel owns the @llvm repo so its own defs.bzl/rules.bzl can reference it. +llvm = use_extension("//:llvm.bzl", "llvm") +use_repo(llvm, llvm = "cppjit_llvm") + +# The @llvm repo also generates a centralized clang C++ toolchain +# (@llvm//:cc_toolchain) so consumers can build with the LLVM tree's clang + +# matching ABI from a single place. It is NOT registered here on purpose: a +# consumer that already has its own C++ toolchain must not have a second one +# forced on it. Standalone consumers opt in by adding +# register_toolchains("@llvm//:cc_toolchain") +# to their own MODULE.bazel (see cppjit and CppInterOp). diff --git a/bazel/ORIGIN.md b/bazel/ORIGIN.md new file mode 100644 index 0000000..04cf996 --- /dev/null +++ b/bazel/ORIGIN.md @@ -0,0 +1,49 @@ +# Runtime path resolution: self-location and the ${ORIGIN} token + +The stack self-locates its own resources (bundled CppInterOp, the cpyrt API +headers, clang's builtin headers) relative to its load path: `libcppjit.so` +calls `dladdr` on itself and joins the baked relative spellings +(`interop/lib/...`, `interop/include`) onto that directory — see +`cppinterop_paths()` in `src/interop/interop_wrapper.cxx`. `staging.bzl` shows +how the Bazel tree recreates the wheel layout that makes this work. The only +paths a consumer must supply are its *own* toolchain args (e.g. +`--gcc-toolchain`) in `CPPINTEROP_EXTRA_INTERPRETER_ARGS`. + +No installer knows its final absolute prefix at build time, and relative paths +break as soon as the process runs from a different cwd (a notebook kernel, a +tool run from $HOME). Args may therefore reference `${ORIGIN}` — the directory +of libcppjit.so itself, mirroring ELF rpath $ORIGIN semantics. + +Bazel consumers: the solib dir sits three levels below the runfiles root, so +sibling repos resolve via ORIGIN_RUNFILES_ROOT (defs.bzl), e.g. +`"--gcc-toolchain=" + ORIGIN_RUNFILES_ROOT + "/" + repo_name("@gcc")`. +The expanded args carry literal `..` components; clang handles them fine. + +`libcppjit.so` expands the token itself, in +`expandOriginInInterpreterArgs()` (`src/interop/interop_wrapper.cxx`): it +rewrites `CPPINTEROP_EXTRA_INTERPRETER_ARGS` before CreateInterpreter reads it. +A consumer therefore passes the token through verbatim and does not expand it. +A process that never loads `libcppjit.so` (a C++ client of CppInterOp alone) +gets no expansion, so it must write absolute or cwd-relative args. + +# Standalone build + +A fresh clone needs one thing from the host: an LLVM/Clang build or install +tree, given by `LLVM_DIR` (the same variable the CMake build takes, but pointed +at the tree root, not at `lib/cmake/llvm`). + +```bash +git clone && cd cppjit +LLVM_DIR=/path/to/llvm bazelisk test //... +``` + +CppInterOp comes from the pinned archive in `MODULE.bazel` — the commit +`CMakeLists.txt` pins as `CPPINTEROP_GIT_TAG` — so no sibling checkout is +needed. To build a local CppInterOp instead (the Bazel equivalent of CMake's +`CPPINTEROP_SOURCE_DIR`), swap the `archive_override` for the commented +`local_path_override` next to it. + +`bazelisk build //:site_packages` writes the installable payload — +the `cppjit/` package with `libcppjit.so` and `cppjit/interop/` holding the bundled CppInterOp +library, headers and clang resource headers — with the same layout and the same +file set that `pip install .` puts into site-packages. diff --git a/bazel/defs.bzl b/bazel/defs.bzl new file mode 100644 index 0000000..7eca1cf --- /dev/null +++ b/bazel/defs.bzl @@ -0,0 +1,218 @@ +"""Shared copts, linkopts, and runtime env for the cppjit stack.""" + +load("@llvm//:defs.bzl", "CLANG_LIB_NAMES", "LLVM_GOLD_PLUGIN", "LLVM_INCLUDE_DIRS", "LLVM_LIB_NAMES", "LLVM_SYSTEM_LIBS", "LLVM_VERSION") + +# Consumer repos aren't deps of cppjit_bazel, so Label() can't resolve them in +# this module's mapping; their single-version deps canonicalize to "name+". Map +# those literals and fall back to Label() for @llvm/anything else. +_CONSUMER_CANON = { + "cppinterop": "cppinterop+", + "cppjit": "cppjit+", +} + +def repo_name(repo): + return _CONSUMER_CANON.get(repo.lstrip("@"), None) or Label(repo).repo_name + +# Build-time (execroot, e.g. -I) and runtime (runfiles) paths to an EXTERNAL dep +# repo. The external assumption breaks for a self-reference (e.g. CppInterOp's +# tests pointing at @cppinterop): as a main repo its files are at the root, not +# external/+. repository_name() can't reveal another repo's canonical name, +# so the caller passes is_self=True for the one reference to its own module. +def repo_loc(repo, is_self = False): + return "." if is_self else "external/" + repo_name(repo) + +def repo_rloc(repo, is_self = False): + return "." if is_self else "../" + repo_name(repo) + +# ${ORIGIN} stands for the directory of libcppjit.so itself (ELF-$ORIGIN +# semantics -- no build-system content in the runtime code). Under Bazel that +# directory is always //python/cppjit (fixed by the solib's +# shared_lib_name), so THREE ups reach the runfiles root where sibling repos +# live. Consumers join this with repo_name() to write cwd-independent +# interpreter args: ORIGIN_RUNFILES_ROOT + "/" + repo_name("@gcc") + "/..." +# libcppjit.so expands the token before CreateInterpreter reads the args, so +# pass it through verbatim; see ORIGIN.md. +ORIGIN_RUNFILES_ROOT = "${ORIGIN}/../../.." + +# True when this module is built standalone (main repo, repository_name() == "@"). +def is_main_repo(current_repo): + return current_repo.lstrip("@") == "" + +# Per-target codegen flags (ABI-critical flags live in the centralized toolchain). +# -fPIC is kept here, not relied on from the toolchain, so objects link into .so's +# even under a consumer toolchain that defaults cc_library to no-PIC (e.g. a +# monorepo consumer). +# -DNDEBUG is mandatory, not a -c opt nicety: the stack ships assert(0) stubs on +# live runtime paths (e.g. Interpreter::toString) that abort the process when a +# consumer builds -c dbg; the CMake/overlay build always sets NDEBUG, so we match. +BASE_COPTS = [ + "-fPIC", + "-fno-exceptions", + "-fno-rtti", + "-ffunction-sections", + "-fdata-sections", + "-fno-common", + "-O3", + "-DNDEBUG", +] + +CPPINTEROP_COPTS = BASE_COPTS + [ + "-DCPPINTEROP_USE_REPL", + "-DLLVM_BINARY_DIR='\"" + repo_loc("@llvm") + "\"'", + "-DCPPINTEROP_VERSION='\"0.1.0-bazel\"'", +] + +def _llvm_L_rpath(): + return [ + "-L" + repo_loc("@llvm") + "/lib", + "-Wl,-rpath," + repo_rloc("@llvm") + "/lib", + ] + +# LTO codegen parallelism for the bitcode-archive link. Fixed (not nproc) for +# reproducibility; 16 is plenty for the cppinterop link's module count. +_LTO_JOBS = 16 + +# LTO opt level for the bitcode-archive link governs the STACK FRAME SIZE of +# clang's own code inside libclangCppInterOp.so: at O0 its recursive constexpr +# evaluator gets fat frames and overflows the 8 MiB stack at JIT time on heavily +# templated code (e.g. Fastor). A per-consumer build setting (:lto_opt_level in +# CppInterOp/BUILD.bazel), not baked in: O0 keeps the standalone link fast, while +# a heavy-JIT consumer raises it (--@cppinterop//:lto_opt_level=2). +LTO_OPT_LEVELS = ["0", "1", "2", "3"] + +# Upstream default: fast link. Insufficient for heavy JIT use; see above. +DEFAULT_LTO_OPT_LEVEL = "0" + +def llvm_lto_opt_linkopts(level): + """The -plugin-opt=O flag for the LTO link, or [] for a non-LTO @llvm. + + Selected per build via the :lto_opt_level flag; emitted only when @llvm + carries IR bitcode (LLVM_GOLD_PLUGIN set), where the level actually drives + codegen. On a non-LTO tree the static link ignores it, so emit nothing. + """ + if LLVM_GOLD_PLUGIN: + return ["-Wl,-plugin-opt=O" + level] + return [] + +# An LTO/bitcode @llvm tree (LLVM_GOLD_PLUGIN set) needs the LLVM plugin to read +# the archives; jobs= keeps its codegen parallel (else serial = minutes). The +# -plugin-opt=O level is appended separately via select() on :lto_opt_level. +# No-op on a non-LTO tree. +def _llvm_plugin(): + if LLVM_GOLD_PLUGIN: + return [ + "-Wl,--plugin=" + repo_loc("@llvm") + "/" + LLVM_GOLD_PLUGIN, + "-Wl,-plugin-opt=jobs=" + str(_LTO_JOBS), + ] + return [] + +# libclangCppInterOp.so links self-contained from the static clang+LLVM archives +# (no libclang-cpp.so dylib), mirroring CMake's DISABLE_LLVM_LINK_LLVM_DYLIB: one +# static copy of every symbol so nothing double-registers when it's dlopen'd. +# --start-group resolves the ~100 archives' mutual refs. The --system-libs +# (zlib/zstd) come separately via the :llvm_system_libs label_flag (so a hermetic +# consumer can ship its own), not from here. +def llvm_linkopts(): + return _llvm_L_rpath() + _llvm_plugin() + [ + "-Wl,--gc-sections", + "-Wl,--start-group", + ] + ["-l" + n for n in CLANG_LIB_NAMES] + \ + ["-l" + n for n in LLVM_LIB_NAMES] + [ + "-Wl,--end-group", + "-ldl", + ] + +# For standalone LLVM tools (cppinterop-tblgen): the static component libs. +# libclang-cpp.so omits TableGen and some cl:: internals, so a tool using +# llvm::TableGen / RecordKeeper must link the components statically. +# tblgen does not run the JIT, so it always uses the fast default LTO level (no +# need for the per-consumer :lto_opt_level the solib carries). +def llvm_tblgen_linkopts(): + return _llvm_L_rpath() + _llvm_plugin() + \ + llvm_lto_opt_linkopts(DEFAULT_LTO_OPT_LEVEL) + \ + ["-l" + n for n in LLVM_LIB_NAMES] + +# The LLVM --system-libs as bare -l flags (e.g. ["-lz", "-lzstd"]). Used as the +# linkopts of the DEFAULT :llvm_system_libs target, which resolves them from the +# host. A hermetic consumer (no host system libs, e.g. remote execution) swaps +# that target for a cc_library that ships the .so files directly. +# +# Wrapped in --no-as-needed/--as-needed: the references to these libs (ZSTD_*, +# inflate, xmlReadMemory) live INSIDE the static LLVM archives, not in the solib's +# own objects, and they're passed after the archive --end-group. Under the linker +# default (--as-needed) ld would drop them as "unused" and the solib would dlopen +# with undefined ZSTD_* symbols. --no-as-needed forces them in; restore the +# default afterwards so it doesn't leak to later libs. +def llvm_system_libs(): + if not LLVM_SYSTEM_LIBS: + return [] + return ["-Wl,--no-as-needed"] + LLVM_SYSTEM_LIBS + ["-Wl,--as-needed"] + +_LLVM_RLOC = repo_rloc("@llvm") + +# clang-repl auto-detects its resource dir from the host binary's location, +# which fails under the test sandbox (the binary isn't beside lib/clang/). +# Point it at @llvm's runfiles so the JIT finds stddef.h etc. Major version +# from LLVM_VERSION ("22.1.8" -> "22"). Also public: it names the versioned +# clang resource dir in the staged wheel layout. +LLVM_MAJOR = LLVM_VERSION.split(".")[0] + +# How the cppinterop gtests find headers at JIT time: the clang builtins dir +# (lib/clang//include) plus every @llvm header root, on CPLUS_INCLUDE_PATH. +# Those tests pass no -resource-dir (it crashed the interpreter in early lexing), +# so the builtins must arrive this way. The cppjit py_tests are the opposite +# case: they DO get a pinned -resource-dir, from the staged bundled headers -- see +# cppjit_base_env below. +_CPLUS_INCLUDE_PATH = ":".join( + [_LLVM_RLOC + "/lib/clang/" + LLVM_MAJOR + "/include"] + + [_LLVM_RLOC + "/" + d for d in LLVM_INCLUDE_DIRS], +) + +# Runtime env for the cppinterop unit tests. A function (not a constant) so the +# @cppinterop runfiles path self-corrects when cppinterop is its own main repo +# (standalone): the caller passes cppinterop_is_self = is_main_repo( +# native.repository_name()). @llvm is always external to cppinterop, so its +# paths never need the fixup. +def cppinterop_base_env(cppinterop_is_self = False): + return { + "CLING_STANDARD_PCH": "none", + "LLVM_LIB_PATH": _LLVM_RLOC + "/lib", + "LD_LIBRARY_PATH": _LLVM_RLOC + "/lib:" + + repo_rloc("@cppinterop", cppinterop_is_self) + "/lib", + "CPLUS_INCLUDE_PATH": _CPLUS_INCLUDE_PATH, + } + +# Runtime env for the cppjit py_tests. libcppjit.so self-locates CppInterOp, the +# cpyrt API headers and clang's builtin headers from its own directory, and pins +# -resource-dir at that bundled copy. The two paths are mutually exclusive: reach +# the clang builtins through CPLUS_INCLUDE_PATH as well and the process dies +# during test collection WITH NO DIAGNOSTIC -- no error, no stack trace, no +# signal, just a truncated log and exit 1. So NO include path belongs here, only +# @llvm's and CppInterOp's shared-library dirs, which Bazel alone knows. +def cppjit_base_env(): + return { + "CLING_STANDARD_PCH": "none", + "LLVM_LIB_PATH": _LLVM_RLOC + "/lib", + "LD_LIBRARY_PATH": _LLVM_RLOC + "/lib:" + repo_rloc("@cppinterop") + "/lib", + } + +# Carries the consumer's JIT interpreter args (--gcc-toolchain / -stdlib++-isystem, +# needed at RUN time; empty standalone where clang-repl autodetects the host) as +# the CPPINTEROP_JIT_CXX_ARGS make-var. The runfiles-relative paths can only be +# formed in a .bzl, not in static MODULE.bazel/.bazelrc text; the test macros +# expand $(...) into the env, and a label_flag swaps the empty default (paired +# with the :jit_cxx_data files flag). See _jit_cxx_env in rules.bzl. +def _jit_cxx_interp_args_impl(ctx): + return [platform_common.TemplateVariableInfo({ + "CPPINTEROP_JIT_CXX_ARGS": ctx.attr.args, + })] + +jit_cxx_interp_args = rule( + implementation = _jit_cxx_interp_args_impl, + attrs = { + "args": attr.string( + doc = "Space-joined interpreter args, exposed as the " + + "CPPINTEROP_JIT_CXX_ARGS make-var. Empty by default (host).", + ), + }, +) diff --git a/bazel/llvm.bzl b/bazel/llvm.bzl new file mode 100644 index 0000000..8a6ea73 --- /dev/null +++ b/bazel/llvm.bzl @@ -0,0 +1,440 @@ +"""Module extension generating the @llvm repo from a local LLVM/Clang tree. + +The tree is selected (highest precedence first) by the LLVM_DIR env var, else +the root module's llvm.config(path=...) tag. No version is pinned: whatever tree +is pointed at is used as-is. + +Header layout: an LLVM *install* tree merges every header under include/, but a +*build* tree splits them across four roots -- source llvm/include and +clang/include (siblings of the build dir) plus generated /include and +/tools/clang/include. We discover all that exist (via llvm-config), symlink +each under a stable name, and record them in LLVM_INCLUDE_DIRS so consumers get +the right -isystem set for either tree shape. +""" + +_BUILD_HEADER = """\ +load(":cc_toolchain.bzl", "cppjit_cc_toolchain") +load("@@rules_cc+//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +filegroup(name = "all_files", srcs = glob([ + "bin/**", "lib/**", "libexec/**", "share/**", {inc_globs} +], allow_empty = True)) + +filegroup(name = "lib_files", srcs = glob(["lib/**"], allow_empty = True)) + +filegroup(name = "include", srcs = glob([{inc_globs}], allow_empty = True)) + +# Clang's builtin (resource-dir) headers alone. cppjit stages these into its own +# tree so the interpreter gets a pinned -resource-dir instead of probing the host. +filegroup(name = "clang_resource_files", srcs = glob([ + "lib/clang/*/include/**", +], allow_empty = True)) + +# A ready-to-use header library: textual_hdrs (LLVM ships .def/.inc and many +# non-self-contained headers, so skip strict standalone-compile validation) plus +# `includes` to emit the -isystem flags for every discovered header root. +cc_library( + name = "headers", + textual_hdrs = glob([{inc_globs}], allow_empty = True), + includes = [{inc_dirs}], +) + +exports_files(["bin/clang", "bin/clang++", "bin/llvm-config"]) + +# The single, centralized C++ toolchain: builds everything with this LLVM tree's +# clang. cppjit_bazel registers @llvm//:cc_toolchain so all consumers inherit it +# with no per-repo CC/CXX wiring. See cc_toolchain.bzl for the captured config. +cppjit_cc_toolchain(name = "cc_toolchain") +""" + +# Generated cc_toolchain.bzl in the @llvm repo: a macro wrapping the stock +# unix cc_toolchain_config with this tree's clang, captured builtin include +# dirs, and the ABI-critical flags LLVM was built with. {placeholders} filled in +# by the repo rule. +_CC_TOOLCHAIN_BZL = '''\ +"""Centralized clang C++ toolchain for the cppjit stack (generated).""" + +load("@bazel_tools//tools/cpp:unix_cc_toolchain_config.bzl", "cc_toolchain_config") +load("@@rules_cc+//cc:defs.bzl", "cc_toolchain") + +_BUILTIN_INCLUDES = {builtin_includes} +_ABI_FLAGS = {abi_flags} + +def cppjit_cc_toolchain(name): + cc_toolchain_config( + name = name + "_config", + cpu = "k8", + compiler = "clang", + toolchain_identifier = "cppjit-clang", + host_system_name = "local", + target_system_name = "local", + target_libc = "local", + abi_version = "local", + abi_libc_version = "local", + cxx_builtin_include_directories = _BUILTIN_INCLUDES, + tool_paths = {tool_paths}, + # ABI-critical flags applied to EVERY C++ TU (incl. third-party gtest) + # so std:: layout / assertion mode is consistent across the .so + # boundary. Per-target codegen flags (-fno-exceptions etc.) stay in + # cppjit_bazel//:defs.bzl BASE_COPTS where they can be overridden. + compile_flags = ["-fPIC"], + cxx_flags = ["-std={cxx_std}"], + link_flags = {link_flags}, + extra_flags_per_feature = {{}}, + opt_compile_flags = [], + dbg_compile_flags = [], + conly_flags = [], + # C++ runtime + libm, as the stock unix toolchain links by default. + link_libs = ["-lstdc++", "-lm"], + opt_link_flags = [], + unfiltered_compile_flags = _ABI_FLAGS, + coverage_compile_flags = [], + coverage_link_flags = [], + # --start-lib/--end-lib is an lld/gold extension; GNU ld rejects it. Only + # claim support when we actually link with lld (see has_lld below). + supports_start_end_lib = {supports_start_end_lib}, + ) + + cc_toolchain( + name = name + "_cc", + toolchain_config = ":" + name + "_config", + all_files = ":all_files", + compiler_files = ":all_files", + dwp_files = ":empty", + linker_files = ":all_files", + objcopy_files = ":all_files", + strip_files = ":all_files", + supports_param_files = 1, + ) + + native.filegroup(name = "empty", srcs = []) + + native.toolchain( + name = name, + toolchain = ":" + name + "_cc", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", + exec_compatible_with = ["@platforms//cpu:x86_64", "@platforms//os:linux"], + target_compatible_with = ["@platforms//cpu:x86_64", "@platforms//os:linux"], + ) +''' + +def _strip_lib_name(fname): + """libLLVMCore.a -> LLVMCore; mirror llvm-config --libnames stripping.""" + name = fname + if name.startswith("lib"): + name = name[len("lib"):] + for suffix in (".a", ".so", ".dylib"): + if name.endswith(suffix): + return name[:-len(suffix)] + return name + +# Candidate header roots, as (repo-relative symlink name, absolute path) given +# the llvm-config includedir and obj-root. An install tree collapses several to +# the same realpath (deduped below) or leaves them absent. +def _header_root_candidates(includedir, obj_root): + # includedir is /llvm/include for a build tree, /include for an + # install tree; its grandparent is the source/prefix root. + src_root = includedir + "/../.." + return [ + ("include", obj_root + "/include"), # generated llvm + ("clang_include", obj_root + "/tools/clang/include"), # generated clang + ("llvm_src_include", includedir), # source llvm + ("clang_src_include", src_root + "/clang/include"), # source clang + ] + +def _llvm_repo_impl(rctx): + # Three ways to point at the LLVM tree, highest precedence first: + # 1. llvm.config(llvm_config_label = "@some_llvm//:bin/llvm-config") -- + # reuse a tree a monorepo consumer already fetched as its own @llvm; no + # second download, and the path is resolved from the label so it's stable. + # 2. LLVM_DIR env var. + # 3. llvm.config(path = ...) string. + # C++ standard the centralized toolchain compiles the stack at. Default + # c++17 (matches the CMake build); a consumer building against a newer + # libstdc++/interpreter overrides via llvm.config(cxx_std = "c++20"|...). + cxx_std = rctx.attr.cxx_std or "c++17" + + if rctx.attr.llvm_config_label: + # Resolving the label materializes the owning repo and gives its real + # on-disk path; the tree root is the grandparent of bin/llvm-config. + cfg = rctx.path(rctx.attr.llvm_config_label) + path = str(cfg.dirname.dirname) + else: + path = rctx.os.environ.get("LLVM_DIR", rctx.attr.path) + if not path: + fail("No LLVM tree configured. Set llvm.config(llvm_config_label = ...) " + + "to reuse an existing @llvm, the LLVM_DIR env var, or " + + "llvm.config(path = ...).") + + llvm_config = rctx.path(path + "/bin/llvm-config") + if not llvm_config.exists: + fail("LLVM tree at '{}' has no bin/llvm-config. Point llvm_config_label, ".format(path) + + "LLVM_DIR, or llvm.config(path = ...) at a valid LLVM build or install tree.") + + # Surface the non-header top-level dirs. + for top in ("bin", "lib", "libexec", "share"): + src = rctx.path(path + "/" + top) + if src.exists: + rctx.symlink(src, top) + + includedir = rctx.execute([llvm_config, "--includedir"]).stdout.strip() + obj_root = rctx.execute([llvm_config, "--obj-root"]).stdout.strip() + + # Symlink each header root that exists, deduping by realpath so an install + # tree (where several candidates resolve to the same dir) yields one entry. + inc_dirs = [] + seen = {} + for name, abspath in _header_root_candidates(includedir, obj_root): + p = rctx.path(abspath) + if not p.exists: + continue + real = str(p.realpath) + if real in seen: + continue + seen[real] = True + rctx.symlink(p, name) + inc_dirs.append(name) + + if not inc_dirs: + fail("@llvm: no header roots found under '{}'. Checked include/, ".format(path) + + "tools/clang/include, and the llvm-config includedir.") + + inc_globs = ", ".join(['"{}/**"'.format(d) for d in inc_dirs]) + inc_dirs_lit = ", ".join(['"{}"'.format(d) for d in inc_dirs]) + rctx.file("BUILD.bazel", _BUILD_HEADER.format( + inc_globs = inc_globs, + inc_dirs = inc_dirs_lit, + )) + + version = rctx.execute([llvm_config, "--version"]).stdout.strip() + + # Capture clang's own builtin include search dirs (libstdc++, the clang + # resource dir, /usr/include) so the centralized toolchain declares exactly + # what an autodetected clang would use -- matching the green build. + clangxx = str(rctx.path(path + "/bin/clang++")) + probe = rctx.execute([clangxx, "-E", "-x", "c++", "/dev/null", "-v"]) + builtin_includes = [] + collecting = False + for line in probe.stderr.split("\n"): + if "#include <...> search starts here:" in line: + collecting = True + continue + if "End of search list." in line: + collecting = False + continue + if collecting: + d = line.strip() + if d: + builtin_includes.append(d) + bindir = str(rctx.path(path + "/bin")) + + # Prefer lld iff this tree ships it: pass -fuse-ld=lld + -B so the + # clang driver finds it. An LLVM *build* tree has bin/ld.lld; many *install* + # trees (e.g. the CI llvm-release recipe) omit it -- forcing -fuse-ld=lld + # there fails ("invalid linker name"). When absent, emit no linker flag and + # let clang use its default linker (the host's ld/gold, or lld if on PATH). + # lld is only strictly required for an LTO/bitcode tree (which also ships it). + has_lld = rctx.path(bindir + "/ld.lld").exists + if has_lld: + link_flags = ["-fuse-ld=lld", "-B" + bindir] + ld_tool = bindir + "/ld.lld" + else: + link_flags = [] + ld_tool = "/usr/bin/ld" + + # --start-lib/--end-lib (which bazel emits when supports_start_end_lib) is an + # lld/gold-only extension; GNU ld rejects it. Tie it to the linker we use. + supports_start_end_lib = "True" if has_lld else "False" + + # The consuming code must compile with the SAME preprocessor defines LLVM + # itself was built with (assertion mode, ABI, STDC macros); a mismatch + # changes the layout/behavior of LLVM/Clang types and breaks the JIT at + # runtime. Capture the -D flags from `llvm-config --cxxflags` verbatim. + cxxflags = rctx.execute([llvm_config, "--cxxflags"]).stdout.replace("\n", " ").split(" ") + llvm_defines = [f for f in cxxflags if f.startswith("-D") or f.startswith("-U")] + + # --link-static forces llvm-config to report the STATIC component archives + + # their system deps even when the tree defaults to shared linkage (e.g. the + # CI llvm-release recipe, which is built shared: plain --libnames there + # returns just "libLLVM-22.so" and --libs "-lLLVM-22", which breaks our + # self-contained static link with undefined refs). The flag is a no-op on a + # static-default tree (a plain local build), so it's safe everywhere. + + # System libs the LLVM static libs were built against (e.g. -lz -lzstd when + # LLVM has compression support, as a prebuilt consumer tree does). Captured from + # llvm-config so the link adapts to whatever the tree needs; without these + # the solib has undefined compressBound/ZSTD_* at dlopen. + system_libs = [ + f + for f in rctx.execute([llvm_config, "--link-static", "--system-libs"]).stdout.replace("\n", " ").split(" ") + if f.startswith("-l") + ] + + raw = rctx.execute([llvm_config, "--link-static", "--libnames", "all"]).stdout.replace("\n", " ") + lib_names = [_strip_lib_name(n) for n in raw.split(" ") if n] + + # llvm-config never lists Polly; append it iff actually present in lib/. + for polly in ("Polly", "PollyISL"): + if rctx.path(path + "/lib/lib" + polly + ".a").exists or \ + rctx.path(path + "/lib/lib" + polly + ".so").exists: + lib_names.append(polly) + + # Clang's static archives aren't covered by llvm-config; enumerate the + # libclang*.a in lib/ (excluding the libclang-cpp dylib). The cppjit stack + # links libclangCppInterOp.so self-contained from these + the LLVM + # components, mirroring CMake's DISABLE_LLVM_LINK_LLVM_DYLIB build. + clang_names = [] + for f in rctx.path(path + "/lib").readdir(): + bn = f.basename + if bn.startswith("libclang") and bn.endswith(".a") and not bn.startswith("libclang-cpp"): + clang_names.append(bn[len("lib"):-len(".a")]) + + # If the LLVM static archives contain IR bitcode (a tree built with LTO, + # like a prebuilt LTO consumer tree), mold/lld must load the LLVM linker plugin to + # read those members; without it mold fails ("failed to load plugin"). + # Expose LLVMgold.so's repo-relative path iff it ships, so link helpers can + # add --plugin only when needed. A non-LTO tree (no LLVMgold.so) leaves it + # empty and the flag is omitted. + gold_plugin = "" + if rctx.path(path + "/lib/LLVMgold.so").exists: + gold_plugin = "lib/LLVMgold.so" + + rctx.file("defs.bzl", ("LLVM_VERSION = {}\nLLVM_LIB_NAMES = {}\n" + + "CLANG_LIB_NAMES = {}\nLLVM_INCLUDE_DIRS = {}\n" + + "LLVM_DEFINES = {}\nLLVM_GOLD_PLUGIN = {}\n" + + "LLVM_SYSTEM_LIBS = {}\nLLVM_CXX_STD = {}\n" + + "LLVM_EXTRA_TEST_TAGS = {}\n").format( + repr(version), + repr(lib_names), + repr(clang_names), + repr(inc_dirs), + repr(llvm_defines), + repr(gold_plugin), + repr(system_libs), + repr(cxx_std), + repr(rctx.attr.extra_test_tags), + )) + + # ABI flags applied to every C++ TU by the centralized toolchain: the LLVM + # build's own defines plus the visibility/codegen flags that must match the + # libraries we interop with (a mismatch trips clang AST asserts / breaks the + # JIT). Per-target flags like -fno-exceptions stay in BASE_COPTS. + abi_flags = llvm_defines + [ + "-fno-semantic-interposition", + "-fvisibility-inlines-hidden", + "-fno-strict-aliasing", + "-funwind-tables", + "-fno-stack-protector", + ] + + # Prefer this LLVM tree's llvm-* binutils, but fall back to the host's GNU + # equivalents when absent: a full LLVM *build* tree ships them all, while a + # minimal *install* tree (e.g. the CI llvm-release recipe) ships only clang + + # a few tools. GNU ar/nm/objcopy/... are ABI-compatible with clang objects + # for a normal (non-LTO) build; an LTO/bitcode tree, which needs llvm-ar to + # read bitcode members, ships its own llvm-* anyway. + def _tool(llvm_name, fallback): + p = bindir + "/" + llvm_name + return p if rctx.path(p).exists else fallback + + tool_paths = { + "gcc": bindir + "/clang", + "cpp": _tool("clang-cpp", "/usr/bin/cpp"), + "ar": _tool("llvm-ar", "/usr/bin/ar"), + "nm": _tool("llvm-nm", "/usr/bin/nm"), + "ld": ld_tool, + "as": bindir + "/clang", + "objcopy": _tool("llvm-objcopy", "/usr/bin/objcopy"), + "objdump": _tool("llvm-objdump", "/usr/bin/objdump"), + "strip": _tool("llvm-strip", "/usr/bin/strip"), + "gcov": _tool("llvm-cov", "/usr/bin/gcov"), + "dwp": _tool("llvm-dwp", "/usr/bin/dwp"), + "llvm-cov": _tool("llvm-cov", "/usr/bin/gcov"), + } + rctx.file("cc_toolchain.bzl", _CC_TOOLCHAIN_BZL.format( + builtin_includes = repr(builtin_includes), + abi_flags = repr(abi_flags), + tool_paths = repr(tool_paths), + link_flags = repr(link_flags), + supports_start_end_lib = supports_start_end_lib, + cxx_std = cxx_std, + )) + +_llvm_repo = repository_rule( + implementation = _llvm_repo_impl, + attrs = { + "path": attr.string(), + "llvm_config_label": attr.label(), + "cxx_std": attr.string(), + "extra_test_tags": attr.string_list(), + }, + environ = ["LLVM_DIR"], + local = True, +) + +_config = tag_class(attrs = { + "path": attr.string( + doc = "Filesystem path to an LLVM build/install tree.", + ), + "llvm_config_label": attr.label( + doc = "Label of a bin/llvm-config in an LLVM repo another module " + + "already provides (e.g. \"@llvm//:bin/llvm-config\"). Reuses " + + "that tree instead of fetching a second one; takes precedence " + + "over path / LLVM_DIR. Resolved in the root module's context.", + ), + "cxx_std": attr.string( + doc = "C++ standard the centralized clang toolchain compiles the " + + "stack at, e.g. \"c++17\" (default), \"c++20\", \"c++2b\". A " + + "consumer building against a newer libstdc++ / running the " + + "interpreter at a higher standard should match it here.", + ), + "extra_test_tags": attr.string_list( + doc = "Extra Bazel tags added to the stack's JIT tests (the cppinterop " + + "and cppjit suites). Empty by default. A consumer running tests on " + + "remote execution should set [\"no-remote-exec\"]: the tests " + + "JIT-compile C++ in-process and need the host toolchain's libc " + + "sysroot headers, which a hermetic remote action does not provide.", + ), +}) + +def _llvm_impl(mctx): + path = "" + llvm_config_label = None + cxx_std = "" + extra_test_tags = [] + + # Only the root module's config tag is honored. + for mod in mctx.modules: + if mod.is_root: + for cfg in mod.tags.config: + if cfg.path: + path = cfg.path + if cfg.llvm_config_label: + llvm_config_label = cfg.llvm_config_label + if cfg.cxx_std: + cxx_std = cfg.cxx_std + if cfg.extra_test_tags: + extra_test_tags = cfg.extra_test_tags + + # Repo name is "cppjit_llvm", NOT "llvm": a monorepo consumer's CI derives + # ASAN_SYMBOLIZER_PATH by globbing the output base for "*+llvm" expecting a + # single LLVM repo. An extension repo named "llvm" canonicalizes to + # "++llvm+llvm", which ends in "+llvm" and collides with that glob, + # breaking LeakSanitizer symbolization (and thus leak suppression) for the + # whole build. "cppjit_llvm" canonicalizes to "++llvm+cppjit_llvm", + # which the glob does not match. Consumers still see it as @llvm via the + # use_repo(llvm, llvm = "cppjit_llvm") alias in their MODULE.bazel. + _llvm_repo( + name = "cppjit_llvm", + path = path, + llvm_config_label = llvm_config_label, + cxx_std = cxx_std, + extra_test_tags = extra_test_tags, + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = {"config": _config}, +) diff --git a/bazel/rules.bzl b/bazel/rules.bzl new file mode 100644 index 0000000..b0ef779 --- /dev/null +++ b/bazel/rules.bzl @@ -0,0 +1,331 @@ +"""Shared Bazel rules for the cppjit stack: CppInterOp tablegen/buildinfo +generation, the c++ shim, and the cc_test / py_test macros. Each macro +translates the corresponding CMake recipe from the consumer repo. + +Repo-mapping note: these macros expand in the CONSUMER package, so apparent +labels like "@cppinterop"/"@cppjit" resolve in the consumer's mapping (each +consumer bazel_deps on them). $(location ...)/$(execpath ...) Make-vars +likewise resolve against the target's own deps. cppjit_bazel itself never +resolves those names. +""" + +load("@llvm//:defs.bzl", "LLVM_CXX_STD", "LLVM_EXTRA_TEST_TAGS") +load("//:defs.bzl", "BASE_COPTS", "CPPINTEROP_COPTS", "cppinterop_base_env", "cppjit_base_env", "is_main_repo", "repo_rloc") + +# Explicit load instead of native.py_test: consumers may build with Starlark +# rule autoloads disabled, where the native symbol no longer exists. +load("@rules_python//python:defs.bzl", "py_test") +load("@@rules_cc+//cc:defs.bzl", "cc_library", "cc_shared_library", "cc_test") + +# The JIT tests need a C++ toolchain at RUN time (clang-repl compiles in-process). +# Standalone uses the host's; a consumer without one (e.g. a hermetic CI) supplies it +# via two paired label_flags per module: :jit_cxx_data stages the headers/clang +# into runfiles, and :jit_cxx_interp_args carries the matching --gcc-toolchain / +# -stdlib++-isystem args as the CPPINTEROP_JIT_CXX_ARGS make-var. A make-var (not +# a MODULE.bazel/.bazelrc arg) because the runfiles-relative paths can only be +# formed in a .bzl, yet the upstream macros must stay host-based. _jit_cxx_env +# appends $(...) to the test env (empty for the default flag -> standalone +# unaffected); each repo adds its own flag to the tests' toolchains so $(...) resolves. +def _jit_cxx_env(env): + existing = env.get("CPPINTEROP_EXTRA_INTERPRETER_ARGS", "") + appended = "$(CPPINTEROP_JIT_CXX_ARGS)" + merged = (existing + " " + appended) if existing else appended + return env | {"CPPINTEROP_EXTRA_INTERPRETER_ARGS": merged} + +# Numeric form of the toolchain C++ standard for BuildInfo's CMAKE_CXX_STANDARD +# (e.g. "c++17" -> "17", "c++2b" -> "2b"); cosmetic, shown by Cpp::GetBuildInfo(). +_CXX_STD_NUM = LLVM_CXX_STD[len("c++"):] if LLVM_CXX_STD.startswith("c++") else LLVM_CXX_STD + +# tablegen action -> output .inc file (lib/CppInterOp/CMakeLists.txt add_custom_command). +_TBLGEN_GENS = [ + ("-gen-cppinterop-api", "CppInterOpAPI.inc"), + ("-gen-cppinterop-decl", "CppInterOpDecl.inc"), + ("-gen-cx-cppinterop-decl", "CXCppInterOpDecl.inc"), + ("-gen-cx-cppinterop-impl", "CXCppInterOpImpl.inc"), +] + +def cppinterop_tblgen_inc_files(): + """Run :cppinterop-tblgen over CppInterOp.td to emit the 4 .inc headers.""" + td = "lib/CppInterOp/CppInterOp.td" + for action, out in _TBLGEN_GENS: + native.genrule( + name = "gen_" + out.replace(".", "_"), + srcs = [td, "lib/CppInterOp/CppInterOpAPI.td", "@llvm//:lib_files"], + outs = ["include/CppInterOp/" + out], + tools = [":cppinterop-tblgen", "@llvm//:bin/llvm-config"], + # tblgen dlopen's LLVM; -I points at the .td's own dir for includes. + cmd = ("LD_LIBRARY_PATH=$$(dirname $(execpath @llvm//:bin/llvm-config))/../lib " + + "$(location :cppinterop-tblgen) " + + "-I$$(dirname $(execpath " + td + ")) " + action + " " + + "$(execpath " + td + ") -o $@"), + ) + +def cppinterop_buildinfo_inc(): + """configure_file(BuildInfo.inc.in -> include/CppInterOp/BuildInfo.inc).""" + native.genrule( + name = "gen_buildinfo_inc", + srcs = ["lib/CppInterOp/BuildInfo.inc.in"], + outs = ["include/CppInterOp/BuildInfo.inc"], + tools = ["@llvm//:bin/clang"], + cmd = _BUILDINFO_CMD.replace("@@CXX_STD_NUM@@", _CXX_STD_NUM), + ) + +# Map COMPILATION_MODE to CMAKE_BUILD_TYPE; parse clang/llvm version from +# `clang --version`; fill uname for the target triple. Done inline in bash so +# the genrule has no extra script file to ship. +_BUILDINFO_CMD = r""" +case "$(COMPILATION_MODE)" in + opt) BT=Release ;; + dbg) BT=Debug ;; + *) BT=RelWithDebInfo ;; +esac +VER=$$($(execpath @llvm//:bin/clang) --version | sed -n '1s/.*version \([0-9.]*\).*/\1/p') +SYSNAME=$$(uname -s) +SYSPROC=$$(uname -m) +sed -e "s|@CMAKE_BUILD_TYPE@|$$BT|g" \ + -e "s|@CMAKE_CXX_STANDARD@|@@CXX_STD_NUM@@|g" \ + -e "s|@CMAKE_CXX_COMPILER_ID@|Clang|g" \ + -e "s|@CMAKE_CXX_COMPILER_VERSION@|$$VER|g" \ + -e "s|@LLVM_PACKAGE_VERSION@|$$VER|g" \ + -e "s|@CPPINTEROP_USE_CLING@|OFF|g" \ + -e "s|@LLVM_USE_SANITIZER@||g" \ + -e "s|@CMAKE_SYSTEM_NAME@|$$SYSNAME|g" \ + -e "s|@CMAKE_SYSTEM_PROCESSOR@|$$SYSPROC|g" \ + -e "s|@CPPINTEROP_CMAKE_INVOCATION@||g" \ + $(location lib/CppInterOp/BuildInfo.inc.in) > $@ +""" + +def cppinterop_cxx_shim(): + """Emit an executable cxx_shim/c++ forwarding to llvm's clang++. + + Cpp::DetectSystemCompilerIncludePaths popen()'s `c++`, which is absent in + hermetic sandboxes; this shim on PATH satisfies it. + """ + native.genrule( + name = "gen_cxx_shim", + outs = ["cxx_shim/c++"], + tools = ["@llvm//:bin/clang++"], + # rlocationpath = the canonical runfiles path of llvm's clang++. + cmd = ( + "echo '#!/bin/bash' > $@ && " + + "echo 'exec \"$$RUNFILES_DIR/" + + "$(rlocationpath @llvm//:bin/clang++)\" \"$$@\"' >> $@ && " + + "chmod +x $@" + ), + executable = True, + ) + +def _cppinterop_test_common(name, srcs, extra_copts, extra_deps, data, env, includes): + return dict( + name = name, + srcs = srcs + ["@cppinterop//:headers", "@cppinterop//:internal_headers"], + # @llvm//:headers supplies the LLVM/Clang -isystem roots; tests include + # them transitively via Utils.h / CppInterOp.h. + deps = ["@googletest//:gtest", "@llvm//:headers"] + extra_deps, + copts = extra_copts, + data = data + [ + "@cppinterop//:data", + "@cppinterop//:solib", + "@cppinterop//:test_solib", + "@cppinterop//:cxx_shim/c++", + "@llvm//:bin/clang++", + # Consumer-supplied JIT C++ toolchain (libstdc++ headers / + # clang) staged into runfiles; empty by default (host). + "@cppinterop//:jit_cxx_data", + ], + env = env, + includes = includes, + ) + +def cppinterop_cc_test(name, srcs, extra_copts = [], extra_deps = [], extra_dynamic_deps = [], data = [], env = {}, extra_tags = []): + """A CppInterOp gtest linking clangCppInterOp via dynamic_deps. + + extra_tags adds to the LLVM tags, so a consumer can select or exclude a + test with --test_tag_filters. + """ + + # This macro only expands in @cppinterop, so when that is the main repo + # (standalone build) the @cppinterop paths are self-references; let them + # resolve to the runfiles root rather than ../cppinterop+ (which only exists + # when cppinterop is an external dep). + cppinterop_is_self = is_main_repo(native.repository_name()) + cxx_shim_dir = repo_rloc("@cppinterop", cppinterop_is_self) + "/cxx_shim" + base = _cppinterop_test_common( + name = name, + srcs = srcs, + # upstream main.cpp provides main(), so gtest (not gtest_main). + extra_copts = CPPINTEROP_COPTS + extra_copts, + extra_deps = extra_deps, + data = data, + env = _jit_cxx_env(cppinterop_base_env(cppinterop_is_self) | {"PATH": cxx_shim_dir + ":/usr/bin:/bin"} | env), + includes = ["include", "unittests/CppInterOp"], + ) + + # Link ONLY the solib (it carries LLVM) -- static LLVM too would give a second + # copy of LLVM's globals and the in-process clang AST asserts. -rdynamic exports + # the test's own symbols so the JIT can resolve template bodies instantiated in + # the test TU (e.g. instantiation_in_host). + cc_test( + dynamic_deps = ["@cppinterop//:solib"] + extra_dynamic_deps, + linkopts = ["-ldl", "-lpthread", "-rdynamic"], + tags = LLVM_EXTRA_TEST_TAGS + extra_tags, + # Resolves $(CPPINTEROP_JIT_CXX_ARGS) in the env (empty by default). + toolchains = ["@cppinterop//:jit_cxx_interp_args"], + **base + ) + +def cppinterop_dispatch_cc_test(name, srcs): + """DispatchTests: dlopen's clangCppInterOp, so NO solib/llvm linkage.""" + cppinterop_is_self = is_main_repo(native.repository_name()) + cppinterop_rloc = repo_rloc("@cppinterop", cppinterop_is_self) + cxx_shim_dir = cppinterop_rloc + "/cxx_shim" + base_env = cppinterop_base_env(cppinterop_is_self) + base = _cppinterop_test_common( + name = name, + srcs = srcs, + # No LLVM link: the tests dlopen the solib at runtime. + extra_copts = BASE_COPTS, + extra_deps = [], + # solib arrives via dlopen at runtime; the common data already ships it. + data = [], + # CPPINTEROP_BIN_DIR is the artifacts prefix the tests dlopen + # /lib/libclangCppInterOp.so from; the solib's shared_lib_name + # gives it that lib/ component under runfiles, whose root is the cwd. + env = _jit_cxx_env(base_env | { + "CPPINTEROP_BIN_DIR": cppinterop_rloc, + "PATH": cxx_shim_dir + ":/usr/bin:/bin", + "LD_LIBRARY_PATH": base_env["LD_LIBRARY_PATH"] + ":lib", + }), + includes = ["include", "unittests/CppInterOp"], + ) + cc_test( + tags = LLVM_EXTRA_TEST_TAGS, + # Resolves $(CPPINTEROP_JIT_CXX_ARGS) in the env (empty by default). + toolchains = ["@cppinterop//:jit_cxx_interp_args"], + **base + ) + +def cppjit_test_dict_sos(sokeys): + """Per key: a cc_library + a cc_shared_library producing test/cpp/Dict.so. + + Mirrors cppjit's test/Makefile, which builds Dict.so next to the + test sources under test/cpp/. + """ + for key in sokeys: + cc_library( + name = key + ".a", + srcs = ["test/cpp/" + key + ".cxx"], + hdrs = native.glob(["test/cpp/*.h"]), + # The test dictionaries have intentional unused/leaky-dtor patterns; + # demote so a consumer toolchain with -Werror still builds. + copts = [ + "-fPIC", + "-Wno-error=unused-but-set-parameter", + "-Wno-unused-but-set-parameter", + "-Wno-error=delete-non-abstract-non-virtual-dtor", + "-Wno-delete-non-abstract-non-virtual-dtor", + ], + deps = ["@rules_python//python/cc:current_py_cc_headers"], + ) + cc_shared_library( + name = key + "Dict.so", + deps = [key + ".a"], + shared_lib_name = "test/cpp/" + key + "Dict.so", + ) + +# The py_test source set shared by every cppjit suite: the test module itself +# plus the helpers every module imports. +def _py_test_srcs(name): + return [ + "test/" + name + ".py", + "test/support.py", + "test/test_main.py", + "test/assert_interactive.py", + ] + +def cppjit_py_test(name, sokeys = [], extra_env = {}): + """A cppjit pytest driven through test/test_main.py. + + The pytest dependency comes from the @cppjit//:pytest label_flag: the + standalone build's hermetic hub by default, or whatever a consumer points it + at (e.g. @cppjit//:ambient_pytest when the interpreter already ships pytest, + as on a consumer's conda toolchain). + """ + dict_sos = [k + "Dict.so" for k in sokeys] + + base_env = cppjit_base_env() + + # test/cpp resolves under the cppjit repo, which is the runfiles cwd + # standalone but external/+ when a consumer runs the suite. + test_dir = repo_rloc("@cppjit", is_main_repo(native.repository_name())) + "/test/cpp" + py_test( + name = name, + main = "test/test_main.py", + tags = LLVM_EXTRA_TEST_TAGS, + srcs = _py_test_srcs(name), + # test_main.py forwards argv to pytest.main(); point it at this test's + # own file via $(rootpath) so the path resolves whether cppjit is the main + # repo (standalone) or an external dep (a consumer's external/cppjit+/). + args = ["$(rootpath test/" + name + ".py)"], + # The test modules do `import support` etc.; put test/ on sys.path. + imports = ["test"], + deps = [ + "@cppjit//:lib", + "@cppjit//:pytest", + "@cppjit_bazel//:sitecustomize", + ], + data = dict_sos + [ + "@cppjit//:cxxh", + "@cppjit//:data", + "@llvm//:all_files", + # Consumer-supplied JIT C++ toolchain (libstdc++ headers / clang) + # staged into runfiles; empty by default (host). + "@cppjit//:jit_cxx_data", + ], + # No library/include path env for cppjit's own resources: the stack + # self-locates all of them from libcppjit.so's own directory, so every + # test exercises that path. + env = _jit_cxx_env(base_env | { + "PYTHONUNBUFFERED": "1", + "CPPJIT_TEST_SKIP_MAKE": "True", + # Some tests load secondary dicts by bare name and the loader + # force-includes the matching header, so put test/cpp on both the + # library and the interpreter include paths. + "LD_LIBRARY_PATH": base_env["LD_LIBRARY_PATH"] + ":" + test_dir, + "CPLUS_INCLUDE_PATH": test_dir, + } | extra_env), + # Resolves $(CPPINTEROP_JIT_CXX_ARGS) in the env (empty by default). + toolchains = ["@cppjit//:jit_cxx_interp_args"], + ) + +def cppjit_selflocation_py_test(name): + """The self-location property test: no lib/include path env vars, no + CPLUS_INCLUDE_PATH, no LD_LIBRARY_PATH, cwd changed away from the runfiles + root before import. Everything must resolve from libcppjit.so's own + location, through the layout :data stages beside it.""" + py_test( + name = name, + main = "test/test_main.py", + tags = LLVM_EXTRA_TEST_TAGS, + srcs = _py_test_srcs(name), + args = ["$(rootpath test/" + name + ".py)"], + imports = ["test"], + deps = [ + "@cppjit//:lib", + "@cppjit//:pytest", + "@cppjit_bazel//:sitecustomize", + ], + data = [ + "@cppjit//:data", + "@llvm//:all_files", + # Consumer-supplied JIT C++ toolchain (libstdc++ headers / clang) + # staged into runfiles; empty by default (host). + "@cppjit//:jit_cxx_data", + ], + env = _jit_cxx_env({ + "PYTHONUNBUFFERED": "1", + "CLING_STANDARD_PCH": "none", + }), + # Resolves $(CPPINTEROP_JIT_CXX_ARGS) in the env (empty by default). + toolchains = ["@cppjit//:jit_cxx_interp_args"], + ) diff --git a/bazel/sitecustomize.py b/bazel/sitecustomize.py new file mode 100644 index 0000000..b2a3ad7 --- /dev/null +++ b/bazel/sitecustomize.py @@ -0,0 +1,46 @@ +"""Startup fixups for the cppjit py_tests (imported by test/test_main.py). + +1. Stale sysconfig include path: rules_python's standalone Python bakes a + build-time INCLUDEPY ("/install/...") into sysconfig, so + sysconfig.get_config_var("INCLUDEPY") points at a path that doesn't exist at + runtime. Some cppjit tests pass that value to cppjit.add_include_path(), which + errors on the missing dir. Repoint it at the real headers under + sys.base_prefix. + +2. Anchor the runfiles-relative library paths to an ABSOLUTE base. The test env + sets LD_LIBRARY_PATH with "../+/..." segments, which only resolve when + the process cwd is the runfiles _main dir. That holds for a local + `bazel test`, but not in every environment (e.g. GitHub Actions, where the + dlopen then fails / doubles the path). RUNFILES_DIR is an absolute path bazel + always sets, and cwd == $RUNFILES_DIR/_main, so "../X" maps to + "$RUNFILES_DIR/X"; rewrite the relative segments accordingly before + `import cppjit` triggers the backend load. +""" + +import os +import sys +import sysconfig + +_real = os.path.join( + sys.base_prefix, + "include", + "python" + sysconfig.get_python_version(), +) +if os.path.isdir(_real): + sysconfig.get_config_vars() # force the cache to populate + sysconfig._CONFIG_VARS["INCLUDEPY"] = _real + +# Rewrite cwd-relative "../" runfiles segments to absolute $RUNFILES_DIR paths. +_runfiles = os.environ.get("RUNFILES_DIR") +if _runfiles: + + def _anchor(value): + parts = [] + for seg in value.split(os.pathsep): + if seg.startswith("../"): + seg = os.path.join(_runfiles, seg[len("../") :]) + parts.append(seg) + return os.pathsep.join(parts) + + if "LD_LIBRARY_PATH" in os.environ: + os.environ["LD_LIBRARY_PATH"] = _anchor(os.environ["LD_LIBRARY_PATH"]) diff --git a/bazel/staging.bzl b/bazel/staging.bzl new file mode 100644 index 0000000..ab25513 --- /dev/null +++ b/bazel/staging.bzl @@ -0,0 +1,48 @@ +"""Stage files from a sibling repo into this package's tree via symlinks. + +A pip wheel bundles its runtime resources inside the package directory, and the +libraries self-locate relative to their own __file__/dladdr path. Under Bazel +those resources live in *sibling* repos, out of reach of any package-relative +lookup. stage_files() re-creates the wheel layout in bazel-out/runfiles with +symlinks (no copies -- the CppInterOp solib is huge), so the same self-location +code works for wheels and Bazel alike. + +Each file lands at /. E.g. srcs = [@cppinterop//:headers], path_anchor = "include", +out_dir = "python/cppjit/interop/include" maps + .../cppinterop+/include/CppInterOp/CppInterOp.h + -> python/cppjit/interop/include/CppInterOp/CppInterOp.h +and works for generated files (the tblgen'd .inc headers) too, since the +anchor is matched in short_path. +""" + +def _stage_files_impl(ctx): + outs = [] + marker = "/" + ctx.attr.path_anchor + "/" + prefix = ctx.attr.path_anchor + "/" + for f in ctx.files.srcs: + sp = f.short_path + idx = sp.find(marker) + if idx >= 0: + rel = sp[idx + len(marker):] + elif sp.startswith(prefix): + rel = sp[len(prefix):] + else: + fail("stage_files: no '%s/' component in %s" % (ctx.attr.path_anchor, sp)) + out = ctx.actions.declare_file(ctx.attr.out_dir + "/" + rel) + ctx.actions.symlink(output = out, target_file = f) + outs.append(out) + return [DefaultInfo( + files = depset(outs), + runfiles = ctx.runfiles(files = outs), + )] + +stage_files = rule( + implementation = _stage_files_impl, + doc = "Symlink srcs into out_dir, keyed by the path remainder after path_anchor.", + attrs = { + "srcs": attr.label_list(allow_files = True, mandatory = True), + "out_dir": attr.string(mandatory = True), + "path_anchor": attr.string(mandatory = True), + }, +) diff --git a/requirements_bazel.txt b/requirements_bazel.txt new file mode 100644 index 0000000..0c2db76 --- /dev/null +++ b/requirements_bazel.txt @@ -0,0 +1,7 @@ +# pytest and its runtime deps, pinned so pip.parse builds a complete hub +# (the pytest package references these as sibling packages). +pytest==9.1.1 +iniconfig==2.3.0 +packaging==26.3 +pluggy==1.6.0 +pygments==2.21.0 diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index cce30fa..e9ee466 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -121,6 +121,55 @@ static bool loadDispatchAPI(const InterOpPaths& Paths) { return true; } +// The directory this library was loaded from, absolutized and lexically +// normalized: dladdr can report a relative or dotted path, which some +// consumers of a derived path reject. Not realpath(): resolving the symlink +// would escape a relocatable install layout. +static std::string self_lib_dir() { +#ifndef _WIN32 + Dl_info info; + if (!dladdr((void*)&self_lib_dir, &info) || !info.dli_fname) + return ""; + std::error_code ec; + std::filesystem::path p(info.dli_fname); + if (p.is_relative()) { + const std::filesystem::path cwd = std::filesystem::current_path(ec); + if (ec) + return ""; + p = cwd / p; + } + return p.parent_path().lexically_normal().string(); +#else + return ""; +#endif +} + +// ${ORIGIN} in CPPINTEROP_EXTRA_INTERPRETER_ARGS stands for this library's own +// directory (see bazel/ORIGIN.md). Rewrite the variable in place: CppInterOp +// re-reads it inside CreateInterpreter. +static void expandOriginInInterpreterArgs() { + static const std::string PH = "${ORIGIN}"; + const char* raw = getenv("CPPINTEROP_EXTRA_INTERPRETER_ARGS"); + if (!raw) + return; + std::string args(raw); + if (args.find(PH) == std::string::npos) + return; + const std::string origin = self_lib_dir(); + if (origin.empty()) { + std::cerr << "[cppjit-backend] cannot resolve ${ORIGIN} in " + "CPPINTEROP_EXTRA_INTERPRETER_ARGS" + << std::endl; + return; + } + for (std::string::size_type pos = 0; + (pos = args.find(PH, pos)) != std::string::npos; pos += origin.size()) + args.replace(pos, PH.size(), origin); +#ifndef _WIN32 + setenv("CPPINTEROP_EXTRA_INTERPRETER_ARGS", args.c_str(), 1); +#endif +} + // CppInterOp itself appends CPPINTEROP_EXTRA_INTERPRETER_ARGS inside // CreateInterpreter, so nothing needs to be forwarded from here. static interop::TInterp_t @@ -128,6 +177,8 @@ acquireOrCreateInterpreter(const InterOpPaths& Paths) { if (auto existingInterp = Cpp::GetInterpreter()) return existingInterp; + expandOriginInInterpreterArgs(); + std::vector args = {"-std=c++17"}; #if !(defined(__arm64__) && defined(__APPLE__)) // apple silicon clang rejects -march=native diff --git a/test/test_main.py b/test/test_main.py index 9e434de..70a65cd 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,6 +1,14 @@ if __name__ == "__main__": import sys + # Bazel-only runtime path fixups. The py_test bootstrap builds sys.path + # in-process, so `site` cannot auto-import them; import them here instead. + # No such module in a pip/CMake run. + try: + import sitecustomize # noqa: F401 + except ImportError: + pass + import pytest sys.exit(pytest.main()) diff --git a/test/test_selflocation.py b/test/test_selflocation.py new file mode 100644 index 0000000..14031ad --- /dev/null +++ b/test/test_selflocation.py @@ -0,0 +1,23 @@ +import os + + +class TestSELFLOCATION: + def test01_jit_from_foreign_cwd(self): + """Import and JIT with cwd '/' and no path env vars: the stack must + locate CppInterOp, the cpyrt API headers and clang's builtin headers + from libcppjit.so's own location alone.""" + + for var in ( + "CPPJIT_API_PATH", + "CPLUS_INCLUDE_PATH", + "LD_LIBRARY_PATH", + "RUNFILES_DIR", + "RUNFILES_MANIFEST_FILE", + ): + os.environ.pop(var, None) + os.chdir("/") + + import cppjit + + cppjit.cppdef("int self_location_add(int a, int b) { return a + b; }") + assert cppjit.gbl.self_location_add(20, 22) == 42