Skip to content

Commit

Permalink
Python 3.12 support (#642)
Browse files Browse the repository at this point in the history
  • Loading branch information
@benfred
benfred authored Oct 12, 2024
1 parent 94e6c90 commit 4eb8cd2
Show file tree
Hide file tree
Showing 13 changed files with 13,214 additions and 166 deletions.
7 changes: 6 additions & 1 deletion .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ jobs:
- name: Install Dependencies
run: sudo apt install libunwind-dev
if: runner.os == 'Linux'
- uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Build
run: cargo build --release --verbose --examples
- uses: actions/setup-python@v4
Expand Down Expand Up @@ -180,7 +183,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [3.6.7, 3.6.15, 3.7.1, 3.7.17, 3.8.0, 3.8.18, 3.9.0, 3.9.20, 3.10.0, 3.10.1, 3.10.2, 3.10.3, 3.10.4, 3.10.5, 3.10.6, 3.10.7, 3.10.8, 3.10.9, 3.10.10, 3.10.11, 3.10.12, 3.10.13, 3.10.14, 3.10.15, 3.11.0, 3.11.1, 3.11.2, 3.11.3, 3.11.4, 3.11.5, 3.11.6, 3.11.7, 3.11.8, 3.11.9, 3.11.10]
python-version: [3.6.7, 3.6.15, 3.7.1, 3.7.17, 3.8.0, 3.8.18, 3.9.0, 3.9.20, 3.10.0, 3.10.1, 3.10.2, 3.10.3, 3.10.4, 3.10.5, 3.10.6, 3.10.7, 3.10.8, 3.10.9, 3.10.10, 3.10.11, 3.10.12, 3.10.13, 3.10.14, 3.10.15, 3.11.0, 3.11.1, 3.11.2, 3.11.3, 3.11.4, 3.11.5, 3.11.6, 3.11.7, 3.11.8, 3.11.9, 3.11.10, 3.12.0]
# TODO: also test windows
os: [ubuntu-20.04, macos-13]
# some versions of python can't be tested on GHA with osx because of SIP:
Expand All @@ -207,6 +210,8 @@ jobs:
python-version: 3.11.9
- os: macos-13
python-version: 3.11.10
- os: macos-13
python-version: 3.12.0

steps:
- uses: actions/checkout@v2
Expand Down
88 changes: 53 additions & 35 deletions generate_bindings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ def build_python(cpython_path, version):
print("Compiling python %s from repo at %s" % (version, cpython_path))
install_path = os.path.abspath(os.path.join(cpython_path, version))

ret = os.system(f"""
ret = os.system(
f"""
cd {cpython_path}
git checkout {version}
Expand All @@ -27,7 +28,8 @@ def build_python(cpython_path, version):
../configure prefix={install_path}
make
make install
""")
"""
)
if ret:
return ret

Expand Down Expand Up @@ -57,8 +59,9 @@ def calculate_pyruntime_offsets(cpython_path, version, configure=False):
size_t interp_head = offsetof(_PyRuntimeState, interpreters.head);
printf("pub static INTERP_HEAD_OFFSET: usize = %i;\n", interp_head);
size_t tstate_current = offsetof(_PyRuntimeState, gilstate.tstate_current);
printf("pub static TSTATE_CURRENT: usize = %i;\n", tstate_current);
// tstate_current has been replaced by a thread-local variable in python 3.12
// size_t tstate_current = offsetof(_PyRuntimeState, gilstate.tstate_current);
// printf("pub static TSTATE_CURRENT: usize = %i;\n", tstate_current);
}
"""

Expand Down Expand Up @@ -88,7 +91,7 @@ def calculate_pyruntime_offsets(cpython_path, version, configure=False):
else:
ret = os.system(f"""gcc {source_filename} -I {cpython_path} -I {cpython_path}/Include -o {exe}""")
if ret:
print("Failed to compile""")
print("Failed to compile")
return ret

ret = os.system(exe)
Expand All @@ -100,19 +103,22 @@ def calculate_pyruntime_offsets(cpython_path, version, configure=False):
def extract_bindings(cpython_path, version, configure=False):
print("Generating bindings for python %s from repo at %s" % (version, cpython_path))

ret = os.system(f"""
ret = os.system(
f"""
cd {cpython_path}
git checkout {version}
# need to run configure on the current branch to generate pyconfig.h sometimes
{("./configure prefix=" + os.path.abspath(os.path.join(cpython_path, version))) if configure else ""}
cat Include/Python.h > bindgen_input.h
cat Include/frameobject.h >> bindgen_input.h
echo "// autogenerated by generate_bindings.py " > bindgen_input.h
echo '#define Py_BUILD_CORE 1\n' >> bindgen_input.h
cat Include/internal/pycore_pystate.h >> bindgen_input.h
cat Include/Python.h >> bindgen_input.h
echo '#undef HAVE_STD_ATOMIC' >> bindgen_input.h
cat Include/frameobject.h >> bindgen_input.h
cat Include/internal/pycore_interp.h >> bindgen_input.h
cat Include/internal/pycore_frame.h >> bindgen_input.h
cat Include/internal/pycore_dict.h >> bindgen_input.h
bindgen bindgen_input.h -o bindgen_output.rs \
--with-derive-default \
Expand All @@ -132,13 +138,12 @@ def extract_bindings(cpython_path, version, configure=False):
--whitelist-type PyFloatObject \
--whitelist-type PyDictObject \
--whitelist-type PyDictKeysObject \
--whitelist-type PyDictKeyEntry \
--whitelist-type PyDictUnicodeEntry \
--whitelist-type PyObject \
--whitelist-type PyTypeObject \
--whitelist-type PyHeapTypeObject \
-- -I . -I ./Include -I ./Include/internal
""")
"""
)
if ret:
return ret

Expand All @@ -152,36 +157,40 @@ def extract_bindings(cpython_path, version, configure=False):
o.write("#![allow(clippy::useless_transmute)]\n")
o.write("#![allow(clippy::default_trait_access)]\n")
o.write("#![allow(clippy::cast_lossless)]\n")
o.write("#![allow(clippy::trivially_copy_pass_by_ref)]\n\n")
o.write("#![allow(clippy::upper_case_acronyms)]\n\n")
o.write("#![allow(clippy::trivially_copy_pass_by_ref)]\n")
o.write("#![allow(clippy::upper_case_acronyms)]\n")
o.write("#![allow(clippy::too_many_arguments)]\n\n")

o.write(open(os.path.join(cpython_path, "bindgen_output.rs")).read())


if __name__ == "__main__":

if sys.platform.startswith("win"):
default_cpython_path = os.path.join(os.getenv("userprofile"), "code", "cpython")
else:
default_cpython_path = os.path.join(os.getenv("HOME"), "code", "cpython")

parser = argparse.ArgumentParser(description="runs bindgen on cpython version",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--cpython", type=str, default=default_cpython_path,
dest="cpython", help="path to cpython repo")
parser.add_argument("--configure",
help="Run configure script prior to generating bindings",
action="store_true")
parser.add_argument("--pyruntime",
help="generate offsets for pyruntime",
action="store_true")
parser.add_argument("--build",
help="Build python for this version",
action="store_true")
parser.add_argument("--all",
help="Build all versions",
action="store_true")

parser.add_argument("versions", type=str, nargs='*', help='versions to extract')
parser = argparse.ArgumentParser(
description="runs bindgen on cpython version",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--cpython",
type=str,
default=default_cpython_path,
dest="cpython",
help="path to cpython repo",
)
parser.add_argument(
"--configure",
help="Run configure script prior to generating bindings",
action="store_true",
)
parser.add_argument("--pyruntime", help="generate offsets for pyruntime", action="store_true")
parser.add_argument("--build", help="Build python for this version", action="store_true")
parser.add_argument("--all", help="Build all versions", action="store_true")

parser.add_argument("versions", type=str, nargs="*", help="versions to extract")

args = parser.parse_args()

Expand All @@ -191,7 +200,16 @@ def extract_bindings(cpython_path, version, configure=False):
sys.exit(1)

if args.all:
versions = ['v3.8.0b4', 'v3.7.0', 'v3.6.6', 'v3.5.5', 'v3.4.8', 'v3.3.7', 'v3.2.6', 'v2.7.15']
versions = [
"v3.8.0b4",
"v3.7.0",
"v3.6.6",
"v3.5.5",
"v3.4.8",
"v3.3.7",
"v3.2.6",
"v2.7.15",
]
else:
versions = args.versions
if not versions:
Expand Down
7 changes: 6 additions & 1 deletion src/coredump.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use crate::binary_parser::{parse_binary, BinaryInfo};
use crate::config::Config;
use crate::dump::print_trace;
use crate::python_bindings::{
v2_7_15, v3_10_0, v3_11_0, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5,
v2_7_15, v3_10_0, v3_11_0, v3_12_0, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5,
};
use crate::python_data_access::format_variable;
use crate::python_interpreters::InterpreterState;
Expand Down Expand Up @@ -303,6 +303,11 @@ impl PythonCoreDump {
minor: 11,
..
} => self._get_stack::<v3_11_0::_is>(config),
Version {
major: 3,
minor: 12,
..
} => self._get_stack::<v3_12_0::_is>(config),
_ => Err(format_err!(
"Unsupported version of Python: {}",
self.version
Expand Down
3 changes: 2 additions & 1 deletion src/python_bindings/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
pub mod v2_7_15;
pub mod v3_10_0;
pub mod v3_11_0;
pub mod v3_12_0;
pub mod v3_3_7;
pub mod v3_5_5;
pub mod v3_6_6;
Expand Down Expand Up @@ -71,7 +72,7 @@ pub mod pyruntime {
} => 32,
Version {
major: 3,
minor: 11,
minor: 11..=12,
..
} => 40,
_ => 24,
Expand Down
Loading

0 comments on commit 4eb8cd2

Please sign in to comment.