From 11eaadcf98514b9f917807aeb59c4ccd6bf93918 Mon Sep 17 00:00:00 2001 From: henry Date: Mon, 9 Oct 2023 12:46:56 -0500 Subject: [PATCH 1/5] CI: support python 3.12 testing --- .github/workflows/tox_matrix.yml | 2 +- ci/docker/windows/tox/chocolatey/python.xml | 1 + tox.ini | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox_matrix.yml b/.github/workflows/tox_matrix.yml index c80e061..dd624aa 100644 --- a/.github/workflows/tox_matrix.yml +++ b/.github/workflows/tox_matrix.yml @@ -6,7 +6,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-2019] - python-version: ['3.8' ,'3.9', '3.10', '3.11'] + python-version: ['3.8' ,'3.9', '3.10', '3.11', '3.12'] include: - os: macos-latest - os: ubuntu-latest diff --git a/ci/docker/windows/tox/chocolatey/python.xml b/ci/docker/windows/tox/chocolatey/python.xml index 5737f34..8927f7e 100644 --- a/ci/docker/windows/tox/chocolatey/python.xml +++ b/ci/docker/windows/tox/chocolatey/python.xml @@ -4,4 +4,5 @@ + diff --git a/tox.ini b/tox.ini index 6070a10..a81e3c8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,3 @@ [tox] -envlist = py{38,39,310,311} +envlist = py{38,39,310,311,312} isolated_build = true \ No newline at end of file From 3e968b0e0663ebf1ee04464431df119d9a648c83 Mon Sep 17 00:00:00 2001 From: henry Date: Mon, 9 Oct 2023 13:06:32 -0500 Subject: [PATCH 2/5] tox env for bandit, pydocstyle, pylint, flake8, and mypy --- tox.ini | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a81e3c8..f331a7f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,40 @@ [tox] envlist = py{38,39,310,311,312} -isolated_build = true \ No newline at end of file +isolated_build = true + +[testenv:mypy] +deps = + mypy + lxml + types-toml +skip_install=True +setenv = MYPY_CACHE_DIR = {temp_dir}/.mypy_cache +commands = mypy {posargs: -p uiucprescon} + +[testenv:flake8] +description = check the code style +deps = + flake8 +skip_install=True +commands = flake8 {posargs: uiucprescon} + +[testenv:pylint] +description = check the code style +deps = + pylint +skip_install=True +commands = pylint {posargs: uiucprescon} + +[testenv:pydocstyle] +skip_install = true +deps= + pydocstyle +commands = + pydocstyle {posargs: {toxinidir}/uiucprescon} + +[testenv:bandit] +skip_install = true +deps= + bandit +commands = + bandit {posargs: --recursive {toxinidir}/uiucprescon} From 0ec2813728d972a0648b9e2840093405eb9f09fc Mon Sep 17 00:00:00 2001 From: henry Date: Mon, 9 Oct 2023 13:14:13 -0500 Subject: [PATCH 3/5] pep8 --- uiucprescon/build/__init__.py | 8 ++++++ uiucprescon/build/compiler_info.py | 16 +++++++----- uiucprescon/build/conan_libs.py | 36 ++++++++++++++++++-------- uiucprescon/build/local_backend.py | 37 +++++++++++++++++++++------ uiucprescon/build/pybind11_builder.py | 16 +++++++++--- 5 files changed, 85 insertions(+), 28 deletions(-) diff --git a/uiucprescon/build/__init__.py b/uiucprescon/build/__init__.py index 1e9b815..b59cd32 100644 --- a/uiucprescon/build/__init__.py +++ b/uiucprescon/build/__init__.py @@ -1,3 +1,11 @@ from .local_backend import build_sdist, \ build_wheel, prepare_metadata_for_build_wheel, get_requires_for_build_sdist + VERSION = "0.1.3.dev0" + +__all__ = [ + 'build_sdist', + 'build_wheel', + 'prepare_metadata_for_build_wheel', + 'get_requires_for_build_sdist' +] diff --git a/uiucprescon/build/compiler_info.py b/uiucprescon/build/compiler_info.py index 8ad70a3..8b60052 100644 --- a/uiucprescon/build/compiler_info.py +++ b/uiucprescon/build/compiler_info.py @@ -48,8 +48,12 @@ def get_visual_studio_version(): for v in possible_versions: try: - winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key % v, 0, - winreg.KEY_ALL_ACCESS) + winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + key % v, + 0, + winreg.KEY_ALL_ACCESS + ) installed_versions.append(v) except Exception as e: pass @@ -76,10 +80,11 @@ def get_clang_version(): # This ensures extension modules are built with correct # compatibility values, specifically LDSHARED which can use # '-undefined dynamic_lookup' which only works on >= 10.3. - cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) + cur_target =\ + os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) + cur_target_split = [int(x) for x in cur_target.split('.')] - if _cfg_target_split[:2] >= [10, 3] and cur_target_split[:2] < [10, - 3]: + if _cfg_target_split[:2] >= [10, 3] > cur_target_split[:2]: my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' 'now "%s" but "%s" during configure;' 'must use 10.3 or later' @@ -160,4 +165,3 @@ def get_compiler_version(): if len(parsed_version) <= 2: return full_version return f"{parsed_version[0]}.{parsed_version[1]}" - diff --git a/uiucprescon/build/conan_libs.py b/uiucprescon/build/conan_libs.py index 7503c40..c4c760a 100644 --- a/uiucprescon/build/conan_libs.py +++ b/uiucprescon/build/conan_libs.py @@ -11,12 +11,16 @@ from distutils import ccompiler from pathlib import Path from uiucprescon.build.deps import get_win_deps -from uiucprescon.build.compiler_info import get_compiler_version, get_compiler_name +from uiucprescon.build.compiler_info import ( + get_compiler_version, + get_compiler_name +) import json from distutils.dist import Distribution import toml from conans.client import conan_api, conf + class ConanBuildInfoParser: def __init__(self, fp): self._fp = fp @@ -76,21 +80,31 @@ def parse(self, filename: str) -> Dict[str, Union[str, List[str]]]: "bindirs": data.get(f"bindirs_{library_name}", []), "resdirs": data.get(f"resdirs_{library_name}", []), "builddirs": data.get(f"builddirs_{library_name}", []), - "system_libs": data.get(f"system_libs_{library_name}", []), - "defines": data.get(f"defines_{library_name}", []), - "cppflags": data.get(f"cppflags_{library_name}", []), - "cxxflags": data.get(f"cxxflags_{library_name}", []), + "system_libs": + data.get(f"system_libs_{library_name}", []), + "defines": + data.get(f"defines_{library_name}", []), + "cppflags": + data.get(f"cppflags_{library_name}", []), + "cxxflags": + data.get(f"cxxflags_{library_name}", []), "cflags": data.get(f"cflags_{library_name}", []), - "sharedlinkflags": data.get(f"sharedlinkflags_{library_name}", []), - "exelinkflags": data.get(f"exelinkflags_{library_name}", []), + "sharedlinkflags": + data.get(f"sharedlinkflags_{library_name}", []), + "exelinkflags": + data.get(f"exelinkflags_{library_name}", []), "sysroot": data.get(f"sysroot_{library_name}", []), - "frameworks": data.get(f"frameworks_{library_name}", []), - "frameworkdirs": data.get(f"frameworkdirs_{library_name}", []), + "frameworks": + data.get(f"frameworks_{library_name}", []), + "frameworkdirs": + data.get(f"frameworkdirs_{library_name}", []), "rootpath": data.get(f"rootpath_{library_name}", []), "name": library_name, "version": version[0] if version else None, - "generatornames": data.get(f"generatornames_{library_name}", []), - "generatorfilenames": data.get(f"generatorfilenames_{library_name}", []), + "generatornames": + data.get(f"generatornames_{library_name}", []), + "generatorfilenames": + data.get(f"generatorfilenames_{library_name}", []), } return { "definitions": definitions, diff --git a/uiucprescon/build/local_backend.py b/uiucprescon/build/local_backend.py index a82a495..a2f1c43 100644 --- a/uiucprescon/build/local_backend.py +++ b/uiucprescon/build/local_backend.py @@ -3,6 +3,7 @@ import setuptools.build_meta from . import conan_libs from pathlib import Path + print("pre pyproject.toml") pyproj_toml = Path('pyproject.toml') print("post pyproject.toml") @@ -13,10 +14,20 @@ def build_sdist(sdist_directory, config_settings=None): return setuptools.build_meta.build_sdist(sdist_directory, config_settings) -def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): - if config_settings is not None and config_settings.get('conan_cache') is not None: - if "CONAN_USER_HOME" in os.environ: - config_settings['conan_cache'] = os.path.join(os.environ["CONAN_USER_HOME"], ".conan") +def build_wheel( + wheel_directory, + config_settings=None, + metadata_directory=None +): + if ( + config_settings is not None and + config_settings.get('conan_cache') is not None and + "CONAN_USER_HOME" in os.environ + ): + config_settings['conan_cache'] = os.path.join( + os.environ["CONAN_USER_HOME"], + ".conan" + ) conan_libs.build_conan( wheel_directory, @@ -28,8 +39,16 @@ def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): try: if config_settings is not None and "conan_cache" in config_settings: - os.environ["CONAN_USER_HOME"] = os.path.normpath(os.path.join(config_settings['conan_cache'], "..")) - return setuptools.build_meta.build_wheel(wheel_directory, config_settings, metadata_directory) + os.environ["CONAN_USER_HOME"] = os.path.normpath( + os.path.join(config_settings['conan_cache'], "..") + ) + + return setuptools.build_meta.build_wheel( + wheel_directory, + config_settings, + metadata_directory + ) + finally: if original_conan_user_home: os.environ["CONAN_USER_HOME"] = original_conan_user_home @@ -48,11 +67,13 @@ def get_requires_for_build_sdist(config_settings=None): print("get_requires_for_build_sdist - Done") - def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): print('prepare_metadata_for_build_wheel') try: - return setuptools.build_meta.prepare_metadata_for_build_wheel(metadata_directory, config_settings) + return setuptools.build_meta.prepare_metadata_for_build_wheel( + metadata_directory, + config_settings + ) finally: print('prepare_metadata_for_build_wheel done') diff --git a/uiucprescon/build/pybind11_builder.py b/uiucprescon/build/pybind11_builder.py index 86cd768..b61eb49 100644 --- a/uiucprescon/build/pybind11_builder.py +++ b/uiucprescon/build/pybind11_builder.py @@ -51,7 +51,10 @@ def find_missing_libraries(self, ext, strategies=None): conan_info_dir = os.environ.get('CONAN_BUILD_INFO_DIR') if conan_info_dir: conanfileinfo_locations.insert(0, conan_info_dir) - conanbuildinfo = conan_libs.locate_conanbuildinfo(conanfileinfo_locations) + + conanbuildinfo =\ + conan_libs.locate_conanbuildinfo(conanfileinfo_locations) + if conanbuildinfo: strategies.insert( 0, @@ -108,8 +111,15 @@ def _add_conan_libs_to_ext(self, ext: Pybind11Extension): else: ext.runtime_library_dirs.append(os.path.abspath(lib_output)) ext.library_dirs.insert(0, os.path.abspath(lib_output)) - ext.library_dirs = list(parse_conan_build_info(conan_build_info, "libdirs")) + ext.library_dirs - ext.include_dirs = list(parse_conan_build_info(conan_build_info, "includedirs")) + ext.include_dirs + + ext.library_dirs = list( + parse_conan_build_info(conan_build_info, "libdirs") + ) + ext.library_dirs + + ext.include_dirs = list( + parse_conan_build_info(conan_build_info, "includedirs") + ) + ext.include_dirs + defines = parse_conan_build_info(conan_build_info, "defines") ext.define_macros = [(d, None) for d in defines] + ext.define_macros From 6485e07502249d03ccd81805bba5a4c83556771c Mon Sep 17 00:00:00 2001 From: henry Date: Mon, 9 Oct 2023 13:24:10 -0500 Subject: [PATCH 4/5] CI: use Ubuntu 22.04 for testing --- ci/docker/linux/jenkins/Dockerfile | 20 ++++++++-- ci/docker/linux/jenkins/apt-packages.txt | 7 ++++ .../linux/shared/update_conan_compiler.py | 37 +++++++++++++++++++ ci/docker/linux/tox/Dockerfile | 18 +++++++-- ci/docker/linux/tox/apt-packages.txt | 7 ++++ 5 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 ci/docker/linux/shared/update_conan_compiler.py diff --git a/ci/docker/linux/jenkins/Dockerfile b/ci/docker/linux/jenkins/Dockerfile index bec139d..99a74ee 100644 --- a/ci/docker/linux/jenkins/Dockerfile +++ b/ci/docker/linux/jenkins/Dockerfile @@ -1,14 +1,28 @@ -FROM ubuntu:20.04 +ARG CONAN_USER_HOME=/conan +ARG PIP_DOWNLOAD_CACHE=/.cache/pip + +FROM ubuntu:22.04 COPY ci/docker/linux/jenkins/apt-packages.txt /tmp/ RUN apt-get update && \ - apt-get install -y software-properties-common && \ + apt-get install -y software-properties-common gpg-agent --no-install-recommends && \ add-apt-repository ppa:deadsnakes/ppa && \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata && \ apt-get update && \ - < /tmp/apt-packages.txt xargs apt-get install -y && \ + < /tmp/apt-packages.txt xargs apt-get install -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + + ARG PIP_EXTRA_INDEX_URL ARG PIP_INDEX_URL COPY requirements/ /tmp/requirements/requirements/ COPY requirements-ci.txt /tmp/requirements/ RUN python3 -m pip install pip --upgrade && pip install -r /tmp/requirements/requirements-ci.txt + +COPY ci/docker/linux/shared/update_conan_compiler.py /tmp/update_conan_compiler.py +ARG CONAN_USER_HOME +RUN conan config init && \ + conan config set general.revisions_enabled=1 && \ + conan profile update settings.compiler.version=$(cc -dumpfullversion -dumpversion | grep -oE "^([0-9]+(\.)?)([0-9]+?)") default && \ + python3 /tmp/update_conan_compiler.py ${CONAN_USER_HOME}/.conan/settings.yml gcc $(cc -dumpfullversion -dumpversion | grep -oE "^([0-9]+(\.)?)([0-9]+?)") +ENV CONAN_USER_HOME=${CONAN_USER_HOME} \ No newline at end of file diff --git a/ci/docker/linux/jenkins/apt-packages.txt b/ci/docker/linux/jenkins/apt-packages.txt index 41233ae..c2ee91c 100644 --- a/ci/docker/linux/jenkins/apt-packages.txt +++ b/ci/docker/linux/jenkins/apt-packages.txt @@ -1,11 +1,18 @@ ca-certificates python3-pip +python3.8-dev +python3.8-distutils +python3.8-venv python3.9-dev python3.9-distutils +python3.9-venv python3.10-dev python3.10-distutils python3.10-venv python3.11-dev python3.11-distutils python3.11-venv +python3.12-dev +python3.12-distutils +python3.12-venv git diff --git a/ci/docker/linux/shared/update_conan_compiler.py b/ci/docker/linux/shared/update_conan_compiler.py new file mode 100644 index 0000000..928b523 --- /dev/null +++ b/ci/docker/linux/shared/update_conan_compiler.py @@ -0,0 +1,37 @@ +import argparse +import yaml + + +def get_arg_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("settings_file") + parser.add_argument("compiler_name") + parser.add_argument("compiler_version") + return parser + + +def update_settings_file(settings_file, compiler_name, compiler_version): + with open(settings_file) as file: + data = yaml.safe_load(file) + if compiler_name not in data['compiler']: + raise ValueError(f"invalid compiler name: {compiler_name}") + + if compiler_version not in data['compiler'][compiler_name]['version']: + data['compiler'][compiler_name]['version'].append(compiler_version) + print("Compiler data added") + print(data['compiler'][compiler_name]['version']) + with open(settings_file, 'w') as file: + yaml.dump(data, file) + +def main(): + args = get_arg_parser().parse_args() + print(args) + update_settings_file( + args.settings_file, + args.compiler_name, + args.compiler_version + + ) + +if __name__ == '__main__': + main() diff --git a/ci/docker/linux/tox/Dockerfile b/ci/docker/linux/tox/Dockerfile index f4b8ea4..1eb98bc 100644 --- a/ci/docker/linux/tox/Dockerfile +++ b/ci/docker/linux/tox/Dockerfile @@ -1,13 +1,16 @@ +ARG CONAN_USER_HOME=/conan ARG PIP_DOWNLOAD_CACHE=/.cache/pip -FROM ubuntu:20.04 +FROM ubuntu:22.04 COPY ci/docker/linux/tox/apt-packages.txt /tmp/ RUN apt-get update && \ - apt-get install -y software-properties-common && \ + apt-get install -y software-properties-common gpg-agent --no-install-recommends && \ add-apt-repository ppa:deadsnakes/ppa && \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata && \ apt-get update && \ - < /tmp/apt-packages.txt xargs apt-get install -y && \ + < /tmp/apt-packages.txt xargs apt-get install -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + ARG PIP_EXTRA_INDEX_URL ARG PIP_INDEX_URL COPY requirements-ci.txt /tmp/python_requirements/ @@ -18,5 +21,14 @@ ARG PIP_DOWNLOAD_CACHE RUN mkdir -p ${PIP_DOWNLOAD_CACHE} && \ chmod -R 777 ${PIP_DOWNLOAD_CACHE} ENV PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE} + +COPY ci/docker/linux/shared/update_conan_compiler.py /tmp/update_conan_compiler.py +ARG CONAN_USER_HOME +RUN conan config init && \ + conan config set general.revisions_enabled=1 && \ + conan profile update settings.compiler.version=$(cc -dumpfullversion -dumpversion | grep -oE "^([0-9]+(\.)?)([0-9]+?)") default && \ + python3 /tmp/update_conan_compiler.py ${CONAN_USER_HOME}/.conan/settings.yml gcc $(cc -dumpfullversion -dumpversion | grep -oE "^([0-9]+(\.)?)([0-9]+?)") +ENV CONAN_USER_HOME=${CONAN_USER_HOME} + WORKDIR /src CMD tox --workdir /tmp/tox --recreate diff --git a/ci/docker/linux/tox/apt-packages.txt b/ci/docker/linux/tox/apt-packages.txt index 15a703e..2cf4b02 100644 --- a/ci/docker/linux/tox/apt-packages.txt +++ b/ci/docker/linux/tox/apt-packages.txt @@ -1,9 +1,16 @@ python3-pip +python3.8-dev +python3.8-distutils +python3.8-venv python3.9-dev python3.9-distutils +python3.9-venv python3.10-dev python3.10-distutils python3.10-venv python3.11-dev python3.11-distutils python3.11-venv +python3.12-dev +python3.12-distutils +python3.12-venv \ No newline at end of file From fc120d5cec41f4ba7a1492ff6aaa9b0f8e758cbd Mon Sep 17 00:00:00 2001 From: henry Date: Mon, 9 Oct 2023 13:20:58 -0500 Subject: [PATCH 5/5] CI: remove deprecated pytest-runner package from ci requirements --- requirements/requirements-ci-freeze.txt | 79 +++++++++++++------------ requirements/requirements-dev.txt | 1 - 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/requirements/requirements-ci-freeze.txt b/requirements/requirements-ci-freeze.txt index 1883fbd..065df80 100644 --- a/requirements/requirements-ci-freeze.txt +++ b/requirements/requirements-ci-freeze.txt @@ -1,61 +1,66 @@ alabaster==0.7.13 -astroid==2.15.6 -Babel==2.12.1 +astroid==3.0.0 +Babel==2.13.0 bottle==0.12.25 -certifi==2023.5.7 -charset-normalizer==3.2.0 +build==1.0.3 +certifi==2023.7.22 +charset-normalizer==3.3.0 +check-manifest==0.49 colorama==0.4.6 -conan==1.60.1 -coverage==7.2.7 -dill==0.3.6 -distlib==0.3.6 +conan==1.61.0 +coverage==7.3.2 +devpi-client==5.2.3 +devpi-common==3.7.2 +dill==0.3.7 +distlib==0.3.7 docutils==0.20.1 -fasteners==0.18 -filelock==3.12.2 -flake8==6.0.0 +fasteners==0.19 +filelock==3.12.4 +flake8==6.1.0 idna==3.4 imagesize==1.4.1 iniconfig==2.0.0 isort==5.12.0 Jinja2==3.1.2 -lazy-object-proxy==1.9.0 +lazy==1.6 lxml==4.9.3 MarkupSafe==2.1.3 mccabe==0.7.0 -mypy==1.4.1 +mypy==1.5.1 mypy-extensions==1.0.0 node-semver==0.6.1 -packaging==23.1 +packaging==21.3 patch-ng==1.17.4 -platformdirs==3.8.1 -pluggy==1.2.0 +pkginfo==1.9.6 +platformdirs==3.11.0 +pluggy==1.3.0 pluginbase==1.0.1 py==1.11.0 -pybind11==2.10.4 -pycodestyle==2.10.0 -pyflakes==3.0.1 -Pygments==2.15.1 -PyJWT==2.7.0 -pylint==2.17.4 -pytest==7.4.0 -pytest-runner==6.0.0 +pybind11==2.11.1 +pycodestyle==2.11.0 +pyflakes==3.1.0 +Pygments==2.16.1 +PyJWT==2.8.0 +pylint==3.0.1 +pyparsing==3.1.1 +pyproject_hooks==1.0.0 +pytest==7.4.2 python-dateutil==2.8.2 -PyYAML==6.0 +PyYAML==6.0.1 requests==2.31.0 six==1.16.0 snowballstemmer==2.2.0 -Sphinx==7.0.1 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 +Sphinx==7.2.6 +sphinxcontrib-applehelp==1.0.7 +sphinxcontrib-devhelp==1.0.5 +sphinxcontrib-htmlhelp==2.0.4 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-qthelp==1.0.6 +sphinxcontrib-serializinghtml==1.1.9 toml==0.10.2 -tomlkit==0.11.8 +tomlkit==0.12.1 tox==3.28.0 -tqdm==4.65.0 -typing_extensions==4.7.1 -urllib3==1.26.16 -virtualenv==20.23.1 -wrapt==1.15.0 +tqdm==4.66.1 +typing_extensions==4.8.0 +urllib3==1.26.17 +virtualenv==20.24.5 diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index d8833f3..2169544 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -3,6 +3,5 @@ flake8 lxml mypy pytest -pytest-runner pylint sphinx