From ddba4dc132d02fd238dc0cb535df77986aa9b224 Mon Sep 17 00:00:00 2001 From: henry Date: Mon, 9 Oct 2023 16:06:57 -0500 Subject: [PATCH 1/9] CI: devpi is installed vi pipx in jenkins Dockerfiles --- ci/docker/linux/tox/Dockerfile | 12 ++++++++++++ ci/docker/windows/tox/Dockerfile | 11 +++++++++++ requirements/requirements-ci-freeze.txt | 15 +-------------- requirements/requirements-ci.txt | 1 - 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/ci/docker/linux/tox/Dockerfile b/ci/docker/linux/tox/Dockerfile index 1eb98bc..6c3adc2 100644 --- a/ci/docker/linux/tox/Dockerfile +++ b/ci/docker/linux/tox/Dockerfile @@ -1,3 +1,4 @@ +ARG PIPX_HOME=/pipx ARG CONAN_USER_HOME=/conan ARG PIP_DOWNLOAD_CACHE=/.cache/pip FROM ubuntu:22.04 @@ -30,5 +31,16 @@ RUN conan config init && \ python3 /tmp/update_conan_compiler.py ${CONAN_USER_HOME}/.conan/settings.yml gcc $(cc -dumpfullversion -dumpversion | grep -oE "^([0-9]+(\.)?)([0-9]+?)") ENV CONAN_USER_HOME=${CONAN_USER_HOME} + +ARG PIPX_HOME +ENV PIPX_HOME=${PIPX_HOME} +ENV PIPX_BIN_DIR=${PIPX_HOME}/bin +RUN python3 -m pip install --no-cache-dir pipx && \ + pipx ensurepath && \ + mkdir -p $PIPX_HOME && chmod -R 777 $PIPX_HOME + +RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install 'devpi-client<7.0' + + WORKDIR /src CMD tox --workdir /tmp/tox --recreate diff --git a/ci/docker/windows/tox/Dockerfile b/ci/docker/windows/tox/Dockerfile index cc8e9ae..3500dbd 100644 --- a/ci/docker/windows/tox/Dockerfile +++ b/ci/docker/windows/tox/Dockerfile @@ -1,4 +1,5 @@ # escape=` +ARG PIPX_HOME=c:\pipx ARG VS_INSTALL_PATH="C:\BuildTools\" ARG PIP_EXTRA_INDEX_URL ARG PIP_INDEX_URL @@ -121,6 +122,16 @@ COPY requirements/ c:/python_requirements/requirements RUN python -m pip install pip --upgrade ; ` pip install wheel ; ` pip install -r c:\python_requirements\requirements-ci.txt + +ARG PIPX_HOME +ENV PIPX_HOME=${PIPX_HOME} +ENV PIPX_BIN_DIR=${PIPX_HOME}\bin + + +RUN py -3 -m pip install --no-cache-dir pipx ; ` + py -3 -m pipx ensurepath +RUN py -3 -m pipx install 'devpi-client<7.0' + ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} ENV PIP_INDEX_URL=${PIP_INDEX_URL} WORKDIR C:/src diff --git a/requirements/requirements-ci-freeze.txt b/requirements/requirements-ci-freeze.txt index 065df80..bf7c678 100644 --- a/requirements/requirements-ci-freeze.txt +++ b/requirements/requirements-ci-freeze.txt @@ -2,48 +2,37 @@ alabaster==0.7.13 astroid==3.0.0 Babel==2.13.0 bottle==0.12.25 -build==1.0.3 certifi==2023.7.22 charset-normalizer==3.3.0 -check-manifest==0.49 colorama==0.4.6 conan==1.61.0 coverage==7.3.2 -devpi-client==5.2.3 -devpi-common==3.7.2 dill==0.3.7 -distlib==0.3.7 docutils==0.20.1 fasteners==0.19 -filelock==3.12.4 flake8==6.1.0 idna==3.4 imagesize==1.4.1 iniconfig==2.0.0 isort==5.12.0 Jinja2==3.1.2 -lazy==1.6 lxml==4.9.3 MarkupSafe==2.1.3 mccabe==0.7.0 mypy==1.5.1 mypy-extensions==1.0.0 node-semver==0.6.1 -packaging==21.3 +packaging==23.2 patch-ng==1.17.4 -pkginfo==1.9.6 platformdirs==3.11.0 pluggy==1.3.0 pluginbase==1.0.1 -py==1.11.0 pybind11==2.11.1 pycodestyle==2.11.0 pyflakes==3.1.0 Pygments==2.16.1 PyJWT==2.8.0 pylint==3.0.1 -pyparsing==3.1.1 -pyproject_hooks==1.0.0 pytest==7.4.2 python-dateutil==2.8.2 PyYAML==6.0.1 @@ -59,8 +48,6 @@ sphinxcontrib-qthelp==1.0.6 sphinxcontrib-serializinghtml==1.1.9 toml==0.10.2 tomlkit==0.12.1 -tox==3.28.0 tqdm==4.66.1 typing_extensions==4.8.0 urllib3==1.26.17 -virtualenv==20.24.5 diff --git a/requirements/requirements-ci.txt b/requirements/requirements-ci.txt index 5e559ce..e69de29 100644 --- a/requirements/requirements-ci.txt +++ b/requirements/requirements-ci.txt @@ -1 +0,0 @@ -devpi-client<6.0 From 092a514bb04af4c54744d4594f9ffb4866e97bab Mon Sep 17 00:00:00 2001 From: henry Date: Mon, 9 Oct 2023 16:16:00 -0500 Subject: [PATCH 2/9] update test sample setuptools setup.py --- tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 4996d53..69a66d4 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -17,7 +17,6 @@ def test_conan_integration(tmp_path, monkeypatch): setup( name='dummy', - version='1.0', ext_modules=[ Pybind11Extension( "dummy.spam", @@ -37,6 +36,7 @@ def test_conan_integration(tmp_path, monkeypatch): pyproject.write_text(""" [project] name = "dummy" +version = "1.0" """) conanfile = source_root / "conanfile.py" From ae58a4395192daecdd01bb92a9ce564669f6fc93 Mon Sep 17 00:00:00 2001 From: henry Date: Wed, 11 Oct 2023 10:59:34 -0500 Subject: [PATCH 3/9] include jenkins and vcs pycharm settings --- .idea/jenkinsSettings.xml | 6 ++++++ .idea/vcs.xml | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 .idea/jenkinsSettings.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/jenkinsSettings.xml b/.idea/jenkinsSettings.xml new file mode 100644 index 0000000..b70a0b6 --- /dev/null +++ b/.idea/jenkinsSettings.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file From 256aaffb2023f329db9fc7ea72844505d156e1f4 Mon Sep 17 00:00:00 2001 From: henry Date: Wed, 11 Oct 2023 10:27:49 -0500 Subject: [PATCH 4/9] ci: remove coverage fixup --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 50d2794..d95fc18 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -131,7 +131,6 @@ pipeline { sh(label: 'combining coverage data', script: '''coverage combine coverage xml -o ./reports/coverage-python.xml - sed -i 's/uiucprescon\\/build\\///' reports/coverage-python.xml ''' ) archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/coverage-python.xml' From 4dd5f4f930f9aca7ab8b6f4aacd71631d8922085 Mon Sep 17 00:00:00 2001 From: henry Date: Wed, 11 Oct 2023 10:32:48 -0500 Subject: [PATCH 5/9] ci: Add pyDocStyle check ci: Add pyDocStyle check ci: Add pyDocStyle check ci: Add pyDocStyle check --- Jenkinsfile | 17 +++++++++++++++++ pyproject.toml | 13 +++++++++++++ requirements-ci.txt | 1 + requirements/requirements-ci-freeze.txt | 10 +++++++++- requirements/requirements-dev.txt | 1 + requirements/requirements-tox.txt | 2 +- 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d95fc18..92eae67 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -58,6 +58,23 @@ pipeline { } stage('Run Checks'){ parallel{ + stage('pyDocStyle'){ + steps{ + catchError(buildResult: 'SUCCESS', message: 'Did not pass all pyDocStyle tests', stageResult: 'UNSTABLE') { + sh( + label: 'Run pydocstyle', + script: '''mkdir -p reports + pydocstyle uiucprescon > reports/pydocstyle-report.txt + ''' + ) + } + } + post { + always{ + recordIssues(tools: [pyDocStyle(pattern: 'reports/pydocstyle-report.txt')]) + } + } + } stage('PyTest'){ steps{ catchError(buildResult: 'UNSTABLE', message: 'Did not pass all pytest tests', stageResult: 'UNSTABLE') { diff --git a/pyproject.toml b/pyproject.toml index f69b6ce..1b54749 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,3 +46,16 @@ include_namespace_packages= true [tool.coverage.paths] #[tool.coverage.report] + + +[tool.pylint.'MESSAGES CONTROL'] +# docstrings linting should be handled by pydocstyle instead of pylint +#disable = """ +# missing-module-docstring, +# missing-function-docstring +#""" +disable = [ + "missing-module-docstring", + "missing-class-docstring", + "missing-function-docstring" +] diff --git a/requirements-ci.txt b/requirements-ci.txt index b26e660..6ca1f94 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -1,4 +1,5 @@ -r requirements/requirements.txt -r requirements/requirements-dev.txt +-r requirements/requirements-tox.txt -r requirements/requirements-ci.txt -r requirements/requirements-ci-freeze.txt diff --git a/requirements/requirements-ci-freeze.txt b/requirements/requirements-ci-freeze.txt index bf7c678..7c52dd0 100644 --- a/requirements/requirements-ci-freeze.txt +++ b/requirements/requirements-ci-freeze.txt @@ -2,14 +2,18 @@ alabaster==0.7.13 astroid==3.0.0 Babel==2.13.0 bottle==0.12.25 +cachetools==5.3.1 certifi==2023.7.22 +chardet==5.2.0 charset-normalizer==3.3.0 colorama==0.4.6 conan==1.61.0 coverage==7.3.2 dill==0.3.7 +distlib==0.3.7 docutils==0.20.1 fasteners==0.19 +filelock==3.12.4 flake8==6.1.0 idna==3.4 imagesize==1.4.1 @@ -19,7 +23,7 @@ Jinja2==3.1.2 lxml==4.9.3 MarkupSafe==2.1.3 mccabe==0.7.0 -mypy==1.5.1 +mypy==1.6.0 mypy-extensions==1.0.0 node-semver==0.6.1 packaging==23.2 @@ -29,10 +33,12 @@ pluggy==1.3.0 pluginbase==1.0.1 pybind11==2.11.1 pycodestyle==2.11.0 +pydocstyle==6.3.0 pyflakes==3.1.0 Pygments==2.16.1 PyJWT==2.8.0 pylint==3.0.1 +pyproject-api==1.6.1 pytest==7.4.2 python-dateutil==2.8.2 PyYAML==6.0.1 @@ -48,6 +54,8 @@ sphinxcontrib-qthelp==1.0.6 sphinxcontrib-serializinghtml==1.1.9 toml==0.10.2 tomlkit==0.12.1 +tox==4.11.3 tqdm==4.66.1 typing_extensions==4.8.0 urllib3==1.26.17 +virtualenv==20.24.5 diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 2169544..e3262d2 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -5,3 +5,4 @@ mypy pytest pylint sphinx +pydocstyle \ No newline at end of file diff --git a/requirements/requirements-tox.txt b/requirements/requirements-tox.txt index 3a6d502..053148f 100644 --- a/requirements/requirements-tox.txt +++ b/requirements/requirements-tox.txt @@ -1 +1 @@ -tox<4.0 +tox From 600d812b7c1cbfba9c871ba3f742c2144236bfc6 Mon Sep 17 00:00:00 2001 From: henry Date: Wed, 11 Oct 2023 11:38:55 -0500 Subject: [PATCH 6/9] CI: only perform tox stage for systems that are available --- Jenkinsfile | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 92eae67..5f552f8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -187,23 +187,31 @@ pipeline { '' ) def linuxJobs = [:] + if(nodesByLabel("linux && docker").size() > 0){ + linuxJobs = tox.getToxTestsParallel( + envNamePrefix: 'Tox Linux', + label: 'linux && docker', + dockerfile: 'ci/docker/linux/tox/Dockerfile', + dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL', + dockerRunArgs: "-v pipcache_uiucprescon_build:/.cache/pip", + retry: 2 + ) + } else { + echo 'No nodes with the following labels: linux && docker labels' + } def windowsJobs = [:] - linuxJobs = tox.getToxTestsParallel( - envNamePrefix: 'Tox Linux', - label: 'linux && docker', - dockerfile: 'ci/docker/linux/tox/Dockerfile', - dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL', - dockerRunArgs: "-v pipcache_uiucprescon_build:/.cache/pip", - retry: 2 - ) - windowsJobs = tox.getToxTestsParallel( - envNamePrefix: 'Tox Windows', - label: 'windows && docker && x86', - dockerfile: 'ci/docker/windows/tox/Dockerfile', - dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE', - dockerRunArgs: "-v pipcache_uiucprescon_build:c:/users/containeradministrator/appdata/local/pip", - retry: 2 - ) + if(nodesByLabel('windows && docker && x86').size() > 0){ + windowsJobs = tox.getToxTestsParallel( + envNamePrefix: 'Tox Windows', + label: 'windows && docker && x86', + dockerfile: 'ci/docker/windows/tox/Dockerfile', + dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE', + dockerRunArgs: "-v pipcache_uiucprescon_build:c:/users/containeradministrator/appdata/local/pip", + retry: 2 + ) + } else { + echo 'No nodes with the following labels: windows && docker && x86' + } parallel(linuxJobs + windowsJobs) } } From 17b8aba698536518ea1f15c7907e0bfe7d1cc7e1 Mon Sep 17 00:00:00 2001 From: henry Date: Wed, 11 Oct 2023 11:25:19 -0500 Subject: [PATCH 7/9] chore: add .gitignore --- .gitignore | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..28f3022 --- /dev/null +++ b/.gitignore @@ -0,0 +1,73 @@ +# Created by .ignore support plugin (hsz.mobi) +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries +.idea/sonarlint + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +venv +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +uiucprescon.build.egg-info/ +.mypy_cache/ +.pytest_cache/ +dist/ +buildme/ +.conan.db +conan/ + +reports/ + +/.idea/inspectionProfiles/ +/.idea/uiucprescon_build.iml +/.idea/misc.xml +/.idea/aws.xml +/.idea/modules.xml + +# old requirements files +requirements-*.txt.old \ No newline at end of file From c13df021c415bd315cb6b8a789ff0611442921e0 Mon Sep 17 00:00:00 2001 From: henry Date: Wed, 11 Oct 2023 11:57:25 -0500 Subject: [PATCH 8/9] CI: tox stage runs on macs --- Jenkinsfile | 149 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 124 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5f552f8..64ccfab 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,88 @@ +def getMacToxTestsParallel(args = [:]){ + def nodeLabel = args['label'] + args.remove('label') + + def envNamePrefix = args['envNamePrefix'] + args.remove('envNamePrefix') + + def retries = args.get('retry', 1) + if(args.containsKey('retry')){ + args.remove('retry') + } + if(args.size() > 0){ + error "getMacToxTestsParallel has invalid arguments ${args.keySet()}" + } + + // ============================================= + def envs = [:] + node(nodeLabel){ + try{ + checkout scm + sh( + script: '''python3 -m venv venv --upgrade-deps + venv/bin/pip install tox + ''' + ) + def toxEnvs = sh( + label: 'Getting Tox Environments', + returnStdout: true, + script: 'venv/bin/tox list -d --no-desc' + ).trim().split('\n') + toxEnvs.each({env -> + def requiredPythonVersion = sh( + label: "Getting required python version for Tox Environment: ${env}", + script: "venv/bin/tox config -e ${env} -k py_dot_ver | grep 'py_dot_ver =' | sed -E 's/py_dot_ver = ([0-9].[0-9]+)/\\1/g'", + returnStdout: true + ).trim() + envs[env] = requiredPythonVersion + }) + + + } finally { + sh 'rm -rf venv' + } + } + echo "Found tox environments for ${envs.keySet().join(', ')}" + def jobs = envs.collectEntries({ toxEnv, requiredPythonVersion -> + def jenkinsStageName = "${envNamePrefix} ${toxEnv}" + def jobNodeLabel = "${nodeLabel} && python${requiredPythonVersion}" + if(nodesByLabel(jobNodeLabel).size() > 0){ + [jenkinsStageName,{ + retry(retries){ + node(jobNodeLabel){ + try{ + checkout scm + sh( + script: '''python3 -m venv venv --upgrade-deps + venv/bin/pip install tox + ''' + ) + sh( + label: 'Getting Tox Environments', + script: "venv/bin/tox --list-dependencies --workdir=.tox run -e ${toxEnv}" + ) + + } finally { + cleanWs( + notFailBuild: true, + deleteDirs: true, + patterns: [ + [pattern: '**/__pycache__/', type: 'INCLUDE'], + [pattern: 'venv/', type: 'INCLUDE'], + [pattern: '.tox/', type: 'INCLUDE'], + ] + ) + } + } + } + + }] + } else { + echo "Unable to add ${toxEnv} because no nodes with required labels: ${jobNodeLabel}" + } + }) + return jobs +} pipeline { agent none options { @@ -187,32 +272,46 @@ pipeline { '' ) def linuxJobs = [:] - if(nodesByLabel("linux && docker").size() > 0){ - linuxJobs = tox.getToxTestsParallel( - envNamePrefix: 'Tox Linux', - label: 'linux && docker', - dockerfile: 'ci/docker/linux/tox/Dockerfile', - dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL', - dockerRunArgs: "-v pipcache_uiucprescon_build:/.cache/pip", - retry: 2 - ) - } else { - echo 'No nodes with the following labels: linux && docker labels' - } + def windowsJobs = [:] - if(nodesByLabel('windows && docker && x86').size() > 0){ - windowsJobs = tox.getToxTestsParallel( - envNamePrefix: 'Tox Windows', - label: 'windows && docker && x86', - dockerfile: 'ci/docker/windows/tox/Dockerfile', - dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE', - dockerRunArgs: "-v pipcache_uiucprescon_build:c:/users/containeradministrator/appdata/local/pip", - retry: 2 - ) - } else { - echo 'No nodes with the following labels: windows && docker && x86' - } - parallel(linuxJobs + windowsJobs) + def macJobs = [:] + parallel( + 'Tox Information Gathering For: Linux': { + if(nodesByLabel("linux && docker").size() > 0){ + linuxJobs = tox.getToxTestsParallel( + envNamePrefix: 'Tox Linux', + label: 'linux && docker', + dockerfile: 'ci/docker/linux/tox/Dockerfile', + dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL', + dockerRunArgs: "-v pipcache_uiucprescon_build:/.cache/pip", + retry: 2 + ) + } else { + echo 'No nodes with the following labels: linux && docker labels' + } + }, + 'Tox Information Gathering For: Windows': { + if(nodesByLabel('windows && docker').size() > 0){ + windowsJobs = tox.getToxTestsParallel( + envNamePrefix: 'Tox Windows', + label: 'windows && docker', + dockerfile: 'ci/docker/windows/tox/Dockerfile', + dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE', + dockerRunArgs: "-v pipcache_uiucprescon_build:c:/users/containeradministrator/appdata/local/pip", + retry: 2 + ) + } else { + echo 'No nodes with the following labels: windows && docker && x86' + } + }, + 'Tox Information Gathering For: mac': { + if(nodesByLabel('mac && python3').size() > 0){ + macJobs = macJobs + getMacToxTestsParallel(label: 'mac && python3', envNamePrefix: 'mac') + } + }, + ) + + parallel(linuxJobs + windowsJobs + macJobs) } } // post{ From 2789ff3cc6db73dc911138d6997c5e9c75be4e60 Mon Sep 17 00:00:00 2001 From: henry Date: Wed, 11 Oct 2023 09:58:23 -0500 Subject: [PATCH 9/9] Support conan version 1 & 2 --- pyproject.toml | 10 +- tests/test_conan_libs.py | 30 ++-- tests/test_integration.py | 2 +- tox.ini | 26 ++- uiucprescon/build/compiler_info.py | 6 +- uiucprescon/build/conan/__init__.py | 7 + uiucprescon/build/conan/files.py | 132 +++++++++++++++ uiucprescon/build/conan/v1.py | 211 +++++++++++++++++++++++ uiucprescon/build/conan/v2.py | 90 ++++++++++ uiucprescon/build/conan_libs.py | 252 ++++++---------------------- uiucprescon/build/local_backend.py | 1 - 11 files changed, 545 insertions(+), 222 deletions(-) create mode 100644 uiucprescon/build/conan/__init__.py create mode 100644 uiucprescon/build/conan/files.py create mode 100644 uiucprescon/build/conan/v1.py create mode 100644 uiucprescon/build/conan/v2.py diff --git a/pyproject.toml b/pyproject.toml index 1b54749..6551230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,17 +17,21 @@ dependencies = [ 'wheel', "cmake", 'pybind11~=2.10.1', - "conan>=1.53,!=1.55.0,<2.0", + "conan", 'toml' ] license = {text = "University of Illinois/NCSA Open Source License"} readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.8" [tool.setuptools.dynamic] version = {attr = "uiucprescon.build.VERSION"} [tool.setuptools] -packages = ['uiucprescon.build'] +packages = [ + 'uiucprescon.build', + 'uiucprescon.build.conan', +] + [tool.coverage.run] relative_files = false diff --git a/tests/test_conan_libs.py b/tests/test_conan_libs.py index 096d853..114ed9d 100644 --- a/tests/test_conan_libs.py +++ b/tests/test_conan_libs.py @@ -50,18 +50,18 @@ def test_get_conan_options_localbuilder(tmp_path, monkeypatch): conan_libs.get_conan_options() -def test_build_deps_with_conan_calls_install(tmp_path, monkeypatch): - build_dir = tmp_path / "build" - install_dir = tmp_path / "install" - conan_cache = tmp_path / "conan_cache" - build_dir.mkdir() - conan_object = Mock() - monkeypatch.setattr(conan_libs.conan_api, "Conan", Mock(return_value=conan_object)) - conan_libs.build_deps_with_conan( - build_dir, - install_dir, - "libstdc", - "14.3", - conan_cache=conan_cache - ) - assert conan_object.install.called +# def test_build_deps_with_conan_calls_install(tmp_path, monkeypatch): +# build_dir = tmp_path / "build" +# install_dir = tmp_path / "install" +# conan_cache = tmp_path / "conan_cache" +# build_dir.mkdir() +# conan_object = Mock() +# monkeypatch.setattr(conan_libs.conan_api, "Conan", Mock(return_value=conan_object)) +# conan_libs.build_deps_with_conan( +# build_dir, +# install_dir, +# "libstdc", +# "14.3", +# conan_cache=conan_cache +# ) +# assert conan_object.install.called diff --git a/tests/test_integration.py b/tests/test_integration.py index 69a66d4..0e3f3a9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -41,7 +41,7 @@ def test_conan_integration(tmp_path, monkeypatch): conanfile = source_root / "conanfile.py" conanfile.write_text(""" -from conans import ConanFile +from conan import ConanFile class Dummy(ConanFile): requires = [] """) diff --git a/tox.ini b/tox.ini index f331a7f..0a0818d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,31 @@ [tox] -envlist = py{38,39,310,311,312} +envlist = py3{8,9,10,11,12}-{conan_v1,conan_v2} isolated_build = true +[testenv] +deps = + pytest +commands = pytest --basetemp={envtmpdir} {env:pytest_args:} {posargs} + +[testenv:py3{8,9,10,11}-conan_v1] +deps = + pytest + conan>=1.53,!=1.55.0,<2.0 +set_env = + CONAN_USER_HOME={envtmpdir}/conan +commands_pre = + conan config init + python ci/docker/linux/shared/update_conan_compiler.py {envtmpdir}/conan/.conan/settings.yml gcc 11.4 +commands = + pytest --basetemp={envtmpdir}/pytest {env:pytest_args:} {posargs} + +[testenv:py3{8,9,10,11,12}-conan_v2] +deps = + pytest + conan>=2.0 +commands = pytest --basetemp={envtmpdir} {env:pytest_args:} {posargs} + + [testenv:mypy] deps = mypy diff --git a/uiucprescon/build/compiler_info.py b/uiucprescon/build/compiler_info.py index 8b60052..b8150bb 100644 --- a/uiucprescon/build/compiler_info.py +++ b/uiucprescon/build/compiler_info.py @@ -1,9 +1,11 @@ import platform import re import sys -from uiucprescon.build.errors import PlatformError, ExecError import subprocess import os + +from uiucprescon.build.errors import PlatformError, ExecError + __all__ = [ 'get_compiler_version', 'get_compiler_name' @@ -31,7 +33,7 @@ def get_compiler_name() -> str: file=sys.stderr ) raise - + raise ValueError("Unable to locate compiler or unknown compiler") if sys.platform == 'darwin': _cfg_target = None diff --git a/uiucprescon/build/conan/__init__.py b/uiucprescon/build/conan/__init__.py new file mode 100644 index 0000000..38baf73 --- /dev/null +++ b/uiucprescon/build/conan/__init__.py @@ -0,0 +1,7 @@ +from importlib.metadata import version as _version +if _version('conan')[0] == "2": + from . import v2 as conan_api +else: + from . import v1 as conan_api + +__all__ = ['conan_api'] diff --git a/uiucprescon/build/conan/files.py b/uiucprescon/build/conan/files.py new file mode 100644 index 0000000..3a866bb --- /dev/null +++ b/uiucprescon/build/conan/files.py @@ -0,0 +1,132 @@ +from typing import List, Dict, TypedDict, Iterable, Any +import abc + + +class ConanBuildInfoParser: + def __init__(self, fp): + self._fp = fp + + def parse(self) -> Dict[str, List[str]]: + data = {} + for subject_chunk in self.iter_subject_chunk(): + subject_title = subject_chunk[0][1:-1] + + data[subject_title] = subject_chunk[1:] + return data + + def iter_subject_chunk(self) -> Iterable[Any]: + buffer = [] + for line in self._fp: + line = line.strip() + if len(line) == 0: + continue + if line.startswith("[") and line.endswith("]") and len(buffer) > 0: + yield buffer + buffer.clear() + buffer.append(line) + yield buffer + buffer.clear() + + +class ConanLibraryMetadata(TypedDict): + libs: List[str] + includedirs: List[str] + libdirs: List[str] + bindirs: List[str] + resdirs: List[str] + builddirs: List[str] + system_libs: List[str] + defines: List[str] + cppflags: List[str] + cxxflags: List[str] + cflags: List[str] + sharedlinkflags: List[str] + exelinkflags: List[str] + sysroot: List[str] + frameworks: List[str] + frameworkdirs: List[str] + rootpath: List[str] + name: List[str] + version: List[str] + generatornames: List[str] + generatorfilenames: List[str] + + +class AbsConanBuildInfo(abc.ABC): + @abc.abstractmethod + def parse(self, filename: str) -> Dict[str, str]: + pass + + +class ConanBuildInfo(TypedDict): + definitions: List[str] + include_paths: List[str] + lib_paths: List[str] + bin_paths: List[str] + libs: List[str] + metadata: Dict[str, ConanLibraryMetadata] + + +class ConanBuildInfoTXT(AbsConanBuildInfo): + + def parse(self, filename: str) -> ConanBuildInfo: + # def parse(self, filename: str) -> Dict[str, Union[str, List[str]]]: + with open(filename, "r", encoding="utf-8") as f: + parser = ConanBuildInfoParser(f) + data = parser.parse() + definitions = data['defines'] + include_paths = data['includedirs'] + lib_paths = data['libdirs'] + bin_paths = data['bindirs'] + libs = data['libs'] + names: List[str] = [ + value.replace("name_", "") + for value in data + if value.startswith("name_") + ] + # print(names) + libsmetadata: Dict[str, ConanLibraryMetadata] = {} + for library_name in names: + version = data.get(f"version_{library_name}", None) + libsmetadata[library_name] = { + "libs": data.get(f"libs_{library_name}", []), + "includedirs": data.get(f"includedirs_{library_name}", []), + "libdirs": data.get(f"libdirs_{library_name}", []), + "bindirs": data.get(f"bindirs_{library_name}", []), + "resdirs": data.get(f"resdirs_{library_name}", []), + "builddirs": data.get(f"builddirs_{library_name}", []), + "system_libs": + data.get(f"system_libs_{library_name}", []), + "defines": + data.get(f"defines_{library_name}", []), + "cppflags": + data.get(f"cppflags_{library_name}", []), + "cxxflags": + data.get(f"cxxflags_{library_name}", []), + "cflags": data.get(f"cflags_{library_name}", []), + "sharedlinkflags": + data.get(f"sharedlinkflags_{library_name}", []), + "exelinkflags": + data.get(f"exelinkflags_{library_name}", []), + "sysroot": data.get(f"sysroot_{library_name}", []), + "frameworks": + data.get(f"frameworks_{library_name}", []), + "frameworkdirs": + data.get(f"frameworkdirs_{library_name}", []), + "rootpath": data.get(f"rootpath_{library_name}", []), + "name": library_name, + "version": version[0] if version else None, + "generatornames": + data.get(f"generatornames_{library_name}", []), + "generatorfilenames": + data.get(f"generatorfilenames_{library_name}", []), + } + return { + "definitions": definitions, + "include_paths": list(include_paths), + "lib_paths": list(lib_paths), + "bin_paths": list(bin_paths), + "libs": list(libs), + "metadata": libsmetadata + + } diff --git a/uiucprescon/build/conan/v1.py b/uiucprescon/build/conan/v1.py new file mode 100644 index 0000000..91fc20f --- /dev/null +++ b/uiucprescon/build/conan/v1.py @@ -0,0 +1,211 @@ +"""Version 1 of conan api. + +This was written before conan has a stable Python API which was introduced in +version 2. This code only works with Conan version 1 because it calls on code +that was removed from conan in version 2. +""" + +import logging +import os +import sys +import re +import platform +import shutil +import subprocess +from pathlib import Path +from typing import List, Optional +from conans.client import conan_api, conf # pylint: disable=import-error + +from uiucprescon.build.compiler_info import ( # pylint: disable=import-error + get_compiler_name +) +from .files import ConanBuildInfoTXT + +__all__ = ['build_deps_with_conan'] + + +def build_deps_with_conan( + conanfile: str, + build_dir: str, + install_dir: str, + compiler_libcxx: str, + compiler_version: str, + conan_cache: Optional[str] = None, + conan_options: Optional[List[str]] = None, + debug: bool = False, + install_libs=True, + build=None, + announce=None +): + + conan = conan_api.Conan(cache_folder=os.path.abspath(conan_cache)) + settings = [] + logger = logging.Logger(__name__) + conan_profile_cache = os.path.join(build_dir, "profiles") + build = build or ['outdated'] + for name, value in conf.detect.detect_defaults_settings( + logger, + conan_profile_cache + ): + settings.append(f"{name}={value}") + if debug is True: + settings.append("build_type=Debug") + else: + settings.append("build_type=Release") + try: + compiler_name = get_compiler_name() + settings.append(f"compiler={compiler_name}") + if compiler_libcxx is not None: + if 'compiler.libcxx=libstdc' in settings: + settings.remove('compiler.libcxx=libstdc') + settings.append(f'compiler.libcxx={compiler_libcxx}') + settings.append(f"compiler.version={compiler_version}") + if compiler_name == 'gcc': + pass + elif compiler_name == "msvc": + settings.append("compiler.cppstd=14") + settings.append("compiler.runtime=dynamic") + elif compiler_name == "Visual Studio": + settings.append("compiler.runtime=MD") + settings.append("compiler.toolset=v142") + except AttributeError: + print( + f"Unable to get compiler information " + f"for {platform.python_compiler()}" + ) + raise + + conanfile_path = os.path.abspath('.') + # conanfile_path = os.path.abspath( + # os.path.join(os.path.dirname(__file__), "..") + # ) + + ninja = shutil.which("ninja") + env = [] + if ninja: + env.append(f"NINJA={ninja}") + + profile_host = conan_api.ProfileData(profiles=None, settings=settings, options=None, + env=env, conf=None) + conan.install( + options=conan_options, + cwd=os.path.abspath(build_dir), + settings=settings, + profile_build=profile_host, + build=build if len(build) > 0 else None, + path=conanfile_path, + env=env, + no_imports=not install_libs, + ) + if install_libs: + import_manifest = os.path.join( + build_dir, + 'conan_imports_manifest.txt' + ) + if os.path.exists(import_manifest): + add_conan_imports( + import_manifest, + path=build_dir, + dest=install_dir + ) +# + conaninfotext = os.path.join(build_dir, "conaninfo.txt") + if os.path.exists(conaninfotext) and announce: + with open(conaninfotext, "r", encoding="utf-8") as r: + announce(r.read(), 5) + build_locations = [ + build_dir, + os.path.join(build_dir, "Release") + ] + conanbuildinfotext = locate_conanbuildinfo(build_locations) + if conanbuildinfotext is None: + raise AssertionError("Missing conanbuildinfo.txt") + metadata_strategy = ConanBuildInfoTXT() + metadata_strategy.parse(conanbuildinfotext) + + +def fixup_library(shared_library): + if sys.platform == "darwin": + otool = shutil.which("otool") + install_name_tool = shutil.which('install_name_tool') + if not all([otool, install_name_tool]): + raise FileNotFoundError( + "Unable to fixed up because required tools are missing. " + "Make sure that otool and install_name_tool are on " + "the PATH." + ) + dylib_regex = re.compile( + r'^(?P([@a-zA-Z./_])+)' + r'/' + r'(?Plib[a-zA-Z/.0-9]+\.dylib)' + ) + for line in subprocess.check_output( + [otool, "-L", shared_library], + encoding="utf8" + ).split("\n"): + if any( + [ + line.strip() == "", # it's an empty line + str(shared_library) in line, # it's the same library + "/usr/lib/" in line, # it's a system library + + ] + ): + continue + value = dylib_regex.match(line.strip()) + try: + original_path = value.group("path") + library_name = value.group("file").strip() + except AttributeError as e: + raise ValueError(f"unable to parse {line}") from e + command = [ + install_name_tool, + "-change", + os.path.join(original_path, library_name), + os.path.join("@loader_path", library_name), + str(shared_library) + ] + subprocess.check_call(command) + + +def add_conan_imports(import_manifest_file: str, path: str, dest: str): + libs = [] + with open(import_manifest_file, "r", encoding="utf8") as f: + for line in f.readlines(): + if ":" not in line: + continue + + try: + file_name, _ = line.strip().split(": ") + except ValueError: + print(f"Failed to parse: {line.strip()}") + raise + libs.append(file_name) + for file_name in libs: + file_path = Path(os.path.join(path, file_name)) + if not file_path.exists(): + raise FileNotFoundError(f"Missing {file_name}") + lib = str(file_path) + fixup_library(lib) + output = Path(os.path.join(dest, file_path.name)) + if output.exists(): + output.unlink() + shutil.copy(file_path, dest, follow_symlinks=False) + if file_path.is_symlink(): + continue + + +def locate_conanbuildinfo(search_locations): + for location in search_locations: + conanbuildinfo = os.path.join(location, "conanbuildinfo.txt") + if os.path.exists(conanbuildinfo): + return conanbuildinfo + return None + + +def locate_conanbuildinfo_json(search_locations): + for location in search_locations: + conanbuildinfo = os.path.join(location, "conanbuildinfo.json") + if os.path.exists(conanbuildinfo): + return conanbuildinfo + return None diff --git a/uiucprescon/build/conan/v2.py b/uiucprescon/build/conan/v2.py new file mode 100644 index 0000000..d3a8f04 --- /dev/null +++ b/uiucprescon/build/conan/v2.py @@ -0,0 +1,90 @@ +from __future__ import annotations +from typing import List, Optional, TYPE_CHECKING +from conans.client.cache.cache import ClientCache +from conan.api.conan_api import ConanAPI +import dataclasses +if TYPE_CHECKING: + from uiucprescon.build.conan_libs import ConanBuildInfo + + +@dataclasses.dataclass +class ProfileArg: + settings_build: Optional[str] = None + options_build: Optional[str] = None + conf_build: Optional[str] = None + profile_host: List[str] = dataclasses.field(default_factory=list) + build_requires: Optional[str] = None + settings_host: Optional[str] = None + options_host: Optional[str] = None + conf_host: Optional[str] = None + profile_build: Optional[str] = None + + +def build_deps_with_conan( + conanfile: str, + build_dir: str, + install_dir: str, + compiler_libcxx: str, + compiler_version: str, + conan_cache: Optional[str] = None, + conan_options: Optional[List[str]] = None, + debug: bool = False, + install_libs=True, + build=None, + announce=None +) -> ConanBuildInfo: + def run(conan_path): + conan_api = ConanAPI() + remotes = conan_api.remotes.list() + profile_host = conan_api.profiles.detect() + profile_host.process_settings(ClientCache(conan_path)) + + profile_build = conan_api.profiles.detect() + profile_build.process_settings(ClientCache(conan_path)) + + deps_graph = conan_api.graph.load_graph_consumer( + path=conanfile, + name="CompressorRecipe", + version="123", + user="args.user", + channel="args.channel", + profile_host=profile_host, + profile_build=profile_build, + lockfile=None, + remotes=remotes, + update=False, + is_build_require=False + ) + conan_api.graph.analyze_binaries( + deps_graph, + build_mode=["missing"], + remotes=remotes + ) + conan_api.install.install_binaries(deps_graph, remotes) + data = {} + for dep in deps_graph.root.dependencies: + conan_file = dep.dst.conanfile + data = {**data, **conan_file.cpp_info.serialize()} + return data + data = run(conan_cache) + + definitions = [] + libs = [] + include_paths = [] + lib_paths = [] + bin_paths = [] + for dep in data.values(): + include_paths += dep['includedirs'] + lib_paths += dep['libdirs'] + bin_paths += dep['bindirs'] + if dep['defines'] is not None: + definitions += dep['defines'] + libs += dep['libs'] + return { + "definitions": definitions, + "include_paths": include_paths, + "lib_paths": lib_paths, + "bin_paths": bin_paths, + "libs": libs, + "metadata": {} + } diff --git a/uiucprescon/build/conan_libs.py b/uiucprescon/build/conan_libs.py index c4c760a..6fa0ad4 100644 --- a/uiucprescon/build/conan_libs.py +++ b/uiucprescon/build/conan_libs.py @@ -1,50 +1,23 @@ -import logging import os import re import subprocess import sys import shutil import abc -from typing import Iterable, Any, Dict, List, Union, Optional +from typing import Dict, List, Optional import setuptools -import platform from distutils import ccompiler from pathlib import Path from uiucprescon.build.deps import get_win_deps from uiucprescon.build.compiler_info import ( get_compiler_version, - get_compiler_name ) import json -from distutils.dist import Distribution +from setuptools.dist import Distribution import toml -from conans.client import conan_api, conf - - -class ConanBuildInfoParser: - def __init__(self, fp): - self._fp = fp - - def parse(self) -> Dict[str, List[str]]: - data = dict() - for subject_chunk in self.iter_subject_chunk(): - subject_title = subject_chunk[0][1:-1] - - data[subject_title] = subject_chunk[1:] - return data - - def iter_subject_chunk(self) -> Iterable[Any]: - buffer = [] - for line in self._fp: - line = line.strip() - if len(line) == 0: - continue - if line.startswith("[") and line.endswith("]") and len(buffer) > 0: - yield buffer - buffer.clear() - buffer.append(line) - yield buffer - buffer.clear() +from uiucprescon.build.conan import conan_api +from uiucprescon.build.conan.files import ConanBuildInfo, ConanBuildInfoParser +__all__ = ['ConanBuildInfoParser'] class AbsConanBuildInfo(abc.ABC): @@ -53,70 +26,6 @@ def parse(self, filename: str) -> Dict[str, str]: pass -class ConanBuildInfoTXT(AbsConanBuildInfo): - - def parse(self, filename: str) -> Dict[str, Union[str, List[str]]]: - with open(filename, "r") as f: - parser = ConanBuildInfoParser(f) - data = parser.parse() - definitions = data['defines'] - include_paths = data['includedirs'] - lib_paths = data['libdirs'] - bin_paths = data['bindirs'] - libs = data['libs'] - names = [] - for value in data.keys(): - if not value.startswith("name_"): - continue - names.append(value.replace("name_", "")) - # print(names) - libsmetadata = {} - for library_name in names: - version = data.get(f"version_{library_name}", None) - libsmetadata[library_name] = { - "libs": data.get(f"libs_{library_name}", []), - "includedirs": data.get(f"includedirs_{library_name}", []), - "libdirs": data.get(f"libdirs_{library_name}", []), - "bindirs": data.get(f"bindirs_{library_name}", []), - "resdirs": data.get(f"resdirs_{library_name}", []), - "builddirs": data.get(f"builddirs_{library_name}", []), - "system_libs": - data.get(f"system_libs_{library_name}", []), - "defines": - data.get(f"defines_{library_name}", []), - "cppflags": - data.get(f"cppflags_{library_name}", []), - "cxxflags": - data.get(f"cxxflags_{library_name}", []), - "cflags": data.get(f"cflags_{library_name}", []), - "sharedlinkflags": - data.get(f"sharedlinkflags_{library_name}", []), - "exelinkflags": - data.get(f"exelinkflags_{library_name}", []), - "sysroot": data.get(f"sysroot_{library_name}", []), - "frameworks": - data.get(f"frameworks_{library_name}", []), - "frameworkdirs": - data.get(f"frameworkdirs_{library_name}", []), - "rootpath": data.get(f"rootpath_{library_name}", []), - "name": library_name, - "version": version[0] if version else None, - "generatornames": - data.get(f"generatornames_{library_name}", []), - "generatorfilenames": - data.get(f"generatorfilenames_{library_name}", []), - } - return { - "definitions": definitions, - "include_paths": list(include_paths), - "lib_paths": list(lib_paths), - "bin_paths": list(bin_paths), - "libs": list(libs), - "metadata": libsmetadata - - } - - class AbsResultTester(abc.ABC): def __init__(self, compiler=None) -> None: self.compiler = compiler or ccompiler.new_compiler() @@ -222,7 +131,7 @@ def update_extension(extension, metadata): extension.define_macros = define_macros + extension.define_macros -def update_extension2(extension, text_md): +def update_extension2(extension, text_md: ConanBuildInfo): include_dirs = text_md['include_paths'] library_dirs = text_md['lib_paths'] define_macros = [(d, None) for d in text_md.get('definitions', [])] @@ -271,6 +180,7 @@ def initialize_options(self): self.conan_cache = None self.compiler_version = None self.compiler_libcxx = None + self.conanfile = None def __init__(self, dist, **kw): self.install_libs = True @@ -351,45 +261,50 @@ def run(self): if not os.path.exists(build_dir_full_path): self.mkpath(build_dir_full_path) self.announce(f"Using {conan_cache} for conan cache", 5) - build_deps_with_conan( - build_dir, + metadata = build_deps_with_conan( + conanfile=self.conanfile, + build_dir=build_dir, install_dir=os.path.abspath(install_dir), compiler_libcxx=self.compiler_libcxx, compiler_version=self.compiler_version, conan_options=get_conan_options(), conan_cache=conan_cache, - install_libs=self.install_libs + install_libs=self.install_libs, + announce=self.announce + ) - conaninfotext = os.path.join(build_dir, "conaninfo.txt") - if os.path.exists(conaninfotext): - with open(conaninfotext) as r: - self.announce(r.read(), 5) - build_locations = [ - build_dir, - os.path.join(build_dir, "Release") - ] - conanbuildinfotext = locate_conanbuildinfo(build_locations) - if conanbuildinfotext is None: - raise AssertionError("Missing conanbuildinfo.txt") - metadata_strategy = ConanBuildInfoTXT() - text_md = metadata_strategy.parse(conanbuildinfotext) build_ext_cmd = self.get_finalized_command("build_ext") for extension in build_ext_cmd.extensions: if build_ext._inplace: extension.runtime_library_dirs.append( os.path.abspath(install_dir) ) - update_extension2(extension, text_md) + update_extension2(extension, metadata) extension.library_dirs.insert(0, install_dir) if sys.platform == "darwin": extension.runtime_library_dirs.append("@loader_path") elif sys.platform == "linux": if "$ORIGIN" not in extension.runtime_library_dirs: extension.runtime_library_dirs.append("$ORIGIN") - # else: - # pprint(text_md) - # raise Exception(text_md) - # if sys.platform == "Windows": + + +def _get_source_root(dist: Distribution): + project_files = ["pyproject.toml", "setup.py"] + path = dist.src_root or os.curdir + for project_file in project_files: + project_file_path = Path(path, project_file) + if os.path.exists(project_file_path): + return os.path.abspath(path) + return os.path.abspath(path) + + +def _find_conanfile(path: str) -> Optional[str]: + conanfile_types = ["conanfile.py"] + for conanfile_type in conanfile_types: + conanfile = os.path.join(path, conanfile_type) + if os.path.exists(conanfile): + return conanfile + return None def build_conan( @@ -400,7 +315,11 @@ def build_conan( ): dist = Distribution() dist.parse_config_files() + + source_root = _get_source_root(dist) + command = BuildConan(dist) + command.conanfile = _find_conanfile(path=source_root) if metadata_directory is not None: build_py = command.get_finalized_command("build_py") build_py.build_lib = wheel_directory @@ -411,6 +330,7 @@ def build_conan( command.install_libs = install_libs build_ext_cmd = command.get_finalized_command("build_ext") conan_cache = None + if config_settings: conan_cache = config_settings.get('conan_cache') command.conan_cache = conan_cache @@ -454,6 +374,7 @@ def get_pyproject_toml_data(): def build_deps_with_conan( + conanfile: str, build_dir: str, install_dir: str, compiler_libcxx: str, @@ -462,75 +383,22 @@ def build_deps_with_conan( conan_options: Optional[List[str]] = None, debug: bool = False, install_libs=True, - build=None + build=None, + announce=None ): - - conan = conan_api.Conan(cache_folder=os.path.abspath(conan_cache)) - settings = [] - logger = logging.Logger(__name__) - conan_profile_cache = os.path.join(build_dir, "profiles") - build = build or ['outdated'] - for name, value in conf.detect.detect_defaults_settings( - logger, - conan_profile_cache - ): - settings.append(f"{name}={value}") - if debug is True: - settings.append("build_type=Debug") - else: - settings.append("build_type=Release") - try: - compiler_name = get_compiler_name() - settings.append(f"compiler={compiler_name}") - if compiler_libcxx is not None: - if 'compiler.libcxx=libstdc' in settings: - settings.remove('compiler.libcxx=libstdc') - settings.append(f'compiler.libcxx={compiler_libcxx}') - settings.append(f"compiler.version={compiler_version}") - if compiler_name == 'gcc': - pass - elif compiler_name == "msvc": - settings.append("compiler.cppstd=14") - settings.append("compiler.runtime=dynamic") - elif compiler_name == "Visual Studio": - settings.append("compiler.runtime=MD") - settings.append("compiler.toolset=v142") - except AttributeError: - print( - f"Unable to get compiler information " - f"for {platform.python_compiler()}" - ) - raise - - conanfile_path = os.path.abspath('.') - # conanfile_path = os.path.abspath( - # os.path.join(os.path.dirname(__file__), "..") - # ) - - ninja = shutil.which("ninja") - env = [] - if ninja: - env.append(f"NINJA={ninja}") - conan.install( - options=conan_options, - cwd=os.path.abspath(build_dir), - settings=settings, - build=build if len(build) > 0 else None, - path=conanfile_path, - env=env, - no_imports=not install_libs, + return conan_api.build_deps_with_conan( + conanfile, + build_dir, + install_dir, + compiler_libcxx, + compiler_version, + conan_cache, + conan_options, + debug, + install_libs, + build, + announce ) - if install_libs: - import_manifest = os.path.join( - build_dir, - 'conan_imports_manifest.txt' - ) - if os.path.exists(import_manifest): - add_conan_imports( - import_manifest, - path=build_dir, - dest=install_dir - ) def fixup_library(shared_library): @@ -602,17 +470,3 @@ def add_conan_imports(import_manifest_file: str, path: str, dest: str): shutil.copy(file_path, dest, follow_symlinks=False) if file_path.is_symlink(): continue - - -def locate_conanbuildinfo(search_locations): - for location in search_locations: - conanbuildinfo = os.path.join(location, "conanbuildinfo.txt") - if os.path.exists(conanbuildinfo): - return conanbuildinfo - - -def locate_conanbuildinfo_json(search_locations): - for location in search_locations: - conanbuildinfo = os.path.join(location, "conanbuildinfo.json") - if os.path.exists(conanbuildinfo): - return conanbuildinfo diff --git a/uiucprescon/build/local_backend.py b/uiucprescon/build/local_backend.py index a2f1c43..1a13aec 100644 --- a/uiucprescon/build/local_backend.py +++ b/uiucprescon/build/local_backend.py @@ -28,7 +28,6 @@ def build_wheel( os.environ["CONAN_USER_HOME"], ".conan" ) - conan_libs.build_conan( wheel_directory, config_settings,