From d43643a13ede5d44c699cbaba634eeb4442f7a9a Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Thu, 24 Oct 2024 14:08:31 -0500 Subject: [PATCH] ci: simplify jenkins Dockerfile --- Jenkinsfile | 441 +++++++------------- ci/docker/python/linux/tox/Dockerfile | 53 --- ci/docker/python/linux/tox/apt-packages.txt | 18 - 3 files changed, 152 insertions(+), 360 deletions(-) delete mode 100644 ci/docker/python/linux/tox/Dockerfile delete mode 100644 ci/docker/python/linux/tox/apt-packages.txt diff --git a/Jenkinsfile b/Jenkinsfile index ac2732050..74608ef99 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -199,292 +199,6 @@ def startup(){ } -def testPythonPackages(){ - script{ - def windowsTests = [:] - SUPPORTED_WINDOWS_VERSIONS.each{ pythonVersion -> - if(params.INCLUDE_WINDOWS-X86_64 == true){ - windowsTests["Windows - Python ${pythonVersion}-x86: sdist"] = { - testPythonPkg( - agent: [ - dockerfile: [ - label: 'windows && docker && x86', - filename: 'ci/docker/python/windows/tox/Dockerfile', - additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg UV_EXTRA_INDEX_URL --build-arg CHOCOLATEY_SOURCE --build-arg chocolateyVersion --build-arg PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip --build-arg UV_CACHE_DIR=c:/users/containeradministrator/appdata/local/uv', - args: '-v pipcache_speedwagon:c:/users/containeradministrator/appdata/local/pip -v uvcache_speedwagon:c:/users/containeradministrator/appdata/local/uv' - ] - ], - retries: 3, - testSetup: { - checkout scm - unstash 'PYTHON_PACKAGES' - }, - testCommand: { - findFiles(glob: 'dist/*.tar.gz,dist/*.zip').each{ - powershell(label: 'Running Tox', script: "tox --installpkg ${it.path} --workdir \$env:TEMP\\tox -e py${pythonVersion.replace('.', '')}-PySide6") - } - - }, - post:[ - cleanup: { - cleanWs( - patterns: [ - [pattern: 'dist/', type: 'INCLUDE'], - [pattern: '**/__pycache__/', type: 'INCLUDE'], - ], - notFailBuild: true, - deleteDirs: true - ) - }, - ] - ) - } - windowsTests["Windows - Python ${pythonVersion}-x86: wheel"] = { - testPythonPkg( - agent: [ - dockerfile: [ - label: 'windows && docker && x86', - filename: 'ci/docker/python/windows/tox/Dockerfile', - additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg UV_EXTRA_INDEX_URL --build-arg CHOCOLATEY_SOURCE --build-arg chocolateyVersion --build-arg PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip --build-arg UV_CACHE_DIR=c:/users/containeradministrator/appdata/local/uv', - args: '-v pipcache_speedwagon:c:/users/containeradministrator/appdata/local/pip -v uvcache_speedwagon:c:/users/containeradministrator/appdata/local/uv' - ] - ], - retries: 3, - testSetup: { - checkout scm - unstash 'PYTHON_PACKAGES' - }, - testCommand: { - findFiles(glob: 'dist/*.whl').each{ - powershell(label: 'Running Tox', script: "tox --installpkg ${it.path} --workdir \$env:TEMP\\tox -e py${pythonVersion.replace('.', '')}-PySide6") - } - - }, - post:[ - cleanup: { - cleanWs( - patterns: [ - [pattern: 'dist/', type: 'INCLUDE'], - [pattern: '**/__pycache__/', type: 'INCLUDE'], - ], - notFailBuild: true, - deleteDirs: true - ) - }, - ] - ) - } - } - } - def linuxTests = [:] - SUPPORTED_LINUX_VERSIONS.each{ pythonVersion -> - def architectures = [] - if(params.INCLUDE_LINUX-X86_64 == true){ - architectures.add('x86_64') - } - if(params.INCLUDE_LINUX-ARM64 == true){ - architectures.add('arm') - } - // As of 12/7/2023 there are no prebuilt binary wheel on ARM64 for - // Python versions prior to 3.11 so there is no reason to run - // tox tests that have a GUI. - def linuxToxEnvironments = [ - "3.8": [ - "x86_64": "py38-PySide6", - "arm": "py38" - ], - "3.9": [ - "x86_64": "py39-PySide6", - "arm": "py39" - ], - "3.10": [ - "x86_64": "py310-PySide6", - "arm": "py310" - ], - "3.11": [ - "x86_64": "py311-PySide6", - "arm": "py311-PySide6" - ], - ] - architectures.each{ processorArchitecture -> - linuxTests["Linux-${processorArchitecture} - Python ${pythonVersion}: sdist"] = { - testPythonPkg( - agent: [ - dockerfile: [ - label: "linux && docker && ${processorArchitecture}", - filename: 'ci/docker/python/linux/tox/Dockerfile', - additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip --build-arg UV_EXTRA_INDEX_URL --build-arg UV_CACHE_DIR=/.cache/uv', - args: '-v pipcache_speedwagon:/.cache/pip -v uvcache_speedwagon:/.cache/uv' - ] - ], - retries: 3, - testSetup: { - checkout scm - unstash 'PYTHON_PACKAGES' - }, - testCommand: { - findFiles(glob: 'dist/*.tar.gz').each{ - sh( - label: 'Running Tox', - script: "tox --installpkg ${it.path} --workdir /tmp/tox -e ${linuxToxEnvironments[pythonVersion][processorArchitecture]}" - ) - } - }, - post:[ - cleanup: { - cleanWs( - patterns: [ - [pattern: 'dist/', type: 'INCLUDE'], - [pattern: '**/__pycache__/', type: 'INCLUDE'], - ], - notFailBuild: true, - deleteDirs: true - ) - }, - ] - ) - } - linuxTests["Linux-${processorArchitecture} - Python ${pythonVersion}: wheel"] = { - testPythonPkg( - agent: [ - dockerfile: [ - label: "linux && docker && ${processorArchitecture}", - filename: 'ci/docker/python/linux/tox/Dockerfile', - additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip --build-arg UV_EXTRA_INDEX_URL --build-arg UV_CACHE_DIR=/.cache/uv', - args: '-v pipcache_speedwagon:/.cache/pip -v uvcache_speedwagon:/.cache/uv' - ] - ], - retries: 3, - testSetup: { - checkout scm - unstash 'PYTHON_PACKAGES' - }, - testCommand: { - findFiles(glob: 'dist/*.whl').each{ - sh( - label: 'Running Tox', - script: "tox --installpkg ${it.path} --workdir /tmp/tox -e ${linuxToxEnvironments[pythonVersion][processorArchitecture]}" - ) - } - }, - post:[ - cleanup: { - cleanWs( - patterns: [ - [pattern: 'dist/', type: 'INCLUDE'], - [pattern: '**/__pycache__/', type: 'INCLUDE'], - ], - notFailBuild: true, - deleteDirs: true - ) - }, - ] - ) - } - } - } - def macTests = [:] - - SUPPORTED_MAC_VERSIONS.each{ pythonVersion -> - def architectures = [] - if(params.INCLUDE_MACOS-X86_64 == true){ - architectures.add('x86_64') - } - if(params.INCLUDE_MACOS-ARM64 == true){ - architectures.add('m1') - } - architectures.each{ processorArchitecture -> - macTests["Mac - ${processorArchitecture} - Python ${pythonVersion}: wheel"] = { - testPythonPkg( - agent: [ - label: "mac && python${pythonVersion} && ${processorArchitecture}", - ], - retries: 3, - testSetup: { - checkout scm - unstash 'PYTHON_PACKAGES' - }, - testCommand: { - findFiles(glob: 'dist/*.whl').each{ - sh(label: 'Running Tox', - script: """python${pythonVersion} -m venv venv - . ./venv/bin/activate - python -m pip install --upgrade pip uv - UV_INDEX_STRATEGY=unsafe-best-match uv pip install -r requirements/requirements_tox.txt tox-uv - UV_INDEX_STRATEGY=unsafe-best-match tox --installpkg ${it.path} -e py${pythonVersion.replace('.', '')}-PySide6""" - ) - } - - }, - post:[ - cleanup: { - cleanWs( - patterns: [ - [pattern: 'dist/', type: 'INCLUDE'], - [pattern: 'venv/', type: 'INCLUDE'], - [pattern: '.tox/', type: 'INCLUDE'], - ], - notFailBuild: true, - deleteDirs: true - ) - }, - ] - ) - } - macTests["Mac - ${processorArchitecture} - Python ${pythonVersion}: sdist"] = { - testPythonPkg( - agent: [ - label: "mac && python${pythonVersion} && ${processorArchitecture}", - ], - retries: 3, - testSetup: { - checkout scm - unstash 'PYTHON_PACKAGES' - }, - testCommand: { - findFiles(glob: 'dist/*.tar.gz').each{ - sh(label: 'Running Tox', - script: """python${pythonVersion} -m venv venv - . ./venv/bin/activate - python -m pip install --upgrade pip uv - UV_INDEX_STRATEGY=unsafe-best-match uv pip install -r requirements/requirements_tox.txt tox-uv - UV_INDEX_STRATEGY=unsafe-best-match tox --installpkg ${it.path} -e py${pythonVersion.replace('.', '')}-PySide6""" - ) - } - - }, - post:[ - cleanup: { - cleanWs( - patterns: [ - [pattern: 'dist/', type: 'INCLUDE'], - [pattern: 'venv/', type: 'INCLUDE'], - [pattern: '.tox/', type: 'INCLUDE'], - ], - notFailBuild: true, - deleteDirs: true - ) - }, - ] - ) - } - } - } - parallel(linuxTests + windowsTests + macTests) - } -} -def buildPackages(){ - timeout(5){ - withEnv(['PIP_NO_CACHE_DIR=off']) { - sh(label: 'Building Python Package', - script: '''python -m venv venv --upgrade-deps - venv/bin/pip install build - venv/bin/python -m build . - ''' - ) - } - } -} def testChocolateyPackage(){ stage('Install'){ @@ -1147,9 +861,158 @@ pipeline { when{ equals expected: true, actual: params.TEST_PACKAGES } - steps{ - testPythonPackages() - } + matrix { + axes { + axis { + name 'PYTHON_VERSION' + values '3.8', '3.9', '3.10', '3.11', '3.12' + } + axis { + name 'OS' + values 'linux', 'macos', 'windows' + } + axis { + name 'ARCHITECTURE' + values 'arm64', 'x86_64' + } + axis { + name 'PACKAGE_TYPE' + values 'wheel', 'sdist' + } + } + excludes { + exclude { + axis { + name 'ARCHITECTURE' + values 'arm64' + } + axis { + name 'OS' + values 'windows' + } + } + } + when{ + expression{ + params.containsKey("INCLUDE_${OS}-${ARCHITECTURE}".toUpperCase()) && params["INCLUDE_${OS}-${ARCHITECTURE}".toUpperCase()] + } + } + options { + retry(conditions: [agent()], count: 2) + } + environment{ + UV_PYTHON="${PYTHON_VERSION}" + TOX_ENV="py${PYTHON_VERSION.replace('.', '')}" + UV_INDEX_STRATEGY='unsafe-best-match' + } + stages { + stage('Test Package in container') { + when{ + expression{['linux', 'windows'].contains(OS)} + beforeAgent true + } + environment{ + PIP_CACHE_DIR="${isUnix() ? '/tmp/pipcache': 'C:\\Users\\ContainerUser\\Documents\\pipcache'}" + UV_TOOL_DIR="${isUnix() ? '/tmp/uvtools': 'C:\\Users\\ContainerUser\\Documents\\uvtools'}" + UV_PYTHON_INSTALL_DIR="${isUnix() ? '/tmp/uvpython': 'C:\\Users\\ContainerUser\\Documents\\uvpython'}" + UV_CACHE_DIR="${isUnix() ? '/tmp/uvcache': 'C:\\Users\\ContainerUser\\Documents\\uvcache'}" + } + agent { + docker { + image 'python' + label "${OS} && ${ARCHITECTURE} && docker" + args "--mount source=python-tmp-speedwagon,target=${['windows'].contains(OS) ? 'C:\\Users\\ContainerUser\\Documents': '/tmp'} ${['windows'].contains(OS) ? '--mount source=msvc-runtime,target=c:\\msvc_runtime\\': ''}" + } + } + steps { + unstash 'PYTHON_PACKAGES' + script{ + withEnv( + ["TOX_INSTALL_PKG=${findFiles(glob: PACKAGE_TYPE == 'wheel' ? 'dist/*.whl' : 'dist/*.tar.gz')[0].path}"] + ) { + if(isUnix()){ + sh( + label: 'Testing with tox', + script: '''python3 -m venv venv + . ./venv/bin/activate + trap "rm -rf venv" EXIT + pip install uv + uvx --with tox-uv tox + ''' + ) + } else { + installMSVCRuntime('c:\\msvc_runtime\\') + bat( + label: 'Install uv', + script: '''python -m venv venv + call venv\\Scripts\\activate.bat + pip install uv + ''' + ) + script{ + retry(3){ + bat( + label: 'Testing with tox', + script: '''call venv\\Scripts\\activate.bat + uvx --with tox-uv tox + ''' + ) + } + } + } + } + } + } + post{ + cleanup{ + cleanWs( + patterns: [ + [pattern: 'dist/', type: 'INCLUDE'], + [pattern: 'venv/', type: 'INCLUDE'], + [pattern: '**/__pycache__/', type: 'INCLUDE'], + ] + ) + } + } + } + stage('Test Package directly on agent') { + when{ + expression{['macos'].contains(OS)} + beforeAgent true + } + agent { + label "${OS} && ${ARCHITECTURE}" + } + steps { + unstash 'PYTHON_PACKAGES' + withEnv( + ["TOX_INSTALL_PKG=${findFiles(glob: PACKAGE_TYPE == 'wheel' ? 'dist/*.whl' : 'dist/*.tar.gz')[0].path}"] + ) { + sh( + label: 'Testing with tox', + script: '''python3 -m venv venv + trap "rm -rf venv" EXIT + . ./venv/bin/activate + pip install uv + uvx --with tox-uv tox + ''' + ) + } + } + post{ + cleanup{ + cleanWs( + patterns: [ + [pattern: 'dist/', type: 'INCLUDE'], + [pattern: 'venv/', type: 'INCLUDE'], + [pattern: '**/__pycache__/', type: 'INCLUDE'], + ] + ) + } + } + } + } + } } } } diff --git a/ci/docker/python/linux/tox/Dockerfile b/ci/docker/python/linux/tox/Dockerfile deleted file mode 100644 index 06120b29d..000000000 --- a/ci/docker/python/linux/tox/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -ARG PIP_DOWNLOAD_CACHE=/.cache/pip -ARG UV_CACHE_DIR=/.cache/uv -ARG UV_EXTRA_INDEX_URL - -ARG PIPX_HOME=/pipx - -FROM ubuntu:22.04 AS wheel_builder_base -COPY ci/docker/python/linux/tox/apt-packages.txt /tmp/ -RUN apt-get update && \ - apt-get install -y software-properties-common gpg-agent --no-install-recommends && \ - add-apt-repository ppa:deadsnakes/ppa && \ - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata && \ - apt-get update && \ - < /tmp/apt-packages.txt xargs apt-get install -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV QT_QPA_PLATFORM="offscreen" - -ARG PIP_EXTRA_INDEX_URL -ARG PIP_INDEX_URL - -COPY requirements/ /tmp/python_requirements/requirements/ -COPY requirements-dev.txt /tmp/python_requirements -ARG UV_EXTRA_INDEX_URL -ENV UV_INDEX_STRATEGY=unsafe-best-match -RUN --mount=type=cache,target=/.cache/python \ - python3 -m pip install --cache-dir=/.cache/python/pip --upgrade pip && \ - python3 -m pip install --cache-dir=/.cache/python/pip --upgrade uv && \ - uv pip install --cache-dir=/.cache/python/uv --system -r /tmp/python_requirements/requirements-dev.txt tox-uv - -ARG PIP_DOWNLOAD_CACHE -ARG UV_CACHE_DIR -RUN mkdir -p ${PIP_DOWNLOAD_CACHE} && \ - chmod -R 777 ${PIP_DOWNLOAD_CACHE} && \ - mkdir -p ${UV_CACHE_DIR} && \ - chmod -R 777 ${UV_CACHE_DIR} - -ARG PIPX_HOME -ENV PIPX_HOME=${PIPX_HOME} -ENV PIPX_BIN_DIR=${PIPX_HOME}/bin -RUN --mount=type=cache,target=/.cache/python \ - python3 -m uv pip install --cache-dir=/.cache/python/uv --system --no-cache-dir pipx && \ - pipx ensurepath && \ - mkdir -p $PIPX_HOME && chmod -R 777 $PIPX_HOME - -ARG UV_CACHE_DIR -ARG PIP_DOWNLOAD_CACHE -ENV UV_CACHE_DIR=${UV_CACHE_DIR} -ENV PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE} - -WORKDIR /src -CMD ["tox", "--workdir", "/tmp/tox", "--recreate", "-p=auto"] diff --git a/ci/docker/python/linux/tox/apt-packages.txt b/ci/docker/python/linux/tox/apt-packages.txt deleted file mode 100644 index 145bc9022..000000000 --- a/ci/docker/python/linux/tox/apt-packages.txt +++ /dev/null @@ -1,18 +0,0 @@ -python3.8-venv -python3.8-dev -python3-pip -python3.9-dev -python3.9-venv -python3.9-distutils -python3.10-dev -python3.10-distutils -python3.10-venv -python3.11-dev -python3.11-distutils -python3.11-venv -libxml2-dev -libxslt-dev -libgl1-mesa-dev -xvfb -libxkbcommon-x11-0 -x11-utils