From afecdda78c392542f006397d638e52265665b2d0 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Thu, 24 Oct 2024 13:53:27 -0500 Subject: [PATCH] ci: simplify jenkins Dockerfile --- Jenkinsfile | 329 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 238 insertions(+), 91 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2da5cfea..ac273205 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -203,7 +203,7 @@ def testPythonPackages(){ script{ def windowsTests = [:] SUPPORTED_WINDOWS_VERSIONS.each{ pythonVersion -> - if(params.INCLUDE_WINDOWS_X86_64 == true){ + if(params.INCLUDE_WINDOWS-X86_64 == true){ windowsTests["Windows - Python ${pythonVersion}-x86: sdist"] = { testPythonPkg( agent: [ @@ -279,10 +279,10 @@ def testPythonPackages(){ def linuxTests = [:] SUPPORTED_LINUX_VERSIONS.each{ pythonVersion -> def architectures = [] - if(params.INCLUDE_LINUX_X86_64 == true){ + if(params.INCLUDE_LINUX-X86_64 == true){ architectures.add('x86_64') } - if(params.INCLUDE_LINUX_ARM == true){ + if(params.INCLUDE_LINUX-ARM64 == true){ architectures.add('arm') } // As of 12/7/2023 there are no prebuilt binary wheel on ARM64 for @@ -387,10 +387,10 @@ def testPythonPackages(){ SUPPORTED_MAC_VERSIONS.each{ pythonVersion -> def architectures = [] - if(params.INCLUDE_MACOS_X86_64 == true){ + if(params.INCLUDE_MACOS-X86_64 == true){ architectures.add('x86_64') } - if(params.INCLUDE_MACOS_ARM == true){ + if(params.INCLUDE_MACOS-ARM64 == true){ architectures.add('m1') } architectures.each{ processorArchitecture -> @@ -509,11 +509,11 @@ def get_props(){ def packageMetadata = readJSON( text: { if (isUnix()){ - return sh(returnStdout: true, script: 'python -c \'import tomllib;print(tomllib.load(open("pyproject.toml", "rb"))["project"])\'').trim() + return sh(returnStdout: true, script: 'python -c \'import tomllib;print(tomllib.load(open("pyproject.toml", "rb"))["project"])\'').trim() } else { return bat(returnStdout: true, script: '@python -c "import tomllib;print(tomllib.load(open(\'pyproject.toml\', \'rb\'))[\'project\'])').trim() } - + }() ) @@ -537,6 +537,33 @@ def get_sonarqube_unresolved_issues(report_task_file){ } } +def installMSVCRuntime(cacheLocation){ + def cachedFile = "${cacheLocation}\\vc_redist.x64.exe".replaceAll(/\\\\+/, '\\\\') + withEnv( + [ + "CACHED_FILE=${cachedFile}", + "RUNTIME_DOWNLOAD_URL=https://aka.ms/vs/17/release/vc_redist.x64.exe" + ] + ){ + lock("${cachedFile}-${env.NODE_NAME}"){ + powershell( + label: 'Ensuring vc_redist runtime installer is available', + script: '''if ([System.IO.File]::Exists("$Env:CACHED_FILE")) + { + Write-Host 'Found installer' + } else { + Write-Host 'No installer found' + Write-Host 'Downloading runtime' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;Invoke-WebRequest "$Env:RUNTIME_DOWNLOAD_URL" -OutFile "$Env:CACHED_FILE" + } + ''' + ) + } + powershell(label: 'Install VC Runtime', script: 'Start-Process -filepath "$Env:CACHED_FILE" -ArgumentList "/install", "/passive", "/norestart" -Passthru | Wait-Process;') + } +} + + def hasSonarCreds(credentialsId){ try{ @@ -559,11 +586,11 @@ pipeline { booleanParam(name: 'BUILD_PACKAGES', defaultValue: false, description: 'Build Packages') booleanParam(name: 'TEST_STANDALONE_PACKAGE_DEPLOYMENT', defaultValue: true, description: 'Test deploying any packages that are designed to be installed without using Python directly') booleanParam(name: 'BUILD_CHOCOLATEY_PACKAGE', defaultValue: false, description: 'Build package for chocolatey package manager') - booleanParam(name: 'INCLUDE_LINUX_ARM', defaultValue: false, description: 'Include ARM architecture for Linux') - booleanParam(name: 'INCLUDE_LINUX_X86_64', defaultValue: true, description: 'Include x86_64 architecture for Linux') - booleanParam(name: 'INCLUDE_MACOS_ARM', defaultValue: false, description: 'Include ARM(m1) architecture for Mac') - booleanParam(name: 'INCLUDE_MACOS_X86_64', defaultValue: false, description: 'Include x86_64 architecture for Mac') - booleanParam(name: 'INCLUDE_WINDOWS_X86_64', defaultValue: true, description: 'Include x86_64 architecture for Windows') + booleanParam(name: 'INCLUDE_LINUX-ARM64', defaultValue: false, description: 'Include ARM architecture for Linux') + booleanParam(name: 'INCLUDE_LINUX-X86_64', defaultValue: true, description: 'Include x86_64 architecture for Linux') + booleanParam(name: 'INCLUDE_MACOS-ARM64', defaultValue: false, description: 'Include ARM(m1) architecture for Mac') + booleanParam(name: 'INCLUDE_MACOS-X86_64', defaultValue: false, description: 'Include x86_64 architecture for Mac') + booleanParam(name: 'INCLUDE_WINDOWS-X86_64', defaultValue: true, description: 'Include x86_64 architecture for Windows') booleanParam(name: 'TEST_PACKAGES', defaultValue: true, description: 'Test Python packages by installing them and running tests on the installed package') booleanParam(name: 'PACKAGE_MAC_OS_STANDALONE_DMG', defaultValue: false, description: 'Create a Apple Application Bundle DMG') booleanParam(name: 'PACKAGE_WINDOWS_STANDALONE_MSI', defaultValue: false, description: 'Create a standalone wix based .msi installer') @@ -575,66 +602,66 @@ pipeline { booleanParam(name: 'DEPLOY_DOCS', defaultValue: false, description: 'Update online documentation') } stages { - stage('Build Sphinx Documentation'){ - agent { - docker{ - image 'sphinxdoc/sphinx-latexpdf' - label 'linux && docker && x86' - } - } - options { - retry(conditions: [agent()], count: 2) - } - environment{ - PIP_CACHE_DIR = '/tmp/pipcache' - UV_INDEX_STRATEGY = 'unsafe-best-match' - UV_TOOL_DIR = '/tmp/uvtools' - UV_PYTHON_INSTALL_DIR = '/tmp/uvpython' - UV_CACHE_DIR = '/tmp/uvcache' - UV_PYTHON = '3.11' - } - steps { - catchError(buildResult: 'UNSTABLE', message: 'Sphinx has warnings', stageResult: 'UNSTABLE') { - sh(label: 'Build docs in html and Latex formats', - script:'''python3 -m venv venv - trap "rm -rf venv" EXIT - . ./venv/bin/activate - pip install uv - uvx --from sphinx --with-editable . --with-requirements requirements-dev.txt sphinx-build -W --keep-going -b html -d build/docs/.doctrees -w logs/build_sphinx_html.log docs/source build/docs/html - uvx --from sphinx --with-editable . --with-requirements requirements-dev.txt sphinx-build -W --keep-going -b latex -d build/docs/.doctrees docs/source build/docs/latex - ''') - sh(label: 'Building PDF docs', - script: '''make -C build/docs/latex - mkdir -p dist/docs - mv build/docs/latex/*.pdf dist/docs/ - ''' - ) - } - } - post{ - always{ - recordIssues(tools: [sphinxBuild(pattern: 'logs/build_sphinx_html.log')]) - } - success{ - stash includes: 'dist/docs/*.pdf', name: 'SPEEDWAGON_DOC_PDF' - zip archive: true, dir: 'build/docs/html', glob: '', zipFile: "dist/${props.name}-${props.version}.doc.zip" - stash includes: 'dist/*.doc.zip,build/docs/html/**', name: 'DOCS_ARCHIVE' - archiveArtifacts artifacts: 'dist/docs/*.pdf' - } - cleanup{ - cleanWs( - notFailBuild: true, - deleteDirs: true, - patterns: [ - [pattern: 'logs/', type: 'INCLUDE'], - [pattern: 'venv/', type: 'INCLUDE'], - [pattern: 'dist/', type: 'INCLUDE'], - [pattern: 'build/', type: 'INCLUDE'], - ] - ) - } - } - } +// stage('Build Sphinx Documentation'){ +// agent { +// docker{ +// image 'sphinxdoc/sphinx-latexpdf' +// label 'linux && docker && x86' +// } +// } +// options { +// retry(conditions: [agent()], count: 2) +// } +// environment{ +// PIP_CACHE_DIR = '/tmp/pipcache' +// UV_INDEX_STRATEGY = 'unsafe-best-match' +// UV_TOOL_DIR = '/tmp/uvtools' +// UV_PYTHON_INSTALL_DIR = '/tmp/uvpython' +// UV_CACHE_DIR = '/tmp/uvcache' +// UV_PYTHON = '3.11' +// } +// steps { +// catchError(buildResult: 'UNSTABLE', message: 'Sphinx has warnings', stageResult: 'UNSTABLE') { +// sh(label: 'Build docs in html and Latex formats', +// script:'''python3 -m venv venv +// trap "rm -rf venv" EXIT +// . ./venv/bin/activate +// pip install uv +// uvx --from sphinx --with-editable . --with-requirements requirements-dev.txt sphinx-build -W --keep-going -b html -d build/docs/.doctrees -w logs/build_sphinx_html.log docs/source build/docs/html +// uvx --from sphinx --with-editable . --with-requirements requirements-dev.txt sphinx-build -W --keep-going -b latex -d build/docs/.doctrees docs/source build/docs/latex +// ''') +// sh(label: 'Building PDF docs', +// script: '''make -C build/docs/latex +// mkdir -p dist/docs +// mv build/docs/latex/*.pdf dist/docs/ +// ''' +// ) +// } +// } +// post{ +// always{ +// recordIssues(tools: [sphinxBuild(pattern: 'logs/build_sphinx_html.log')]) +// } +// success{ +// stash includes: 'dist/docs/*.pdf', name: 'SPEEDWAGON_DOC_PDF' +// zip archive: true, dir: 'build/docs/html', glob: '', zipFile: "dist/${props.name}-${props.version}.doc.zip" +// stash includes: 'dist/*.doc.zip,build/docs/html/**', name: 'DOCS_ARCHIVE' +// archiveArtifacts artifacts: 'dist/docs/*.pdf' +// } +// cleanup{ +// cleanWs( +// notFailBuild: true, +// deleteDirs: true, +// patterns: [ +// [pattern: 'logs/', type: 'INCLUDE'], +// [pattern: 'venv/', type: 'INCLUDE'], +// [pattern: 'dist/', type: 'INCLUDE'], +// [pattern: 'build/', type: 'INCLUDE'], +// ] +// ) +// } +// } +// } stage('Checks'){ stages{ stage('Code Quality'){ @@ -894,39 +921,159 @@ pipeline { equals expected: true, actual: params.TEST_RUN_TOX } parallel{ - stage('Linux'){ + stage('Linux') { when{ - expression {return nodesByLabel('linux && docker && x86').size() > 0} + expression {return nodesByLabel('linux && docker').size() > 0} + } + environment{ + PIP_CACHE_DIR='/tmp/pipcache' + UV_INDEX_STRATEGY='unsafe-best-match' + UV_TOOL_DIR='/tmp/uvtools' + UV_PYTHON_INSTALL_DIR='/tmp/uvpython' + UV_CACHE_DIR='/tmp/uvcache' } steps{ script{ + def envs = [] + node('docker && linux'){ + docker.image('python').inside('--mount source=python-tmp-speedwagon,target=/tmp'){ + try{ + checkout scm + sh(script: 'python3 -m venv venv && venv/bin/pip install uv') + envs = sh( + label: 'Get tox environments', + script: './venv/bin/uvx --quiet --with tox-uv tox list -d --no-desc', + returnStdout: true, + ).trim().split('\n') + } finally{ + cleanWs( + patterns: [ + [pattern: 'venv/', type: 'INCLUDE'], + [pattern: '.tox', type: 'INCLUDE'], + [pattern: '**/__pycache__/', type: 'INCLUDE'], + ] + ) + } + } + } parallel( - getToxTestsParallel( - envNamePrefix: 'Tox Linux', - label: 'linux && docker && x86', - dockerfile: 'ci/docker/python/linux/tox/Dockerfile', - dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip --build-arg UV_EXTRA_INDEX_URL --build-arg UV_CACHE_DIR=/.cache/uv', - dockerRunArgs: '-v pipcache_speedwagon:/.cache/pip -v uvcache_speedwagon:/.cache/uv', - retry: 2 - ) + envs.collectEntries{toxEnv -> + def version = toxEnv.replaceAll(/py(\d)(\d+).*/, '$1.$2') + [ + "Tox Environment: ${toxEnv}", + { + node('docker && linux && x86_64'){ + checkout scm + def image = docker.build(UUID.randomUUID().toString(), '-f ci/docker/python/linux/jenkins/Dockerfile .') + try{ + image.inside('--mount source=python-tmp-speedwagon,target=/tmp'){ + try{ + sh( label: 'Running Tox', + script: """python3 -m venv venv && venv/bin/pip install uv + . ./venv/bin/activate + uv python install cpython-${version} + uvx -p ${version} --with tox-uv tox run -e ${toxEnv} + """ + ) + } catch(e) { + sh(script: '''. ./venv/bin/activate + uv python list + ''' + ) + throw e + } finally{ + cleanWs( + patterns: [ + [pattern: 'venv/', type: 'INCLUDE'], + [pattern: '.tox', type: 'INCLUDE'], + [pattern: '**/__pycache__/', type: 'INCLUDE'], + ] + ) + } + } + } finally { + sh "docker image rm --force ${image.imageName()}" + } + } + } + ] + } ) } - } + } } - stage('Windows'){ + stage('Windows') { when{ expression {return nodesByLabel('windows && docker && x86').size() > 0} } + environment{ + UV_INDEX_STRATEGY='unsafe-best-match' + PIP_CACHE_DIR='C:\\Users\\ContainerUser\\Documents\\pipcache' + UV_TOOL_DIR='C:\\Users\\ContainerUser\\Documents\\uvtools' + UV_PYTHON_INSTALL_DIR='C:\\Users\\ContainerUser\\Documents\\uvpython' + UV_CACHE_DIR='C:\\Users\\ContainerUser\\Documents\\uvcache' + VC_RUNTIME_INSTALLER_LOCATION='c:\\msvc_runtime\\' + } steps{ script{ + def envs = [] + node('docker && windows'){ + docker.image('python').inside('--mount source=python-tmp-speedwagon,target=C:\\Users\\ContainerUser\\Documents'){ + try{ + checkout scm + bat(script: 'python -m venv venv && venv\\Scripts\\pip install uv') + envs = bat( + label: 'Get tox environments', + script: '@.\\venv\\Scripts\\uvx --quiet --with tox-uv tox list -d --no-desc', + returnStdout: true, + ).trim().split('\r\n') + } finally{ + cleanWs( + patterns: [ + [pattern: 'venv/', type: 'INCLUDE'], + [pattern: '.tox', type: 'INCLUDE'], + [pattern: '**/__pycache__/', type: 'INCLUDE'], + ] + ) + } + } + } parallel( - getToxTestsParallel( - envNamePrefix: 'Tox Windows', - label: 'windows && docker && x86', - dockerfile: 'ci/docker/python/windows/tox/Dockerfile', - dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg UV_EXTRA_INDEX_URL --build-arg CHOCOLATEY_SOURCE --build-arg chocolateyVersion --build-arg PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip --build-arg UV_CACHE_DIR=c:/users/containeradministrator/appdata/local/uv', - retry: 2 - ) + envs.collectEntries{toxEnv -> + def version = toxEnv.replaceAll(/py(\d)(\d+).*/, '$1.$2') + [ + "Tox Environment: ${toxEnv}", + { + node('docker && windows'){ + docker.image('python').inside('--mount source=python-tmp-speedwagon,target=C:\\Users\\ContainerUser\\Documents --mount source=msvc-runtime,target=$VC_RUNTIME_INSTALLER_LOCATION'){ + installMSVCRuntime(env.VC_RUNTIME_INSTALLER_LOCATION) + checkout scm + try{ + bat(label: 'Install uv', + script: 'python -m venv venv && venv\\Scripts\\pip install uv' + ) + retry(3){ + bat(label: 'Running Tox', + script: """call venv\\Scripts\\activate.bat + uv python install cpython-${version} + uvx -p ${version} --with tox-uv tox run -e ${toxEnv} + """ + ) + } + } finally{ + cleanWs( + patterns: [ + [pattern: 'venv/', type: 'INCLUDE'], + [pattern: '.tox', type: 'INCLUDE'], + [pattern: '**/__pycache__/', type: 'INCLUDE'], + ] + ) + } + } + } + } + ] + } ) } }