From d32c969cee5c06b02efaa36ba9ddee7908f92076 Mon Sep 17 00:00:00 2001 From: henry Date: Wed, 11 Oct 2023 14:16:12 -0500 Subject: [PATCH] CI: tox stage runs on macs --- Jenkinsfile | 136 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 104 insertions(+), 32 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8107d1f..64ccfab 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,24 +1,86 @@ def getMacToxTestsParallel(args = [:]){ def nodeLabel = args['label'] args.remove('label') - def jobs = [:] - node(label){ + + def envNamePrefix = args['envNamePrefix'] + args.remove('envNamePrefix') + + def retries = args.get('retry', 1) + if(args.containsKey('retry')){ + args.remove('retry') + } + if(args.size() > 0){ + error "getMacToxTestsParallel has invalid arguments ${args.keySet()}" + } + + // ============================================= + def envs = [:] + node(nodeLabel){ try{ + checkout scm sh( script: '''python3 -m venv venv --upgrade-deps venv/bin/pip install tox ''' ) - def envs = sh( - label: "Getting Tox Environments", + def toxEnvs = sh( + label: 'Getting Tox Environments', returnStdout: true, - script: "tox -l" + script: 'venv/bin/tox list -d --no-desc' ).trim().split('\n') + toxEnvs.each({env -> + def requiredPythonVersion = sh( + label: "Getting required python version for Tox Environment: ${env}", + script: "venv/bin/tox config -e ${env} -k py_dot_ver | grep 'py_dot_ver =' | sed -E 's/py_dot_ver = ([0-9].[0-9]+)/\\1/g'", + returnStdout: true + ).trim() + envs[env] = requiredPythonVersion + }) + } finally { sh 'rm -rf venv' } } + echo "Found tox environments for ${envs.keySet().join(', ')}" + def jobs = envs.collectEntries({ toxEnv, requiredPythonVersion -> + def jenkinsStageName = "${envNamePrefix} ${toxEnv}" + def jobNodeLabel = "${nodeLabel} && python${requiredPythonVersion}" + if(nodesByLabel(jobNodeLabel).size() > 0){ + [jenkinsStageName,{ + retry(retries){ + node(jobNodeLabel){ + try{ + checkout scm + sh( + script: '''python3 -m venv venv --upgrade-deps + venv/bin/pip install tox + ''' + ) + sh( + label: 'Getting Tox Environments', + script: "venv/bin/tox --list-dependencies --workdir=.tox run -e ${toxEnv}" + ) + + } finally { + cleanWs( + notFailBuild: true, + deleteDirs: true, + patterns: [ + [pattern: '**/__pycache__/', type: 'INCLUDE'], + [pattern: 'venv/', type: 'INCLUDE'], + [pattern: '.tox/', type: 'INCLUDE'], + ] + ) + } + } + } + + }] + } else { + echo "Unable to add ${toxEnv} because no nodes with required labels: ${jobNodeLabel}" + } + }) return jobs } pipeline { @@ -210,35 +272,45 @@ pipeline { '' ) def linuxJobs = [:] - if(nodesByLabel("linux && docker").size() > 0){ - linuxJobs = tox.getToxTestsParallel( - envNamePrefix: 'Tox Linux', - label: 'linux && docker', - dockerfile: 'ci/docker/linux/tox/Dockerfile', - dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL', - dockerRunArgs: "-v pipcache_uiucprescon_build:/.cache/pip", - retry: 2 - ) - } else { - echo 'No nodes with the following labels: linux && docker labels' - } + def windowsJobs = [:] - if(nodesByLabel('windows && docker && x86').size() > 0){ - windowsJobs = tox.getToxTestsParallel( - envNamePrefix: 'Tox Windows', - label: 'windows && docker && x86', - dockerfile: 'ci/docker/windows/tox/Dockerfile', - dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE', - dockerRunArgs: "-v pipcache_uiucprescon_build:c:/users/containeradministrator/appdata/local/pip", - retry: 2 - ) - } else { - echo 'No nodes with the following labels: windows && docker && x86' - } def macJobs = [:] - if(nodesByLabel('mac && python3 && x86').size() > 0){ - macJobs = getMacToxTestsParallel(label: 'mac && python3 && x86') - } + parallel( + 'Tox Information Gathering For: Linux': { + if(nodesByLabel("linux && docker").size() > 0){ + linuxJobs = tox.getToxTestsParallel( + envNamePrefix: 'Tox Linux', + label: 'linux && docker', + dockerfile: 'ci/docker/linux/tox/Dockerfile', + dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL', + dockerRunArgs: "-v pipcache_uiucprescon_build:/.cache/pip", + retry: 2 + ) + } else { + echo 'No nodes with the following labels: linux && docker labels' + } + }, + 'Tox Information Gathering For: Windows': { + if(nodesByLabel('windows && docker').size() > 0){ + windowsJobs = tox.getToxTestsParallel( + envNamePrefix: 'Tox Windows', + label: 'windows && docker', + dockerfile: 'ci/docker/windows/tox/Dockerfile', + dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE', + dockerRunArgs: "-v pipcache_uiucprescon_build:c:/users/containeradministrator/appdata/local/pip", + retry: 2 + ) + } else { + echo 'No nodes with the following labels: windows && docker && x86' + } + }, + 'Tox Information Gathering For: mac': { + if(nodesByLabel('mac && python3').size() > 0){ + macJobs = macJobs + getMacToxTestsParallel(label: 'mac && python3', envNamePrefix: 'mac') + } + }, + ) + parallel(linuxJobs + windowsJobs + macJobs) } }