From 9283ea1a0373c0d670e1bd22762ac90cccac9d76 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 8 Oct 2024 15:22:51 -0500 Subject: [PATCH] ci: remove devpi support --- Jenkinsfile | 466 +------------------------ ci/docker/linux/tox/Dockerfile | 3 +- ci/docker/windows/tox/Dockerfile | 3 +- ci/docker/windows/tox_no_vs/Dockerfile | 3 +- ci/jenkins/scripts/devpi.groovy | 314 ----------------- 5 files changed, 5 insertions(+), 784 deletions(-) delete mode 100644 ci/jenkins/scripts/devpi.groovy diff --git a/Jenkinsfile b/Jenkinsfile index c9ca2481..4402de3b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,22 +17,6 @@ def getPypiConfig() { } } -def getDevpiConfig() { - node(){ - configFileProvider([configFile(fileId: 'devpi_config', variable: 'CONFIG_FILE')]) { - def configProperties = readProperties(file: CONFIG_FILE) - configProperties.stagingIndex = { - if (env.TAG_NAME?.trim()){ - return 'tag_staging' - } else{ - return "${env.BRANCH_NAME}_staging" - } - }() - return configProperties - } - } -} -def DEVPI_CONFIG = getDevpiConfig() def getToxStages(){ script{ def windowsJobs = [:] @@ -70,176 +54,6 @@ def getToxStages(){ } } -def getMacDevpiTestStages(packageName, packageVersion, pythonVersions, devpiServer, devpiCredentialsId, devpiIndex) { - node(){ - checkout scm - devpi = load('ci/jenkins/scripts/devpi.groovy') - } - def macPackageStages = [:] - def architectures = [] - if(params.INCLUDE_MACOS_X86_64 == true){ - architectures.add('x86_64') - } - if(params.INCLUDE_MACOS_ARM == true){ - architectures.add("arm64") - } - pythonVersions.each{pythonVersion -> - architectures.each{ processorArchitecture -> - macPackageStages["MacOS-${processorArchitecture} - Python ${pythonVersion}: wheel"] = { - withEnv(['PATH+EXTRA=./venv/bin']) { - devpi.testDevpiPackage( - agent: [ - label: "mac && python${pythonVersion} && ${processorArchitecture} && devpi-access" - ], - devpi: [ - index: devpiIndex, - server: devpiServer, - credentialsId: devpiCredentialsId, - devpiExec: 'venv/bin/devpi' - ], - package:[ - name: packageName, - version: packageVersion, - selector: "(${pythonVersion.replace('.','')}).*(-*macosx_*).*(${processorArchitecture}\\.whl)" - ], - test:[ - setup: { - checkout scm - sh( - label:'Installing Devpi client', - script: '''python3 -m venv venv - . ./venv/bin/activate - python -m pip install pip --upgrade - python -m pip install 'devpi-client<7.0' -r requirements/requirements-tox.txt - ''' - ) - }, - toxEnv: "py${pythonVersion}".replace('.',''), - teardown: { - sh( label: 'Remove Devpi client', script: 'rm -r venv') - } - ] - ) - } - } - macPackageStages["MacOS-${processorArchitecture} - Python ${pythonVersion}: sdist"]= { - withEnv(['PATH+EXTRA=./venv/bin']) { - devpi.testDevpiPackage( - agent: [ - label: "mac && python${pythonVersion} && ${processorArchitecture} && devpi-access" - ], - devpi: [ - index: devpiIndex, - server: devpiServer, - credentialsId: devpiCredentialsId, - devpiExec: 'venv/bin/devpi' - ], - package:[ - name: packageName, - version: packageVersion, - selector: 'tar.gz' - ], - test:[ - setup: { - checkout scm - sh( - label:'Installing Devpi client', - script: '''python3 -m venv venv - . ./venv/bin/activate - python -m pip install pip --upgrade - python -m pip install 'devpi-client<7.0' -r requirements/requirements-tox.txt - ''' - ) - }, - toxEnv: "py${pythonVersion}".replace('.',''), - teardown: { - sh( label: 'Remove Devpi client', script: 'rm -r venv') - } - ] - ) - } - } - } - if(params.INCLUDE_MACOS_X86_64 && params.INCLUDE_MACOS_ARM && pythonVersion != '3.8'){ - macPackageStages["MacOS x86_64 - Python ${pythonVersion}: universal2 wheel"] = { - withEnv(['PATH+EXTRA=./venv/bin']) { - devpi.testDevpiPackage( - agent: [ - label: "mac && python${pythonVersion} && x86 && devpi-access" - ], - devpi: [ - index: devpiIndex, - server: devpiServer, - credentialsId: devpiCredentialsId, - devpiExec: 'venv/bin/devpi' - ], - package:[ - name: packageName, - version: packageVersion, - selector: "(${pythonVersion.replace('.','')}).*(-*macosx_*).*(universal2\\.whl)" - ], - test:[ - setup: { - checkout scm - sh( - label:'Installing Devpi client', - script: '''python3 -m venv venv - . ./venv/bin/activate - python -m pip install pip --upgrade - python -m pip install 'devpi-client<7.0' -r requirements/requirements-tox.txt - ''' - ) - }, - toxEnv: "py${pythonVersion}".replace('.',''), - teardown: { - sh( label: 'Remove Devpi client', script: 'rm -r venv') - } - ] - ) - } - } - macPackageStages["MacOS m1 - Python ${pythonVersion}: universal2 wheel"] = { - withEnv(['PATH+EXTRA=./venv/bin']) { - devpi.testDevpiPackage( - agent: [ - label: "mac && python${pythonVersion} && m1 && devpi-access" - ], - devpi: [ - index: devpiIndex, - server: devpiServer, - credentialsId: devpiCredentialsId, - devpiExec: 'venv/bin/devpi' - ], - package:[ - name: packageName, - version: packageVersion, - selector: "(${pythonVersion.replace('.','')}).*(-*macosx_*).*(universal2\\.whl)" - ], - test:[ - setup: { - checkout scm - sh( - label:'Installing Devpi client', - script: '''python3 -m venv venv - . ./venv/bin/activate - python -m pip install pip --upgrade - python -m pip install 'devpi-client<7.0' -r requirements/requirements-tox.txt - ''' - ) - }, - toxEnv: "py${pythonVersion}".replace('.',''), - teardown: { - sh( label: 'Remove Devpi client', script: 'rm -r venv') - } - ] - ) - } - } - } - } - return macPackageStages; -} - wheelStashes = [] @@ -951,8 +765,6 @@ pipeline { booleanParam(name: 'INCLUDE_LINUX_X86_64', defaultValue: true, description: 'Include x86_64 architecture for Linux') booleanParam(name: 'INCLUDE_WINDOWS_X86_64', defaultValue: true, description: 'Include x86_64 architecture for Windows') booleanParam(name: 'TEST_PACKAGES', defaultValue: true, description: 'Test Python packages by installing them and running tests on the installed package') - booleanParam(name: 'DEPLOY_DEVPI', defaultValue: false, description: "Deploy to devpi on ${DEVPI_CONFIG.server}/${DEVPI_CONFIG.devpiUserName}/${env.BRANCH_NAME}") - booleanParam(name: 'DEPLOY_DEVPI_PRODUCTION', defaultValue: false, description: "Deploy to ${DEVPI_CONFIG.server}/production/release") booleanParam(name: 'DEPLOY_PYPI', defaultValue: false, description: 'Deploy to pypi') booleanParam(name: 'DEPLOY_DOCS', defaultValue: false, description: 'Update online documentation') } @@ -962,7 +774,6 @@ pipeline { anyOf{ equals expected: true, actual: params.RUN_CHECKS equals expected: true, actual: params.TEST_RUN_TOX - equals expected: true, actual: params.DEPLOY_DEVPI } } stages{ @@ -975,10 +786,7 @@ pipeline { } } when{ - anyOf{ - equals expected: true, actual: params.RUN_CHECKS - equals expected: true, actual: params.DEPLOY_DEVPI - } + equals expected: true, actual: params.RUN_CHECKS beforeAgent true } stages{ @@ -1277,11 +1085,7 @@ pipeline { } stage('Python Packaging'){ when{ - anyOf{ - equals expected: true, actual: params.BUILD_PACKAGES - equals expected: true, actual: params.DEPLOY_DEVPI - equals expected: true, actual: params.DEPLOY_DEVPI_PRODUCTION - } + equals expected: true, actual: params.BUILD_PACKAGES } failFast true parallel{ @@ -1550,272 +1354,6 @@ pipeline { } } } - stage('Deploy to DevPi') { - agent none - options{ - lock('uiucprescon.ocr-devpi') - } - when { - allOf{ - equals expected: true, actual: params.DEPLOY_DEVPI - anyOf { - equals expected: 'master', actual: env.BRANCH_NAME - equals expected: 'dev', actual: env.BRANCH_NAME - tag '*' - } - } - beforeAgent true - beforeOptions true - } - stages{ - stage('Upload to DevPi Staging'){ - agent { - dockerfile { - filename 'ci/docker/linux/tox/Dockerfile' - label 'linux && docker && x86 && devpi-access' - additionalBuildArgs '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip --build-arg UV_CACHE_DIR=/.cache/uv' - args '-v pipcache_tesseractglue:/.cache/pip -v uvcache_tesseractglue:/.cache/uv' - } - } - options{ - retry(3) - } - steps { - script{ - unstash 'DOCS_ARCHIVE' - wheelStashes.each{ - unstash it - } - load('ci/jenkins/scripts/devpi.groovy').upload( - server: DEVPI_CONFIG.server, - credentialsId: DEVPI_CONFIG.credentialsId, - index: DEVPI_CONFIG.stagingIndex, - ) - } - } - post{ - cleanup{ - cleanWs( - notFailBuild: true - ) - } - } - } - stage('Test DevPi packages') { - steps{ - script{ - def devpi = null - node(){ - checkout scm - devpi = load('ci/jenkins/scripts/devpi.groovy') - } - def macPackages = getMacDevpiTestStages(props.Name, props.Version, SUPPORTED_MAC_VERSIONS, DEVPI_CONFIG.server, DEVPI_CONFIG.credentialsId, DEVPI_CONFIG.stagingIndex) - def windowsPackages = [:] - SUPPORTED_WINDOWS_VERSIONS.each{pythonVersion -> - if(params.INCLUDE_WINDOWS_X86_64 == true){ - windowsPackages["Windows - Python ${pythonVersion}: sdist"] = { - retry(3){ - devpi.testDevpiPackage( - agent: [ - dockerfile: [ - filename: 'ci/docker/windows/tox/Dockerfile', - additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE --build-arg chocolateyVersion --build-arg PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip --build-arg UV_CACHE_DIR=c:/users/ContainerUser/appdata/local/uv', - label: 'windows && docker && x86 && devpi-access' - ] - ], - dockerImageName: "${currentBuild.fullProjectName}_devpi_with_msvc".replaceAll('-', '_').replaceAll('/', '_').replaceAll(' ', '').toLowerCase(), - devpi: [ - index: DEVPI_CONFIG.stagingIndex, - server: DEVPI_CONFIG.server, - credentialsId: DEVPI_CONFIG.credentialsId, - ], - package:[ - name: props.Name, - version: props.Version, - selector: 'tar.gz' - ], - test:[ - toxEnv: "py${pythonVersion}".replace('.',''), - ] - ) - } - } - windowsPackages["Test Python ${pythonVersion}: wheel Windows"] = { - retry(3){ - devpi.testDevpiPackage( - agent: [ - dockerfile: [ - filename: 'ci/docker/windows/tox_no_vs/Dockerfile', - additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE --build-arg chocolateyVersion --build-arg PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip --build-arg UV_CACHE_DIR=c:/users/ContainerUser/appdata/local/uv', - label: 'windows && docker && x86 && devpi-access' - ] - ], - devpi: [ - index: DEVPI_CONFIG.stagingIndex, - server: DEVPI_CONFIG.server, - credentialsId: DEVPI_CONFIG.credentialsId, - ], - dockerImageName: "${currentBuild.fullProjectName}_devpi_without_msvc".replaceAll('-', '_').replaceAll('/', '_').replaceAll(' ', '').toLowerCase(), - package:[ - name: props.Name, - version: props.Version, - selector: "(${pythonVersion.replace('.','')}).*(win_amd64\\.whl)" - ], - test:[ - toxEnv: "py${pythonVersion}".replace('.',''), - ] - ) - } - } - } - } - def linuxPackages = [:] - SUPPORTED_LINUX_VERSIONS.each{pythonVersion -> - if(params.INCLUDE_LINUX_X86_64 == true){ - linuxPackages["Linux - Python ${pythonVersion}: sdist"] = { - retry(3){ - devpi.testDevpiPackage( - agent: [ - dockerfile: [ - filename: 'ci/docker/linux/tox/Dockerfile', - additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip --build-arg UV_CACHE_DIR=/.cache/uv', - label: 'linux && docker && x86 && devpi-access', - args: '-v pipcache_tesseractglue:/.cache/pip -v uvcache_tesseractglue:/.cache/uv', - ] - ], - devpi: [ - index: DEVPI_CONFIG.stagingIndex, - server: DEVPI_CONFIG.server, - credentialsId: DEVPI_CONFIG.credentialsId, - ], - package:[ - name: props.Name, - version: props.Version, - selector: 'tar.gz' - ], - test:[ - toxEnv: "py${pythonVersion}".replace('.',''), - ] - ) - } - } - linuxPackages["Linux - Python ${pythonVersion}: wheel"] = { - retry(3){ - devpi.testDevpiPackage( - agent: [ - dockerfile: [ - filename: 'ci/docker/linux/tox/Dockerfile', - additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip --build-arg UV_CACHE_DIR=/.cache/uv', - label: 'linux && docker && x86 && devpi-access', - args: '-v pipcache_tesseractglue:/.cache/pip -v uvcache_tesseractglue:/.cache/uv', - ] - ], - devpi: [ - index: DEVPI_CONFIG.stagingIndex, - server: DEVPI_CONFIG.server, - credentialsId: DEVPI_CONFIG.credentialsId, - ], - package:[ - name: props.Name, - version: props.Version, - selector: "(${pythonVersion.replace('.','')}).+(manylinux).+x86" - ], - test:[ - toxEnv: "py${pythonVersion}".replace('.',''), - ] - ) - } - } - } - } - parallel(windowsPackages + linuxPackages + macPackages) - } - } - } - stage('Deploy to DevPi Production') { - when { - allOf{ - equals expected: true, actual: params.DEPLOY_DEVPI_PRODUCTION - anyOf { - branch 'master' - tag '*' - } - } - beforeAgent true - beforeInput true - } - options{ - timeout(time: 1, unit: 'DAYS') - } - input { - message 'Release to DevPi Production?' - } - agent { - dockerfile { - filename 'ci/docker/linux/tox/Dockerfile' - label 'linux && docker && devpi-access' - additionalBuildArgs '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip --build-arg UV_CACHE_DIR=/.cache/uv' - args '-v pipcache_tesseractglue:/.cache/pip -v uvcache_tesseractglue:/.cache/uv' - } - } - steps { - script{ - echo 'Pushing to production/release index' - load('ci/jenkins/scripts/devpi.groovy').pushPackageToIndex( - pkgName: props.Name, - pkgVersion: props.Version, - server: DEVPI_CONFIG.server, - indexSource: "${DEVPI_CONFIG.devpiUserName}/${DEVPI_CONFIG.stagingIndex}", - indexDestination: 'production/release', - credentialsId: DEVPI_CONFIG.credentialsId - ) - } - } - } - } - post { - success{ - node('linux && docker && devpi-access') { - script{ - if (!env.TAG_NAME?.trim()){ - checkout scm - def dockerImage = docker.build('ocr:devpi','-f ./ci/docker/linux/tox/Dockerfile --build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip .') - dockerImage.inside{ - load('ci/jenkins/scripts/devpi.groovy').pushPackageToIndex( - pkgName: props.Name, - pkgVersion: props.Version, - server: DEVPI_CONFIG.server, - indexSource: "${DEVPI_CONFIG.devpiUserName}/${DEVPI_CONFIG.stagingIndex}", - indexDestination: "${DEVPI_CONFIG.devpiUserName}/${env.BRANCH_NAME}", - credentialsId: DEVPI_CONFIG.credentialsId - ) - } - sh script: "docker image rm --no-prune ${dockerImage.imageName()}" - } - } - } - } - cleanup{ - node('linux && docker && devpi-access') { - script{ - checkout scm - def dockerImage = docker.build('ocr:devpi','-f ./ci/docker/linux/tox/Dockerfile --build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip .') - dockerImage.inside{ - load('ci/jenkins/scripts/devpi.groovy').removePackage( - pkgName: props.Name, - pkgVersion: props.Version, - index: "${DEVPI_CONFIG.devpiUserName}/${DEVPI_CONFIG.stagingIndex}", - server: DEVPI_CONFIG.server, - credentialsId: DEVPI_CONFIG.credentialsId, - - ) - } - sh script: "docker image rm --no-prune ${dockerImage.imageName()}" - } - } - } - } - } stage('Deploy'){ parallel{ stage('Deploy to pypi') { diff --git a/ci/docker/linux/tox/Dockerfile b/ci/docker/linux/tox/Dockerfile index 22c9523d..6ecffa81 100644 --- a/ci/docker/linux/tox/Dockerfile +++ b/ci/docker/linux/tox/Dockerfile @@ -73,8 +73,7 @@ RUN python3 -m pip install --no-cache-dir pipx && \ pipx ensurepath && \ mkdir -p $PIPX_HOME && chmod -R 777 $PIPX_HOME -RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install 'devpi-client<7.0' && \ - PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install 'tox' && \ +RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install 'tox' && \ PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx inject tox tox-uv WORKDIR /src diff --git a/ci/docker/windows/tox/Dockerfile b/ci/docker/windows/tox/Dockerfile index 2592247d..e1e7a97f 100644 --- a/ci/docker/windows/tox/Dockerfile +++ b/ci/docker/windows/tox/Dockerfile @@ -132,8 +132,7 @@ ENV PIPX_BIN_DIR=${PIPX_HOME}\bin RUN py -3 -m pip install --no-cache-dir pipx ; ` py -3 -m pipx ensurepath -RUN py -3 -m pipx install 'devpi-client<7.0' ; ` - py -3 -m pipx install 'tox' ; ` +RUN py -3 -m pipx install 'tox' ; ` py -3 -m pipx inject tox tox-uv diff --git a/ci/docker/windows/tox_no_vs/Dockerfile b/ci/docker/windows/tox_no_vs/Dockerfile index d8dc09b6..5fa5279b 100644 --- a/ci/docker/windows/tox_no_vs/Dockerfile +++ b/ci/docker/windows/tox_no_vs/Dockerfile @@ -71,8 +71,7 @@ ENV PIPX_BIN_DIR=${PIPX_HOME}\bin RUN py -3 -m pip install --no-cache-dir pipx ; ` py -3 -m pipx ensurepath -RUN py -3 -m pipx install 'devpi-client<7.0' ; ` - py -3 -m pipx install 'tox' ; ` +RUN py -3 -m pipx install 'tox' ; ` py -3 -m pipx inject tox tox-uv diff --git a/ci/jenkins/scripts/devpi.groovy b/ci/jenkins/scripts/devpi.groovy deleted file mode 100644 index afb01a4e..00000000 --- a/ci/jenkins/scripts/devpi.groovy +++ /dev/null @@ -1,314 +0,0 @@ -def upload(args = [:]){ - def credentialsId = args['credentialsId'] - def clientDir = args['clientDir'] ? args['clientDir']: './devpi' - def index = args['index'] - def devpiExec = args['devpiExec'] ? args['devpiExec']: "devpi" - - withEnv([ - "DEVPI_SERVER=${args['server']}", - "CLIENT_DIR=${clientDir}", - "DEVPI=${devpiExec}", - "INDEX=${index}" - ]) { - withCredentials( - [ - usernamePassword( - credentialsId: credentialsId, - passwordVariable: 'DEVPI_PASSWORD', - usernameVariable: 'DEVPI_USERNAME' - ) - ] - ) - { - if(isUnix()){ - sh(label: 'Logging into DevPi', - script: '''$DEVPI use $DEVPI_SERVER --clientdir $CLIENT_DIR - $DEVPI login $DEVPI_USERNAME --password=$DEVPI_PASSWORD --clientdir $CLIENT_DIR - ''' - ) - sh(label: 'Uploading to DevPi Staging', - script: '''$DEVPI use --clientdir $CLIENT_DIR --debug - $DEVPI upload --from-dir dist --clientdir $CLIENT_DIR --debug --index $DEVPI_USERNAME/$INDEX - ''' - ) - } else { - bat(label: 'Logging into DevPi', - script: '''%DEVPI% use %DEVPI_SERVER% --clientdir %CLIENT_DIR% - %DEVPI% login %DEVPI_USERNAME% --password %DEVPI_PASSWORD% --clientdir %CLIENT_DIR% - ''' - ) - bat(label: 'Uploading to DevPi Staging', - script: '''%DEVPI% use %INDEX% --clientdir %CLIENT_DIR% - %DEVPI% upload --from-dir dist --clientdir %CLIENT_DIR% --index %DEVPI_USERNAME%/%INDEX% - ''' - ) - } - } - } -} - -def pushPackageToIndex(args = [:]){ - def sourceIndex = args['indexSource'] - def destinationIndex = args['indexDestination'] - def pkgName = args['pkgName'] - def pkgVersion = args['pkgVersion'] - def clientDir = args['clientDir'] ? args['clientDir']: './devpi' - def devpi = args['devpiExec'] ? args['devpiExec']: "devpi" - def server = args['server'] - - withCredentials( - [usernamePassword( - credentialsId: args['credentialsId'], - passwordVariable: 'DEVPI_PASSWORD', - usernameVariable: 'DEVPI_USERNAME' - )]) - { - withEnv([ - "DEVPI_SERVER=${server}", - "CLIENT_DIR=${clientDir}", - "DEVPI=${devpi}" - ]){ - if(isUnix()){ - sh(label: "Logging into DevPi", - script: '''$DEVPI use $DEVPI_SERVER --clientdir $CLIENT_DIR - $DEVPI login $DEVPI_USERNAME --password=$DEVPI_PASSWORD --clientdir $CLIENT_DIR - ''' - ) - - } else { - bat(label: "Logging into DevPi Staging", - script: '''%DEVPI% use %DEVPI_SERVER% --clientdir %CLIENT_DIR% - %DEVPI% login %DEVPI_USERNAME% --password=%DEVPI_PASSWORD% --clientdir %CLIENT_DIR% - ''' - ) - - } - } - } - if(isUnix()){ - sh( - label: "Pushing DevPi package from ${sourceIndex} to ${destinationIndex}", - script: "${devpi} push --index ${sourceIndex} ${pkgName}==${pkgVersion} ${destinationIndex} --clientdir ${clientDir}" - ) - } -} - -def removePackage(args = [:]){ - def clientDir = args['clientDir'] ? args['clientDir']: './devpi' - def devpi = args['devpiExec'] ? args['devpiExec']: "devpi" - def server = args['server'] - def pkgName = args['pkgName'] - def pkgVersion = args['pkgVersion'] - def index = args['index'] - withEnv([ - "DEVPI=${devpi}", - "DEVPI_SERVER=${server}", - "CLIENT_DIR=${clientDir}" - ]){ - withCredentials( - [usernamePassword( - credentialsId: args['credentialsId'], - passwordVariable: 'DEVPI_PASSWORD', - usernameVariable: 'DEVPI_USERNAME' - )]){ - if(isUnix()){ - sh(label: "Logging into DevPi", - script: '''$DEVPI use $DEVPI_SERVER --clientdir $CLIENT_DIR - $DEVPI login $DEVPI_USERNAME --password=$DEVPI_PASSWORD --clientdir $CLIENT_DIR - ''' - ) - - } else { - bat(label: "Logging into DevPi Staging", - script: '''%DEVPI% use %DEVPI_SERVER% --clientdir %CLIENT_DIR% - %DEVPI% login %DEVPI_USERNAME% --password=%DEVPI_PASSWORD% --clientdir %CLIENT_DIR% - ''' - ) - - } - } - } - if(isUnix()){ - sh(label: "Removing Package from DevPi ${index} index", - script: """${devpi} use ${index} --clientdir ${clientDir} - ${devpi} remove -y --index ${index} ${pkgName}==${pkgVersion} --clientdir ${clientDir} - """ - ) - } else{ - bat(label: "Removing Package from DevPi ${index} index", - script: """${devpi} use ${index}--clientdir ${clientDir} - ${devpi} remove -y --index ${index} ${pkgName}==${pkgVersion} --clientdir ${clientDir} - """ - ) - } -} - -def getNodeLabel(agent){ - def label - if (agent.containsKey("dockerfile")){ - return agent.dockerfile.label - } - return label -} -def getAgentLabel(args){ - if (args.agent.containsKey("label")){ - return args.agent['label'] - } - if (args.agent.containsKey("dockerfile")){ - return args.agent.dockerfile.label - } - error('Invalid agent type, expect [dockerfile,label]') -} - -def getAgent(args){ - if (args.agent.containsKey("label")){ - return { inner -> - node(args.agent.label){ - ws{ - inner() - } - } - } - - } - if (args.agent.containsKey("dockerfile")){ - def runArgs = args.agent.dockerfile.containsKey('args') ? args.agent.dockerfile['args']: '' - return { inner -> - node(args.agent.dockerfile.label){ - ws{ - checkout scm - def dockerImage - def dockerImageName = "${currentBuild.fullProjectName}_devpi_${UUID.randomUUID().toString()}".replaceAll("-", "_").replaceAll('/', "_").replaceAll(' ', "").toLowerCase() - lock("docker build-${env.NODE_NAME}"){ - dockerImage = docker.build(dockerImageName, "-f ${args.agent.dockerfile.filename} ${args.agent.dockerfile.additionalBuildArgs} .") - } - try{ - dockerImage.inside(runArgs){ - inner() - } - } finally { - if(isUnix()){ - sh( - label: "Untagging Docker Image used", - script: "docker image rm --no-prune ${dockerImage.imageName()}", - returnStatus: true - ) - } else { - powershell( - label: "Untagging Docker Image used", - script: "docker image rm --no-prune ${dockerImage.imageName()}", - returnStatus: true - ) - } - } - } - } - } - } - error('Invalid agent type, expect [dockerfile,label]') -} - -def logIntoDevpiServer(devpiExec, serverUrl, credentialsId, clientDir){ - withEnv([ - "DEVPI=${devpiExec}", - "DEVPI_SERVER=${serverUrl}", - "CLIENT_DIR=${clientDir}" - ]){ - withCredentials( - [usernamePassword( - credentialsId: credentialsId, - passwordVariable: 'DEVPI_PASSWORD', - usernameVariable: 'DEVPI_USERNAME' - )]){ - if(isUnix()){ - sh(label: "Logging into DevPi", - script: '''$DEVPI use $DEVPI_SERVER --clientdir $CLIENT_DIR - $DEVPI login $DEVPI_USERNAME --password=$DEVPI_PASSWORD --clientdir $CLIENT_DIR - ''' - ) - - } else { - bat(label: "Logging into DevPi Staging", - script: '''%DEVPI% use %DEVPI_SERVER% --clientdir %CLIENT_DIR% - %DEVPI% login %DEVPI_USERNAME% --password=%DEVPI_PASSWORD% --clientdir %CLIENT_DIR% - ''' - ) - - } - } - } -} - -def runDevpiTest(devpiExec, devpiIndex, pkgName, pkgVersion, pkgSelector, clientDir, toxEnv){ - withEnv([ - "DEVPI=${devpiExec}", - "_DEVPI_INDEX=${devpiIndex}", - "CLIENT_DIR=${clientDir}", - "PACKAGE_NAME=${pkgName}", - "PACKAGE_VERSION=${pkgVersion}", - "TOX_ENV=${toxEnv}", - "TOX_PACKAGE_SELECTOR=${pkgSelector}", - ]){ - if(isUnix()){ - sh( - label: 'Running tests on Packages on DevPi', - script: '$DEVPI test --index $_DEVPI_INDEX $PACKAGE_NAME==$PACKAGE_VERSION -s $TOX_PACKAGE_SELECTOR --clientdir $CLIENT_DIR -e $TOX_ENV -v' - ) - } else{ - bat( - label: 'Running tests on Packages on DevPi', - script: '%DEVPI% test --index %_DEVPI_INDEX% %PACKAGE_NAME%==%PACKAGE_VERSION% -s %TOX_PACKAGE_SELECTOR% --clientdir %CLIENT_DIR% -e %TOX_ENV% -v' - ) - } - } -} - -def getToxEnvName(args){ - try{ - def pythonVersion = args.pythonVersion.replace(".", "") - return "py${pythonVersion}" - } catch(e){ - return "py" - } -} - -def testDevpiPackage2(args=[:]){ - def agent = getAgent(args) - def devpiExec = args.devpi['devpiExec'] ? args.devpi['devpiExec'] : "devpi" - def devpiIndex = args.devpi.index - def devpiServerUrl = args.devpi.server - def credentialsId = args.devpi.credentialsId - def clientDir = args['clientDir'] ? args['clientDir']: './devpi' - def pkgName = args.package.name - def pkgVersion = args.package.version - def pkgSelector = args.package.selector - def toxEnv = args.test.toxEnv - def testSetup = args.test['setup'] ? args.test['setup'] : {} - def testTeardown = args.test['teardown'] ? args.test['teardown'] : {} - def retries = args['retries'] ? args['retries'] : 1 - def attempt = 1 - retry(retries){ - agent{ - testSetup() - try{ - logIntoDevpiServer(devpiExec, devpiServerUrl, credentialsId, clientDir) - runDevpiTest(devpiExec, devpiIndex, pkgName, pkgVersion, pkgSelector, clientDir, toxEnv) - } catch(Exception e){ - if (attempt < retries) { - echo 'Waiting 5 seconds' - sleep 5; - } - throw e; - }finally { - testTeardown() - attempt += 1 - } - } - } -} -return [ - testDevpiPackage: this.&testDevpiPackage2, - removePackage: this.&removePackage, - pushPackageToIndex: this.&pushPackageToIndex, - upload: this.&upload -]