Skip to content

Commit

Permalink
Merge pull request #148 from UIUCLibrary/jenkins-use-shared-library
Browse files Browse the repository at this point in the history 
CI: Jenkinsfile - use JenkinsPythonHelperLibrary shared library
  • Loading branch information
@henryborchers
henryborchers authored Jan 9, 2024
2 parents 41373c7 + e19d207 commit 5b0280b
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 215 deletions.
99 changes: 35 additions & 64 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
library identifier: 'JenkinsPythonHelperLibrary@2024.1.1', retriever: modernSCM(
[$class: 'GitSCMSource',
remote: 'https://github.com/UIUCLibrary/JenkinsPythonHelperLibrary.git',
])

SONARQUBE_CREDENTIAL_ID = 'sonarcloud_token'
SUPPORTED_MAC_VERSIONS = ['3.8', '3.9', '3.10', '3.11', '3.12']
SUPPORTED_LINUX_VERSIONS = ['3.8', '3.9', '3.10', '3.11', '3.12']
Expand Down Expand Up @@ -30,19 +35,12 @@ def getDevpiConfig() {
def DEVPI_CONFIG = getDevpiConfig()
def getToxStages(){
script{
def tox = fileLoader.fromGit(
'tox',
'https://github.com/UIUCLibrary/jenkins_helper_scripts.git',
'8',
null,
''
)
def windowsJobs = [:]
def linuxJobs = [:]
stage('Scanning Tox Environments'){
parallel(
'Linux':{
linuxJobs = tox.getToxTestsParallel(
linuxJobs = getToxTestsParallel(
envNamePrefix: 'Tox Linux',
label: 'linux && docker && x86',
dockerfile: 'ci/docker/linux/tox/Dockerfile',
Expand All @@ -53,7 +51,7 @@ def getToxStages(){
},
'Windows':{
timeout(240){
windowsJobs = tox.getToxTestsParallel(
windowsJobs = getToxTestsParallel(
envNamePrefix: 'Tox Windows',
label: 'windows && docker && x86',
dockerfile: 'ci/docker/windows/tox/Dockerfile',
Expand Down Expand Up @@ -248,11 +246,6 @@ defaultParameterValues = [
]

def linux_wheels(){
def packages
node(){
checkout scm
packages = load 'ci/jenkins/scripts/packaging.groovy'
}
def wheelStages = [failFast: true]
SUPPORTED_LINUX_VERSIONS.each{ pythonVersion ->
wheelStages["Python ${pythonVersion} - Linux"] = {
Expand All @@ -262,7 +255,7 @@ def linux_wheels(){
archs["Linux x86_64 - Python ${pythonVersion}: wheel"] = {
stage("Linux x86_64 - Python ${pythonVersion}: wheel"){
stage("Build Wheel (${pythonVersion} Linux x86_64)"){
packages.buildPkg(
buildPythonPkg(
agent: [
dockerfile: [
label: 'linux && docker && x86',
Expand Down Expand Up @@ -307,7 +300,7 @@ def linux_wheels(){
}
if(params.TEST_PACKAGES == true){
stage("Test Wheel (${pythonVersion} Linux x86_64)"){
packages.testPkg(
testPythonPkg(
agent: [
dockerfile: [
label: 'linux && docker && x86',
Expand Down Expand Up @@ -355,7 +348,7 @@ def linux_wheels(){
archs["Linux arm64 - Python ${pythonVersion}: wheel"] = {
stage("Linux arm64 - Python ${pythonVersion}: wheel"){
stage("Build Wheel (${pythonVersion} Linux arm64)"){
packages.buildPkg(
buildPythonPkg(
agent: [
dockerfile: [
label: 'linux && docker && arm64',
Expand Down Expand Up @@ -399,7 +392,7 @@ def linux_wheels(){
)
}
stage("Test Wheel (${pythonVersion} Linux arm64)"){
packages.testPkg(
testPythonPkg(
agent: [
dockerfile: [
label: 'linux && docker && arm64',
Expand Down Expand Up @@ -450,19 +443,14 @@ def linux_wheels(){
}

def windows_wheels(){
def packages
node(){
checkout scm
packages = load 'ci/jenkins/scripts/packaging.groovy'
}
def wheelStages = [failFast: true]
SUPPORTED_WINDOWS_VERSIONS.each{ pythonVersion ->
if(params.INCLUDE_WINDOWS_X86_64 == true){
wheelStages["Windows - Python ${pythonVersion}"] = {
stage("Windows - Python ${pythonVersion}"){
stage("Windows - Python ${pythonVersion} x86_64: wheel"){
stage("Build Wheel (${pythonVersion} Windows)"){
packages.buildPkg(
buildPythonPkg(
agent: [
dockerfile: [
label: 'windows && docker && x86',
Expand Down Expand Up @@ -500,15 +488,15 @@ def windows_wheels(){
}
if(params.TEST_PACKAGES == true){
stage("Test Wheel (${pythonVersion} Windows x86_64)"){
packages.testPkg(
testPythonPkg(
agent: [
dockerfile: [
label: 'windows && docker && x86',
filename: 'ci/docker/windows/tox_no_vs/Dockerfile',
additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE'
additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE',
dockerImageName: "${currentBuild.fullProjectName}_test_no_msvc".replaceAll('-', '_').replaceAll('/', '_').replaceAll(' ', '').toLowerCase(),
]
],
dockerImageName: "${currentBuild.fullProjectName}_test_no_msvc".replaceAll('-', '_').replaceAll('/', '_').replaceAll(' ', '').toLowerCase(),
testSetup: {
checkout scm
unstash "python${pythonVersion} windows wheel"
Expand Down Expand Up @@ -546,11 +534,6 @@ def windows_wheels(){
}

def mac_wheels(){
def packages
node(){
checkout scm
packages = load 'ci/jenkins/scripts/packaging.groovy'
}
def wheelStages = [failFast: true]
SUPPORTED_MAC_VERSIONS.each{ pythonVersion ->
wheelStages["Python ${pythonVersion} - Mac"] = {
Expand All @@ -561,7 +544,7 @@ def mac_wheels(){
archBuilds["Python ${pythonVersion} MacOS x86_64"] = {
stage("Python ${pythonVersion} MacOS x86_64"){
stage("Build Wheel (${pythonVersion} MacOS x86_64)"){
packages.buildPkg(
buildPythonPkg(
agent: [
label: "mac && python${pythonVersion} && x86",
],
Expand Down Expand Up @@ -613,7 +596,7 @@ def mac_wheels(){
}
if(params.TEST_PACKAGES == true){
stage("Test Wheel (${pythonVersion} MacOS x86_64)"){
packages.testPkg(
testPythonPkg(
agent: [
label: "mac && python${pythonVersion} && x86_64",
],
Expand Down Expand Up @@ -658,7 +641,7 @@ def mac_wheels(){
archBuilds["Python ${pythonVersion} MacOS m1 wheel"] = {
stage("Python ${pythonVersion} MacOS m1 wheel"){
stage("Build Wheel (${pythonVersion} MacOS m1)"){
packages.buildPkg(
buildPythonPkg(
agent: [
label: "mac && python${pythonVersion} && m1",
],
Expand Down Expand Up @@ -712,7 +695,7 @@ def mac_wheels(){
}
if(params.TEST_PACKAGES == true){
stage("Test Wheel (${pythonVersion} MacOS m1)"){
packages.testPkg(
testPythonPkg(
agent: [
label: "mac && python${pythonVersion} && m1",
],
Expand Down Expand Up @@ -801,15 +784,15 @@ def mac_wheels(){
parallel(
"Test Python ${pythonVersion} universal2 Wheel on x86_64 mac": {
stage("Test Python ${pythonVersion} universal2 Wheel on x86_64 mac"){
packages.testPkg(
testPythonPkg(
agent: [
label: "mac && python${pythonVersion} && x86_64",
],
testSetup: {
checkout scm
unstash "python${pythonVersion} mac-universal2 wheel"
},
retry: 3,
retries: 3,
testCommand: {
findFiles(glob: 'dist/*.whl').each{
sh(label: 'Running Tox',
Expand Down Expand Up @@ -843,15 +826,15 @@ def mac_wheels(){
},
"Test Python ${pythonVersion} universal2 Wheel on M1 Mac": {
stage("Test Python ${pythonVersion} universal2 Wheel on M1 Mac"){
packages.testPkg(
testPythonPkg(
agent: [
label: "mac && python${pythonVersion} && m1",
],
testSetup: {
checkout scm
unstash "python${pythonVersion} mac-universal2 wheel"
},
retry: 3,
retries: 3,
testCommand: {
findFiles(glob: 'dist/*.whl').each{
sh(label: 'Running Tox',
Expand Down Expand Up @@ -1411,11 +1394,6 @@ pipeline {
}
steps{
script{
def packages
node(){
checkout scm
packages = load 'ci/jenkins/scripts/packaging.groovy'
}
def testSdistStages = [
failFast: true
]
Expand All @@ -1430,7 +1408,7 @@ pipeline {
arches.each{arch ->
testSdistStages["Test sdist (MacOS ${arch} - Python ${pythonVersion})"] = {
stage("Test sdist (MacOS ${arch} - Python ${pythonVersion})"){
packages.testPkg(
testPythonPkg(
agent: [
label: "mac && python${pythonVersion} && ${arch}",
],
Expand Down Expand Up @@ -1472,16 +1450,16 @@ pipeline {
testSdistStages["Test sdist (Windows x86_64 - Python ${pythonVersion})"] = {
stage("Test sdist (Windows x86_64 - Python ${pythonVersion})"){
retry(2){
packages.testPkg(
testPythonPkg(
agent: [
dockerfile: [
label: 'windows && docker && x86',
filename: 'ci/docker/windows/tox/Dockerfile',
additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE --build-arg PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip',
args: '-v pipcache_pykdu:c:/users/containeradministrator/appdata/local/pip'
args: '-v pipcache_pykdu:c:/users/containeradministrator/appdata/local/pip',
dockerImageName: "${currentBuild.fullProjectName}_test_with_msvc".replaceAll('-', '_').replaceAll('/', '_').replaceAll(' ', "").toLowerCase(),
]
],
dockerImageName: "${currentBuild.fullProjectName}_test_with_msvc".replaceAll('-', '_').replaceAll('/', '_').replaceAll(' ', "").toLowerCase(),
testSetup: {
checkout scm
unstash 'python sdist'
Expand Down Expand Up @@ -1513,7 +1491,7 @@ pipeline {
if(params.INCLUDE_LINUX_X86_64 == true){
testSdistStages["Test sdist (Linux x86_64 - Python ${pythonVersion})"] = {
stage("Test sdist (Linux x86_64 - Python ${pythonVersion})"){
packages.testPkg(
testPythonPkg(
agent: [
dockerfile: [
label: 'linux && docker && x86',
Expand Down Expand Up @@ -1552,15 +1530,15 @@ pipeline {
if(params.INCLUDE_LINUX_ARM == true){
testSdistStages["Test sdist (Linux ARM64 - Python ${pythonVersion})"] = {
stage("Test sdist (Linux ARM64 - Python ${pythonVersion})"){
packages.testPkg(
testPythonPkg(
agent: [
dockerfile: [
label: 'linux && docker && arm64',
filename: 'ci/docker/linux/tox/Dockerfile',
additionalBuildArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg PIP_DOWNLOAD_CACHE=/.cache/pip'
]
],
retry: 3,
retries: 3,
testSetup: {
checkout scm
unstash 'python sdist'
Expand Down Expand Up @@ -1901,19 +1879,12 @@ pipeline {
wheelStashes.each{
unstash it
}
def pypi = fileLoader.fromGit(
'pypi',
'https://github.com/UIUCLibrary/jenkins_helper_scripts.git',
'2',
null,
''
)
pypi.pypiUpload(
credentialsId: 'jenkins-nexus',
repositoryUrl: SERVER_URL,
glob: 'dist/*'
)
}
pypiUpload(
credentialsId: 'jenkins-nexus',
repositoryUrl: SERVER_URL,
glob: 'dist/*'
)
}
post{
cleanup{
Expand Down
Loading

0 comments on commit 5b0280b

Please sign in to comment.