From fb5ec457ed19db87936a256a85a1a0139b44964f Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Mon, 9 Oct 2023 11:52:50 -0400 Subject: [PATCH 1/3] [release/public-v2.2.0] Update Jenkinsfile and Jenkins test scripts to allow the simultaneous run of all tests on all machines (#931) With the addition of Gaea C5 and Hercules to the Jenkinsfile, file contention between Gaea and Gaea C5, as well as Orion and Hercules, caused testing to fail for one of these two pairs of machines. While requeuing the failed machine would allow it to work, the automated tests should run automatically on all machines, simultaneously. The Platform team recommended adding the dir command to the stages' step sections in the Jenkinsfile. This has been completed and the Jenkins tests are now running on all machines simultaneously. --- .cicd/Jenkinsfile | 40 ++++++++++++------- .cicd/scripts/qsub_srw_ftest.sh | 2 +- .cicd/scripts/sbatch_srw_ftest.sh | 2 +- .cicd/scripts/srw_build.sh | 8 +--- .cicd/scripts/srw_ftest.sh | 8 +--- .cicd/scripts/srw_metric_example.sh | 4 +- .cicd/scripts/srw_test.sh | 8 +--- .cicd/scripts/srw_unittest.sh | 4 +- .cicd/scripts/wrapper_srw_ftest.sh | 14 +++---- ...et_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml | 2 + 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index f4e0294441..6231fd7e11 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -117,32 +117,38 @@ pipeline { // Clean the workspace, checkout the repository, and run checkout_externals stage('Initialize') { steps { - echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" + dir ("${env.SRW_PLATFORM}") { + echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" cleanWs() checkout scm - sh '"${WORKSPACE}/manage_externals/checkout_externals"' + sh '"${WORKSPACE}/${SRW_PLATFORM}/manage_externals/checkout_externals"' + } } } // Run the unittest functional tests that require an HPC platform stage('Functional UnitTests') { steps { + dir ("${env.SRW_PLATFORM}") { echo "Running unittest on retrieve_data.py" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_unittest.sh"' + sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_unittest.sh"' + } } } // Run the unified build script; if successful create a tarball of the build and upload to S3 stage('Build') { steps { - echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"' + dir ("${env.SRW_PLATFORM}") { + echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" + sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_build.sh"' + } } post { success { - sh 'cd "${WORKSPACE}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" *' - s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + sh 'cd "${WORKSPACE}/${SRW_PLATFORM}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${SRW_PLATFORM}/${BUILD_NAME}.tgz" *' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } } } @@ -150,22 +156,25 @@ pipeline { // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage stage('Functional WorkflowTaskTests') { steps { - echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/wrapper_srw_ftest.sh"' + dir ("${env.SRW_PLATFORM}") { + echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" + sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/wrapper_srw_ftest.sh"' + } } } // Run the unified test script stage('Test') { environment { - SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/expt_dirs" + SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/${env.SRW_PLATFORM}/expt_dirs" } steps { - echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" + dir ("${env.SRW_PLATFORM}") { + echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" // Remove the following line to re-enable comprehensive tests - sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"' + sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' // Uncomment the following block to re-enable comprehensive tests /* @@ -183,18 +192,19 @@ pipeline { } } - sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"' + sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' } */ + } } post { always { // Archive the test log files - sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/tests/WE2E/WE2E_tests_*yaml WE2E_summary*txt ${WORKSPACE}/tests/WE2E/log.*' + sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/${SRW_PLATFORM}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/${SRW_PLATFORM}/tests/WE2E/WE2E_tests_*yaml WE2E_summary*txt ${WORKSPACE}/${SRW_PLATFORM}/tests/WE2E/log.*' // Remove the data sets from the experiments directory to conserve disk space sh 'find "${SRW_WE2E_EXPERIMENT_BASE_DIR}" -regextype posix-extended -regex "^.*(orog|[0-9]{10})$" -type d | xargs rm -rf' - s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: '*_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/*_test_results-*-*.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/we2e_test_logs-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } } } diff --git a/.cicd/scripts/qsub_srw_ftest.sh b/.cicd/scripts/qsub_srw_ftest.sh index 2a669fe61a..e9f0170a05 100644 --- a/.cicd/scripts/qsub_srw_ftest.sh +++ b/.cicd/scripts/qsub_srw_ftest.sh @@ -12,4 +12,4 @@ #PBS -o log_wrap.%j.log #PBS -e err_wrap.%j.err -bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh +bash ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_ftest.sh diff --git a/.cicd/scripts/sbatch_srw_ftest.sh b/.cicd/scripts/sbatch_srw_ftest.sh index 1655148068..5add5368b5 100644 --- a/.cicd/scripts/sbatch_srw_ftest.sh +++ b/.cicd/scripts/sbatch_srw_ftest.sh @@ -13,4 +13,4 @@ #SBATCH -o log_wrap.%j.log #SBATCH -e err_wrap.%j.err -bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh +bash ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_ftest.sh diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh index 7d8e70b321..196d984a05 100755 --- a/.cicd/scripts/srw_build.sh +++ b/.cicd/scripts/srw_build.sh @@ -10,8 +10,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi @@ -24,10 +24,6 @@ else platform="${SRW_PLATFORM}" fi -if [[ "${SRW_PLATFORM}" = jet-epic ]]; then - platform='jet' -fi - # Build and install cd ${workspace}/tests set +e diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index 52c0cc45f5..7a787a7137 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -28,8 +28,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" cd $workspace else workspace="$(cd -- "${script_dir}/../.." && pwd)" @@ -43,10 +43,6 @@ else platform="${SRW_PLATFORM}" fi -if [[ "${SRW_PLATFORM}" = jet-epic ]]; then - platform='jet' -fi - # Test directories we2e_experiment_base_dir="${workspace}/expt_dirs" we2e_test_dir="${workspace}/tests/WE2E" diff --git a/.cicd/scripts/srw_metric_example.sh b/.cicd/scripts/srw_metric_example.sh index 67d795f11f..2018505735 100755 --- a/.cicd/scripts/srw_metric_example.sh +++ b/.cicd/scripts/srw_metric_example.sh @@ -18,8 +18,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh index 2b0041a847..1bffe083bd 100755 --- a/.cicd/scripts/srw_test.sh +++ b/.cicd/scripts/srw_test.sh @@ -11,8 +11,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi @@ -25,10 +25,6 @@ else platform="${SRW_PLATFORM}" fi -if [[ "${SRW_PLATFORM}" = jet-epic ]]; then - platform='jet' -fi - # Test directories we2e_experiment_base_dir="${workspace}/expt_dirs" we2e_test_dir="${workspace}/tests/WE2E" diff --git a/.cicd/scripts/srw_unittest.sh b/.cicd/scripts/srw_unittest.sh index 9424fd4f68..cfc1719de4 100755 --- a/.cicd/scripts/srw_unittest.sh +++ b/.cicd/scripts/srw_unittest.sh @@ -10,8 +10,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, # set relative to script directory. declare workspace -if [[ -n "${WORKSPACE:-}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi diff --git a/.cicd/scripts/wrapper_srw_ftest.sh b/.cicd/scripts/wrapper_srw_ftest.sh index 02ece830f9..403233354a 100755 --- a/.cicd/scripts/wrapper_srw_ftest.sh +++ b/.cicd/scripts/wrapper_srw_ftest.sh @@ -24,18 +24,18 @@ fi # Customize wrapper scripts if [[ "${SRW_PLATFORM}" == gaea ]]; then - sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh - sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh fi if [[ "${SRW_PLATFORM}" == gaea-c5 ]]; then - sed -i '15i #SBATCH --clusters=c5' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh - sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i '15i #SBATCH --clusters=c5' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh fi # Call job card and return job_id -echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh" -job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh) +echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh" +job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh) echo "Waiting ten seconds for node to initialize" sleep 10 @@ -51,7 +51,7 @@ do echo "Job has completed." # Return exit code and check for results file first - results_file="${WORKSPACE}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt" + results_file="${WORKSPACE}/${SRW_PLATFORM}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt" if [ ! -f "$results_file" ]; then echo "Missing results file! \nexit 1" exit 1 diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml b/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml index 338edeb186..cc07f37bc8 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml @@ -21,3 +21,5 @@ task_get_extrn_lbcs: EXTRN_MDL_NAME_LBCS: FV3GFS LBC_SPEC_INTVL_HRS: 3 FV3GFS_FILE_FMT_LBCS: netcdf +task_run_fcst: + DT_ATMOS: 60 From d94b6e4d84fc3f9ed8d12ea1c200e81506223b7a Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Wed, 11 Oct 2023 09:36:20 -0600 Subject: [PATCH 2/3] [release/public-v2.2.0] Fix crontab bug for Cheyenne and Derecho, update PR template for new platforms (#939) The option to create an experiment with the option USE_CRON_TO_RELAUNCH=True is currently broken on Cheyenne and Derecho due to some bad python logic. This fixes that issue. Also took the opportunity to update the PR template to include the new supported platforms (Derecho, Hercules, and Gaea C5) Contains an additional fix for removing old crontab entries on Cheyenne and Derecho --- .github/PULL_REQUEST_TEMPLATE | 3 +++ ush/get_crontab_contents.py | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 7b1cdbf109..1c363c651f 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -32,9 +32,12 @@ - [ ] hera.intel - [ ] orion.intel +- [ ] hercules.intel - [ ] cheyenne.intel - [ ] cheyenne.gnu +- [ ] derecho.intel - [ ] gaea.intel +- [ ] gaeac5.intel - [ ] jet.intel - [ ] wcoss2.intel - [ ] NOAA Cloud (indicate which platform) diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 5e9ccd6fa8..58a17bf35c 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -35,11 +35,10 @@ def get_crontab_contents(called_from_cron, machine, debug): # themselves being called as cron jobs. In that case, we must instead # call the system version of crontab at /usr/bin/crontab. # - if machine == "CHEYENNE" or machine == "DERECHO": + crontab_cmd = "crontab" + if machine.upper() == "CHEYENNE" or machine.upper() == "DERECHO": if called_from_cron: crontab_cmd = "/usr/bin/crontab" - else: - crontab_cmd = "crontab" print_info_msg( f""" From d989d63a0e99918dccf51448b5f2f14a41f80f95 Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:09:26 -0400 Subject: [PATCH 3/3] [release/public-v2.2.0] Reactivate comprehensive tests and fix Hera GNU pipeline issues (#936) Changes were made to reactivate the comprehensive tests in the SRW App Jenkins pipeline via the Jenkinsfile. A fix was introduced to wrapper_srw_ftest.sh to correct occasional failures of the Functional Workflow Task Tests stage on Hera GNU. --- .cicd/Jenkinsfile | 9 +-------- .cicd/scripts/wrapper_srw_ftest.sh | 6 ++++++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 6231fd7e11..5d82ab875f 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -17,8 +17,7 @@ pipeline { choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') - // Uncomment the following line to re-enable comprehensive tests - // booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests' + booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests' } stages { @@ -173,11 +172,6 @@ pipeline { dir ("${env.SRW_PLATFORM}") { echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" - // Remove the following line to re-enable comprehensive tests - sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' - - // Uncomment the following block to re-enable comprehensive tests - /* // If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set, // override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter script { @@ -194,7 +188,6 @@ pipeline { sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' } - */ } } diff --git a/.cicd/scripts/wrapper_srw_ftest.sh b/.cicd/scripts/wrapper_srw_ftest.sh index 403233354a..2e35b43b89 100755 --- a/.cicd/scripts/wrapper_srw_ftest.sh +++ b/.cicd/scripts/wrapper_srw_ftest.sh @@ -33,6 +33,12 @@ if [[ "${SRW_PLATFORM}" == gaea-c5 ]]; then sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh fi +if [[ "${SRW_PLATFORM}" == hera ]]; then + if [[ "${SRW_COMPILER}" == gnu ]]; then + sed -i 's|00:30:00|00:45:00|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + fi +fi + # Call job card and return job_id echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh" job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh)