diff --git a/.github/scripts/release_create/create_tag_release.sh b/.github/scripts/release_create/create_tag_release.sh new file mode 100755 index 000000000..dd3e00f69 --- /dev/null +++ b/.github/scripts/release_create/create_tag_release.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -ex + +echo "Create a tag release for ${TARGET_VERSION_TAG} in ${REPOSITORY}" + +RELEASE_REPO_DIR=$(dirname ${WORKING_DIR})/repo_dir +git clone \ + --depth=1 \ + --branch=${RELEASE_BRANCH} \ + https://${GH_USER_NAME}:${GH_TOKEN}@github.com/${REPOSITORY} \ + ${RELEASE_REPO_DIR} +cd ${RELEASE_REPO_DIR} + +gh release create ${TARGET_VERSION_TAG} --target ${RELEASE_BRANCH} --generate-notes --notes-start-tag ${PREVIOUS_VERSION_TAG} + +cat <> /tmp/release-notes.md + +This is a release comprising of multiple repos: +* DSP component for ${TARGET_VERSION_TAG} can be found [here](https://github.com/${GH_ORG}/data-science-pipelines/releases/tag/${TARGET_VERSION_TAG}) +* DSPO component for ${TARGET_VERSION_TAG} can be found [here](https://github.com/${GH_ORG}/data-science-pipelines-operator/releases/tag/${TARGET_VERSION_TAG}) + +Version Table for components can be found [here](https://github.com/${GH_ORG}/data-science-pipelines-operator/blob/main/docs/release/compatibility.md) +EOF + +echo "$(gh release view ${TARGET_VERSION_TAG} --json body --jq .body)" >> /tmp/release-notes.md + +echo "Release notes to be created:" +cat /tmp/release-notes.md + +gh release edit ${TARGET_VERSION_TAG} --notes-file /tmp/release-notes.md +rm /tmp/release-notes.md diff --git a/.github/scripts/release_create/notify.sh b/.github/scripts/release_create/notify.sh new file mode 100755 index 000000000..7045daac6 --- /dev/null +++ b/.github/scripts/release_create/notify.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -ex + +cat <> /tmp/body-file.txt +Release created successfully: + +https://github.com/${GH_ORG}/data-science-pipelines-operator/releases/tag/${TARGET_VERSION_TAG} + +https://github.com/${GH_ORG}/data-science-pipelines/releases/tag/${TARGET_VERSION_TAG} +EOF + +gh pr comment ${PR_NUMBER} --body-file /tmp/body-file.txt + +echo "::notice:: DSPO Release: https://github.com/${GH_ORG}/data-science-pipelines-operator/releases/tag/${TARGET_VERSION_TAG}" +echo "::notice:: DSP Release: https://github.com/${GH_ORG}/data-science-pipelines/releases/tag/${TARGET_VERSION_TAG}" +echo "::notice:: Feedback sent to PR." diff --git a/.github/scripts/release_create/validate_pr.sh b/.github/scripts/release_create/validate_pr.sh new file mode 100755 index 000000000..f2553e919 --- /dev/null +++ b/.github/scripts/release_create/validate_pr.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -ex + +echo "::notice:: Performing Release PR Validation for: ${PR_NUMBER}" + +# Retrive PR Author: +PR_AUTHOR=$(gh pr view ${PR_NUMBER} --json author -q .author.login) + +echo "Current OWNERS:" +cat ./OWNERS + +echo "::notice:: Checking if PR author ${PR_AUTHOR} is DSPO Owner..." + +is_owner=$(cat ./OWNERS | var=${PR_AUTHOR} yq '[.approvers] | contains([env(var)])') +if [[ $is_owner == "false" ]]; then + echo "::error:: PR author ${PR_AUTHOR} is not an approver in OWNERS file. Only approvers can create releases." + exit 1 +fi + +echo "::notice:: PR author ${PR_AUTHOR} is an approver in DSPO OWNERS." + +echo "::notice:: Validation successful." diff --git a/.github/scripts/release_create/vars.sh b/.github/scripts/release_create/vars.sh new file mode 100755 index 000000000..3c1682ce8 --- /dev/null +++ b/.github/scripts/release_create/vars.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex + +cat ./config.yaml +target_version_tag=$(yq .target_version_tag ./config.yaml) +previous_version_tag=$(yq .previous_release_tag ./config.yaml) +release_branch=$(yq .release_branch ./config.yaml) +odh_org=$(yq .odh_org ./config.yaml) +pr_number=$(cat ./pr_number) + +echo "pr_number=${pr_number}" >> $GITHUB_OUTPUT +echo "target_version_tag=${target_version_tag}" >> $GITHUB_OUTPUT +echo "previous_version_tag=${previous_version_tag}" >> $GITHUB_OUTPUT +echo "release_branch=${release_branch}" >> $GITHUB_OUTPUT +echo "odh_org=${odh_org}" >> $GITHUB_OUTPUT diff --git a/.github/scripts/release_prep/create_branches.sh b/.github/scripts/release_prep/create_branches.sh new file mode 100755 index 000000000..885b21084 --- /dev/null +++ b/.github/scripts/release_prep/create_branches.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -ex + +echo "Cut branch ${MINOR_RELEASE_BRANCH} from main/master" + +echo "Current branches in ${DSPO_REPOSITORY_FULL}" +git branch -r + +git checkout -B ${MINOR_RELEASE_BRANCH} +git push origin ${MINOR_RELEASE_BRANCH} +echo "::notice:: Created DSPO ${MINOR_RELEASE_BRANCH} branch" + +echo "Current branches in ${DSP_REPOSITORY_FULL}" +DSP_DIR=$(dirname ${WORKING_DIR})/data-science-pipelines +git clone \ + --depth=1 \ + --branch=master \ + https://${GH_USER_NAME}:${GH_TOKEN}@github.com/${DSP_REPOSITORY_FULL} \ + ${DSP_DIR} +cd ${DSP_DIR} +git checkout -B ${MINOR_RELEASE_BRANCH} +git push origin ${MINOR_RELEASE_BRANCH} +echo "::notice:: Created DSP ${MINOR_RELEASE_BRANCH} branch" diff --git a/.github/scripts/release_prep/generate_pr.sh b/.github/scripts/release_prep/generate_pr.sh new file mode 100755 index 000000000..057b49b83 --- /dev/null +++ b/.github/scripts/release_prep/generate_pr.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# Note: The yaml in the body of the PR is used to feed inputs into the release workflow +# since there's no easy way to communicate information between the pr closing, and then triggering the +# release creation workflow. +# Therefore, take extra care when adding new code blocks in the PR body, or updating the existing one. +# Ensure any changes are compatible with the release_create workflow. + +set -ex +set -o pipefail + +echo "Retrieve the sha images from the resulting workflow (check quay.io for the digests)." +echo "Using [release-tools] generate a params.env and submit a new pr to vx.y+1.**x** branch." +echo "For images pulled from registry, ensure latest images are upto date" + +BRANCH_NAME="release-${TARGET_RELEASE}" +git config --global user.email "${GH_USER_EMAIL}" +git config --global user.name "${GH_USER_NAME}" +git remote add ${GH_USER_NAME} https://${GH_USER_NAME}:${GH_TOKEN}@github.com/${GH_USER_NAME}/${DSPO_REPOSITORY}.git +git checkout -B ${BRANCH_NAME} + +echo "Created branch: ${BRANCH_NAME}" + +python ./scripts/release/release.py params --quay_org ${QUAY_ORG} --tag ${MINOR_RELEASE_TAG} --out_file ./config/base/params.env \ + --override="IMAGES_OAUTHPROXY=registry.redhat.io/openshift4/ose-oauth-proxy@sha256:ab112105ac37352a2a4916a39d6736f5db6ab4c29bad4467de8d613e80e9bb33" + +git add . +git commit -m "Generate params for ${TARGET_RELEASE}" +git push ${GH_USER_NAME} $BRANCH_NAME -f + +# Used to feed inputs to release creation workflow. +# target_version is used as the GH TAG +tmp_config="/tmp/body-config.txt" +body_txt="/tmp/body-text.txt" +cp $CONFIG_TEMPLATE $tmp_config + +var=${GH_ORG} yq -i '.odh_org=env(var)' $tmp_config +var=${MINOR_RELEASE_BRANCH} yq -i '.release_branch=env(var)' $tmp_config +var=${MINOR_RELEASE_TAG} yq -i '.target_version_tag=env(var)' $tmp_config +var=${PREVIOUS_RELEASE_TAG} yq -i '.previous_release_tag=env(var)' $tmp_config + +cat <<"EOF" > $body_txt +This is an automated PR to prep Data Science Pipelines Operator for release. +```yaml + +``` +EOF + +sed -i "//{ + s///g + r ${tmp_config} +}" $body_txt + +pr_url=$(gh pr create \ + --repo https://github.com/${DSPO_REPOSITORY_FULL} \ + --body-file $body_txt \ + --title "Release ${MINOR_RELEASE_TAG}" \ + --head "${GH_USER_NAME}:$BRANCH_NAME" \ + --label "release-automation" \ + --base "${MINOR_RELEASE_BRANCH}") + +echo "::notice:: PR successfully created: ${pr_url}" diff --git a/.github/scripts/release_prep/prereqs.sh b/.github/scripts/release_prep/prereqs.sh new file mode 100755 index 000000000..3100c0008 --- /dev/null +++ b/.github/scripts/release_prep/prereqs.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -ex + +check_branch_exists(){ + branch_exists=$(git ls-remote --heads https://github.com/${1}.git refs/heads/${2}) + echo "Checking for existence of branch ${2} in GH Repo ${1}" + if [[ $branch_exists ]]; then + echo "::error:: Branch ${2} already exist in GH Repo ${1}" + exit 1 + fi + echo "::notice:: Confirmed Branch ${2} does not exist in GH Repo ${1}" +} + +check_branch_exists ${DSPO_REPOSITORY_FULL} ${MINOR_RELEASE_BRANCH} +check_branch_exists ${DSP_REPOSITORY_FULL} ${MINOR_RELEASE_BRANCH} + +echo "Ensure compatibility.yaml is upto date, and generate a new compatibility.md. Use [release-tools] to accomplish this" + +BRANCH_NAME="compatibility-doc-generate-${TARGET_RELEASE}" + +git config --global user.email "${GH_USER_EMAIL}" +git config --global user.name "${GH_USER_NAME}" +git remote add ${GH_USER_NAME} https://${GH_USER_NAME}:${GH_TOKEN}@github.com/${GH_USER_NAME}/${DSPO_REPOSITORY}.git +git checkout -B ${BRANCH_NAME} + +echo "Created branch: ${BRANCH_NAME}" +echo "Checking if compatibility.yaml contains ${TARGET_RELEASE} release...." + +contains_rel=$(cat docs/release/compatibility.yaml | rel=${MINOR_RELEASE_WILDCARD} yq '[.[].dsp] | contains([env(rel)])') + +if [[ "$contains_rel" == "false" ]]; then + +cat <> /tmp/error.txt +compatibility.yaml has NOT been updated with target release. + +Please add ${MINOR_RELEASE_WILDCARD} dsp row in compatibility.yaml, + +then regenerate the compatibility.md by following the instructions here: +https://github.com/opendatahub-io/data-science-pipelines-operator/tree/main/scripts/release#compatibility-doc-generation +EOF + +echo ::error::$(cat /tmp/error.txt) +exit 1 + +fi + +echo "::notice:: Confirmed existence of ${MINOR_RELEASE_BRANCH} in compatibility.yaml." + +echo "Confirming that compatibility.md is upto date." +python ./scripts/release/release.py version_doc --input_file docs/release/compatibility.yaml --out_file docs/release/compatibility.md + +git status + +prereqs_successful=true + +if [[ `git status --porcelain` ]]; then + echo "::notice:: Compatibility.md is not up to date with Compatibility.yaml, creating pr to synchronize." + + git add . + git commit -m "Update DSPO to $TARGET_RELEASE" + git push ${GH_USER_NAME} $BRANCH_NAME -f + gh pr create \ + --repo https://github.com/${DSPO_REPOSITORY_FULL} \ + --body "This is an automated PR to update Data Science Pipelines Operator version compatibility doc." \ + --title "Update DSP version compatibility doc." \ + --head "${GH_USER_NAME}:$BRANCH_NAME" \ + --base "main" + + echo "::notice:: PR to update compatibility doc has been created, please re-run this workflow once this PR is merged." + prereqs_successful=false +else + echo "::notice:: Compatibility.md doc is up to date with Compatibility.yaml, continuing with workflow..." +fi + +# Save step outputs +echo "prereqs_successful=${prereqs_successful}" +echo "prereqs_successful=${prereqs_successful}" >> $GITHUB_OUTPUT diff --git a/.github/scripts/release_prep/templates/config.yaml b/.github/scripts/release_prep/templates/config.yaml new file mode 100644 index 000000000..7a4301196 --- /dev/null +++ b/.github/scripts/release_prep/templates/config.yaml @@ -0,0 +1,4 @@ +odh_org: placeholder +release_branch: placeholder +target_version_tag: placeholder +previous_release_tag: placeholder diff --git a/.github/scripts/release_trigger/upload-data.sh b/.github/scripts/release_trigger/upload-data.sh new file mode 100755 index 000000000..e6c974f76 --- /dev/null +++ b/.github/scripts/release_trigger/upload-data.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -ex +set -o pipefail + +mkdir -p ./pr + +cat <> /tmp/body-file-raw.txt +${PR_BODY} +EOF + +sed -n '/^```yaml/,/^```/ p' < /tmp/body-file-raw.txt | sed '/^```/ d' > ./pr/config.yaml +echo Parsed config from PR body: +yq ./pr/config.yaml + +# Also store pr details +echo ${PR_NUMBER} >> ./pr/pr_number +echo ${PR_STATE} >> ./pr/pr_state +echo ${PR_HEAD_SHA} >> ./pr/head_sha diff --git a/.github/scripts/tests/tests.sh b/.github/scripts/tests/tests.sh new file mode 100755 index 000000000..a48aed0f5 --- /dev/null +++ b/.github/scripts/tests/tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -ex + +echo "Perform any tests on the branch, confirm stability. If issues are found, they should be corrected in `main/master` and be cherry-picked into this branch." diff --git a/.github/workflows/build-tags.yml b/.github/workflows/build-tags.yml index c06c7f623..000a45425 100644 --- a/.github/workflows/build-tags.yml +++ b/.github/workflows/build-tags.yml @@ -1,6 +1,33 @@ name: Build images from sources. run-name: Build images from sources. on: + workflow_call: + inputs: + src_branch: + type: string + default: 'v1.0.x' + description: 'Source branch to build DSPO/DSP from' + required: true + target_tag: + type: string + default: 'vx.y.z' + description: 'Target Image Tag' + required: true + quay_org: + type: string + default: 'opendatahub' + description: 'Quay Organization' + required: true + dsp_org_repo: + type: string + default: 'opendatahub-io/data-science-pipelines' + description: 'DSP org/repo' + required: true + overwrite_imgs: + type: string + default: 'true' + description: 'Overwrite images in quay if they already exist for this release.' + required: true workflow_dispatch: inputs: src_branch: @@ -19,6 +46,11 @@ on: default: 'opendatahub-io/data-science-pipelines' description: 'DSP org/repo' required: true + overwrite_imgs: + type: string + default: 'false' + description: 'Overwrite images in quay if they already exist for this release.' + required: true env: IMAGE_REPO_DSPO: data-science-pipelines-operator IMAGE_REPO_SERVER: ds-pipelines-api-server @@ -36,8 +68,9 @@ env: QUAY_ID: ${{ secrets.QUAY_ID }} QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} TARGET_IMAGE_TAG: ${{ inputs.target_tag }} + OVERWRITE_IMAGES: ${{ inputs.overwrite_imgs }} jobs: - dspo-build: + DSPO-build: runs-on: ubuntu-latest permissions: contents: read @@ -49,8 +82,9 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_DSPO }} DOCKERFILE: Dockerfile GH_REPO: ${{ github.repository }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} - server-build: + SERVER-build: runs-on: ubuntu-latest permissions: contents: read @@ -62,8 +96,9 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_SERVER }} DOCKERFILE: backend/Dockerfile GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} - ui-build: + UI-build: runs-on: ubuntu-latest permissions: contents: read @@ -75,8 +110,9 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_UI }} DOCKERFILE: frontend/Dockerfile GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} - cache-build: + CACHE-build: runs-on: ubuntu-latest permissions: contents: read @@ -88,6 +124,7 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_CACHE }} DOCKERFILE: backend/Dockerfile.cacheserver GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} PA-build: runs-on: ubuntu-latest @@ -101,6 +138,7 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_PA }} DOCKERFILE: backend/Dockerfile.persistenceagent GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} SWF-build: runs-on: ubuntu-latest @@ -114,6 +152,7 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_SWF }} DOCKERFILE: backend/Dockerfile.scheduledworkflow GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} VC-build: runs-on: ubuntu-latest @@ -127,6 +166,7 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_VC }} DOCKERFILE: backend/Dockerfile.viewercontroller GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} ARTIFACT-build: runs-on: ubuntu-latest @@ -140,6 +180,7 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_ARTIFACT }} DOCKERFILE: backend/artifact_manager/Dockerfile GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} MLMD_WRITER-build: runs-on: ubuntu-latest @@ -153,8 +194,9 @@ jobs: IMAGE_REPO: ${{ env.IMAGE_REPO_MLMD_WRITER }} DOCKERFILE: backend/metadata_writer/Dockerfile GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} - MLMD_ENVOY-build: + MLMD_GRPC-build: runs-on: ubuntu-latest permissions: contents: read @@ -163,11 +205,12 @@ jobs: - uses: ./.github/actions/build name: Build Image with: - IMAGE_REPO: ${{ env.IMAGE_REPO_MLMD_ENVOY }} - DOCKERFILE: third-party/metadata_envoy/Dockerfile + IMAGE_REPO: ${{ env.IMAGE_REPO_MLMD_GRPC }} + DOCKERFILE: third-party/ml-metadata/Dockerfile GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} - MLMD_GRPC-build: + MLMD_ENVOY-build: runs-on: ubuntu-latest permissions: contents: read @@ -176,6 +219,7 @@ jobs: - uses: ./.github/actions/build name: Build Image with: - IMAGE_REPO: ${{ env.IMAGE_REPO_MLMD_GRPC }} - DOCKERFILE: third-party/ml-metadata/Dockerfile + IMAGE_REPO: ${{ env.IMAGE_REPO_MLMD_ENVOY }} + DOCKERFILE: third-party/metadata_envoy/Dockerfile GH_REPO: ${{ inputs.dsp_org_repo }} + OVERWRITE: ${{ env.OVERWRITE_IMAGES }} diff --git a/.github/workflows/odh-manifests-PR-sync.yml b/.github/workflows/odh-manifests-PR-sync.yml index 3ba6e85bb..19b029a48 100644 --- a/.github/workflows/odh-manifests-PR-sync.yml +++ b/.github/workflows/odh-manifests-PR-sync.yml @@ -2,6 +2,24 @@ name: odh-manifests sync run-name: Sync manifests in odh-manifests on: + workflow_call: + inputs: + src_branch: + type: string + default: 'v1.x.x' + description: 'Source branch to build DSPO/DSP from (for example: v1.0.x)' + required: true + target_tag: + type: string + default: 'v1.x.x' + description: 'DSPO version tag to be selected to sync manifests from (for example: v1.0.0)' + required: true + # This is included for dev testing this workflow. + odh_manifest_org: + type: string + default: 'opendatahub-io' + description: 'The GH org for odh-manifest.' + required: true workflow_dispatch: inputs: src_branch: @@ -27,7 +45,7 @@ jobs: pull-requests: write steps: - name: Checkout data-science-pipelines-operator repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{ inputs.src_branch }} @@ -36,7 +54,7 @@ jobs: - name: Send pull-request env: - GH_TOKEN: ${{ secrets.ACCESS_TOKEN }} + GH_TOKEN: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} run: | TARGET_TAG=${{ inputs.target_tag }} @@ -48,7 +66,7 @@ jobs: git clone \ --depth=1 \ --branch=master \ - https://${{ env.GH_USER_NAME}}:${{ secrets.ACCESS_TOKEN }}@github.com/$MANIFESTS_REPOSITORY \ + https://${{ env.GH_USER_NAME}}:${{ secrets.DSP_DEVS_ACCESS_TOKEN }}@github.com/$MANIFESTS_REPOSITORY \ $ODH_MANIFESTS_DIR cd $ODH_MANIFESTS_DIR @@ -69,15 +87,24 @@ jobs: echo Rsync Complete # Commit the changes and push the feature branch to origin + echo "Changes Summary:" git status + if [[ `git status --porcelain` ]]; then + git add . git commit -m "Update DSPO to $TARGET_TAG" git push origin $BRANCH_NAME -f - gh pr create \ + pr_url=$(gh pr create \ --repo https://github.com/${{ inputs.odh_manifest_org }}/odh-manifests \ --body "This is an automated PR to update Data Science Pipelines Operator manifests to $TARGET_TAG" \ --title "Update DSP Operator manifests to $TARGET_TAG" \ --head "${{ env.GH_USER_NAME}}:$BRANCH_NAME" \ - --base "master" + --base "master") + + echo "::notice:: Changes detected in manifests, PR To ODH-Manifest Repo created: ${pr_url}" + + else + echo "::notice:: Changes No changes to manifests requested, no pr required to odh-manifests." + fi diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 554090bab..b50598dc3 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -14,7 +14,7 @@ jobs: volumes: - /cache steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Activate cache uses: actions/cache@v2 with: diff --git a/.github/workflows/release_create.yaml b/.github/workflows/release_create.yaml new file mode 100644 index 000000000..ca78aa10f --- /dev/null +++ b/.github/workflows/release_create.yaml @@ -0,0 +1,141 @@ +name: "Release Create" +run-name: Create Release +on: + workflow_run: + workflows: ["Release Trigger Create"] + types: + - completed +env: + DSPO_REPOSITORY: data-science-pipelines-operator + DSP_REPOSITORY: data-science-pipelines + GH_USER_NAME: dsp-developers +jobs: + fetch-data: + name: Fetch workflow payload + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + outputs: + target_version_tag: ${{ steps.vars.outputs.target_version_tag }} + previous_version_tag: ${{ steps.vars.outputs.previous_version_tag }} + release_branch: ${{ steps.vars.outputs.release_branch }} + odh_org: ${{ steps.vars.outputs.odh_org }} + pr_number: ${{ steps.vars.outputs.pr_number }} + steps: + - name: checkout + uses: actions/checkout@v3 + - name: 'Download artifact' + uses: actions/github-script@v3.1.0 + with: + script: | + var artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{github.event.workflow_run.id }}, + }); + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "pr" + })[0]; + var download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{github.workspace}}/pr.zip', Buffer.from(download.data)); + - run: unzip pr.zip + - shell: bash + id: vars + run: ./.github/scripts/release_create/vars.sh + + validate_pr: + name: Validate PR + runs-on: ubuntu-latest + needs: fetch-data + steps: + - name: checkout + uses: actions/checkout@v3 + - name: validate + env: + PR_NUMBER: ${{ needs.fetch-data.outputs.pr_number }} + TARGET_VERSION_TAG: ${{ needs.fetch-data.outputs.target_version_tag }} + DSPO_REPOSITORY: data-science-pipelines-operator + ODH_ORG: ${{ needs.fetch-data.outputs.odh_org }} + GH_TOKEN: ${{ github.token }} + run: ./.github/scripts/release_create/validate_pr.sh + + create_dspo_tag_release: + name: Create DSPO Release + runs-on: ubuntu-latest + needs: + - fetch-data + - validate_pr + steps: + - name: checkout + uses: actions/checkout@v3 + - name: Creates a DSPO release in GitHub + env: + GITHUB_TOKEN: ${{ github.token }} + GH_USER_NAME: ${{ env.GH_USER_NAME }} + TARGET_VERSION_TAG: ${{ needs.fetch-data.outputs.target_version_tag }} + PREVIOUS_VERSION_TAG: ${{ needs.fetch-data.outputs.previous_version_tag }} + RELEASE_BRANCH: ${{ needs.fetch-data.outputs.release_branch }} + REPOSITORY: ${{ needs.fetch-data.outputs.odh_org }}/${{ env.DSPO_REPOSITORY }} + WORKING_DIR: ${{ github.workspace }} + shell: bash + run: ./.github/scripts/release_create/create_tag_release.sh + + create_dsp_tag_release: + name: Create DSP Release + runs-on: ubuntu-latest + needs: + - fetch-data + - validate_pr + steps: + - name: checkout + uses: actions/checkout@v3 + with: + token: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + - name: Creates a DSPO release in GitHub + env: + GITHUB_TOKEN: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + GH_USER_NAME: ${{ env.GH_USER_NAME }} + TARGET_VERSION_TAG: ${{ needs.fetch-data.outputs.target_version_tag }} + PREVIOUS_VERSION_TAG: ${{ needs.fetch-data.outputs.previous_version_tag }} + RELEASE_BRANCH: ${{ needs.fetch-data.outputs.release_branch }} + REPOSITORY: ${{ needs.fetch-data.outputs.odh_org }}/${{ env.DSP_REPOSITORY }} + GH_ORG: ${{ needs.fetch-data.outputs.odh_org }} + WORKING_DIR: ${{ github.workspace }} + shell: bash + run: ./.github/scripts/release_create/create_tag_release.sh + + sync_manifests: + name: Sync ODH Manifests + uses: ./.github/workflows/odh-manifests-PR-sync.yml + needs: fetch-data + with: + src_branch: ${{ needs.fetch-data.outputs.release_branch }} + target_tag: ${{ needs.fetch-data.outputs.target_version_tag }} + odh_manifest_org: ${{ needs.fetch-data.outputs.odh_org }} + secrets: inherit + + notify_pr: + name: Report Feedback + runs-on: ubuntu-latest + needs: + - validate_pr + - fetch-data + - sync_manifests + - create_dspo_tag_release + - create_dsp_tag_release + steps: + - uses: actions/checkout@v3 + - name: Notify in Pull Request + env: + GITHUB_TOKEN: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + PR_NUMBER: ${{ needs.fetch-data.outputs.pr_number }} + TARGET_VERSION_TAG: ${{ needs.fetch-data.outputs.target_version_tag }} + GH_ORG: ${{ needs.fetch-data.outputs.odh_org }} + run: ./.github/scripts/release_create/notify.sh diff --git a/.github/workflows/release_prep.yaml b/.github/workflows/release_prep.yaml new file mode 100644 index 000000000..40709f5ca --- /dev/null +++ b/.github/workflows/release_prep.yaml @@ -0,0 +1,145 @@ +# Workflow Secrets required to be setup in repo: +# QUAY_ID +# QUAY_TOKEN +# DSP_DEVS_ACCESS_TOKEN +name: "Release Prep" +run-name: Prep Release +on: + workflow_dispatch: + inputs: + previous_release_tag: + default: 'v1.x.y' + description: 'Previous Release tag' + required: true + target_release: + default: '1.x' + description: 'Target Minor Release (e.g. 1.2, 1.3, etc.)' + required: true + gh_org: + default: 'opendatahub-io' + description: 'DSPO GitHub Org' + required: true + quay_org: + type: string + default: 'opendatahub' + description: 'Quay Organization to push builds.' + required: true + overwrite_imgs: + default: 'true' + description: 'Overwrite images in quay if they already exist for this release.' + required: true +env: + GH_USER_EMAIL: 140449482+dsp-developers@users.noreply.github.com + GH_USER_NAME: dsp-developers + TARGET_RELEASE: ${{ inputs.target_release }} + MINOR_RELEASE_WILDCARD: ${{ inputs.target_release }}.x + MINOR_RELEASE_BRANCH: v${{ inputs.target_release }}.x + MINOR_RELEASE_TAG: v${{ inputs.target_release }}.0 + QUAY_ORG: ${{ inputs.quay_org }} + GH_ORG: ${{ inputs.gh_org }} + DSPO_REPOSITORY: data-science-pipelines-operator + DSPO_REPOSITORY_FULL: ${{ inputs.gh_org }}/data-science-pipelines-operator + DSP_REPOSITORY: data-science-pipelines + DSP_REPOSITORY_FULL: ${{ inputs.gh_org }}/data-science-pipelines + PREVIOUS_RELEASE_TAG: ${{ inputs.previous_release_tag }} + OVERWRITE_IMAGES: ${{ inputs.overwrite_imgs }} + CONFIG_TEMPLATE: "./.github/scripts/release_prep/templates/config.yaml" +jobs: + prereqs: + name: Prerequisites + runs-on: ubuntu-latest + outputs: + prereqs_successful: ${{ steps.vars.outputs.prereqs_successful }} + steps: + - name: checkout + uses: actions/checkout@v3 + with: + token: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + - name: compatibility doc + id: vars + env: + GH_TOKEN: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + GH_USER_NAME: ${{ env.GH_USER_NAME }} + GH_USER_EMAIL: ${{ env.GH_USER_EMAIL }} + TARGET_RELEASE: ${{ env.TARGET_RELEASE }} + MINOR_RELEASE_WILDCARD: ${{ env.MINOR_RELEASE_WILDCARD }} + MINOR_RELEASE_BRANCH: ${{ env.MINOR_RELEASE_BRANCH }} + DSPO_REPOSITORY_FULL: ${{ env.DSPO_REPOSITORY_FULL }} + DSPO_REPOSITORY: ${{ env.DSPO_REPOSITORY }} + run: ./.github/scripts/release_prep/prereqs.sh + + create_branches: + name: Create Release Branches + runs-on: ubuntu-latest + needs: prereqs + if: needs.prereqs.outputs.prereqs_successful == 'true' + steps: + - uses: actions/checkout@v3 + name: checkout + with: + token: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + fetch-depth: 0 + - name: create branches + env: + GH_TOKEN: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + GH_USER_NAME: ${{ env.GH_USER_NAME }} + MINOR_RELEASE_BRANCH: ${{ env.MINOR_RELEASE_BRANCH }} + DSPO_REPOSITORY_FULL: ${{ env.DSPO_REPOSITORY_FULL }} + DSP_REPOSITORY_FULL: ${{ env.DSP_REPOSITORY_FULL }} + WORKING_DIR: ${{ github.workspace }} + run: ./.github/scripts/release_prep/create_branches.sh + + # NOTE: env from current workflow does not extend to re-usable workflows + # so "with:" commands do not get "env.*" context, but "needs.*.outputs.*" works + # this is a workaround. + # More Context: https://github.com/orgs/community/discussions/26671 + get-env-vars: + name: Get Re-Usable Env Vars + runs-on: ubuntu-latest + outputs: + MINOR_RELEASE_BRANCH: ${{ env.MINOR_RELEASE_BRANCH }} + MINOR_RELEASE_TAG: ${{ env.MINOR_RELEASE_TAG }} + QUAY_ORG: ${{ env.QUAY_ORG }} + DSPO_GH_ORG: ${{ env.DSP_REPOSITORY_FULL }} + OVERWRITE_IMAGES: ${{ env.OVERWRITE_IMAGES }} + steps: + - run: echo "Storing env vars for re-usable workflow." + + build_images: + name: Build/Push Images + needs: [create_branches, get-env-vars] + if: needs.prereqs.outputs.prereqs_successful == 'true' + uses: ./.github/workflows/build-tags.yml + with: + src_branch: ${{ needs.get-env-vars.outputs.MINOR_RELEASE_BRANCH }} + target_tag: ${{ needs.get-env-vars.outputs.MINOR_RELEASE_TAG }} + quay_org: ${{ needs.get-env-vars.outputs.QUAY_ORG }} + dsp_org_repo: ${{ needs.get-env-vars.outputs.DSPO_GH_ORG }} + overwrite_imgs: ${{ needs.get-env-vars.outputs.OVERWRITE_IMAGES }} + secrets: inherit + + generate_pr: + name: Generate Release PR + runs-on: ubuntu-latest + needs: build_images + steps: + - uses: actions/checkout@v3 + name: checkout + with: + token: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + ref: ${{ env.MINOR_RELEASE_BRANCH }} + - name: generate pr + env: + GH_TOKEN: ${{ secrets.DSP_DEVS_ACCESS_TOKEN }} + GH_USER_NAME: ${{ env.GH_USER_NAME }} + GH_USER_EMAIL: ${{ env.GH_USER_EMAIL }} + TARGET_RELEASE: ${{ env.TARGET_RELEASE }} + MINOR_RELEASE_BRANCH: ${{ env.MINOR_RELEASE_BRANCH }} + DSPO_REPOSITORY: ${{ env.DSPO_REPOSITORY }} + DSPO_REPOSITORY_FULL: ${{ env.DSPO_REPOSITORY_FULL }} + GH_ORG: ${{ env.GH_ORG }} + QUAY_ORG: ${{ env.QUAY_ORG }} + MINOR_RELEASE_TAG: ${{ env.MINOR_RELEASE_TAG }} + PREVIOUS_RELEASE_TAG: ${{ env.PREVIOUS_RELEASE_TAG }} + CONFIG_TEMPLATE: ${{ env.CONFIG_TEMPLATE }} + run: ./.github/scripts/release_prep/generate_pr.sh diff --git a/.github/workflows/release_tests.yaml b/.github/workflows/release_tests.yaml new file mode 100644 index 000000000..646a7161d --- /dev/null +++ b/.github/workflows/release_tests.yaml @@ -0,0 +1,15 @@ +name: "Release Tests" +run-name: Run Release Test Suite +on: + pull_request: + branches: + - v** +jobs: + tests: + name: Release Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + name: checkout + - name: sometests + run: ./.github/scripts/tests/tests.sh diff --git a/.github/workflows/release_trigger.yaml b/.github/workflows/release_trigger.yaml new file mode 100644 index 000000000..1ae551f1a --- /dev/null +++ b/.github/workflows/release_trigger.yaml @@ -0,0 +1,28 @@ +name: "Release Trigger Create" # This is used by release_create.yaml on.workflow_run.workflows, change with caution +on: + pull_request: + types: + - closed + paths: + - config/base/params.env +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + upload-data: + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'release-automation') && github.event.pull_request.merged + steps: + - uses: actions/checkout@v3 + - name: Save PR payload + shell: bash + env: + PR_BODY: ${{github.event.pull_request.body}} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_STATE: ${{ github.event.pull_request.state }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: ./.github/scripts/release_trigger/upload-data.sh + - uses: actions/upload-artifact@v2 + with: + name: pr + path: pr/ diff --git a/OWNERS b/OWNERS index 8b67037f1..72b639d36 100644 --- a/OWNERS +++ b/OWNERS @@ -2,6 +2,7 @@ approvers: - accorvin - anishasthana - DharmitD + - dsp-developers - gmfrasca - gregsheremeta - harshad16 diff --git a/docs/release/compatibility.md b/docs/release/compatibility.md index 1fef5e2f4..2ef05657d 100644 --- a/docs/release/compatibility.md +++ b/docs/release/compatibility.md @@ -25,6 +25,7 @@ registries, this is true for the following: | 1.0.x | 1.5.1 | 1.5.0 | 1.8.4 | v4.10 | v4.12 | 1 | 8.8 | 8.8 | 4.10,4.11,4.12 | | 1.1.x | 1.5.1 | 1.5.0 | 1.8.4 | v4.10 | v4.12 | 1 | 8.8 | 8.8 | 4.10,4.11,4.12 | | 1.2.x | 1.5.1 | 1.5.0 | 1.8.4 | v4.10 | v4.10 | 1 | 8.8 | 8.8 | 4.10,4.11,4.12 | +| 1.3.x | 1.5.1 | 1.5.0 | 1.9.2 | v4.10 | v4.10 | 1 | 8.8 | 8.8 | 4.10,4.11,4.12 | diff --git a/docs/release/compatibility.yaml b/docs/release/compatibility.yaml index b774bc56b..35fd3ecc6 100644 --- a/docs/release/compatibility.yaml +++ b/docs/release/compatibility.yaml @@ -28,3 +28,13 @@ ubi-minimal: 8.8 ubi-micro: 8.8 openshift: 4.10,4.11,4.12 +- dsp: 1.3.x + kfp-tekton: 1.5.1 + ml-metadata: 1.5.0 + envoy: 1.8.4 + ocp-pipelines: v4.10 + oauth-proxy: v4.10 + mariadb-103: 1 + ubi-minimal: 8.8 + ubi-micro: 8.8 + openshift: 4.10,4.11,4.12 diff --git a/scripts/release/params.py b/scripts/release/params.py index 7d3f64f91..5eba14b66 100644 --- a/scripts/release/params.py +++ b/scripts/release/params.py @@ -100,11 +100,11 @@ def generate_params(args): image_repo = QUAY_REPOS[image_env_var] digest = fetch_quay_repo_tag_digest(image_repo, quay_org, tag) image_repo_with_digest = f"{image_repo}@{digest}" - images.append(f"{image_env_var}=quay.io/opendatahub/{image_repo_with_digest}") + images.append(f"{image_env_var}=quay.io/{quay_org}/{image_repo_with_digest}") # Fetch RH Registry images rh_registry_images = { - RH_REGISTRY_IO: [ + RH_REGISTRY_ACCESS: [ { "repo": REPO_UBI_MINIMAL, "tag": ubi_minimal_tag, @@ -116,7 +116,7 @@ def generate_params(args): "env": IMAGES_MOVERESULTSIMAGE }, ], - RH_REGISTRY_ACCESS: [ + RH_REGISTRY_IO: [ { "repo": REPO_MARIADB, "tag": mariadb_tag, diff --git a/scripts/release/version_doc.py b/scripts/release/version_doc.py index 2fe134378..a4c2aae55 100644 --- a/scripts/release/version_doc.py +++ b/scripts/release/version_doc.py @@ -1,3 +1,5 @@ +import os + import yaml @@ -37,7 +39,9 @@ def version_doc(args): with open(input_file, 'r') as f: rows = yaml.safe_load(f) - with open('template/version_doc.md', 'r') as vd: + dirname = os.path.dirname(__file__) + template_file = os.path.join(dirname, 'template/version_doc.md') + with open(template_file, 'r') as vd: final_md = vd.read() table_md = table(rows)