ci: Release automation (#282)

Signed-off-by: Oliver Koenig <okoenig@nvidia.com>
NVIDIA · Sep 24, 2024 · b2165bd · b2165bd
1 parent 298efaf
commit b2165bd
Show file tree

Hide file tree

Showing 4 changed files with 238 additions and 49 deletions.
diff --git a/.github/workflows/_build_container.yml b/.github/workflows/_build_container.yml
@@ -0,0 +1,78 @@
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: Build NeMo Aligner container
+on: 
+  workflow_call:
+    inputs:
+      ref: 
+        description: Git ref to checkout
+        default: ${{ github.sha }}
+        required: false
+        type: string
+
+defaults:
+  run:
+    shell: bash -x -e -u -o pipefail {0}
+
+jobs:
+  main:
+    runs-on: self-hosted-builder
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+          ref: ${{ inputs.sha }}
+
+      - name: Clean runner cache
+        run: | 
+          docker system prune --filter "until=24h" --force
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with: 
+          # We use `docker` driver as this speeds things up for 
+          # trivial (non-multi-stage) builds.
+          driver: docker
+
+      - name: Pull cache images
+        run: |
+          docker pull nemoci.azurecr.io/nemo_aligner_container:${{ github.event.pull_request.number || 'buildcache' }} || true
+          docker pull nemoci.azurecr.io/nemo_aligner_container:buildcache || true
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile
+          push: true
+          build-args: |
+            MAX_JOBS=32
+            ALIGNER_COMMIT=${{ github.event.pull_request.head.sha || github.sha }}
+          cache-from: |
+            nemoci.azurecr.io/nemo_aligner_container:${{ github.event.pull_request.number || 'buildcache' }}
+            nemoci.azurecr.io/nemo_aligner_container:buildcache
+          cache-to: type=inline
+          tags: nemoci.azurecr.io/nemo_aligner_container:${{ github.run_id }}
+
+      - name: Update PR image
+        if: github.event_name == 'pull_request'
+        run: |
+          docker tag nemoci.azurecr.io/nemo_aligner_container:${{ github.run_id }} nemoci.azurecr.io/nemo_aligner_container:${{ github.event.pull_request.number }}
+          docker push nemoci.azurecr.io/nemo_aligner_container:${{ github.event.pull_request.number }}
+
+      - name: Update buildcache image
+        if: github.ref == 'refs/heads/main'
+        run: |
+          docker tag nemoci.azurecr.io/nemo_aligner_container:${{ github.run_id }} nemoci.azurecr.io/nemo_aligner_container:buildcache
+          docker push nemoci.azurecr.io/nemo_aligner_container:buildcache
diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml
@@ -28,53 +28,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
-    runs-on: self-hosted-builder
+  build-container:
     if: ${{ github.event.label.name == 'Run CICD' || github.ref == 'refs/heads/main' }}
-    steps:
-      - name: Clean runner cache
-        run: | 
-          docker system prune --filter "until=24h" --force
-
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          path: ${{ github.run_id }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with: 
-          # We use `docker` driver as this speeds things up for 
-          # trivial (non-multi-stage) builds.
-          driver: docker
-
-      - name: Pull cache images
-        run: |
-          docker pull nemoci.azurecr.io/nemo_aligner_container:${{ github.event.pull_request.number || 'buildcache' }} || true
-          docker pull nemoci.azurecr.io/nemo_aligner_container:buildcache || true
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          file: Dockerfile
-          push: true
-          build-args: |
-            MAX_JOBS=32
-            ALIGNER_COMMIT=${{ github.event.pull_request.head.sha || github.sha }}
-          cache-from: |
-            nemoci.azurecr.io/nemo_aligner_container:${{ github.event.pull_request.number || 'buildcache' }}
-            nemoci.azurecr.io/nemo_aligner_container:buildcache
-          cache-to: type=inline
-          tags: nemoci.azurecr.io/nemo_aligner_container:${{ github.run_id }}
-
-      - name: Update PR image
-        if: github.event_name == 'pull_request'
-        run: |
-          docker tag nemoci.azurecr.io/nemo_aligner_container:${{ github.run_id }} nemoci.azurecr.io/nemo_aligner_container:${{ github.event.pull_request.number }}
-          docker push nemoci.azurecr.io/nemo_aligner_container:${{ github.event.pull_request.number }}
-
-      - name: Update buildcache image
-        if: github.ref == 'refs/heads/main'
-        run: |
-          docker tag nemoci.azurecr.io/nemo_aligner_container:${{ github.run_id }} nemoci.azurecr.io/nemo_aligner_container:buildcache
-          docker push nemoci.azurecr.io/nemo_aligner_container:buildcache
+    uses: ./.github/workflows/_build_container.yml
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -0,0 +1,157 @@
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: "Release"
+
+defaults:
+  run:
+    shell: bash -x -e -u -o pipefail {0}
+
+on:
+  workflow_dispatch:
+
+jobs: 
+  build-container:
+    uses: ./.github/workflows/_build_container.yml
+
+  build-and-test-wheel:
+    needs: build-container
+    runs-on: self-hosted-builder
+    container: nemoci.azurecr.io/nemo_aligner_container:${{ github.run_id }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+
+      - name: Build and test wheel
+        run: |
+          cd ${{ github.run_id }}
+          EXPECTED_VERSION=$(python -c 'import nemo_aligner; print(nemo_aligner.__version__)')
+
+          python3 -m pip install --upgrade build
+          python3 -m build
+
+          pip install dist/*.whl
+
+          cd ../
+
+          INSTALLED_VERSION=$(python -c 'import nemo_aligner; print(nemo_aligner.__version__)')
+          
+          if [[ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]]; then
+            echo 'Wheel has an outdated version, mission abort immediately!'
+            exit 1
+          fi
+        
+      - name: Upload wheel 
+        uses: actions/upload-artifact@v4
+        with:
+          name: pip-wheel
+          path: ${{ github.run_id }}/dist/
+
+  publish-wheel:
+    needs: build-and-test-wheel
+    if: contains(fromJSON('["terryk", "gshennvm", "ko3n1g"]'), github.actor)
+    runs-on: ubuntu-latest
+    environment: 
+      name: main
+    steps:  
+      - name: Download wheel
+        uses: actions/download-artifact@v4
+        with:
+          name: pip-wheel
+          path: ${{ github.run_id }}/dist/
+
+      - name: Release wheel
+        env:
+          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+        run: |
+          python3 -m pip install --upgrade twine
+          python3 -m twine upload --repository pypi ${{ github.run_id }}/dist/*
+
+  create-gh-release:
+    needs: publish-wheel
+    runs-on: ubuntu-latest
+    if: contains(fromJSON('["terryk", "gshennvm", "ko3n1g"]'), github.actor)
+    environment: 
+      name: main
+    outputs:
+      version-number: ${{ steps.version-number.outputs.version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+
+      - name: Create release
+        id: version-number
+        run: |
+          cd ${{ github.run_id }}
+          VERSION=$(python -c "import nemo_aligner; print(nemo_aligner.__version__)")
+          echo "version=$VERSION" | tee -a "$GITHUB_OUTPUT"
+          
+          NAME="NVIDIA NeMo-Aligner ${VERSION}"
+          CHANGELOG=$(awk '/^## '"$NAME"'/{flag=1; next} /^## /{flag=0} flag' CHANGELOG.md)
+          CHANGELOG=$(echo "$CHANGELOG" | sed '/./,$!d' | sed ':a;N;$!ba;s/\n$//')
+          SHA=${{ github.sha }}
+
+          PAYLOAD=$(jq \
+                      -n \
+                      -c \
+                      --arg TAG_NAME "v${VERSION}" \
+                      --arg CI_COMMIT_BRANCH "$SHA" \
+                      --arg NAME "$NAME" \
+                      --arg BODY "$CHANGELOG" \
+                      '{
+                        "tag_name": $TAG_NAME,
+                        "target_commitish": $CI_COMMIT_BRANCH,
+                        "name": $NAME,
+                        "body": $BODY,
+                        "draft": false,
+                        "prerelease": false,
+                        "generate_release_notes": false
+                      }'
+                  )
+          GH_TOKEN=${{ secrets.PAT }}
+          curl -L \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer $GH_TOKEN" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/NVIDIA/NeMo-Aligner/releases \
+            -d "$PAYLOAD"
+
+  notify:
+    needs: create-gh-release
+    runs-on: ubuntu-latest
+    if: contains(fromJSON('["terryk", "gshennvm", "ko3n1g"]'), github.actor)
+    steps:
+      - name: notify
+        run: |
+          VERSION=${{ needs.create-gh-release.outputs.version-number }}
+          URL="https://github.com/NVIDIA/NeMo-Aligner/releases/tag/v$VERSION"
+          WEBHOOK=${{ secrets.SLACK_WEBHOOK }}
+          MESSAGE='{
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Releasebot 🤖: NeMo-Aligner released <'$URL'|v'$VERSION'> 🚀"
+                }
+              }
+            ]
+          }'
+
+          curl -X POST -H "Content-type: application/json" --data "$MESSAGE" $WEBHOOK
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1 @@
+include setup/requirements.txt