From cd176f0c58ad03cab25107e6828abb0ca39ad58b Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 18 Oct 2024 11:35:10 +0800 Subject: [PATCH 1/4] ci(engine-image): adjust token and build by action --- .github/workflows/build-dev-image.yml | 35 ++++++++++++++++++--------- .github/workflows/build-image.yml | 34 +++++++++++++++++--------- .github/workflows/stable-release.yml | 34 +++++++++++++++++--------- 3 files changed, 68 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index d1bfdf27b..d2efe1bdc 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -24,28 +24,39 @@ jobs: - name: Build run: | ./mvnw clean install -B -DskipTests -P exec-jar + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/canner/wren-engine + tags: | + type=sha + type=raw,value=nightly - name: Login to ghcr uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build Docker image + - name: Prepare + id: prepare run: | + cp ./wren-server/target/wren-server-${WREN_VERSION}-executable.jar ./docker WREN_VERSION=$(./mvnw --quiet help:evaluate -Dexpression=project.version -DforceStdout) - LATEST_COMMIT=$(git log -1 --pretty=%h) - cd ./docker - cp ../wren-server/target/wren-server-${WREN_VERSION}-executable.jar ./ - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag ghcr.io/canner/wren-engine:main-${LATEST_COMMIT} \ - --tag ghcr.io/canner/wren-engine:nightly \ - --push -f ./Dockerfile \ - --build-arg "WREN_VERSION=${WREN_VERSION}" . + echo "WREN_VERSION=$WREN_VERSION" >> "$GITHUB_OUTPUT" + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: ./docker + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + build-args: | + WREN_VERSION=${{ steps.prepare.outputs.WREN_VERSION }} + push: true build-ibis-image: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 4d6968b03..7db49cfad 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -37,28 +37,38 @@ jobs: - name: Build run: | ./mvnw clean install -B -DskipTests -P exec-jar + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/canner/wren-engine + tags: | + type=raw,value=${{ needs.prepare-tag.outputs.tag_name }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build Docker image - env: - TAG_NAME: ${{ needs.prepare-tag.outputs.tag_name }} + - name: Prepare + id: prepare run: | + cp ./wren-server/target/wren-server-${WREN_VERSION}-executable.jar ./docker WREN_VERSION=$(./mvnw --quiet help:evaluate -Dexpression=project.version -DforceStdout) - cd ./docker - cp ../wren-server/target/wren-server-${WREN_VERSION}-executable.jar ./ - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag ghcr.io/canner/wren-engine:$TAG_NAME \ - --push -f ./Dockerfile \ - --build-arg "WREN_VERSION=${WREN_VERSION}" . + echo "WREN_VERSION=$WREN_VERSION" >> "$GITHUB_OUTPUT" + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: ./docker + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + build-args: | + WREN_VERSION=${{ steps.prepare.outputs.WREN_VERSION }} + push: true build-ibis-image: needs: prepare-tag runs-on: ubuntu-latest diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 5b07ed046..3d4c15d1a 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -80,27 +80,39 @@ jobs: - name: Build run: | ./mvnw clean install -B -DskipTests -P exec-jar + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/canner/wren-engine + tags: | + type=raw,value=${{ needs.prepare-version.outputs.maven_version }} + type=raw,value=latest - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build Docker image + - name: Prepare + id: prepare run: | + cp ./wren-server/target/wren-server-${WREN_VERSION}-executable.jar ./docker WREN_VERSION=$(./mvnw --quiet help:evaluate -Dexpression=project.version -DforceStdout) - cd ./docker - cp ../wren-server/target/wren-server-${WREN_VERSION}-executable.jar ./ - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag ghcr.io/canner/wren-engine:${{ needs.prepare-version.outputs.maven_version }} \ - --tag ghcr.io/canner/wren-engine:latest \ - --push -f ./Dockerfile \ - --build-arg "WREN_VERSION=${WREN_VERSION}" . + echo "WREN_VERSION=$WREN_VERSION" >> "$GITHUB_OUTPUT" + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: ./docker + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + build-args: | + WREN_VERSION=${{ steps.prepare.outputs.WREN_VERSION }} + push: true stable-release-ibis: needs: prepare-version runs-on: ubuntu-latest From 6b9ed7ccca5fe1118ed795412e8d6b0faa68f54b Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 18 Oct 2024 11:35:38 +0800 Subject: [PATCH 2/4] ci: remove unused --- .github/workflows/deploy.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index c570ec652..000000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Deploy - -on: - workflow_dispatch - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 21 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: '21' - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - ./mvnw clean install -B -DskipTests - ./mvnw deploy -B -DskipTests From aa8e492d48eca7bf18f550194d24b7532e6a4d2d Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 18 Oct 2024 11:40:21 +0800 Subject: [PATCH 3/4] ci: merge test ci --- .github/workflows/build.yml | 33 ------------------- ...{integration-tests.yml => maven-tests.yml} | 20 +++++------ 2 files changed, 8 insertions(+), 45 deletions(-) delete mode 100644 .github/workflows/build.yml rename .github/workflows/{integration-tests.yml => maven-tests.yml} (67%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index ac8b568ac..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: PreCommit - -on: - pull_request: - types: [ labeled ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.number }} - cancel-in-progress: true - -jobs: - build: - if: ${{ github.event.label.name == 'v1-engine-changed' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 21 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: '21' - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: build - run: | - ./mvnw clean install -B -DskipTests - - name: unit tests - run: | - ./mvnw test -B --fail-at-end -pl !:wren-tests diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/maven-tests.yml similarity index 67% rename from .github/workflows/integration-tests.yml rename to .github/workflows/maven-tests.yml index 101a76590..954821c19 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/maven-tests.yml @@ -1,5 +1,4 @@ -# Run secret-dependent integration tests -name: Integration tests +name: Maven tests on: pull_request: @@ -16,26 +15,23 @@ concurrency: cancel-in-progress: true jobs: - # Branch-based in origin repo pull request - integration-trusted: + maven-tests: runs-on: ubuntu-latest - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.label.name == 'v1-engine-changed' }} + if: ${{ github.event.label.name == 'v1-engine-changed' }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK 21 uses: actions/setup-java@v2 with: distribution: 'temurin' java-version: '21' - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + cache: 'maven' - name: build run: | ./mvnw clean install -B -DskipTests + - name: unit tests + run: | + ./mvnw test -B --fail-at-end -pl !:wren-tests - name: integration tests env: TEST_BIG_QUERY_PROJECT_ID: ${{ secrets.TEST_BIG_QUERY_PROJECT_ID }} From 300c517723bebbe1d904b16d4e4573466c1061c7 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 18 Oct 2024 11:41:17 +0800 Subject: [PATCH 4/4] ci: update name --- .github/workflows/build-dev-image.yml | 2 +- .github/workflows/build-image.yml | 2 +- .github/workflows/modeling-py-mult-platform-ci.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index d2efe1bdc..d673bf50d 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -1,4 +1,4 @@ -name: Build dev image +name: Build dev image on push to main automatically on: push: diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 7db49cfad..533fe6dcc 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -1,4 +1,4 @@ -name: Build image +name: Build image manually on: workflow_dispatch: diff --git a/.github/workflows/modeling-py-mult-platform-ci.yml b/.github/workflows/modeling-py-mult-platform-ci.yml index a6a80ea76..1c6ccc547 100644 --- a/.github/workflows/modeling-py-mult-platform-ci.yml +++ b/.github/workflows/modeling-py-mult-platform-ci.yml @@ -3,7 +3,7 @@ # # maturin generate-ci --pytest github # -name: Modeling Python CI +name: Modeling Python CI (multi-platform) # We don't ready to run this workflow on stable release flow yet on: