Fix/clickhouse sink issue with collapsing merge tree (#2477) #1520
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: [release, release-dev, release-test, main] | |
| tags: | |
| - "v*.*.*" | |
| env: | |
| CARGO_TERM_COLOR: always | |
| BUCKET_NAME: "dozer-releases" | |
| ECR_REGISTRY: public.ecr.aws/k7k6x1d4 | |
| ECR_REPOSITORY: dozer | |
| DOCKERHUB_REGISTRY: getdozer | |
| DOCKERHUB_REPOSITORY: dozer | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| contents: write # This is required for actions/checkout | |
| jobs: | |
| # https://github.com/orhun/git-cliff/blob/main/.github/workflows/cd.yml | |
| prepare: | |
| name: Prepare | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| outputs: | |
| release_body: ${{ steps.release.outputs.release_body }} | |
| version: ${{ steps.version.outputs.version }} | |
| prerelease: ${{ steps.version.outputs.prerelease }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: 'recursive' | |
| - name: Generate a changelog | |
| uses: orhun/git-cliff-action@v1 | |
| id: git-cliff | |
| with: | |
| config: .github/config/cliff.toml | |
| args: -vv --latest --strip header | |
| env: | |
| OUTPUT: CHANGES.md | |
| - name: Set the release body | |
| id: release | |
| shell: bash | |
| run: | | |
| r=$(cat ${{ steps.git-cliff.outputs.changelog }}) | |
| r="$(printf "$r" | tail -n +3)" | |
| r="${r//'%'/'%25'}" | |
| r="${r//$'\n'/'%0A'}" | |
| r="${r//$'\r'/'%0D'}" | |
| echo "::set-output name=release_body::$r" | |
| - name: Set release version | |
| id: version | |
| run: | | |
| tag=$(printf "%q" ${{ github.ref_name }}) | |
| if [[ $tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| echo "::set-output name=version::$tag" | |
| echo "::set-output name=prerelease::false" | |
| else | |
| echo "::set-output name=version::dev" | |
| echo "::set-output name=prerelease::true" | |
| fi | |
| release-linux-aarch64: | |
| name: Release Linux binary for aarch64 | |
| runs-on: ubuntu-20.04 | |
| needs: prepare | |
| env: | |
| CARGO_TARGET: aarch64-unknown-linux-gnu | |
| DEB_NAME: dozer-linux-aarch64 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Rust cache | |
| uses: swatinem/rust-cache@v2 | |
| - name: Install toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| target: ${{ env.CARGO_TARGET }} | |
| - name: Install cross | |
| uses: baptiste0928/cargo-install@v1 | |
| with: | |
| crate: cross | |
| cache-key: '${{ env.CARGO_TARGET }}' | |
| - name: Build dozer | |
| run: cross build --package=dozer-cli --profile=release --target ${{ env.CARGO_TARGET }} --bin dozer | |
| - name: Install cargo-deb | |
| uses: baptiste0928/cargo-install@v1 | |
| with: | |
| crate: cargo-deb | |
| cache-key: '${{ env.CARGO_TARGET }}' | |
| - name: Compile deb file | |
| run: cargo deb -p dozer-cli --target ${{ env.CARGO_TARGET }} --no-build --no-strip --output ./deb/${{ env.DEB_NAME }}.deb | |
| - name: Prepare release assets | |
| shell: bash | |
| run: | | |
| mkdir -p release | |
| cp {LICENSE,README.md,CHANGELOG.md} release/ 2> /dev/null || echo "Copy Failed...Ignoring.." | |
| cp target/${{ env.CARGO_TARGET }}/release/dozer release/ | |
| mv release/ ${{ env.DEB_NAME }}/ | |
| tar -czvf ${{ env.DEB_NAME }}.tar.gz ${{ env.DEB_NAME }}/ | |
| cp deb/${{ env.DEB_NAME }}.deb ./ | |
| ls -l ${{ env.DEB_NAME }}* | |
| - name: Upload the release | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: ${{ env.DEB_NAME }}* | |
| file_glob: true | |
| overwrite: true | |
| tag: ${{ needs.prepare.outputs.version }} | |
| release_name: "Development Release - ${{ needs.prepare.outputs.version }}" | |
| prerelease: ${{ needs.prepare.outputs.prerelease }} | |
| body: "${{ needs.prepare.outputs.release_body }}" | |
| - name: Set env variables | |
| env: | |
| VERSION: ${{ needs.prepare.outputs.version }} | |
| RELEASE_NAME: ${{ env.DEB_NAME }}.tar.gz | |
| run: | | |
| echo "RELEASE_NAME=${{env.RELEASE_NAME}}" >> $GITHUB_ENV | |
| echo "DEB_NAME=${{ env.DEB_NAME }}.deb" >> $GITHUB_ENV | |
| echo "VERSION=${{env.VERSION}}" >> $GITHUB_ENV | |
| echo "ARTIFACT_URL=https://${{ env.BUCKET_NAME }}.s3.ap-southeast-1.amazonaws.com/${{ env.VERSION }}/${{ env.RELEASE_NAME }}" >> $GITHUB_ENV | |
| - name: List deb output files | |
| run: ls -lR ./deb | |
| - name: configure aws credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| role-session-name: deployer | |
| aws-region: ap-southeast-1 | |
| - name: Upload release to S3 | |
| id: upload_s3 | |
| run: | | |
| aws s3 cp $RELEASE_NAME s3://${{ env.BUCKET_NAME }}/$VERSION/$RELEASE_NAME | |
| - name: Upload release deb to S3 | |
| id: upload_s3_deb | |
| run: | | |
| aws s3 cp deb/$DEB_NAME s3://${{ env.BUCKET_NAME }}/$VERSION/$DEB_NAME | |
| release-macos-apple-silicon: | |
| name: Release binary for macOS silicon | |
| runs-on: ${{ matrix.os }} | |
| needs: prepare | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: macos-12 | |
| target: aarch64-apple-darwin | |
| file_name: dozer | |
| asset_name: dozer-macos-aarch64 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Installing Rust toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| profile: minimal | |
| target: ${{ matrix.target }} | |
| override: true | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Rust cache | |
| uses: swatinem/rust-cache@v2 | |
| - name: Cargo build | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: build | |
| args: --release --target ${{ matrix.target }} --bin ${{ matrix.file_name }} | |
| - name: List target output files | |
| run: ls -lR ./target | |
| - name: Prepare release assets | |
| shell: bash | |
| run: | | |
| mkdir -p release | |
| cp {LICENSE,README.md,CHANGELOG.md} release/ 2> /dev/null || echo "Copy Failed...Ignoring.." | |
| cp target/${{ matrix.target }}/release/${{matrix.file_name}} release/ | |
| mv release/ ${{matrix.asset_name}}/ | |
| tar -czvf ${{matrix.asset_name}}.tar.gz ${{matrix.asset_name}}/ | |
| - name: Upload the release | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: ${{matrix.asset_name}}* | |
| file_glob: true | |
| overwrite: true | |
| tag: ${{ needs.prepare.outputs.version }} | |
| release_name: "Development Release - ${{ needs.prepare.outputs.version }}" | |
| prerelease: ${{ needs.prepare.outputs.prerelease }} | |
| body: "${{ needs.prepare.outputs.release_body }}" | |
| - name: Set env variables | |
| env: | |
| VERSION: ${{ needs.prepare.outputs.version }} | |
| RELEASE_NAME: ${{matrix.asset_name}}.tar.gz | |
| run: | | |
| echo "RELEASE_NAME=${{env.RELEASE_NAME}}" >> $GITHUB_ENV | |
| echo "VERSION=${{env.VERSION}}" >> $GITHUB_ENV | |
| echo "ARTIFACT_URL=https://${{ env.BUCKET_NAME }}.s3.ap-southeast-1.amazonaws.com/${{ env.VERSION }}/${{ env.RELEASE_NAME }}" >> $GITHUB_ENV | |
| - name: configure aws credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| role-session-name: deployer | |
| aws-region: ap-southeast-1 | |
| - name: Upload release to S3 | |
| id: upload_s3 | |
| run: | | |
| aws s3 cp $RELEASE_NAME s3://${{ env.BUCKET_NAME }}/$VERSION/$RELEASE_NAME | |
| release: | |
| name: Release | |
| runs-on: | |
| labels: ${{ matrix.os }} | |
| needs: prepare | |
| strategy: | |
| matrix: | |
| os: [ubuntu-20.04] | |
| include: | |
| - os: ubuntu-20.04 | |
| file_name: dozer | |
| target: x86_64-unknown-linux-gnu | |
| asset_name: dozer-linux-amd64 | |
| - os: macos-12 | |
| file_name: dozer | |
| target: x86_64-apple-darwin | |
| asset_name: dozer-macos-amd64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: stable | |
| target: ${{ matrix.target }} | |
| components: rustfmt, clippy | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Rust cache | |
| uses: swatinem/rust-cache@v2 | |
| - name: Install cargo-deb | |
| if: matrix.os == 'ubuntu-20.04' | |
| run: cargo install cargo-deb | |
| - name: Compile deb file | |
| if: matrix.os == 'ubuntu-20.04' | |
| run: cargo-deb -p dozer-cli --output ./deb/${{matrix.asset_name}}.deb | |
| - name: Build package | |
| if: matrix.os != 'ubuntu-20.04' | |
| run: cargo build --release --bin ${{ matrix.file_name }} | |
| - name: Build package for ubuntu (with kafka & snowflake) | |
| if: matrix.os == 'ubuntu-20.04' | |
| run: cargo build --release --bin ${{ matrix.file_name }} --features "kafka snowflake" | |
| - name: Prepare release assets | |
| shell: bash | |
| run: | | |
| mkdir -p release | |
| cp {LICENSE,README.md,CHANGELOG.md} release/ 2> /dev/null || echo "Copy Failed...Ignoring.." | |
| cp target/release/${{matrix.file_name}} release/ | |
| mv release/ ${{matrix.asset_name}}/ | |
| tar -czvf ${{matrix.asset_name}}.tar.gz \ | |
| ${{matrix.asset_name}}/ | |
| cp deb/${{matrix.asset_name}}.deb ./ 2>/dev/null || : | |
| ls -l ${{matrix.asset_name}}* | |
| - name: Upload the release | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: ${{matrix.asset_name}}* | |
| file_glob: true | |
| overwrite: true | |
| tag: ${{ needs.prepare.outputs.version }} | |
| release_name: "Development Release - ${{ needs.prepare.outputs.version }}" | |
| prerelease: ${{ needs.prepare.outputs.prerelease }} | |
| body: "${{ needs.prepare.outputs.release_body }}" | |
| - name: Set env variables | |
| env: | |
| VERSION: ${{ needs.prepare.outputs.version }} | |
| RELEASE_NAME: ${{matrix.asset_name}}.tar.gz | |
| run: | | |
| echo "RELEASE_NAME=${{env.RELEASE_NAME}}" >> $GITHUB_ENV | |
| echo "DEB_NAME=${{matrix.asset_name}}.deb" >> $GITHUB_ENV | |
| echo "VERSION=${{env.VERSION}}" >> $GITHUB_ENV | |
| echo "ARTIFACT_URL=https://${{ env.BUCKET_NAME }}.s3.ap-southeast-1.amazonaws.com/${{ env.VERSION }}/${{ env.RELEASE_NAME }}" >> $GITHUB_ENV | |
| - name: configure aws credentials | |
| if: matrix.os == 'ubuntu-20.04' | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| role-session-name: deployer | |
| aws-region: ap-southeast-1 | |
| - name: Upload release to S3 | |
| id: upload_s3 | |
| if: matrix.os == 'ubuntu-20.04' | |
| run: | | |
| aws s3 cp $RELEASE_NAME s3://${{ env.BUCKET_NAME }}/$VERSION/$RELEASE_NAME | |
| - name: Upload release deb to S3 | |
| id: upload_s3_deb | |
| if: matrix.os == 'ubuntu-20.04' | |
| run: | | |
| aws s3 cp deb/$DEB_NAME s3://${{ env.BUCKET_NAME }}/$VERSION/$DEB_NAME | |
| - name: Build, tag, and push image to Amazon ECR | |
| id: build_push_ecr | |
| if: matrix.os == 'ubuntu-20.04' | |
| env: | |
| IMAGE_TAG: ${{ needs.prepare.outputs.version }} | |
| run: | | |
| aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ECR_REGISTRY | |
| docker build -f ci/Dockerfile -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:$GITHUB_SHA . | |
| docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags | |
| - name: Update latest image if releasing | |
| if: (needs.prepare.outputs.prerelease == 'false') && (matrix.os == 'ubuntu-20.04') | |
| env: | |
| IMAGE_TAG: ${{ needs.prepare.outputs.version }} | |
| run: | | |
| docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest | |
| docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest | |
| - name: Log in to Docker Hub | |
| if: (github.event_name == 'release') && (needs.prepare.outputs.prerelease == 'false') && (matrix.os == 'ubuntu-20.04') | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and push Docker image to Docker Hub | |
| if: (github.event_name == 'release') && (needs.prepare.outputs.prerelease == 'false') && (matrix.os == 'ubuntu-20.04') | |
| env: | |
| IMAGE_TAG: ${{ needs.prepare.outputs.version }} | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| file: ./ci/Dockerfile | |
| push: true | |
| tags: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKERHUB_REPOSITORY }}:latest,${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKERHUB_REPOSITORY }}:${{ env.IMAGE_TAG }} | |
| - name: Release notification | |
| if: ${{ env.VERSION != 'dev' && matrix.os == 'ubuntu-20.04'}} | |
| env: | |
| DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_HOOK }} | |
| DISCORD_EMBEDS: '[ { | |
| "title": "New version `${{env.VERSION}}` released", | |
| "author": { "icon_url": "https://avatars.githubusercontent.com/${{ github.actor }}", "name": "${{ github.actor }}", "url": "https://github.com/${{github.actor}}" }, | |
| "fields": [ | |
| { "name": "Repository", "value": "[getdozer/dozer](https://github.com/getdozer/dozer)", "inline": true }, | |
| { "name": "Binary", "value": "[${{ env.RELEASE_NAME }}](${{ env.ARTIFACT_URL }})", "inline": true }, | |
| { "name": "Using Binary", "value": "`dozer -h`"}, | |
| { "name": "Release Notes", "value": "Release notes can be found [here](https://github.com/getdozer/dozer/releases/tag/${{env.VERSION}})"} | |
| ], | |
| "color": 990099 | |
| }]' | |
| uses: Ilshidur/action-discord@master |