From 725cf418360777da1b7a4c53386fc4f1bde48ff0 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Mon, 6 Nov 2023 23:27:23 -0600 Subject: [PATCH] Release engineering --- .github/workflows/release.yml | 398 +++++++++++++++++++--------------- .github/workflows/test.yml | 4 +- README.md | 8 + pyproject.toml | 17 ++ 4 files changed, 254 insertions(+), 173 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd6d1dd..2106e8d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,202 +1,260 @@ -# Copyright 2022-2023, axodotdev -# SPDX-License-Identifier: MIT or Apache-2.0 +# Publish to crates.io and pypi.org, also cross-compile binaries which are uploaded to GitHub releases. # -# CI that: -# -# * checks for a Git Tag that looks like a release -# * builds artifacts with cargo-dist (archives, installers, hashes) -# * uploads those artifacts to temporary workflow zip -# * on success, uploads the artifacts to a Github Release™ -# -# Note that the Github Release™ will be created with a generated -# title/body based on your changelogs. -name: Release +# Notes +# - not using https://github.com/taiki-e/upload-rust-binary-action because it builds after release, we prefer build before release +# - not using https://opensource.axo.dev/cargo-dist/ because +# - https://github.com/axodotdev/cargo-dist/issues/551 +# - no ppc64le support, https://github.com/axodotdev/cargo-dist/issues/74 +# - no deb support, no pypi support -permissions: - contents: write +name: Release -# This task will run whenever you push a git tag that looks like a version -# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. -# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where -# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION -# must be a Cargo-style SemVer Version (must have at least major.minor.patch). -# -# If PACKAGE_NAME is specified, then the release will be for that -# package (erroring out if it doesn't have the given version or isn't cargo-dist-able). -# -# If PACKAGE_NAME isn't specified, then the release will be for all -# (cargo-dist-able) packages in the workspace with that version (this mode is -# intended for workspaces with only one dist-able package, or with all dist-able -# packages versioned/released in lockstep). -# -# If you push multiple tags at once, separate instances of this workflow will -# spin up, creating an independent Github Release™ for each one. However Github -# will hard limit this to 3 tags per commit, as it will assume more tags is a -# mistake. -# -# If there's a prerelease-style suffix to the version, then the Github Release™ -# will be marked as a prerelease. on: push: tags: - - '**[0-9]+.[0-9]+.[0-9]+*' + - v[0-9]+.* jobs: - # Run 'cargo dist plan' to determine what tasks we need to do - plan: + + ################################################# + # # + # CRATES.IO BUILD # + # # + ################################################# + + crate: runs-on: ubuntu-latest - outputs: - val: ${{ steps.plan.outputs.manifest }} - tag: ${{ !github.event.pull_request && github.ref_name || '' }} - tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} - publishing: ${{ !github.event.pull_request }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Publish to crates.io + steps: + - name: Git checkout + uses: actions/checkout@v4 + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + - name: Publish to crates.io + run: cargo publish + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + + ################################################# + # # + # CROSS COMPILE BINARIES # + # # + ################################################# + + build: + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest , target: armv7-unknown-linux-gnueabihf , deb: false , use-cross: use-cross } + - { os: ubuntu-latest , target: aarch64-unknown-linux-gnu , deb: true , use-cross: use-cross } + - { os: ubuntu-latest , target: aarch64-unknown-linux-musl , deb: false , use-cross: use-cross } + - { os: ubuntu-latest , target: powerpc-unknown-linux-gnu , deb: true , use-cross: use-cross } + - { os: ubuntu-latest , target: powerpc64-unknown-linux-gnu , deb: true , use-cross: use-cross } + - { os: ubuntu-latest , target: powerpc64le-unknown-linux-gnu , deb: true , use-cross: use-cross } + - { os: ubuntu-latest , target: riscv64gc-unknown-linux-gnu , deb: false , use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , deb: true , use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-linux-musl , deb: false , use-cross: use-cross, convenient: true } + + - { os: macos-latest , target: x86_64-apple-darwin } + - { os: macos-latest , target: aarch64-apple-darwin } + - { os: macos-latest , target: universal-apple-darwin } + - { os: windows-latest , target: x86_64-pc-windows-gnu } + - { os: windows-latest , target: x86_64-pc-windows-msvc } + runs-on: ${{ matrix.os }} + steps: + + # Build + # ------------------------------------------------------------ + - name: Git checkout + uses: actions/checkout@v4 + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - name: Install cross-compilation tools + if: matrix.use-cross + uses: taiki-e/setup-cross-toolchain-action@v1 + with: + target: ${{ matrix.target }} + - name: Setup rust cache + uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.target }} + - name: Build + run: cargo build --release + + # Bundle + # ------------------------------------------------------------ + - name: Extract crate information + id: crate-metadata + run: | + name="$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].name')" + version="$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version')" + echo "name=$name" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "bin_name=$name-${{ matrix.target }}-$version" >> "$GITHUB_OUTPUT" + - name: Bundle release (Windows) + if: matrix.os == 'windows-latest' + shell: bash + run: | + mkdir '${{ steps.crate-metadata.outputs.bin_name }}' + cp -v 'target/${{ matrix.target }}/release/${{ steps.crate-metadata.outputs.name }}.exe' '${{ steps.crate-metadata.outputs.bin_name }}' + 7z a '${{ steps.crate-metadata.outputs.bin_name }}.zip' '${{ steps.crate-metadata.outputs.bin_name }}' + echo "ASSET='${{ steps.crate-metadata.outputs.bin_name }}'.zip" >> $GITHUB_ENV + - name: Bundle release (Linux and macOS) + if: matrix.os != 'windows-latest' + shell: bash + run: | + mkdir -v '${{ steps.crate-metadata.outputs.bin_name }}' + cp -v 'target/${{ matrix.target }}/release/${{ steps.crate-metadata.outputs.name }}' '${{ steps.crate-metadata.outputs.bin_name }}' + tarball='${{ steps.crate-metadata.outputs.bin_name }}.tgz' + tar -czvf "$tarball" '${{ steps.crate-metadata.outputs.bin_name }}' + echo "ASSET=$tarball" >> $GITHUB_ENV + - name: Create release directory for artifact, move file + shell: bash + run: | + mkdir -v release + mv -v '${{ env.ASSET }}' release/ + - name: Copy "convenience" binary to release directory + if: matrix.convenient + shell: bash + run: cp -v 'target/${{ matrix.triple.target }}/release/${{ steps.crate-metadata.outputs.name }}' release/ + + - name: Build Debian release + id: deb + if: matrix.deb + run: | + cargo install cargo-deb --version 2.0.0 --locked + cargo deb --no-build --target ${{ matrix.target }} + cp -v ./target/${{ matrix.target }}/debian/*.deb release/ + - name: Test installation of .deb + if: steps.deb.conclusion != 'skipped' + run: sudo dpkg -i release/*.deb + + - name: Save release as artifact + uses: actions/upload-artifact@v3 + with: + retention-days: 3 + name: release + path: release + + + upload-release: + name: upload-release + runs-on: ubuntu-latest + needs: [ build ] + steps: + + # Create Release + # ------------------------------------------------------------ + - name: Get release artifacts + uses: actions/download-artifact@v3 + with: + name: release + path: release + - name: Print out all release files + run: | + echo "Generated $(ls ./release | wc -l) files:" + ls ./release + - name: Upload all saved release files + uses: softprops/action-gh-release@c9b46fe7aad9f02afd89b12450b780f52dacfb2d + with: + draft: true + fail_on_unmatched_files: true + files: | + ./release/* + + ################################################# + # # + # BUILD FOR PYPI USING MATURIN # + # # + ################################################# + maturin-linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64, aarch64, armv7] steps: - uses: actions/checkout@v4 + - name: Build wheels + uses: PyO3/maturin-action@v1 with: - submodules: recursive - - name: Install cargo-dist - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.4.2/cargo-dist-installer.sh | sh" - - id: plan - run: | - cargo dist plan ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} --output-format=json > dist-manifest.json - echo "cargo dist plan ran successfully" - cat dist-manifest.json - echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" - - name: "Upload dist-manifest.json" + target: ${{ matrix.target }} + args: --release --strip --locked --out dist + sccache: 'true' + manylinux: auto + - name: Upload wheels uses: actions/upload-artifact@v3 with: - name: artifacts - path: dist-manifest.json - - # Build and packages all the platform-specific things - upload-local-artifacts: - # Let the initial task tell us to not run (currently very blunt) - needs: plan - if: ${{ fromJson(needs.plan.outputs.val).releases != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + name: wheels + path: dist + + maturin-windows: + runs-on: windows-latest strategy: - fail-fast: false - # Target platforms/runners are computed by cargo-dist in create-release. - # Each member of the matrix has the following arguments: - # - # - runner: the github runner - # - dist-args: cli flags to pass to cargo dist - # - install-dist: expression to run to install cargo-dist on the runner - # - # Typically there will be: - # - 1 "global" task that builds universal installers - # - N "local" tasks that build each platform's binaries and platform-specific installers - matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} - runs-on: ${{ matrix.runner }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + matrix: + target: [x64] steps: - uses: actions/checkout@v4 + - name: Build wheels + uses: PyO3/maturin-action@v1 with: - submodules: recursive - - uses: swatinem/rust-cache@v2 - - name: Install cargo-dist - run: ${{ matrix.install_dist }} - - name: Install dependencies - run: | - ${{ matrix.packages_install }} - - name: Build artifacts - run: | - # Actually do builds and make zips and whatnot - cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json - echo "cargo dist ran successfully" - - id: cargo-dist - name: Post-build - # We force bash here just because github makes it really hard to get values up - # to "real" actions without writing to env-vars, and writing to env-vars has - # inconsistent syntax between shell and powershell. - shell: bash - run: | - # Parse out what we just built and upload it to the Github Release™ - echo "paths<> "$GITHUB_OUTPUT" - jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - - cp dist-manifest.json "$BUILD_MANIFEST_NAME" - - name: "Upload artifacts" + target: ${{ matrix.target }} + args: --release --strip --locked --out dist + sccache: 'true' + - name: Upload wheels uses: actions/upload-artifact@v3 with: - name: artifacts - path: | - ${{ steps.cargo-dist.outputs.paths }} - ${{ env.BUILD_MANIFEST_NAME }} - - # Build and package all the platform-agnostic(ish) things - upload-global-artifacts: - needs: [plan, upload-local-artifacts] - runs-on: "ubuntu-20.04" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: wheels + path: dist + + maturin-macos: + runs-on: macos-latest + strategy: + matrix: + target: [x86_64, aarch64] steps: - uses: actions/checkout@v4 + - name: Build wheels + uses: PyO3/maturin-action@v1 with: - submodules: recursive - - name: Install cargo-dist - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.4.2/cargo-dist-installer.sh | sh" - # Get all the local artifacts for the global tasks to use (for e.g. checksums) - - name: Fetch local artifacts - uses: actions/download-artifact@v3 - with: - name: artifacts - path: target/distrib/ - - id: cargo-dist - shell: bash - run: | - cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json - echo "cargo dist ran successfully" - - # Parse out what we just built and upload it to the Github Release™ - echo "paths<> "$GITHUB_OUTPUT" - jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - - name: "Upload artifacts" + target: ${{ matrix.target }} + args: --release --strip --locked --out dist + sccache: 'true' + - name: Upload wheels uses: actions/upload-artifact@v3 with: - name: artifacts - path: ${{ steps.cargo-dist.outputs.paths }} - - should-publish: - needs: - - plan - - upload-local-artifacts - - upload-global-artifacts - if: ${{ needs.plan.outputs.publishing == 'true' }} - runs-on: ubuntu-latest - steps: - - name: print tag - run: echo "ok we're publishing!" + name: wheels + path: dist - # Create a Github Release with all the results once everything is done - publish-release: - needs: [plan, should-publish] + maturin-sdist: runs-on: ubuntu-latest - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 with: - submodules: recursive - - name: "Download artifacts" - uses: actions/download-artifact@v3 + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + maturin-release: + name: Release + runs-on: ubuntu-latest + needs: [maturin-linux, maturin-windows, maturin-macos, maturin-sdist] + steps: + - uses: actions/download-artifact@v3 with: - name: artifacts - path: artifacts - - name: Cleanup - run: | - # Remove the granular manifests - rm artifacts/*-dist-manifest.json - - name: Create Release - uses: ncipollo/release-action@v1 + name: wheels + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} with: - tag: ${{ needs.plan.outputs.tag }} - name: ${{ fromJson(needs.plan.outputs.val).announcement_title }} - body: ${{ fromJson(needs.plan.outputs.val).announcement_github_body }} - prerelease: ${{ fromJson(needs.plan.outputs.val).announcement_is_prerelease }} - artifacts: "artifacts/*" + command: upload + args: --skip-existing * diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d1e87a..6612d47 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,11 +1,9 @@ -name: CI +name: Test on: push: branches: [ master ] - tags: - - "v?[0-9]+.[0-9]+.[0-9]+*" pull_request: jobs: diff --git a/README.md b/README.md index 6d7970b..be309b3 100644 --- a/README.md +++ b/README.md @@ -33,3 +33,11 @@ mni2mz3 thickness.txt thickness.mz3 - Output file will be gzip compressed. - For surfaces, only triangle meshes are supported. - For data, only 32-bit single-precision "float" is supported. + +## Testing + +It is recommended to install [cargo-nextest](https://nexte.st/). + +```shell +cargo nextest run +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6878da1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["maturin>=1.3,<2.0"] +build-backend = "maturin" + +[project] +name = "mni2mz3" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Rust", + "License :: OSI Approved :: MIT License", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Bio-Informatics", + "Topic :: Scientific/Engineering :: Medical Science Apps." +] + +[tool.maturin] +bindings = "bin"