Setup tweaks #5475
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test, Build and Deploy Images | |
| env: | |
| DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| schedule: | |
| # Run every 6 days to keep our caches alive | |
| - cron: '0 0 */6 * *' | |
| jobs: | |
| python-tests: | |
| runs-on: ubuntu-latest | |
| env: | |
| python-version: 3.11 # Our base image has Python 3.11 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ env.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install --no-install-recommends --assume-yes shellcheck parallel | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| - name: Run tests | |
| run: | | |
| .hooks/pre-push | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage | |
| path: htmlcov | |
| frontend-tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: oven-sh/setup-bun@v1 | |
| with: | |
| bun-version: 1.0.3 | |
| - name: Bun install | |
| run: bun install --cwd ./core/frontend | |
| - name: Bun lint | |
| run: bun --cwd ./core/frontend lint | |
| - name: Bun build | |
| run: bun run --cwd ./core/frontend build | |
| deploy-docker-images: | |
| runs-on: ubuntu-latest | |
| needs: [frontend-tests, python-tests] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| docker: [bootstrap, core] | |
| project: [blueos] | |
| platforms: ["linux/arm/v7,linux/arm64/v8,linux/amd64"] | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: false | |
| docker-images: false | |
| swap-storage: true | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 #Number of commits to fetch. 0 indicates all history for all branches and tags. | |
| submodules: recursive | |
| - name: Prepare | |
| id: prepare | |
| run: | | |
| # Deploy image with the name of the branch, if the build is a git tag, replace tag with the tag name. | |
| # If git tag matches semver, append latest tag to the push. | |
| DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }} | |
| VERSION=${GITHUB_REF##*/} | |
| if [[ $GITHUB_REF == refs/tags/* ]]; then | |
| VERSION=${GITHUB_REF#refs/tags/} | |
| fi | |
| TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" | |
| if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then | |
| TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" | |
| fi | |
| echo "docker_image=${DOCKER_IMAGE}" >> $GITHUB_OUTPUT | |
| echo "version=${VERSION}" >> $GITHUB_OUTPUT | |
| echo "buildx_args=\ | |
| --build-arg GIT_DESCRIBE_TAGS=$(git describe --tags --long --always) \ | |
| --build-arg VITE_APP_GIT_DESCRIBE=$(git describe --long --always --dirty --all) \ | |
| --cache-from 'type=local,src=/tmp/.buildx-cache' \ | |
| --cache-to 'type=local,dest=/tmp/.buildx-cache' \ | |
| ${TAGS} \ | |
| --file ${{ matrix.docker }}/Dockerfile ./${{ matrix.docker }}" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: all | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| version: latest | |
| - name: Cache Docker layers | |
| uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }} | |
| ${{ runner.os }}-buildx-${{ matrix.docker }} | |
| - name: Docker Buildx (build) | |
| run: | | |
| # Pull latest version of image to help with build speed | |
| for platform in $(echo ${{ matrix.platforms }} | tr ',' '\n'); do | |
| docker pull --platform ${platform} ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}:master || true | |
| done | |
| docker buildx build \ | |
| --output "type=image,push=false" \ | |
| --platform ${{ matrix.platforms }} \ | |
| ${{ steps.prepare.outputs.buildx_args }} | |
| - name: Check core size | |
| if: ${{ matrix.docker == 'core' }} | |
| run: | | |
| # Check if the image size is lower than our limit | |
| docker image list | |
| IMAGE_ID=$(docker images -q ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }} | head -n 1) | |
| LIMIT_SIZE_MB=700 | |
| IMAGE_SIZE_MB=$(( $(docker inspect $IMAGE_ID --format {{.Size}})/(2**20) )) | |
| echo "Core size is: $IMAGE_SIZE_MB MB" | |
| ((IMAGE_SIZE_MB < LIMIT_SIZE_MB)) | |
| - name: Login to DockerHub | |
| if: success() && github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker Buildx (push) | |
| if: success() && github.event_name != 'pull_request' | |
| run: | | |
| docker buildx build \ | |
| --output "type=image,push=true" \ | |
| --platform ${{ matrix.platforms }} \ | |
| ${{ steps.prepare.outputs.buildx_args }} | |
| - name: Inspect image | |
| if: always() && github.event_name != 'pull_request' | |
| run: | | |
| docker buildx imagetools \ | |
| inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} | |
| - name: Create image artifact | |
| if: success() && matrix.docker == 'core' | |
| run: | | |
| DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }} | |
| GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA") | |
| docker buildx build \ | |
| ${{ steps.prepare.outputs.buildx_args }} \ | |
| --platform "linux/arm64/v8" \ | |
| --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \ | |
| --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm64-v8.tar" \ | |
| - name: Upload artifact arm64-v8 | |
| uses: actions/upload-artifact@v4 | |
| if: success() && matrix.docker == 'core' | |
| with: | |
| name: BlueOS-core-docker-image-arm64-v8 | |
| path: '*arm64-v8.tar' | |
| - name: Create image artifact | |
| if: success() && matrix.docker == 'core' | |
| run: | | |
| DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }} | |
| GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA") | |
| docker buildx build \ | |
| ${{ steps.prepare.outputs.buildx_args }} \ | |
| --platform "linux/arm/v7" \ | |
| --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \ | |
| --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm-v7.tar" \ | |
| - name: Upload artifact arm-v7 | |
| uses: actions/upload-artifact@v4 | |
| if: success() && matrix.docker == 'core' | |
| with: | |
| name: BlueOS-core-docker-image-arm-v7 | |
| path: '*arm-v7.tar' | |
| - name: Create image artifact | |
| if: success() && matrix.docker == 'core' | |
| run: | | |
| DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }} | |
| GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA") | |
| docker buildx build \ | |
| ${{ steps.prepare.outputs.buildx_args }} \ | |
| --platform "linux/amd64" \ | |
| --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \ | |
| --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-amd64.tar" \ | |
| - name: Upload artifact amd64 | |
| uses: actions/upload-artifact@v4 | |
| if: success() && matrix.docker == 'core' | |
| with: | |
| name: BlueOS-core-docker-image-amd64 | |
| path: '*amd64.tar' | |
| - name: Upload docker image for release | |
| uses: svenstaro/upload-release-action@v2 | |
| if: startsWith(github.ref, 'refs/tags/') && success() && matrix.docker == 'core' | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: '*.tar' | |
| tag: ${{ github.ref }} | |
| overwrite: true | |
| prerelease: true | |
| file_glob: true | |
| deploy-raspberry-image: | |
| needs: deploy-docker-images | |
| if: github.event_name != 'pull_request' && github.repository_owner == 'bluerobotics' | |
| timeout-minutes: 180 # Detect if it gets into an infinite loop or some unexpected state | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: blueos-ci | |
| platform: "linux/arm/v7" | |
| os: "bookworm" | |
| image: "raspios_lite_armhf/images/raspios_lite_armhf-2024-07-04/2024-07-04-raspios-bookworm-armhf-lite.img.xz" | |
| - runner: pi4-builder2 | |
| platform: "linux/arm/v7" | |
| os: "bullseye" | |
| image: "raspios_lite_armhf/images/raspios_lite_armhf-2022-01-28/2022-01-28-raspios-bullseye-armhf-lite.zip" | |
| - runner: pi5-builder | |
| platform: "linux/arm64/v8" | |
| os: "bookworm" | |
| image: "raspios_lite_arm64/images/raspios_lite_arm64-2024-07-04/2024-07-04-raspios-bookworm-arm64-lite.img.xz" | |
| runs-on: ${{ matrix.runner }} | |
| # The runners for this job are: | |
| # - pi4-builders: A Raspberry Pi OS Bullseye (32-bit). | |
| # - pi5-builder: A Raspberry Pi OS Bookworm lite 64-bit. | |
| # For pi5-ci, install docker (curl -sSL https://get.docker.com/ | sh) and follow the instructions for setting up a new runner in | |
| # https://github.com/bluerobotics/BlueOS-docker/settings/actions/runners/new | |
| steps: | |
| - name: Install git | |
| run: sudo apt install -y git | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| # We use our own pimod as upstream doesn't provide armv7 images | |
| - name: Pimod Build | |
| run: | | |
| VERSION=$GITHUB_REPOSITORY | |
| VERSION=${VERSION:-master} | |
| wget https://raw.githubusercontent.com//Nature40/pimod/master/pimod.sh && chmod +x pimod.sh | |
| docker run --rm --privileged \ | |
| -v $PWD:/files \ | |
| -e PATH=/pimod:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ | |
| -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY \ | |
| -e VERSION=$GITHUB_REF_NAME \ | |
| -e BASE_IMAGE=${{ matrix.image }} \ | |
| --workdir=/files \ | |
| --platform ${{ matrix.platform }} nature40/pimod:latest pimod.sh deploy/pimod/blueos.Pifile | |
| - name: Add /boot additions | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y parted kpartx | |
| # Create mount point if it doesn't exist | |
| sudo mkdir -p /mnt/piboot | |
| # Map the image file | |
| LOOP_DEVICE=$(sudo kpartx -avs deploy/pimod/blueos.img | sed -E 's/.*(loop[0-9]+)p[0-9]+.*/\1/g' | head -1) | |
| # Mount the boot partition | |
| sudo mount "/dev/mapper/${LOOP_DEVICE}p1" /mnt/piboot | |
| # Create ssh and userconf files | |
| sudo cp install/boards/config.toml /mnt/piboot/custom.toml | |
| sudo umount /mnt/piboot | |
| sudo kpartx -d deploy/pimod/blueos.img | |
| echo "Boot partition updated successfully." | |
| - name: Sanitize platform name | |
| run: echo "SANITIZED_PLATFORM=$(echo ${{ matrix.platform }} | tr '/' '-')" >> $GITHUB_ENV | |
| - name: Zip image | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| sudo apt install zip | |
| zip BlueOS-raspberry-${{ env.SANITIZED_PLATFORM }}-${{ matrix.os }}.zip deploy/pimod/blueos.img | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| timeout-minutes: 120 | |
| with: | |
| name: BlueOS-raspberry-${{ env.GITHUB_REF_NAME }}${{ env.SANITIZED_PLATFORM }}-${{ matrix.os }} | |
| path: deploy/pimod/blueos.img | |
| if-no-files-found: error | |
| retention-days: 7 | |
| - name: Upload raspberry image for release | |
| uses: svenstaro/upload-release-action@v2 | |
| if: startsWith(github.ref, 'refs/tags/') | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: BlueOS-raspberry-${{ env.SANITIZED_PLATFORM }}-${{ matrix.os }}.zip | |
| tag: ${{ github.ref }} | |
| overwrite: true | |
| prerelease: true | |
| # This is required because docker has root permissions, which means the runner is unable to clear this cache normally | |
| - name: Cleanup | |
| if: ${{ always() }} | |
| run: sudo rm -rf .cache |