From 722d8626ffe051ba33aa470fabb047476cf6d4d8 Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Fri, 25 Aug 2023 17:31:49 -0700 Subject: [PATCH] feat(build): eliminate latest-web-dev image `docker-compose` and `docker compose` are substantially different. The former corresponds with "Compose V1", which is deprecated, and the later is a rewrite that, significantly, uses the same build system as the other `docker/*` actions and, thereby, the same cache. Once done, it became apparent that the development images are unnecessary. - Pushing to `localhost:5000` in workflows isn't needed, because images produced by earlier steps are now available in the `buildx` cache primed by the `build-push-action` and used by `docker compose`. - With `localhost:5000` removed, `DJANGO_DOCKER_IMAGE` and `CELERY_DOCKER_IMAGE` environment variables are no longer used. - With no consumption of development images outside of the local environment, there's no reason to publish these images or have a `Makefile` targets to build them. This change switches from `docker-compose` (V1) to `docker compose` (V2) and changes the configuration of the `cl-django` and `cl-celery` to replace `image` with `build`. It then removes all development image related code, including: - `.github/workflows/docker-build-dev.yml` workflow - local registry in the `.github/workflows/tests.yml` - `development` targets of `docker/django/Makefile` For debugging it also separates the docker operations of `pull` and `build` into separate steps in the `tests.yml` workflow. After this change, where implementers would run: ``` % make development --file docker/django/Makefile VERSION=$RANDOM ``` They should use `docker compose` with the `--build` argument: ``` docker compose up --build ``` --- .github/workflows/docker-build-dev.yml | 18 ------------- .github/workflows/tests.yml | 34 ++++++++++++------------- docker/courtlistener/docker-compose.yml | 14 ++++++++-- docker/django/Dockerfile | 1 - docker/django/Makefile | 17 ------------- 5 files changed, 28 insertions(+), 56 deletions(-) delete mode 100644 .github/workflows/docker-build-dev.yml diff --git a/.github/workflows/docker-build-dev.yml b/.github/workflows/docker-build-dev.yml deleted file mode 100644 index 625018c695..0000000000 --- a/.github/workflows/docker-build-dev.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Push new dev images -on: - push: - branches: [ "main" ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and Push - run: | - make multiarch_push_development --file docker/django/Makefile -e VERSION=$(git rev-parse --short HEAD) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a673df1da1..00c9f6298a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,11 +11,6 @@ on: jobs: build: runs-on: ubuntu-latest - services: - registry: - image: registry:2 - ports: - - 5000:5000 strategy: fail-fast: false matrix: @@ -66,11 +61,10 @@ jobs: with: context: ./courtlistener file: ./courtlistener/docker/django/Dockerfile - push: true + load: true target: web-dev build-args: | BUILD_ENV=dev - tags: localhost:5000/freelawproject/courtlistener:latest-web-dev-${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max - name: Build latest docker celery image @@ -78,30 +72,34 @@ jobs: with: context: ./courtlistener file: ./courtlistener/docker/django/Dockerfile - push: true + load: true target: celery build-args: | BUILD_ENV=dev - tags: localhost:5000/freelawproject/courtlistener:latest-celery-${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max + # Prepare Docker images + - name: Pull docker images + working-directory: courtlistener/docker/courtlistener + run: docker compose pull --quiet --ignore-buildable + - name: Build docker images + working-directory: courtlistener/docker/courtlistener + run: docker compose build + - name: List docker images + run: docker image ls -a --no-trunc + # Docker images are ready. Start them up. - name: Create docker network run: docker network create -d bridge --attachable cl_net_overlay - - name: Start docker compose + - name: Start docker containers working-directory: courtlistener/docker/courtlistener - run: docker-compose -f docker-compose.yml -f docker-compose.tmpfs.yml up -d - env: - CELERY_DOCKER_IMAGE: "localhost:5000/freelawproject/courtlistener:latest-celery-${{ github.sha }}" - DJANGO_DOCKER_IMAGE: "localhost:5000/freelawproject/courtlistener:latest-web-dev-${{ github.sha }}" - - name: List docker images - run: docker image ls -a --no-trunc - - name: List docker container statuses + run: docker compose -f docker-compose.yml -f docker-compose.tmpfs.yml up -d --no-build --pull=never + - name: List docker containers run: docker ps -a --no-trunc - name: Show the docker startup logs working-directory: courtlistener/docker/courtlistener - run: docker-compose logs + run: docker compose logs # Run the checks and tests - name: Check if migrations are missing diff --git a/docker/courtlistener/docker-compose.yml b/docker/courtlistener/docker-compose.yml index 52abf64bd4..f998faa4f7 100644 --- a/docker/courtlistener/docker-compose.yml +++ b/docker/courtlistener/docker-compose.yml @@ -66,7 +66,12 @@ services: # Task Server cl-celery: container_name: cl-celery - image: ${CELERY_DOCKER_IMAGE:-freelawproject/courtlistener:latest-celery-dev} + build: + context: "../../" + dockerfile: "./docker/django/Dockerfile" + args: + BUILD_ENV: dev + target: celery depends_on: - cl-postgresql - cl-redis @@ -82,7 +87,12 @@ services: cl-django: container_name: cl-django - image: ${DJANGO_DOCKER_IMAGE:-freelawproject/courtlistener:latest-web-dev} + build: + context: "../../" + dockerfile: "./docker/django/Dockerfile" + args: + BUILD_ENV: dev + target: web-dev depends_on: - cl-postgresql - cl-redis diff --git a/docker/django/Dockerfile b/docker/django/Dockerfile index d0b19ab363..d6a6c39036 100644 --- a/docker/django/Dockerfile +++ b/docker/django/Dockerfile @@ -105,7 +105,6 @@ CMD celery \ --concurrency=${CELERY_PREFORK_CONCURRENCY:-0} \ --prefetch-multiplier=${CELERY_PREFETCH_MULTIPLIER:-1} -#freelawproject/courtlistener:latest-web-dev FROM python-base as web-dev USER www-data diff --git a/docker/django/Makefile b/docker/django/Makefile index f496126fc2..24d5a66ee1 100644 --- a/docker/django/Makefile +++ b/docker/django/Makefile @@ -10,13 +10,10 @@ endif REPO ?= freelawproject/courtlistener # The various tags available -DOCKER_TAG_DEV = $(VERSION)-web-dev DOCKER_TAG_PROD = $(VERSION)-web-prod WEB_PROD ?= latest-web-prod -WEB_DEV ?= latest-web-dev CELERY_TAG = $(VERSION)-celery CELERY_LATEST ?= latest-celery -CELERY_DEV ?= latest-celery-dev RSS_TAG ?= $(VERSION)-scrape-rss RSS_LATEST ?= latest-scrape-rss WEBHOOKS_TAG ?= $(VERSION)-webhooks-retry @@ -28,10 +25,6 @@ UNAME := $(shell uname -m) all: image -development: - docker build --target web-dev -t $(REPO):$(WEB_DEV) --build-arg BUILD_ENV=dev --file docker/django/Dockerfile . - docker build --target celery -t $(REPO):$(CELERY_DEV) --build-arg BUILD_ENV=dev --file docker/django/Dockerfile . - image: docker build --target web-prod -t $(REPO):$(DOCKER_TAG_PROD) -t $(REPO):$(WEB_PROD) --file docker/django/Dockerfile . docker build --target celery -t $(REPO):$(CELERY_TAG) -t $(REPO):$(CELERY_LATEST) --file docker/django/Dockerfile . @@ -79,13 +72,3 @@ x86_push: docker buildx build --push --platform linux/amd64 -t $(REPO):$(CELERY_LATEST) -t $(REPO):$(CELERY_TAG) --file docker/django/Dockerfile . docker buildx build --push --platform linux/amd64 -t $(REPO):$(RSS_TAG) -t $(REPO):$(RSS_LATEST) --file docker/django/Dockerfile . docker buildx build --push --platform linux/amd64 -t $(REPO):$(WEBHOOKS_TAG) -t $(REPO):$(WEBHOOKS_LATEST) --file docker/django/Dockerfile . - -multiarch_push_development: - export DOCKER_CLI_EXPERIMENTAL=enabled - # Fix for #2116 as per https://github.com/docker/buildx/issues/495#issuecomment-761562905 - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name flp-builder --driver docker-container --use - # Wait for the builder to boot - docker buildx inspect --bootstrap - docker buildx build --target web-dev --push --platform linux/amd64,linux/arm64 -t $(REPO):$(WEB_DEV) --build-arg BUILD_ENV=dev --file docker/django/Dockerfile . - docker buildx build --target celery --push --platform linux/amd64,linux/arm64 -t $(REPO):$(CELERY_DEV) --build-arg BUILD_ENV=dev --file docker/django/Dockerfile .