diff --git a/.github/workflows/deploy.yaml b/.github/workflows/cd.yaml similarity index 71% rename from .github/workflows/deploy.yaml rename to .github/workflows/cd.yaml index f6cb2a97..3b398a08 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/cd.yaml @@ -1,5 +1,5 @@ --- -name: Deployment +name: CD on: pull_request: types: [closed] @@ -15,13 +15,14 @@ env: IMAGE_NAME: ghcr.io/basedosdados/queries-basedosdados jobs: build-container: - if: (github.event.pull_request.merged == true && github.event.pull_request.base.ref - == 'main') + if: | + github.event.pull_request.merged == true + && github.event.pull_request.base.ref == 'main' name: Deployment runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Google Cloud CLI uses: google-github-actions/setup-gcloud@v0.2.1 with: @@ -62,37 +63,39 @@ jobs: uses: tj-actions/changed-files@v35 with: separator: ',' - - name: Setup Python 3.9 - uses: actions/setup-python@v2 + - name: Set up poetry + run: pipx install poetry + - name: Set up python + uses: actions/setup-python@v4 with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + cache: poetry + python-version: '3.10' + - name: Install requirements + run: poetry install --only=dev - name: Run script for approving table run: | - python .github/workflows/scripts/table_approve.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }} --source-bucket-name ${{ secrets.SOURCE_BUCKET_NAME }} --destination-bucket-name ${{ secrets.DESTINATION_BUCKET_NAME }} --backup-bucket-name ${{ secrets.BACKUP_BUCKET_NAME }} --prefect-backend-token ${{ secrets.PREFECT_BACKEND_TOKEN }} --materialization-mode ${{ secrets.MATERIALIZATION_MODE }} --materialization-label ${{ secrets.MATERIALIZATION_LABEL }} + poetry run python .github/workflows/scripts/table_approve.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }} --source-bucket-name ${{ secrets.SOURCE_BUCKET_NAME }} --destination-bucket-name ${{ secrets.DESTINATION_BUCKET_NAME }} --backup-bucket-name ${{ secrets.BACKUP_BUCKET_NAME }} --prefect-backend-token ${{ secrets.PREFECT_BACKEND_TOKEN }} --materialization-mode ${{ secrets.MATERIALIZATION_MODE }} --materialization-label ${{ secrets.MATERIALIZATION_LABEL }} change-metadata-status: needs: table-approve name: Change metadata status to "production" runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get all changed files using a comma separator id: changed-files uses: tj-actions/changed-files@v35 with: separator: ',' - - name: Setup Python 3.9 - uses: actions/setup-python@v2 + - name: Set up poetry + run: pipx install poetry + - name: Set up python + uses: actions/setup-python@v4 with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + cache: poetry + python-version: '3.10' + - name: Install requirements + run: poetry install --only=dev - name: Run script for changing metadata status run: |- python .github/workflows/scripts/change_metadata_status.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }} --status published --email ${{ secrets.BACKEND_EMAIL }} --password ${{ secrets.BACKEND_PASSWORD }} diff --git a/.github/workflows/ci-dbt.yaml b/.github/workflows/ci-dbt.yaml index f687ebd7..cda880c9 100644 --- a/.github/workflows/ci-dbt.yaml +++ b/.github/workflows/ci-dbt.yaml @@ -10,14 +10,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up poetry + run: pipx install poetry - name: Set up python uses: actions/setup-python@v4 with: - cache: pip + cache: poetry python-version: '3.10' - name: Install requirements - run: pip install -r requirements-dev.txt + run: poetry install --only=dev - name: Lint sql - run: sqlfmt --diff . + run: poetry run sqlfmt --diff . - name: Lint yaml - run: yamlfix --exclude ".kubernetes/**/*" . + run: poetry run yamlfix --exclude ".kubernetes/**/*" . diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b689a7db..0d1b4575 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -7,30 +7,30 @@ env: DBT_SA: ${{ secrets.DBT_SA }} jobs: docs: - if: github.event.pull_request.merged == true && github.event.pull_request.base.ref - == 'main' + if: | + github.event.pull_request.merged == true + && github.event.pull_request.base.ref == 'main' name: Deploy docs runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/checkout@v4 + - name: Set up poetry + run: pipx install poetry + - name: Set up python + uses: actions/setup-python@v4 with: - python-version: 3.9.x - - name: Upgrade pip - run: | - pip install -U pip - - name: Install dependencies - run: | - pip3 install -r requirements-docs.txt -r requirements.txt + cache: poetry + python-version: '3.10' + - name: Install requirements + run: poetry install --with=dev - name: Setup credentials run: | echo $DBT_SA | base64 --decode > dbt-sa.json - python3 .github/workflows/scripts/modify_profiles.py + poetry run python .github/workflows/scripts/modify_profiles.py - name: Generate docs run: | - dbt docs generate --profiles-dir . + poetry run dbt docs generate --profiles-dir . - name: Delete credentials run: | rm dbt-sa.json diff --git a/.github/workflows/sync-dbt-schema.yaml b/.github/workflows/sync-dbt-schema.yaml index 5b6873f1..3d00a0f1 100644 --- a/.github/workflows/sync-dbt-schema.yaml +++ b/.github/workflows/sync-dbt-schema.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} - name: Get all changed files using a comma separator @@ -20,17 +20,18 @@ jobs: uses: tj-actions/changed-files@v35 with: separator: ',' - - name: Setup Python 3.9 - uses: actions/setup-python@v2 + - name: Set up poetry + run: pipx install poetry + - name: Set up python + uses: actions/setup-python@v4 with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + cache: poetry + python-version: '3.10' + - name: Install requirements + run: poetry install --only=dev - name: Run script for syncing DBT schema run: | - python .github/workflows/scripts/sync_dbt_schema.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }} + poetry run python .github/workflows/scripts/sync_dbt_schema.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }} - name: Commit changed files uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/Dockerfile b/Dockerfile index e1f199e8..30231f69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,30 @@ -FROM python:3.8-slim +# Builder Image -# Setup virtual environment -ENV VIRTUAL_ENV=/opt/venv -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" +FROM python:3.9-bookworm AS builder -# Copy and install dependencies -WORKDIR /tmp -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt && rm requirements.txt +RUN pip install --no-cache-dir poetry==1.7.0 + +ENV POETRY_VIRTUALENVS_CREATE=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + POETRY_CACHE_DIR=/tmp/pypoetry + +WORKDIR /app +COPY pyproject.toml poetry.lock dbt_project.yml packages.yml ./ +RUN poetry install --no-root && poetry run dbt deps && rm -rf $POETRY_CACHE_DIR + +# Runner Image + +FROM python:3.9-slim-bookworm AS runner + +ENV VIRTUAL_ENV=/app/.venv \ + DBT_PACKAGES=/app/dbt_packages \ + PATH="/app/.venv/bin:$PATH" + +WORKDIR /app + +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} +COPY --from=builder ${DBT_PACKAGES} ${DBT_PACKAGES} -# Copy dbt project and profiles -WORKDIR /dbt COPY . . -# Run dbt deps and dbt rpc -CMD ["/dbt/start-server.sh"] +CMD ["dbt-rpc", "serve", "--profiles-dir", ".", "--host", "0.0.0.0", "--port", "8580"] diff --git a/start-server.sh b/start-server.sh deleted file mode 100755 index ea2a6a74..00000000 --- a/start-server.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env sh - -dbt deps -dbt-rpc serve --profiles-dir "." --host "0.0.0.0" --port "8580"