From a83a44a46d4e6a8c3968d2557c34ea3e199ef67a Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Fri, 18 Aug 2023 13:49:34 -0500 Subject: [PATCH 1/3] Use rsync instead of mv to update dbt cache dir in CI --- .github/workflows/build_and_test_dbt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test_dbt.yaml b/.github/workflows/build_and_test_dbt.yaml index e0be5a3e3..c2fdb993c 100644 --- a/.github/workflows/build_and_test_dbt.yaml +++ b/.github/workflows/build_and_test_dbt.yaml @@ -82,6 +82,6 @@ jobs: shell: bash - name: Move dbt manifest directory to update cache - run: mv "$TARGET_DIR" "$STATE_DIR" + run: rsync -av --delete "$TARGET_DIR/" "$STATE_DIR" working-directory: ${{ env.PROJECT_DIR }} shell: bash From 64fcdf2cb41a014528be34d466bab943095a490a Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Fri, 18 Aug 2023 13:49:51 -0500 Subject: [PATCH 2/3] Make sure to configure dbt env vars before generating docs in deploy-dbt-docs --- .github/workflows/deploy_dbt_docs.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy_dbt_docs.yaml b/.github/workflows/deploy_dbt_docs.yaml index 647872c57..13c20e2a9 100644 --- a/.github/workflows/deploy_dbt_docs.yaml +++ b/.github/workflows/deploy_dbt_docs.yaml @@ -31,6 +31,9 @@ jobs: role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }} aws-region: us-east-1 + - name: Configure dbt environment + uses: ./.github/actions/configure_dbt_environment + - name: Generate docs run: dbt docs generate --target "$TARGET" working-directory: ${{ env.PROJECT_DIR }} From 068ddad1985b2c5c7bf93ab25b2d4cf92b3a535b Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Fri, 18 Aug 2023 14:18:44 -0500 Subject: [PATCH 3/3] Better caching to support restore-keys flow in build_and_test_dbt workflow --- .github/workflows/build_and_test_dbt.yaml | 31 +++++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_and_test_dbt.yaml b/.github/workflows/build_and_test_dbt.yaml index c2fdb993c..9d1f9be76 100644 --- a/.github/workflows/build_and_test_dbt.yaml +++ b/.github/workflows/build_and_test_dbt.yaml @@ -33,21 +33,35 @@ jobs: - name: Configure dbt environment uses: ./.github/actions/configure_dbt_environment - - name: Cache dbt state directory + # We have to use the separate `restore`/`save` actions instead of the + # unified `cache` action because only `restore` provides access to the + # `cache-matched-key` and `cache-primary-key` outputs as of v3 + - name: Restore dbt state cache id: cache - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: ${{ env.PROJECT_DIR }}/${{ env.STATE_DIR }} key: ${{ env.CACHE_NAME }}-${{ env.CACHE_KEY }} restore-keys: | ${{ env.CACHE_NAME }}-master - - if: ${{ steps.cache.outputs.cache-hit == 'true' }} + # If we restore the cache from the `restore-keys` key, the `cache-hit` + # output will be 'false' but the `cache-matched-key` output will be + # the name of the `restore-keys` key; we want to count this case as a hit + - if: | + steps.cache.outputs.cache-hit == 'true' || + steps.cache.outputs.cache-matched-key == format( + '{0}-master', env.CACHE_NAME + ) name: Set command args to build/test modified resources run: echo "MODIFIED_RESOURCES_ONLY=true" >> "$GITHUB_ENV" shell: bash - - if: ${{ steps.cache.outputs.cache-hit != 'true' }} + - if: | + steps.cache.outputs.cache-hit != 'true' && + steps.cache.outputs.cache-matched-key != format( + '{0}-master', env.CACHE_NAME + ) name: Set command args to build/test all resources run: echo "MODIFIED_RESOURCES_ONLY=false" >> "$GITHUB_ENV" shell: bash @@ -81,7 +95,8 @@ jobs: working-directory: ${{ env.PROJECT_DIR }} shell: bash - - name: Move dbt manifest directory to update cache - run: rsync -av --delete "$TARGET_DIR/" "$STATE_DIR" - working-directory: ${{ env.PROJECT_DIR }} - shell: bash + - name: Update dbt state cache + uses: actions/cache/save@v3 + with: + path: ${{ env.PROJECT_DIR }}/${{ env.TARGET_DIR }} + key: ${{ env.CACHE_NAME }}-${{ env.CACHE_KEY }}