Better caching to support restore-keys flow in build_and_test_dbt wor…

…kflow
ccao-data · Aug 18, 2023 · e31e62e · e31e62e
1 parent 64fcdf2
commit e31e62e
Showing 1 changed file with 19 additions and 8 deletions.
diff --git a/.github/workflows/build_and_test_dbt.yaml b/.github/workflows/build_and_test_dbt.yaml
@@ -33,21 +33,31 @@ jobs:
       - name: Configure dbt environment
         uses: ./.github/actions/configure_dbt_environment
 
-      - name: Cache dbt state directory
+      # We have to use the separate `restore`/`save` actions instead of the
+      # unified `cache` action because only `restore` provides access to the
+      # `cache-matched-key` and `cache-primary-key` outputs as of v3
+      - name: Restore dbt state cache
         id: cache
-        uses: actions/cache@v3
+        uses: actions/cache/restore@v3
         with:
           path: ${{ env.PROJECT_DIR }}/${{ env.STATE_DIR }}
           key: ${{ env.CACHE_NAME }}-${{ env.CACHE_KEY }}
           restore-keys: |
             ${{ env.CACHE_NAME }}-master
 
-      - if: ${{ steps.cache.outputs.cache-hit == 'true' }}
+      # If we restore the cache from the `restore-keys` key, the `cache-hit`
+      # output will be 'false' but the `cache-matched-key` output will be
+      # the name of the `restore-keys` key; we want to count this case as a hit
+      - if: |
+          steps.cache.outputs.cache-hit == 'true' ||
+          steps.cache.outputs.cache-matched-key == env.CACHE_NAME + '-master'
         name: Set command args to build/test modified resources
         run: echo "MODIFIED_RESOURCES_ONLY=true" >> "$GITHUB_ENV"
         shell: bash
 
-      - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      - if: |
+          steps.cache.outputs.cache-hit != 'true' &&
+          steps.cache.outputs.cache-matched-key != env.CACHE_NAME + '-master'
         name: Set command args to build/test all resources
         run: echo "MODIFIED_RESOURCES_ONLY=false" >> "$GITHUB_ENV"
         shell: bash
@@ -81,7 +91,8 @@ jobs:
         working-directory: ${{ env.PROJECT_DIR }}
         shell: bash
 
-      - name: Move dbt manifest directory to update cache
-        run: rsync -av --delete "$TARGET_DIR/" "$STATE_DIR"
-        working-directory: ${{ env.PROJECT_DIR }}
-        shell: bash
+      - name: Update dbt state cache
+        uses: actions/cache/save@v3
+        with:
+          path: ${{ env.PROJECT_DIR }}/${{ env.TARGET_DIR }}
+          key: ${{ env.CACHE_NAME }}-${{ env.CACHE_KEY }}