Add conditional uniqueness tests for iasWorld tables (#98)

* Add source freshness definitions for iasworld tables

* Add test_dbt_source_freshness workflow

* Consolidate shared dbt setup steps into setup_dbt composite action on CI

* Temporarily add config to run test_dbt_source_freshness on PRs for testing

* Revert "Temporarily add config to run test_dbt_source_freshness on PRs for testing"

This reverts commit 913db16.

* Adjust iasworld freshness checks to include prior year's data

* Use YAML anchors for error/warning thresholds in iasworld source freshness config

* Add `where` flag to unique_combination_of_columns macro and use it in iasworld test

* Add uniqueness tests for htpar and sales tables

* Add additional filter condition to asmt_all test that was missing from sheet

* Move where cluase in unique_combination_of_columns macro and add it to docstring

* Switch to using builtin config.where filter for iasworld tests

dbt/models/iasworld/schema.yml

@@ -30,6 +30,20 @@ sources:
           filter: &latest_taxyr taxyr >= date_format(current_date - interval '1' year, '%Y')
           warn_after: &24_hours {count: 24, period: hour}
           error_after: &48_hours {count: 48, period: hour}
+        tests:
+          - unique_combination_of_columns:
+              name: asmt_all_unique_by_parid_procname_and_taxyr
+              combination_of_columns:
+                - parid
+                - procname
+                - taxyr
+              config:
+                where: >-
+                  cur = 'Y' and
+                  deactivat is null and
+                  procname in ('CCAOVALUE', 'CCAOFINAL', 'BORVALUE') and
+                  valclass is null
+                error_if: ">125"
       - name: asmt_hist
       - name: cname
       - name: comdat
@@ -73,6 +87,17 @@ sources:
           filter: *latest_taxyr
           warn_after: *24_hours
           error_after: *48_hours
+        tests:
+          - unique_combination_of_columns:
+              name: htpar_unique_by_parid_caseno_taxyr_subkey
+              combination_of_columns:
+                - parid
+                - caseno
+                - taxyr
+                - subkey
+              config:
+                where: cur = 'Y' and deactivat is null
+                error_if: ">2"
       - name: land
         description: '{{ doc("land") }}'
         tests:
@@ -89,7 +114,7 @@ sources:
               name: legdat_unique_by_parid_taxyr
               combination_of_columns:
                 - parid
-                - taxyr      
+                - taxyr
       - name: lpmod
       - name: lpnbhd
       - name: oby
@@ -109,21 +134,29 @@ sources:
               name: owndat_unique_by_parid_taxyr
               combination_of_columns:
                 - parid
-                - taxyr      
+                - taxyr
       - name: pardat
         description: '{{ doc("pardat") }}'
         tests:
           - unique_combination_of_columns:
               name: pardat_unique_by_parid_taxyr
               combination_of_columns:
                 - parid
-                - taxyr   
+                - taxyr
       - name: permit
         freshness:
           filter: date_format(date_parse(permdt, '%Y-%m-%d %H:%i:%s.0'), '%Y') >= date_format(current_date - interval '1' year, '%Y')
           warn_after: *48_hours
           error_after: &72_hours {count: 72, period: hour}
       - name: rcoby
       - name: sales
+        tests:
+          - unique_combination_of_columns:
+              name: sales_unique_by_parid_instruno
+              combination_of_columns:
+                - parid
+                - instruno
+              config:
+                where: substr(saledt, 1, 4) >= '2023'
       - name: splcom
       - name: valclass