diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c9b83fe2..23ef5275 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,6 +8,11 @@ on: - "tests/unit/**" - ".github/workflows/main.yml" - ".github/workflows/stale.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: run-tox-tests-uc-cluster: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 39168877..4c4e5b2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,16 @@ - Fix places where we were not properly closing cursors, and other test warnings ([713](https://github.com/databricks/dbt-databricks/pull/713)) - Upgrade databricks-sql-connector dependency to 3.4.0 ([790](https://github.com/databricks/dbt-databricks/pull/790)) -## dbt-databricks 1.8.7 (TBD) +## dbt-databricks 1.8.7 (October 10, 2024) + +### Features + +- Add config for generating unique tmp table names for enabling parralel replace-where (thanks @huangxingyi-git!) ([811](https://github.com/databricks/dbt-databricks/pull/811)) ### Fixes - Stop setting cluster by to None. If you want to drop liquid clustering, you will need to full-refresh ([806]https://github.com/databricks/dbt-databricks/pull/806) +- Don't define table properties on snapshot staging views (thanks @jelmerk!) ([820](https://github.com/databricks/dbt-databricks/pull/820)) ## dbt-databricks 1.8.6 (September 18, 2024) diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py index 8fe27739..192c2fde 100644 --- a/dbt/adapters/databricks/__version__.py +++ b/dbt/adapters/databricks/__version__.py @@ -1 +1 @@ -version: str = "1.8.6" +version: str = "1.8.7" diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 4c3e3b8d..24117c13 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -22,6 +22,7 @@ from typing import Type from typing import TYPE_CHECKING from typing import Union +from uuid import uuid4 from dbt.adapters.base import AdapterConfig from dbt.adapters.base import PythonJobHelper @@ -124,6 +125,7 @@ class DatabricksConfig(AdapterConfig): databricks_tags: Optional[Dict[str, str]] = None tblproperties: Optional[Dict[str, str]] = None zorder: Optional[Union[List[str], str]] = None + unique_tmp_table_suffix: bool = False skip_non_matched_step: Optional[bool] = None skip_matched_step: Optional[bool] = None matched_condition: Optional[str] = None @@ -711,6 +713,10 @@ def get_config_from_model(self, model: RelationConfig) -> DatabricksRelationConf f"Materialization {model.config.materialized} is not supported." ) + @available + def generate_unique_temporary_table_suffix(self, suffix_initial: str = "__dbt_tmp") -> str: + return f"{suffix_initial}_{str(uuid4())}" + @dataclass(frozen=True) class RelationAPIBase(ABC, Generic[DatabricksRelationConfig]): diff --git a/dbt/include/databricks/macros/materializations/incremental/incremental.sql b/dbt/include/databricks/macros/materializations/incremental/incremental.sql index faa59b0a..6096be4a 100644 --- a/dbt/include/databricks/macros/materializations/incremental/incremental.sql +++ b/dbt/include/databricks/macros/materializations/incremental/incremental.sql @@ -5,6 +5,7 @@ {%- set grant_config = config.get('grants') -%} {%- set tblproperties = config.get('tblproperties') -%} {%- set tags = config.get('databricks_tags') -%} + {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%} {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%} {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%} @@ -18,7 +19,11 @@ {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} {%- set target_relation = this.incorporate(type='table') -%} {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%} - + {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%} + {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%} + {%- else -%} + {%- set temp_relation_suffix = '__dbt_tmp' -%} + {%- endif -%} {#-- Set Overwrite Mode to STATIC for initial replace --#} {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%} @@ -69,7 +74,7 @@ {%- set _existing_config = adapter.get_relation_config(existing_relation) -%} {%- set model_config = adapter.get_config_from_model(config.model) -%} {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%} - {%- set temp_relation = databricks__make_temp_relation(target_relation, as_table=language != 'sql') -%} + {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%} {%- call statement('create_temp_relation', language=language) -%} {{ create_table_as(True, temp_relation, compiled_code, language) }} {%- endcall -%} diff --git a/dbt/include/databricks/macros/materializations/snapshot.sql b/dbt/include/databricks/macros/materializations/snapshot.sql index c1b8055c..3d1236a1 100644 --- a/dbt/include/databricks/macros/materializations/snapshot.sql +++ b/dbt/include/databricks/macros/materializations/snapshot.sql @@ -10,7 +10,9 @@ {# needs to be a non-temp view so that its columns can be ascertained via `describe` #} {% call statement('build_snapshot_staging_relation') %} - {{ create_view_as(tmp_relation, select) }} + create or replace view {{ tmp_relation }} + as + {{ select }} {% endcall %} {% do return(tmp_relation) %}