From c733aa5db46deaecff4fa444a5ddb76cd427c0d7 Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:20:04 -0700 Subject: [PATCH 1/6] Cancel in progress runs of the same branch (#816) --- .github/workflows/integration.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c9b83fe2..23ef5275 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,6 +8,11 @@ on: - "tests/unit/**" - ".github/workflows/main.yml" - ".github/workflows/stale.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: run-tox-tests-uc-cluster: runs-on: ubuntu-latest From 6bcd4eadcc5bef37719b869048e1f1dfa312f6ae Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Thu, 3 Oct 2024 16:39:49 -0700 Subject: [PATCH 2/6] Fixing failing tests from newer dbt versions (#817) --- dbt/adapters/databricks/connections.py | 7 ++++--- dev-requirements.txt | 2 +- tests/functional/adapter/constraints/fixtures.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/databricks/connections.py b/dbt/adapters/databricks/connections.py index cf6f600a..c4a297b4 100644 --- a/dbt/adapters/databricks/connections.py +++ b/dbt/adapters/databricks/connections.py @@ -75,6 +75,7 @@ from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError from dbt_common.exceptions import DbtRuntimeError +from dbt_common.exceptions import DbtDatabaseError from dbt_common.utils import cast_to_str from requests import Session @@ -508,7 +509,7 @@ def exception_handler(self, sql: str) -> Iterator[None]: except Error as exc: logger.debug(QueryError(log_sql, exc)) - raise DbtRuntimeError(str(exc)) from exc + raise DbtDatabaseError(str(exc)) from exc except Exception as exc: logger.debug(QueryError(log_sql, exc)) @@ -518,9 +519,9 @@ def exception_handler(self, sql: str) -> Iterator[None]: thrift_resp = exc.args[0] if hasattr(thrift_resp, "status"): msg = thrift_resp.status.errorMessage - raise DbtRuntimeError(msg) from exc + raise DbtDatabaseError(msg) from exc else: - raise DbtRuntimeError(str(exc)) from exc + raise DbtDatabaseError(str(exc)) from exc # override/overload def set_connection_name( diff --git a/dev-requirements.txt b/dev-requirements.txt index 46c95b6e..0c3ba458 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -15,4 +15,4 @@ types-requests types-mock pre-commit -dbt-tests-adapter~=1.8.0 +dbt-tests-adapter>=1.10.0, <2.0 diff --git a/tests/functional/adapter/constraints/fixtures.py b/tests/functional/adapter/constraints/fixtures.py index 1a9d1013..7a1c3ee3 100644 --- a/tests/functional/adapter/constraints/fixtures.py +++ b/tests/functional/adapter/constraints/fixtures.py @@ -81,7 +81,7 @@ - type: foreign_key name: fk_example__child_table_1 columns: ["parent_id"] - to: parent_table + to: ref('parent_table') to_columns: ["id"] columns: - name: id From d2f3f824728e02cbbb741a239a446176ddec24be Mon Sep 17 00:00:00 2001 From: huangxingyi <99020960+huangxingyi-git@users.noreply.github.com> Date: Sat, 5 Oct 2024 02:50:44 +0900 Subject: [PATCH 3/6] Set unique temp table suffix to allow parallel incremental executions (#811) Signed-off-by: huang xingyi Co-authored-by: Ben Cassell --- CHANGELOG.md | 6 +++++- dbt/adapters/databricks/impl.py | 6 ++++++ .../macros/materializations/incremental/incremental.sql | 9 +++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eac8689..df65df75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ ## dbt-databricks 1.8.7 (TBD) +### Features + +- Add config for generating unique tmp table names for enabling parralel replace-where (thanks @huangxingyi-git!) ([811](https://github.com/databricks/dbt-databricks/pull/811)) + ### Fixes -- Stop setting cluster by to None. If you want to drop liquid clustering, you will need to full-refresh ([806]https://github.com/databricks/dbt-databricks/pull/806) +- Stop setting cluster by to None. If you want to drop liquid clustering, you will need to full-refresh ([806](https://github.com/databricks/dbt-databricks/pull/806)) ## dbt-databricks 1.8.6 (September 18, 2024) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 46cb11dd..a2184014 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING from typing import TypeVar from typing import Union +from uuid import uuid4 from dbt.adapters.base import AdapterConfig from dbt.adapters.base import PythonJobHelper @@ -103,6 +104,7 @@ class DatabricksConfig(AdapterConfig): databricks_tags: Optional[Dict[str, str]] = None tblproperties: Optional[Dict[str, str]] = None zorder: Optional[Union[List[str], str]] = None + unique_tmp_table_suffix: bool = False def check_not_found_error(errmsg: str) -> bool: @@ -685,6 +687,10 @@ def get_config_from_model(self, model: RelationConfig) -> DatabricksRelationConf f"Materialization {model.config.materialized} is not supported." ) + @available + def generate_unique_temporary_table_suffix(self, suffix_initial: str = "__dbt_tmp") -> str: + return f"{suffix_initial}_{str(uuid4())}" + @dataclass(frozen=True) class RelationAPIBase(ABC, Generic[DatabricksRelationConfig]): diff --git a/dbt/include/databricks/macros/materializations/incremental/incremental.sql b/dbt/include/databricks/macros/materializations/incremental/incremental.sql index faa59b0a..6096be4a 100644 --- a/dbt/include/databricks/macros/materializations/incremental/incremental.sql +++ b/dbt/include/databricks/macros/materializations/incremental/incremental.sql @@ -5,6 +5,7 @@ {%- set grant_config = config.get('grants') -%} {%- set tblproperties = config.get('tblproperties') -%} {%- set tags = config.get('databricks_tags') -%} + {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%} {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%} {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%} @@ -18,7 +19,11 @@ {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} {%- set target_relation = this.incorporate(type='table') -%} {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%} - + {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%} + {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%} + {%- else -%} + {%- set temp_relation_suffix = '__dbt_tmp' -%} + {%- endif -%} {#-- Set Overwrite Mode to STATIC for initial replace --#} {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%} @@ -69,7 +74,7 @@ {%- set _existing_config = adapter.get_relation_config(existing_relation) -%} {%- set model_config = adapter.get_config_from_model(config.model) -%} {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%} - {%- set temp_relation = databricks__make_temp_relation(target_relation, as_table=language != 'sql') -%} + {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%} {%- call statement('create_temp_relation', language=language) -%} {{ create_table_as(True, temp_relation, compiled_code, language) }} {%- endcall -%} From fa5f89e41d657bc3c0ae931ba565fca1cdeb611f Mon Sep 17 00:00:00 2001 From: Jelmer Kuperus Date: Fri, 4 Oct 2024 20:38:52 +0200 Subject: [PATCH 4/6] Fix for: https://github.com/databricks/dbt-databricks/issues/819 (#820) Co-authored-by: Ben Cassell <98852248+benc-db@users.noreply.github.com> --- CHANGELOG.md | 3 ++- dbt/include/databricks/macros/materializations/snapshot.sql | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df65df75..ce1b1339 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ ### Fixes -- Stop setting cluster by to None. If you want to drop liquid clustering, you will need to full-refresh ([806](https://github.com/databricks/dbt-databricks/pull/806)) +- Stop setting cluster by to None. If you want to drop liquid clustering, you will need to full-refresh ([806]https://github.com/databricks/dbt-databricks/pull/806) +- Don't define table properties on snapshot staging views (thanks @jelmerk!) ([820](https://github.com/databricks/dbt-databricks/pull/820)) ## dbt-databricks 1.8.6 (September 18, 2024) diff --git a/dbt/include/databricks/macros/materializations/snapshot.sql b/dbt/include/databricks/macros/materializations/snapshot.sql index c1b8055c..3d1236a1 100644 --- a/dbt/include/databricks/macros/materializations/snapshot.sql +++ b/dbt/include/databricks/macros/materializations/snapshot.sql @@ -10,7 +10,9 @@ {# needs to be a non-temp view so that its columns can be ascertained via `describe` #} {% call statement('build_snapshot_staging_relation') %} - {{ create_view_as(tmp_relation, select) }} + create or replace view {{ tmp_relation }} + as + {{ select }} {% endcall %} {% do return(tmp_relation) %} From 7e6b45090ff05b572b983614fa55db0ac26f1c37 Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Fri, 4 Oct 2024 15:26:29 -0700 Subject: [PATCH 5/6] fix merge issue --- dbt/adapters/databricks/connections.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/databricks/connections.py b/dbt/adapters/databricks/connections.py index 48583bf2..4eb292eb 100644 --- a/dbt/adapters/databricks/connections.py +++ b/dbt/adapters/databricks/connections.py @@ -77,7 +77,6 @@ from dbt_common.exceptions import DbtDatabaseError from dbt_common.exceptions import DbtInternalError from dbt_common.exceptions import DbtRuntimeError -from dbt_common.exceptions import DbtDatabaseError from dbt_common.utils import cast_to_str from requests import Session From 52e9c7a379ccd3a0496c9e60e1493706596b1bf1 Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Thu, 10 Oct 2024 12:39:14 -0700 Subject: [PATCH 6/6] Prep for 1.8.7 release (#818) --- CHANGELOG.md | 2 +- dbt/adapters/databricks/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce1b1339..b476424c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## dbt-databricks 1.8.7 (TBD) +## dbt-databricks 1.8.7 (October 10, 2024) ### Features diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py index 8fe27739..192c2fde 100644 --- a/dbt/adapters/databricks/__version__.py +++ b/dbt/adapters/databricks/__version__.py @@ -1 +1 @@ -version: str = "1.8.6" +version: str = "1.8.7"