Merge branch 'main' into 1.9.latest

.github/workflows/main.yml

@@ -65,7 +65,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4.3.0
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install python dependencies
         run: |
@@ -86,7 +86,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     env:
       TOXENV: "unit"
@@ -138,7 +138,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4.3.0
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install python dependencies
         run: |
@@ -183,7 +183,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
@@ -207,9 +207,6 @@ jobs:
       - name: Install wheel distributions
         run: |
           find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Install dbt-core
-        run: |
-          python -m pip install dbt-core==1.8.0rc2
       - name: Check wheel distributions
         run: |
           dbt --version

CHANGELOG.md

@@ -20,8 +20,10 @@
 
 ### Under the Hood
 
+- Significant refactoring and increased testing of python_submissions ([830](https://github.com/databricks/dbt-databricks/pull/830))
 - Fix places where we were not properly closing cursors, and other test warnings ([713](https://github.com/databricks/dbt-databricks/pull/713))
-- Upgrade databricks-sql-connector dependency to 3.4.0 ([790](https://github.com/databricks/dbt-databricks/pull/790))
+- Drop support for Python 3.8 ([713](https://github.com/databricks/dbt-databricks/pull/713))
+- Upgrade databricks-sql-connector dependency to 3.5.0 ([833](https://github.com/databricks/dbt-databricks/pull/833))
 
 ## dbt-databricks 1.8.7 (October 10, 2024)
 

dbt/adapters/databricks/python_models/python_config.py

@@ -0,0 +1,57 @@
+from typing import Any, Dict, List, Optional
+import uuid
+from pydantic import BaseModel, Field
+
+
+DEFAULT_TIMEOUT = 60 * 60 * 24
+
+
+class PythonJobConfig(BaseModel):
+    """Pydantic model for config found in python_job_config."""
+
+    name: Optional[str] = None
+    grants: Dict[str, List[Dict[str, str]]] = Field(exclude=True, default_factory=dict)
+    existing_job_id: str = Field("", exclude=True)
+    post_hook_tasks: List[Dict[str, Any]] = Field(exclude=True, default_factory=list)
+    additional_task_settings: Dict[str, Any] = Field(exclude=True, default_factory=dict)
+
+    class Config:
+        extra = "allow"
+
+
+class PythonModelConfig(BaseModel):
+    """
+    Pydantic model for a Python model configuration.
+    Includes some job-specific settings that are not yet part of PythonJobConfig.
+    """
+
+    user_folder_for_python: bool = False
+    timeout: int = Field(DEFAULT_TIMEOUT, gt=0)
+    job_cluster_config: Dict[str, Any] = Field(default_factory=dict)
+    access_control_list: List[Dict[str, str]] = Field(default_factory=list)
+    packages: List[str] = Field(default_factory=list)
+    index_url: Optional[str] = None
+    additional_libs: List[Dict[str, Any]] = Field(default_factory=list)
+    python_job_config: PythonJobConfig = Field(default_factory=lambda: PythonJobConfig(**{}))
+    cluster_id: Optional[str] = None
+    http_path: Optional[str] = None
+    create_notebook: bool = False
+
+
+class ParsedPythonModel(BaseModel):
+    """Pydantic model for a Python model parsed from a dbt manifest"""
+
+    catalog: str = Field("hive_metastore", alias="database")
+
+    # Schema is a reserved name in Pydantic
+    schema_: str = Field("default", alias="schema")
+
+    identifier: str = Field(alias="alias")
+    config: PythonModelConfig
+
+    @property
+    def run_name(self) -> str:
+        return f"{self.catalog}-{self.schema_}-{self.identifier}-{uuid.uuid4()}"
+
+    class Config:
+        allow_population_by_field_name = True