fix: added minor changes + fix testcase

datahub-project · Oct 17, 2024 · c4fe5c5 · c4fe5c5
1 parent bd1ad49
commit c4fe5c5
Show file tree

Hide file tree

Showing 5 changed files with 2,025 additions and 221 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
@@ -300,6 +300,7 @@ def community_get_formatted_tables(
         dataset_list = []
         column_dictionary: Dict[str, List[Dict]] = defaultdict(list)
 
+        ordinal_position = 0
         for record in tables_and_columns:
             if not record.get("COLUMN_NAME"):
                 continue
@@ -311,13 +312,17 @@ def community_get_formatted_tables(
             column_dictionary[table_full_path].append(
                 {
                     "name": record["COLUMN_NAME"],
-                    "ordinal_position": record["ORDINAL_POSITION"],
+                    "ordinal_position": record.get(
+                        "ORDINAL_POSITION", ordinal_position
+                    ),
                     "is_nullable": record["IS_NULLABLE"],
                     "data_type": record["DATA_TYPE"],
                     "column_size": record["COLUMN_SIZE"],
                 }
             )
 
+            ordinal_position += 1
+
             if record.get("TABLE_SCHEMA") not in schema_list:
                 schema_list.append(record.get("TABLE_SCHEMA"))
 
@@ -412,6 +417,7 @@ def get_pattern_condition(
         all_tables_and_columns = []
 
         for schema in containers:
+            formatted_query = ""
             try:
                 formatted_query = query_template.format(
                     schema_pattern=schema_condition,
@@ -428,7 +434,7 @@ def get_pattern_condition(
                 )
             except Exception as exc:
                 logger.warning(
-                    f"{schema.subclass} {schema.container_name} had no tables or views"
+                    f"{schema.subclass} {schema.container_name} had no tables or views {formatted_query}"
                 )
                 logger.debug(exc)
 

diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py
@@ -224,13 +224,14 @@ def __init__(
         else:
             self.dataset_type = DremioDatasetType.TABLE
 
+        self.owner = dataset_details.get("OWNER")
+        self.owner_type = dataset_details.get("OWNER_TYPE")
+
         if api_operations.edition in (
             DremioEdition.ENTERPRISE,
             DremioEdition.CLOUD,
         ):
             self.created = dataset_details.get("CREATED", "")
-            self.owner = dataset_details.get("OWNER")
-            self.owner_type = dataset_details.get("OWNER_TYPE")
             self.format_type = dataset_details.get("FORMAT_TYPE")
 
         self.description = api_operations.get_description_for_resource(

diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py
@@ -13,11 +13,11 @@ class DremioSQLQueries:
         C.COLUMN_SIZE
     FROM
         INFORMATION_SCHEMA."TABLES" T
-        LEFT JOININFORMATION_SCHEMA.VIEWS V ON
+        LEFT JOIN INFORMATION_SCHEMA.VIEWS V ON
         V.TABLE_CATALOG = T.TABLE_CATALOG
         AND V.TABLE_SCHEMA = T.TABLE_SCHEMA
         AND V.TABLE_NAME = T.TABLE_NAME
-        INNER JOININFORMATION_SCHEMA.COLUMNS C ON
+        INNER JOIN INFORMATION_SCHEMA.COLUMNS C ON
         C.TABLE_CATALOG = T.TABLE_CATALOG
         AND C.TABLE_SCHEMA = T.TABLE_SCHEMA
         AND C.TABLE_NAME = T.TABLE_NAME