From c4fe5c5a446b33d7607b2d4d2b3a6d6398852601 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Thu, 17 Oct 2024 23:21:56 +0530 Subject: [PATCH] fix: added minor changes + fix testcase --- .../ingestion/source/dremio/dremio_api.py | 10 +- .../source/dremio/dremio_entities.py | 5 +- .../source/dremio/dremio_sql_queries.py | 4 +- .../dremio/dremio_mces_golden.json | 2225 +++++++++++++++-- .../tests/integration/dremio/test_dremio.py | 2 +- 5 files changed, 2025 insertions(+), 221 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py index da71900a72ce29..c1e5fa5b34efc9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py @@ -300,6 +300,7 @@ def community_get_formatted_tables( dataset_list = [] column_dictionary: Dict[str, List[Dict]] = defaultdict(list) + ordinal_position = 0 for record in tables_and_columns: if not record.get("COLUMN_NAME"): continue @@ -311,13 +312,17 @@ def community_get_formatted_tables( column_dictionary[table_full_path].append( { "name": record["COLUMN_NAME"], - "ordinal_position": record["ORDINAL_POSITION"], + "ordinal_position": record.get( + "ORDINAL_POSITION", ordinal_position + ), "is_nullable": record["IS_NULLABLE"], "data_type": record["DATA_TYPE"], "column_size": record["COLUMN_SIZE"], } ) + ordinal_position += 1 + if record.get("TABLE_SCHEMA") not in schema_list: schema_list.append(record.get("TABLE_SCHEMA")) @@ -412,6 +417,7 @@ def get_pattern_condition( all_tables_and_columns = [] for schema in containers: + formatted_query = "" try: formatted_query = query_template.format( schema_pattern=schema_condition, @@ -428,7 +434,7 @@ def get_pattern_condition( ) except Exception as exc: logger.warning( - f"{schema.subclass} {schema.container_name} had no tables or views" + f"{schema.subclass} {schema.container_name} had no tables or views {formatted_query}" ) logger.debug(exc) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py index 895d6cbc0281c9..3cb48b578324cf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py @@ -224,13 +224,14 @@ def __init__( else: self.dataset_type = DremioDatasetType.TABLE + self.owner = dataset_details.get("OWNER") + self.owner_type = dataset_details.get("OWNER_TYPE") + if api_operations.edition in ( DremioEdition.ENTERPRISE, DremioEdition.CLOUD, ): self.created = dataset_details.get("CREATED", "") - self.owner = dataset_details.get("OWNER") - self.owner_type = dataset_details.get("OWNER_TYPE") self.format_type = dataset_details.get("FORMAT_TYPE") self.description = api_operations.get_description_for_resource( diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py index 0d46d0ad357b35..1db03b3c04af68 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py @@ -13,11 +13,11 @@ class DremioSQLQueries: C.COLUMN_SIZE FROM INFORMATION_SCHEMA."TABLES" T - LEFT JOININFORMATION_SCHEMA.VIEWS V ON + LEFT JOIN INFORMATION_SCHEMA.VIEWS V ON V.TABLE_CATALOG = T.TABLE_CATALOG AND V.TABLE_SCHEMA = T.TABLE_SCHEMA AND V.TABLE_NAME = T.TABLE_NAME - INNER JOININFORMATION_SCHEMA.COLUMNS C ON + INNER JOIN INFORMATION_SCHEMA.COLUMNS C ON C.TABLE_CATALOG = T.TABLE_CATALOG AND C.TABLE_SCHEMA = T.TABLE_SCHEMA AND C.TABLE_NAME = T.TABLE_NAME diff --git a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json index b6a03761fa4c5f..3969ee4b68b718 100644 --- a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json +++ b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json @@ -1,14 +1,14 @@ [ { "entityType": "container", - "entityUrn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1", + "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": {}, - "name": "my_space", - "qualifiedName": "my_space", + "name": "my_folder", + "qualifiedName": "my_space.my_folder", "description": "", "env": "PROD" } @@ -21,12 +21,17 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1", + "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "browsePathsV2", "aspect": { "json": { - "platform": "urn:li:dataPlatform:dremio" + "path": [ + { + "id": "my_space", + "urn": "urn:li:container:19724f46e67c95babcd35b49cd200397" + } + ] } }, "systemMetadata": { @@ -37,16 +42,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b", + "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "container", "aspect": { "json": { - "customProperties": {}, - "name": "@admin", - "qualifiedName": "@admin", - "description": "# Wikis & Labels\n\n![Gnarly Catalog](https://d33wubrfki0l68.cloudfront.net/c1a54376c45a9276c080f3d10ed25ce61c17bcd2/2b946/img/home/open-source-for-everyone.svg)\n\nYou are reading the wiki for your home space! You can create and edit this information for any source, space, or folder.\n\nThis sidebar always shows the wiki for the current source, space or folder you are browsing.\n\nWhen browsing or previewing datasets, click on the `Open details panel` button to create a wiki or add labels to that dataset.\n\n**Tip:** You can hide the wiki by clicking on the sidebar icon on upper right hand side.", - "env": "PROD" + "container": "urn:li:container:5492d77b2954324ad07fff10d741e2d1" } }, "systemMetadata": { @@ -57,12 +58,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1", + "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:dremio" } }, "systemMetadata": { @@ -73,13 +74,13 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1", + "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Dremio Space" + "Dremio Folder" ] } }, @@ -91,12 +92,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b", + "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:dremio" + "removed": false } }, "systemMetadata": { @@ -107,14 +108,16 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b", + "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "containerProperties", "aspect": { "json": { - "typeNames": [ - "Dremio Space" - ] + "customProperties": {}, + "name": "catalog_page", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page", + "description": "", + "env": "PROD" } }, "systemMetadata": { @@ -125,12 +128,25 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b", + "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "Samples", + "urn": "urn:li:container:3acf5b43ce20cbc5b60abbdb57bb2a1a" + }, + { + "id": "samples.dremio.com", + "urn": "urn:li:container:c370d9075a8f1b3a28dbc3603717d466" + }, + { + "id": "tpcds_sf1000", + "urn": "urn:li:container:0c593828953ba6a54f29a97639e300e5" + } + ] } }, "systemMetadata": { @@ -141,12 +157,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", + "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:0c593828953ba6a54f29a97639e300e5" + "container": "urn:li:container:201af44510e34d8245f5515ee71a751f" } }, "systemMetadata": { @@ -157,14 +173,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", + "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "typeNames": [ - "Dremio Folder" - ] + "platform": "urn:li:dataPlatform:dremio" } }, "systemMetadata": { @@ -175,12 +189,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", + "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Dremio Folder" + ] } }, "systemMetadata": { @@ -191,12 +207,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", + "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:dremio" + "removed": false } }, "systemMetadata": { @@ -207,14 +223,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", + "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": {}, - "name": "catalog_page", - "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page", + "name": "tpcds_sf1000", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000", "description": "", "env": "PROD" } @@ -227,7 +243,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", + "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -235,15 +251,11 @@ "path": [ { "id": "Samples", - "urn": "urn:li:container:3acf5b43ce20cbc5b60abbdb57bb2a1a" + "urn": "urn:li:container:ef36cbff27a47ed33a76d0232fc3295c" }, { "id": "samples.dremio.com", - "urn": "urn:li:container:c370d9075a8f1b3a28dbc3603717d466" - }, - { - "id": "tpcds_sf1000", - "urn": "urn:li:container:0c593828953ba6a54f29a97639e300e5" + "urn": "urn:li:container:201af44510e34d8245f5515ee71a751f" } ] } @@ -256,16 +268,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", + "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "container", "aspect": { "json": { - "customProperties": {}, - "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00", - "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00", - "description": "", - "env": "PROD" + "container": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" } }, "systemMetadata": { @@ -276,29 +284,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", + "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "path": [ - { - "id": "Samples", - "urn": "urn:li:container:be5a17b95b6903826193185a545fdb38" - }, - { - "id": "samples.dremio.com", - "urn": "urn:li:container:cd673c4bfa428a6c3e4157eed8f4efdd" - }, - { - "id": "tpcds_sf1000", - "urn": "urn:li:container:ad2959fe35f5f1baa446ebb3a9a90d27" - }, - { - "id": "catalog_page", - "urn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d" - } - ] + "platform": "urn:li:dataPlatform:dremio" } }, "systemMetadata": { @@ -309,12 +300,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", + "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:201af44510e34d8245f5515ee71a751f" + "typeNames": [ + "Dremio Folder" + ] } }, "systemMetadata": { @@ -325,12 +318,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", + "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:dremio" + "removed": false } }, "systemMetadata": { @@ -386,16 +379,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", + "entityUrn": "urn:li:container:973eabd34940a8012ff93f357510bc1d", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "container", "aspect": { "json": { - "customProperties": {}, - "name": "s3", - "qualifiedName": "s3", - "description": "", - "env": "PROD" + "container": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" } }, "systemMetadata": { @@ -406,14 +395,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", + "entityUrn": "urn:li:container:973eabd34940a8012ff93f357510bc1d", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "typeNames": [ - "Dremio Folder" - ] + "platform": "urn:li:dataPlatform:dremio" } }, "systemMetadata": { @@ -426,10 +413,12 @@ "entityType": "container", "entityUrn": "urn:li:container:973eabd34940a8012ff93f357510bc1d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" + "typeNames": [ + "Dremio Folder" + ] } }, "systemMetadata": { @@ -442,10 +431,10 @@ "entityType": "container", "entityUrn": "urn:li:container:973eabd34940a8012ff93f357510bc1d", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:dremio" + "removed": false } }, "systemMetadata": { @@ -456,14 +445,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": {}, - "name": "samples.dremio.com", - "qualifiedName": "Samples.samples.dremio.com", + "name": "Samples", + "qualifiedName": "Samples", "description": "", "env": "PROD" } @@ -476,17 +465,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "path": [ - { - "id": "Samples", - "urn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" - } - ] + "platform": "urn:li:dataPlatform:dremio" } }, "systemMetadata": { @@ -497,16 +481,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": {}, - "name": "my_folder", - "qualifiedName": "my_space.my_folder", - "description": "", - "env": "PROD" + "typeNames": [ + "Dremio Source" + ] } }, "systemMetadata": { @@ -517,17 +499,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "my_space", - "urn": "urn:li:container:19724f46e67c95babcd35b49cd200397" - } - ] + "removed": false } }, "systemMetadata": { @@ -538,12 +515,16 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:0c593828953ba6a54f29a97639e300e5", + "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "containerProperties", "aspect": { "json": { - "removed": false + "customProperties": {}, + "name": "s3", + "qualifiedName": "s3", + "description": "", + "env": "PROD" } }, "systemMetadata": { @@ -554,14 +535,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:973eabd34940a8012ff93f357510bc1d", + "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "typeNames": [ - "Dremio Folder" - ] + "platform": "urn:li:dataPlatform:dremio" } }, "systemMetadata": { @@ -572,14 +551,16 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", + "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:5492d77b2954324ad07fff10d741e2d1" - } - }, + "typeNames": [ + "Dremio Source" + ] + } + }, "systemMetadata": { "lastObserved": 1697353200000, "runId": "dremio-2023_10_15-07_00_00", @@ -588,12 +569,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", + "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:dremio" + "removed": false } }, "systemMetadata": { @@ -606,10 +587,14 @@ "entityType": "container", "entityUrn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "containerProperties", "aspect": { "json": { - "container": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + "customProperties": {}, + "name": "samples.dremio.com", + "qualifiedName": "Samples.samples.dremio.com", + "description": "", + "env": "PROD" } }, "systemMetadata": { @@ -620,13 +605,16 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", + "entityUrn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Dremio Folder" + "path": [ + { + "id": "Samples", + "urn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" + } ] } }, @@ -638,12 +626,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:19724f46e67c95babcd35b49cd200397", + "entityUrn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" } }, "systemMetadata": { @@ -670,12 +658,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:973eabd34940a8012ff93f357510bc1d", + "entityUrn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Dremio Folder" + ] } }, "systemMetadata": { @@ -688,12 +678,10 @@ "entityType": "container", "entityUrn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Dremio Folder" - ] + "removed": false } }, "systemMetadata": { @@ -704,12 +692,16 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe", + "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "containerProperties", "aspect": { "json": { - "removed": false + "customProperties": {}, + "name": "@admin", + "qualifiedName": "@admin", + "description": "# Wikis & Labels\n\n![Gnarly Catalog](https://d33wubrfki0l68.cloudfront.net/c1a54376c45a9276c080f3d10ed25ce61c17bcd2/2b946/img/home/open-source-for-everyone.svg)\n\nYou are reading the wiki for your home space! You can create and edit this information for any source, space, or folder.\n\nThis sidebar always shows the wiki for the current source, space or folder you are browsing.\n\nWhen browsing or previewing datasets, click on the `Open details panel` button to create a wiki or add labels to that dataset.\n\n**Tip:** You can hide the wiki by clicking on the sidebar icon on upper right hand side.", + "env": "PROD" } }, "systemMetadata": { @@ -720,14 +712,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", + "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "typeNames": [ - "Dremio Source" - ] + "platform": "urn:li:dataPlatform:dremio" } }, "systemMetadata": { @@ -738,12 +728,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", + "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "subTypes", "aspect": { "json": { - "platform": "urn:li:dataPlatform:dremio" + "typeNames": [ + "Dremio Space" + ] } }, "systemMetadata": { @@ -754,7 +746,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", + "entityUrn": "urn:li:container:3ace723c787c49d7966071cc89d06c9b", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -770,14 +762,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "entityUrn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": {}, - "name": "Samples", - "qualifiedName": "Samples", + "name": "my_space", + "qualifiedName": "my_space", "description": "", "env": "PROD" } @@ -790,7 +782,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "entityUrn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -806,12 +798,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a", + "entityUrn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:63a316133b08a091e919dc8c7a828a4d" + "typeNames": [ + "Dremio Space" + ] } }, "systemMetadata": { @@ -822,7 +816,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "entityUrn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -838,14 +832,16 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "entityUrn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "containerProperties", "aspect": { "json": { - "typeNames": [ - "Dremio Source" - ] + "customProperties": {}, + "name": "warehouse", + "qualifiedName": "s3.warehouse", + "description": "", + "env": "PROD" } }, "systemMetadata": { @@ -858,11 +854,14 @@ "entityType": "container", "entityUrn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Dremio Folder" + "path": [ + { + "id": "s3", + "urn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a" + } ] } }, @@ -876,10 +875,10 @@ "entityType": "container", "entityUrn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:63a316133b08a091e919dc8c7a828a4d" } }, "systemMetadata": { @@ -908,12 +907,46 @@ "entityType": "container", "entityUrn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a", "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", + "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": {}, - "name": "warehouse", - "qualifiedName": "s3.warehouse", + "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00", "description": "", "env": "PROD" } @@ -926,15 +959,27 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a", + "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "s3", - "urn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a" + "id": "Samples", + "urn": "urn:li:container:be5a17b95b6903826193185a545fdb38" + }, + { + "id": "samples.dremio.com", + "urn": "urn:li:container:cd673c4bfa428a6c3e4157eed8f4efdd" + }, + { + "id": "tpcds_sf1000", + "urn": "urn:li:container:ad2959fe35f5f1baa446ebb3a9a90d27" + }, + { + "id": "catalog_page", + "urn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d" } ] } @@ -947,12 +992,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", + "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" + "container": "urn:li:container:0c593828953ba6a54f29a97639e300e5" } }, "systemMetadata": { @@ -963,7 +1008,23 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", + "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -981,7 +1042,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", + "entityUrn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -996,13 +1057,21 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "datasetProperties", "aspect": { "json": { - "platform": "urn:li:dataPlatform:dremio" + "customProperties": {}, + "externalUrl": "http://localhost:9047/api/v3/source/\"Samples\"/\"samples.dremio.com\".\"NYC-weather.csv\"", + "name": "NYC-weather.csv", + "qualifiedName": "Samples.samples.dremio.com.NYC-weather.csv", + "description": "", + "created": { + "time": 0 + }, + "tags": [] } }, "systemMetadata": { @@ -1012,17 +1081,15 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": {}, - "name": "tpcds_sf1000", - "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000", - "description": "", - "env": "PROD" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -1032,8 +1099,24 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:201af44510e34d8245f5515ee71a751f", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1041,11 +1124,1725 @@ "path": [ { "id": "Samples", - "urn": "urn:li:container:ef36cbff27a47ed33a76d0232fc3295c" + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" }, { "id": "samples.dremio.com", - "urn": "urn:li:container:201af44510e34d8245f5515ee71a751f" + "urn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.NYC-weather.csv", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "F", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "G", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "H", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "I", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "A", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "B", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "C", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "D", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "E", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/samples.dremio.com/NYC-weather.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/api/v3/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"googleplaystore.csv\"", + "name": "googleplaystore.csv", + "qualifiedName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "samples.dremio.com", + "urn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" + }, + { + "id": "Dremio University", + "urn": "urn:li:container:973eabd34940a8012ff93f357510bc1d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:973eabd34940a8012ff93f357510bc1d" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "A", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "B", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "C", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "D", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "E", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "F", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "G", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "H", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "I", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "J", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "K", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "L", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "M", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/samples.dremio.com/Dremio University/googleplaystore.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/api/v3/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"oracle-departments.xlsx\"", + "name": "oracle-departments.xlsx", + "qualifiedName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "samples.dremio.com", + "urn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" + }, + { + "id": "Dremio University", + "urn": "urn:li:container:973eabd34940a8012ff93f357510bc1d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:973eabd34940a8012ff93f357510bc1d" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "DEPARTMENT_NAME", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "MANAGER_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "DEPARTMENT_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LOCATION_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/api/v3/space/\"my_space\"/\"my_folder\".\"raw\"", + "name": "raw", + "qualifiedName": "my_space.my_folder.raw", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "my_space", + "urn": "urn:li:container:5492d77b2954324ad07fff10d741e2d1" + }, + { + "id": "my_folder", + "urn": "urn:li:container:19724f46e67c95babcd35b49cd200397" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:19724f46e67c95babcd35b49cd200397" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "my_space.my_folder.raw", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "age", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "salary", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/api/v3/source/\"Samples\"/\"samples.dremio.com\".\"tpcds_sf1000\".\"catalog_page\".\"1ab266d5-18eb-4780-711d-0fa337fa6c00\".\"0_0_0.parquet\"", + "name": "0_0_0.parquet", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "samples.dremio.com", + "urn": "urn:li:container:e247722e4a5ae3879de75a2a478ecbbe" + }, + { + "id": "tpcds_sf1000", + "urn": "urn:li:container:201af44510e34d8245f5515ee71a751f" + }, + { + "id": "catalog_page", + "urn": "urn:li:container:0c593828953ba6a54f29a97639e300e5" + }, + { + "id": "1ab266d5-18eb-4780-711d-0fa337fa6c00", + "urn": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1da1928d2528f84a5e928fca4c3bc75d" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "cp_start_date_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_page_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_page_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_end_date_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_department", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_number", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_page_number", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_description", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_type", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/api/v3/source/\"s3\"/\"warehouse\".\"sample.parquet\"", + "name": "sample.parquet", + "qualifiedName": "s3.warehouse.sample.parquet", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "s3", + "urn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d" + }, + { + "id": "warehouse", + "urn": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b6e7d0c364ccc53ccb02b438999fda9a" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "s3.warehouse.sample.parquet", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "age", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "salary", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/warehouse/sample.parquet,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,s3.warehouse.sample.parquet,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.my_space.my_folder.raw%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.my_space.my_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM s3.warehouse.\"sample.parquet\"", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.my_space.my_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,s3.warehouse.sample.parquet,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.my_space.my_folder.raw,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.my_space.my_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/warehouse/sample.parquet,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/samples.dremio.com/Dremio University/googleplaystore.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/samples.dremio.com/NYC-weather.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,/samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)", + "type": "COPY" } ] } diff --git a/metadata-ingestion/tests/integration/dremio/test_dremio.py b/metadata-ingestion/tests/integration/dremio/test_dremio.py index 3aa83ea0e73490..6a3b8b2c828497 100644 --- a/metadata-ingestion/tests/integration/dremio/test_dremio.py +++ b/metadata-ingestion/tests/integration/dremio/test_dremio.py @@ -226,7 +226,7 @@ def add_datasets_to_space(dremio_token): response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to create view: {response.text}" - time.sleep(5) + time.sleep(10) url = f"{DREMIO_HOST}/apiv2/dataset/tmp.UNTITLED/version/{sql_version_number}/save?as=%22my_space%22.%22my_folder%22.raw" response = requests.post(url, headers=headers) assert response.status_code == 200, f"Failed to add view in folder: {response.text}"