From 4d69cea4724109640d67554f00069b35cc0ec741 Mon Sep 17 00:00:00 2001 From: Robert Espinoza Date: Thu, 28 Mar 2024 05:15:03 -0700 Subject: [PATCH 1/5] add row type for athena types (#10131) Co-authored-by: Robert Espinoza --- metadata-ingestion/src/datahub/ingestion/source/sql/athena.py | 4 ++-- .../src/datahub/ingestion/source/sql/sql_types.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index c3759875b2769..ae17cff60fedd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -138,8 +138,8 @@ def _get_column_type( args = [array_type] - elif type_name in ["struct", "record"]: - # STRUCT is not part of the SQLalchemy types selection + elif type_name in ["struct", "record", "row"]: + # STRUCT and ROW are not part of the SQLalchemy types selection # but is provided by another official SQLalchemy library and # compatible with the other SQLalchemy types detected_col_type = STRUCT diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py index 3b4a7e1dc0287..dedb6eedd5ee4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py @@ -387,6 +387,7 @@ def resolve_vertica_modified_type(type_string: str) -> Any: "struct": RecordType, "map": MapType, "array": ArrayType, + "row": RecordType, } # https://www.vertica.com/docs/11.1.x/HTML/Content/Authoring/SQLReferenceManual/DataTypes/SQLDataTypes.htm From 140c0f1d13b7e1ae4e544027401aad85f845a0a7 Mon Sep 17 00:00:00 2001 From: trialiya <41265764+trialiya@users.noreply.github.com> Date: Thu, 28 Mar 2024 20:23:44 +0300 Subject: [PATCH 2/5] fix(setup): fix postgres setup to create temp table with no data (#10154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Хазиев Ленар --- docker/postgres-setup/init.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/postgres-setup/init.sql b/docker/postgres-setup/init.sql index 72b2f73192e00..f2bda8b2e1dd9 100644 --- a/docker/postgres-setup/init.sql +++ b/docker/postgres-setup/init.sql @@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS metadata_aspect_v2 ( create index timeIndex ON metadata_aspect_v2 (createdon); -- create default records for datahub user if not exists -CREATE TEMP TABLE temp_metadata_aspect_v2 AS TABLE metadata_aspect_v2; +CREATE TEMP TABLE temp_metadata_aspect_v2 AS TABLE metadata_aspect_v2 WITH NO DATA; INSERT INTO temp_metadata_aspect_v2 (urn, aspect, version, metadata, createdon, createdby) VALUES( 'urn:li:corpuser:datahub', 'corpUserInfo', From 4e328c38a76ca8a2293c2557a1e96e77c2793f4c Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Fri, 29 Mar 2024 00:27:43 +0530 Subject: [PATCH 3/5] feat(ingest/looker): update browse paths to align with looker UI (#10147) --- docs/how/updating-datahub.md | 3 +- .../ingestion/source/looker/looker_common.py | 1 + .../ingestion/source/looker/looker_config.py | 4 +- .../ingestion/source/looker/looker_source.py | 61 +++-- .../looker/golden_looker_mces.json | 101 ++++--- .../looker/golden_test_allow_ingest.json | 52 ++-- ...olden_test_external_project_view_mces.json | 193 ++++++++++--- .../looker/golden_test_file_path_ingest.json | 152 +++++++++-- .../golden_test_independent_look_ingest.json | 189 +++++++++++-- .../looker/golden_test_ingest.json | 193 ++++++++++--- .../looker/golden_test_ingest_joins.json | 193 ++++++++++--- .../golden_test_ingest_unaliased_joins.json | 52 ++-- .../looker_mces_golden_deleted_stateful.json | 214 ++++++++++++--- .../looker/looker_mces_usage_history.json | 79 ++++-- .../tests/integration/looker/test_looker.py | 5 +- .../integration/lookml/expected_output.json | 246 ++++++++++------- .../lookml/lookml_mces_api_bigquery.json | 255 +++++++++++------- .../lookml/lookml_mces_api_hive2.json | 255 +++++++++++------- .../lookml/lookml_mces_badsql_parser.json | 255 +++++++++++------- .../lookml/lookml_mces_offline.json | 255 +++++++++++------- .../lookml_mces_offline_deny_pattern.json | 108 +++++--- ...lookml_mces_offline_platform_instance.json | 255 +++++++++++------- .../lookml_mces_with_external_urls.json | 255 +++++++++++------- .../lookml/lookml_reachable_views.json | 72 +++-- .../refinement_include_order_golden.json | 82 ++++-- .../tests/integration/lookml/test_lookml.py | 6 +- 26 files changed, 2463 insertions(+), 1073 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 8ab80af594e99..60504aaa7b80a 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -25,7 +25,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #10026 - The dbt `use_compiled_code` option has been removed, because we now support capturing both source and compiled dbt SQL. This can be configured using `include_compiled_code`, which will be default enabled in 0.13.1. - #10055 - Assertion entities generated by dbt are now associated with the dbt dataset entity, and not the entity in the data warehouse. - #10090 - For Redshift ingestion, `use_lineage_v2` is now enabled by default. - +- #10147 - For looker ingestion, the browse paths for looker Dashboard, Chart, View, Explore have been updated to align with Looker UI. This does not affect URNs or lineage but primarily affects (improves) browsing experience. +- ### Potential Downtime ### Deprecations diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py index 41accd64ae59e..3ac59962fffe6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py @@ -1199,6 +1199,7 @@ class LookerDashboardElement: type: Optional[str] = None description: Optional[str] = None input_fields: Optional[List[InputFieldElement]] = None + folder_path: Optional[str] = None # for independent looks. def url(self, base_url: str) -> str: # A dashboard element can use a look or just a raw query against an explore diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py index 87835ddae085b..e501ad4813288 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py @@ -108,14 +108,14 @@ class LookerCommonConfig(EnvConfigMixin, PlatformInstanceConfigMixin): ) explore_browse_pattern: LookerNamingPattern = pydantic.Field( description=f"Pattern for providing browse paths to explores. {LookerNamingPattern.allowed_docstring()}", - default=LookerNamingPattern(pattern="/{env}/{platform}/{project}/explores"), + default=LookerNamingPattern(pattern="/Explore/{project}/{model}"), ) view_naming_pattern: LookerViewNamingPattern = Field( LookerViewNamingPattern(pattern="{project}.view.{name}"), description=f"Pattern for providing dataset names to views. {LookerViewNamingPattern.allowed_docstring()}", ) view_browse_pattern: LookerViewNamingPattern = Field( - LookerViewNamingPattern(pattern="/{env}/{platform}/{project}/views"), + LookerViewNamingPattern(pattern="/Develop/{project}/{file_path}"), description=f"Pattern for providing browse paths to views. {LookerViewNamingPattern.allowed_docstring()}", ) tag_measures_and_dimensions: bool = Field( diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index c63a845633825..208f412613cf2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -452,6 +452,12 @@ def _get_looker_dashboard_element( # noqa: C901 else: slug = "" + look_folder_path = None + if element.look.folder is not None: + look_folder_path = self._get_folder_path( + element.look.folder, self.looker_api + ) + return LookerDashboardElement( id=element.id, title=title, @@ -464,6 +470,7 @@ def _get_looker_dashboard_element( # noqa: C901 for exp in explores ], input_fields=input_fields, + folder_path=look_folder_path, ) # Failing the above two approaches, pick out details from result_maker @@ -524,10 +531,12 @@ def _get_looker_dashboard_element( # noqa: C901 type=element.type, description=element.subtitle_text, look_id=element.look_id, - query_slug=element.result_maker.query.slug - if element.result_maker.query is not None - and element.result_maker.query.slug is not None - else "", + query_slug=( + element.result_maker.query.slug + if element.result_maker.query is not None + and element.result_maker.query.slug is not None + else "" + ), upstream_explores=[ LookerExplore(model_name=model, name=exp) for exp in explores ], @@ -603,18 +612,29 @@ def _make_chart_metadata_events( chartUrl=dashboard_element.url(self.source_config.external_base_url or ""), inputs=dashboard_element.get_view_urns(self.source_config), customProperties={ - "upstream_fields": ",".join( - sorted(set(field.name for field in dashboard_element.input_fields)) + "upstream_fields": ( + ",".join( + sorted( + set(field.name for field in dashboard_element.input_fields) + ) + ) + if dashboard_element.input_fields + else "" ) - if dashboard_element.input_fields - else "" }, ) chart_snapshot.aspects.append(chart_info) if dashboard and dashboard.folder_path is not None: browse_path = BrowsePathsClass( - paths=[f"/looker/{dashboard.folder_path}/{dashboard.title}"] + paths=[f"/Folders/{dashboard.folder_path}/{dashboard.title}"] + ) + chart_snapshot.aspects.append(browse_path) + elif ( + dashboard is None and dashboard_element.folder_path is not None + ): # independent look + browse_path = BrowsePathsClass( + paths=[f"/Folders/{dashboard_element.folder_path}"] ) chart_snapshot.aspects.append(browse_path) @@ -673,7 +693,7 @@ def _make_dashboard_metadata_events( dashboard_snapshot.aspects.append(dashboard_info) if looker_dashboard.folder_path is not None: browse_path = BrowsePathsClass( - paths=[f"/looker/{looker_dashboard.folder_path}"] + paths=[f"/Folders/{looker_dashboard.folder_path}"] ) dashboard_snapshot.aspects.append(browse_path) @@ -1084,10 +1104,12 @@ def process_dashboard( looker_dashboard = self._get_looker_dashboard(dashboard_object, self.looker_api) mces = self._make_dashboard_and_chart_mces(looker_dashboard) workunits = [ - MetadataWorkUnit(id=f"looker-{mce.proposedSnapshot.urn}", mce=mce) - if isinstance(mce, MetadataChangeEvent) - else MetadataWorkUnit( - id=f"looker-{mce.aspectName}-{mce.entityUrn}", mcp=mce + ( + MetadataWorkUnit(id=f"looker-{mce.proposedSnapshot.urn}", mce=mce) + if isinstance(mce, MetadataChangeEvent) + else MetadataWorkUnit( + id=f"looker-{mce.aspectName}-{mce.entityUrn}", mcp=mce + ) ) for mce in mces ] @@ -1177,12 +1199,7 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: self.reporter.report_stage_start("extract_independent_looks") logger.debug("Extracting looks not part of Dashboard") - look_fields: List[str] = [ - "id", - "title", - "description", - "query_id", - ] + look_fields: List[str] = ["id", "title", "description", "query_id", "folder"] query_fields: List[str] = [ "id", "view", @@ -1227,8 +1244,8 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: subtitle_text=look.description, look_id=look.id, dashboard_id=None, # As this is independent look - look=LookWithQuery(query=query), - ) + look=LookWithQuery(query=query, folder=look.folder), + ), ) if dashboard_element is not None: diff --git a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json index 1da42b94e320c..4390499248206 100644 --- a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json +++ b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json @@ -34,7 +34,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +50,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -109,7 +111,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -145,7 +148,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -225,18 +229,19 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/bogus data" ] } }, @@ -251,7 +256,7 @@ "looker.explore.label": "My Explore View", "looker.explore.file": "test_source_file.lkml" }, - "externalUrl": "https://looker.company.com/explore/data/my_view", + "externalUrl": "https://looker.company.com/explore/bogus data/my_view", "name": "My Explore View", "description": "lorem ipsum", "tags": [] @@ -321,12 +326,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -338,55 +344,61 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", "changeType": "UPSERT", "aspectName": "embed", "aspect": { "json": { - "renderUrl": "https://looker.company.com/embed/explore/data/my_view" + "renderUrl": "https://looker.company.com/embed/explore/bogus data/my_view" } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "bogus data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/data" ] } }, @@ -401,7 +413,7 @@ "looker.explore.label": "My Explore View", "looker.explore.file": "test_source_file.lkml" }, - "externalUrl": "https://looker.company.com/explore/bogus data/my_view", + "externalUrl": "https://looker.company.com/explore/data/my_view", "name": "My Explore View", "description": "lorem ipsum", "tags": [] @@ -471,12 +483,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -488,44 +501,50 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", "changeType": "UPSERT", "aspectName": "embed", "aspect": { "json": { - "renderUrl": "https://looker.company.com/embed/explore/bogus data/my_view" + "renderUrl": "https://looker.company.com/embed/explore/data/my_view" } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -544,7 +563,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -563,7 +583,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +603,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -597,7 +619,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -612,7 +635,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -627,7 +651,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +667,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -657,7 +683,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json index 685a606a57c33..a3788f52459ae 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json @@ -34,7 +34,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +50,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -109,7 +111,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -169,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -180,7 +184,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/data" ] } }, @@ -265,7 +269,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -282,7 +287,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -297,7 +303,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -308,18 +315,22 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -338,7 +349,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -357,7 +369,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -376,7 +389,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -391,7 +405,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -406,7 +421,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -421,7 +437,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -436,7 +453,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json index 069788cb088ac..7e570e7eccd23 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json @@ -1,4 +1,99 @@ [ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,dashboard_elements.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "upstream_fields": "calc,dim1" + }, + "title": "", + "description": "Some text", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "https://looker.company.com/x/", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared/foo" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Look" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + }, + { + "id": "foo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9,7 +104,9 @@ "customProperties": {}, "title": "foo", "description": "lorem ipsum", - "charts": [], + "charts": [ + "urn:li:chart:(looker,dashboard_elements.2)" + ], "datasets": [], "lastModified": { "created": { @@ -24,6 +121,13 @@ "dashboardUrl": "https://looker.company.com/dashboards/1" } }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared" + ] + } + }, { "com.linkedin.pegasus2avro.common.Status": { "removed": false @@ -34,7 +138,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +154,31 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -109,7 +238,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -169,7 +299,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -180,7 +311,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/looker_hub/explores" + "/Explore/looker_hub/data" ] } }, @@ -265,7 +396,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -282,7 +414,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -297,7 +430,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -308,18 +442,22 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "looker_hub" }, { - "id": "explores" + "id": "data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -338,7 +476,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -357,7 +496,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -376,22 +516,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -406,7 +532,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -421,7 +548,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -436,7 +564,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json index f1c932ebd5a70..96ef028f580c3 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json @@ -1,4 +1,99 @@ [ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,dashboard_elements.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "upstream_fields": "calc,dim1" + }, + "title": "", + "description": "Some text", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "https://looker.company.com/x/", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared/foo" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Look" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + }, + { + "id": "foo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9,7 +104,9 @@ "customProperties": {}, "title": "foo", "description": "lorem ipsum", - "charts": [], + "charts": [ + "urn:li:chart:(looker,dashboard_elements.2)" + ], "datasets": [], "lastModified": { "created": { @@ -24,6 +121,13 @@ "dashboardUrl": "https://looker.company.com/dashboards/1" } }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared" + ] + } + }, { "com.linkedin.pegasus2avro.common.Status": { "removed": false @@ -54,6 +158,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", @@ -184,7 +311,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/looker_hub/explores" + "/Explore/looker_hub/data" ] } }, @@ -315,11 +442,14 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "looker_hub" }, { - "id": "explores" + "id": "data" } ] } @@ -390,22 +520,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", diff --git a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json index e110246605974..bd2a6ae74efbb 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json @@ -1,4 +1,99 @@ [ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,dashboard_elements.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "upstream_fields": "calc,dim1" + }, + "title": "", + "description": "Some text", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "https://looker.company.com/x/", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared/foo" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Look" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + }, + { + "id": "foo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9,7 +104,9 @@ "customProperties": {}, "title": "foo", "description": "lorem ipsum", - "charts": [], + "charts": [ + "urn:li:chart:(looker,dashboard_elements.2)" + ], "datasets": [], "lastModified": { "created": { @@ -24,6 +121,13 @@ "dashboardUrl": "https://looker.company.com/dashboards/1" } }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared" + ] + } + }, { "com.linkedin.pegasus2avro.common.Status": { "removed": false @@ -54,6 +158,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", @@ -210,6 +337,13 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared" + ] + } } ] } @@ -270,6 +404,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.looks_1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -278,7 +435,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/data" ] } }, @@ -409,11 +566,14 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "data" } ] } @@ -432,7 +592,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/sales_model" ] } }, @@ -563,11 +723,14 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "sales_model" } ] } @@ -638,22 +801,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json index dbacd52fe83de..f4e6ff83fa92d 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json @@ -1,4 +1,99 @@ [ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,dashboard_elements.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "upstream_fields": "calc,dim1" + }, + "title": "", + "description": "Some text", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "https://looker.company.com/x/", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared/foo" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Look" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + }, + { + "id": "foo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9,7 +104,9 @@ "customProperties": {}, "title": "foo", "description": "lorem ipsum", - "charts": [], + "charts": [ + "urn:li:chart:(looker,dashboard_elements.2)" + ], "datasets": [], "lastModified": { "created": { @@ -24,6 +121,13 @@ "dashboardUrl": "https://looker.company.com/dashboards/1" } }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared" + ] + } + }, { "com.linkedin.pegasus2avro.common.Status": { "removed": false @@ -34,7 +138,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +154,31 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -109,7 +238,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -169,7 +299,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -180,7 +311,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/data" ] } }, @@ -265,7 +396,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -282,7 +414,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -297,7 +430,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -308,18 +442,22 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -338,7 +476,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -357,7 +496,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -376,22 +516,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -406,7 +532,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -421,7 +548,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -436,7 +564,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json index aaa874d9ff348..0b8ac8afb631b 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json @@ -1,4 +1,99 @@ [ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,dashboard_elements.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "upstream_fields": "calc,dim1" + }, + "title": "", + "description": "Some text", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "https://looker.company.com/x/", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared/foo" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Look" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + }, + { + "id": "foo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9,7 +104,9 @@ "customProperties": {}, "title": "foo", "description": "lorem ipsum", - "charts": [], + "charts": [ + "urn:li:chart:(looker,dashboard_elements.2)" + ], "datasets": [], "lastModified": { "created": { @@ -24,6 +121,13 @@ "dashboardUrl": "https://looker.company.com/dashboards/1" } }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared" + ] + } + }, { "com.linkedin.pegasus2avro.common.Status": { "removed": false @@ -34,7 +138,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +154,31 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -109,7 +238,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -169,7 +299,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -180,7 +311,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/data" ] } }, @@ -289,7 +420,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -306,7 +438,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -321,7 +454,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -332,18 +466,22 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -362,7 +500,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -381,7 +520,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -400,22 +540,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -430,7 +556,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -445,7 +572,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -460,7 +588,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json index be8db0722aea3..f378129b08ea4 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json @@ -34,7 +34,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +50,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -109,7 +111,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -169,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -180,7 +184,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/data" ] } }, @@ -281,7 +285,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -298,7 +303,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -313,7 +319,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -324,18 +331,22 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -354,7 +365,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -373,7 +385,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -392,7 +405,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -407,7 +421,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -422,7 +437,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -437,7 +453,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -452,7 +469,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json index 05b74f163ad45..1821d931c7c7a 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json @@ -1,4 +1,99 @@ [ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,dashboard_elements.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "upstream_fields": "calc,dim1" + }, + "title": "", + "description": "Some text", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "https://looker.company.com/x/", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared/foo" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Look" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + }, + { + "id": "foo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9,7 +104,9 @@ "customProperties": {}, "title": "foo", "description": "lorem ipsum", - "charts": [], + "charts": [ + "urn:li:chart:(looker,dashboard_elements.2)" + ], "datasets": [], "lastModified": { "created": { @@ -24,6 +121,13 @@ "dashboardUrl": "https://looker.company.com/dashboards/1" } }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Folders/Shared" + ] + } + }, { "com.linkedin.pegasus2avro.common.Status": { "removed": false @@ -34,7 +138,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +154,31 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + }, + { + "id": "Shared" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -109,7 +238,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -169,7 +299,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -180,7 +311,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/data" ] } }, @@ -265,7 +396,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -282,7 +414,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -297,7 +430,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -308,18 +442,22 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -338,7 +476,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -357,7 +496,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -376,22 +516,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -406,7 +532,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -421,7 +548,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -436,12 +564,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.11)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -451,12 +580,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -466,12 +596,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(looker,dashboards.11)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -481,7 +612,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json index 0778aa0050b00..a7a7aff801781 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json @@ -34,7 +34,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +50,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -85,7 +87,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -121,7 +124,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -132,7 +136,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/explores" + "/Explore/lkml_samples/data" ] } }, @@ -217,7 +221,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -234,7 +239,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -249,7 +255,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -260,18 +267,22 @@ "aspect": { "json": { "path": [ + { + "id": "Explore" + }, { "id": "lkml_samples" }, { - "id": "explores" + "id": "data" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -290,7 +301,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -309,7 +321,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -328,7 +341,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -350,7 +364,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -389,7 +404,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -414,7 +430,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -447,7 +464,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -467,7 +485,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -501,7 +520,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -525,7 +545,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -549,7 +570,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -564,7 +586,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -579,7 +602,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -594,7 +618,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -609,7 +634,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -624,7 +650,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py index 3e614dbdbc860..8e167328e0ba2 100644 --- a/metadata-ingestion/tests/integration/looker/test_looker.py +++ b/metadata-ingestion/tests/integration/looker/test_looker.py @@ -11,6 +11,7 @@ from looker_sdk.sdk.api40.models import ( Dashboard, DashboardElement, + FolderBase, Look, LookmlModelExplore, LookmlModelExploreField, @@ -285,10 +286,11 @@ def setup_mock_dashboard(mocked_client): created_at=datetime.utcfromtimestamp(time.time()), updated_at=datetime.utcfromtimestamp(time.time()), description="lorem ipsum", + folder=FolderBase(name="Shared", id="shared-folder-id"), dashboard_elements=[ DashboardElement( id="2", - type="", + type="vis", subtitle_text="Some text", query=Query( model="data", @@ -308,6 +310,7 @@ def setup_mock_look(mocked_client): title="Outer Look", description="I am not part of any Dashboard", query_id="1", + folder=FolderBase(name="Shared", id="shared-folder-id"), ) ] diff --git a/metadata-ingestion/tests/integration/lookml/expected_output.json b/metadata-ingestion/tests/integration/lookml/expected_output.json index b53d5857f1d66..a1ac1ecb0e8ea 100644 --- a/metadata-ingestion/tests/integration/lookml/expected_output.json +++ b/metadata-ingestion/tests/integration/lookml/expected_output.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo.view.lkml" ] } }, @@ -176,7 +176,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -193,7 +194,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -210,7 +212,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -222,20 +225,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -246,7 +250,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/bar.view.lkml" ] } }, @@ -394,7 +398,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -411,7 +416,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -428,7 +434,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -440,20 +447,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "bar.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -464,7 +472,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/included_view_file.view.lkml" ] } }, @@ -501,7 +509,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -518,7 +527,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -535,7 +545,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -547,20 +558,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "included_view_file.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -571,7 +583,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -608,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -625,7 +638,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +656,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -654,20 +669,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -678,7 +694,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -760,7 +776,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -777,7 +794,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -794,7 +812,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -806,20 +825,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +850,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -867,7 +887,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -884,7 +905,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -901,7 +923,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -913,20 +936,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -937,7 +961,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -974,7 +998,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -991,7 +1016,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1008,7 +1034,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1020,20 +1047,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1044,7 +1072,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/nested/fragment_derived.view.lkml" ] } }, @@ -1156,7 +1184,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1173,7 +1202,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1190,7 +1220,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1202,20 +1233,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "nested" + }, + { + "id": "fragment_derived.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1226,7 +1261,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/liquid.view.lkml" ] } }, @@ -1263,7 +1298,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1280,7 +1316,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1297,7 +1334,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1309,20 +1347,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "liquid.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1333,7 +1372,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/ability.view.lkml" ] } }, @@ -1449,7 +1488,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1466,7 +1506,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1483,7 +1524,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1495,20 +1537,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "ability.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1519,7 +1562,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/owners.view.lkml" ] } }, @@ -1648,7 +1691,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1665,7 +1709,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1682,7 +1727,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1694,20 +1740,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "owners.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1718,7 +1765,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/native_derived_table.view.lkml" ] } }, @@ -1887,7 +1934,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1904,7 +1952,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1921,7 +1970,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1933,20 +1983,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "native_derived_table.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1957,7 +2008,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/flights.view.lkml" ] } }, @@ -2054,7 +2105,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2071,7 +2123,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2088,7 +2141,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2100,20 +2154,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "flights.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2128,7 +2183,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2143,7 +2199,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2158,7 +2215,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json index 5a0bd4e12fd3a..3fc639c0b4030 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo.view.lkml" ] } }, @@ -176,7 +176,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -193,7 +194,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -210,7 +212,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -222,20 +225,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -246,7 +250,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/bar.view.lkml" ] } }, @@ -394,7 +398,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -411,7 +416,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -428,7 +434,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -440,20 +447,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "bar.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -464,7 +472,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/included_view_file.view.lkml" ] } }, @@ -501,7 +509,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -518,7 +527,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -535,7 +545,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -547,20 +558,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "included_view_file.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -571,7 +583,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -608,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -625,7 +638,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +656,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -654,20 +669,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -678,7 +694,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -760,7 +776,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -777,7 +794,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -794,7 +812,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -806,20 +825,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +850,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -867,7 +887,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -884,7 +905,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -901,7 +923,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -913,20 +936,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -937,7 +961,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -974,7 +998,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -991,7 +1016,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1008,7 +1034,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1020,20 +1047,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1044,7 +1072,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/nested/fragment_derived.view.lkml" ] } }, @@ -1156,7 +1184,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1173,7 +1202,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1190,7 +1220,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1202,20 +1233,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "nested" + }, + { + "id": "fragment_derived.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1226,7 +1261,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/liquid.view.lkml" ] } }, @@ -1263,7 +1298,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1280,7 +1316,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1297,7 +1334,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1309,20 +1347,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "liquid.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1333,7 +1372,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/ability.view.lkml" ] } }, @@ -1449,7 +1488,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1466,7 +1506,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1483,7 +1524,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1495,20 +1537,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "ability.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1519,7 +1562,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/owners.view.lkml" ] } }, @@ -1648,7 +1691,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1665,7 +1709,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1682,7 +1727,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1694,20 +1740,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "owners.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1718,7 +1765,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/native_derived_table.view.lkml" ] } }, @@ -1887,7 +1934,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1904,7 +1952,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1921,7 +1970,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1933,20 +1983,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "native_derived_table.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1957,7 +2008,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/flights.view.lkml" ] } }, @@ -2054,7 +2105,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2071,7 +2123,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2088,7 +2141,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2100,20 +2154,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "flights.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2132,7 +2187,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2151,7 +2207,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2170,7 +2227,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2185,7 +2243,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2200,7 +2259,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2215,7 +2275,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json index 1b0ee3216383c..ec4df7ced01a3 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo.view.lkml" ] } }, @@ -176,7 +176,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -193,7 +194,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -210,7 +212,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -222,20 +225,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -246,7 +250,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/bar.view.lkml" ] } }, @@ -394,7 +398,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -411,7 +416,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -428,7 +434,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -440,20 +447,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "bar.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -464,7 +472,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/included_view_file.view.lkml" ] } }, @@ -501,7 +509,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -518,7 +527,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -535,7 +545,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -547,20 +558,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "included_view_file.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -571,7 +583,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -608,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -625,7 +638,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +656,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -654,20 +669,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -678,7 +694,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -760,7 +776,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -777,7 +794,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -794,7 +812,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -806,20 +825,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +850,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -867,7 +887,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -884,7 +905,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -901,7 +923,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -913,20 +936,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -937,7 +961,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -974,7 +998,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -991,7 +1016,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1008,7 +1034,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1020,20 +1047,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1044,7 +1072,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/nested/fragment_derived.view.lkml" ] } }, @@ -1156,7 +1184,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1173,7 +1202,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1190,7 +1220,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1202,20 +1233,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "nested" + }, + { + "id": "fragment_derived.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1226,7 +1261,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/liquid.view.lkml" ] } }, @@ -1263,7 +1298,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1280,7 +1316,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1297,7 +1334,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1309,20 +1347,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "liquid.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1333,7 +1372,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/ability.view.lkml" ] } }, @@ -1449,7 +1488,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1466,7 +1506,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1483,7 +1524,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1495,20 +1537,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "ability.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1519,7 +1562,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/owners.view.lkml" ] } }, @@ -1648,7 +1691,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1665,7 +1709,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1682,7 +1727,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1694,20 +1740,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "owners.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1718,7 +1765,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/native_derived_table.view.lkml" ] } }, @@ -1887,7 +1934,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1904,7 +1952,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1921,7 +1970,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1933,20 +1983,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "native_derived_table.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1957,7 +2008,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/flights.view.lkml" ] } }, @@ -2054,7 +2105,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2071,7 +2123,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2088,7 +2141,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2100,20 +2154,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "flights.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2132,7 +2187,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2151,7 +2207,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2170,7 +2227,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2185,7 +2243,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2200,7 +2259,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2215,7 +2275,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json index b960ba581e6b5..4f85bbf9022f6 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo.view.lkml" ] } }, @@ -162,7 +162,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -179,7 +180,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -196,7 +198,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -208,20 +211,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -232,7 +236,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/bar.view.lkml" ] } }, @@ -366,7 +370,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -383,7 +388,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -400,7 +406,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -412,20 +419,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "bar.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -436,7 +444,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/included_view_file.view.lkml" ] } }, @@ -473,7 +481,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -490,7 +499,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -507,7 +517,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -519,20 +530,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "included_view_file.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -543,7 +555,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -580,7 +592,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -597,7 +610,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +628,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -626,20 +641,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -650,7 +666,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -732,7 +748,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -749,7 +766,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -766,7 +784,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -778,20 +797,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -802,7 +822,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -839,7 +859,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -856,7 +877,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -873,7 +895,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -885,20 +908,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -909,7 +933,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -946,7 +970,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -963,7 +988,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -980,7 +1006,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -992,20 +1019,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1016,7 +1044,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/nested/fragment_derived.view.lkml" ] } }, @@ -1039,7 +1067,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1056,7 +1085,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1073,7 +1103,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1085,20 +1116,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "nested" + }, + { + "id": "fragment_derived.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1109,7 +1144,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/liquid.view.lkml" ] } }, @@ -1146,7 +1181,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1163,7 +1199,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1180,7 +1217,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1192,20 +1230,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "liquid.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1216,7 +1255,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/ability.view.lkml" ] } }, @@ -1332,7 +1371,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1349,7 +1389,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1366,7 +1407,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1378,20 +1420,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "ability.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1402,7 +1445,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/owners.view.lkml" ] } }, @@ -1531,7 +1574,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1548,7 +1592,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1565,7 +1610,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1577,20 +1623,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "owners.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1601,7 +1648,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/native_derived_table.view.lkml" ] } }, @@ -1770,7 +1817,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1787,7 +1835,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1804,7 +1853,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1816,20 +1866,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "native_derived_table.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1840,7 +1891,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/flights.view.lkml" ] } }, @@ -1937,7 +1988,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1954,7 +2006,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1971,7 +2024,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1983,20 +2037,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "flights.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2015,7 +2070,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2034,7 +2090,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2053,7 +2110,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2068,7 +2126,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2083,7 +2142,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2098,7 +2158,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json index e29292a44c949..a4e37303730a9 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo.view.lkml" ] } }, @@ -176,7 +176,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -193,7 +194,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -210,7 +212,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -222,20 +225,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -246,7 +250,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/bar.view.lkml" ] } }, @@ -394,7 +398,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -411,7 +416,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -428,7 +434,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -440,20 +447,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "bar.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -464,7 +472,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/included_view_file.view.lkml" ] } }, @@ -501,7 +509,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -518,7 +527,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -535,7 +545,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -547,20 +558,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "included_view_file.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -571,7 +583,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -608,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -625,7 +638,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +656,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -654,20 +669,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -678,7 +694,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -760,7 +776,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -777,7 +794,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -794,7 +812,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -806,20 +825,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +850,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -867,7 +887,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -884,7 +905,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -901,7 +923,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -913,20 +936,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -937,7 +961,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -974,7 +998,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -991,7 +1016,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1008,7 +1034,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1020,20 +1047,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1044,7 +1072,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/nested/fragment_derived.view.lkml" ] } }, @@ -1156,7 +1184,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1173,7 +1202,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1190,7 +1220,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1202,20 +1233,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "nested" + }, + { + "id": "fragment_derived.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1226,7 +1261,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/liquid.view.lkml" ] } }, @@ -1263,7 +1298,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1280,7 +1316,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1297,7 +1334,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1309,20 +1347,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "liquid.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1333,7 +1372,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/ability.view.lkml" ] } }, @@ -1449,7 +1488,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1466,7 +1506,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1483,7 +1524,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1495,20 +1537,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "ability.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1519,7 +1562,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/owners.view.lkml" ] } }, @@ -1648,7 +1691,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1665,7 +1709,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1682,7 +1727,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1694,20 +1740,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "owners.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1718,7 +1765,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/native_derived_table.view.lkml" ] } }, @@ -1887,7 +1934,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1904,7 +1952,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1921,7 +1970,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1933,20 +1983,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "native_derived_table.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1957,7 +2008,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/flights.view.lkml" ] } }, @@ -2054,7 +2105,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2071,7 +2123,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2088,7 +2141,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2100,20 +2154,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "flights.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2132,7 +2187,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2151,7 +2207,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2170,7 +2227,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2185,7 +2243,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2200,7 +2259,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2215,7 +2275,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json index 04ecaecbd4afb..998c5ef3df9ee 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/included_view_file.view.lkml" ] } }, @@ -44,7 +44,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -61,7 +62,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -78,7 +80,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -90,20 +93,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "included_view_file.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -114,7 +118,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -151,7 +155,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -168,7 +173,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -185,7 +191,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -197,20 +204,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -221,7 +229,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -303,7 +311,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -320,7 +329,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -337,7 +347,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -349,20 +360,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -373,7 +385,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -410,7 +422,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -427,7 +440,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -444,7 +458,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -456,20 +471,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -480,7 +496,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -517,7 +533,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -534,7 +551,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -551,7 +569,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -563,20 +582,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -595,7 +615,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +635,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -633,7 +655,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -648,7 +671,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -663,7 +687,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -678,7 +703,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json index 080931ae637bc..6115f8353190b 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo.view.lkml" ] } }, @@ -176,7 +176,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -193,7 +194,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -210,7 +212,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -222,20 +225,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -246,7 +250,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/bar.view.lkml" ] } }, @@ -394,7 +398,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -411,7 +416,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -428,7 +434,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -440,20 +447,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "bar.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -464,7 +472,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/included_view_file.view.lkml" ] } }, @@ -501,7 +509,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -518,7 +527,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -535,7 +545,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -547,20 +558,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "included_view_file.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -571,7 +583,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -608,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -625,7 +638,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +656,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -654,20 +669,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -678,7 +694,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -760,7 +776,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -777,7 +794,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -794,7 +812,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -806,20 +825,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +850,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -867,7 +887,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -884,7 +905,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -901,7 +923,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -913,20 +936,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -937,7 +961,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -974,7 +998,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -991,7 +1016,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1008,7 +1034,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1020,20 +1047,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1044,7 +1072,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/nested/fragment_derived.view.lkml" ] } }, @@ -1156,7 +1184,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1173,7 +1202,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1190,7 +1220,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1202,20 +1233,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "nested" + }, + { + "id": "fragment_derived.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1226,7 +1261,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/liquid.view.lkml" ] } }, @@ -1263,7 +1298,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1280,7 +1316,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1297,7 +1334,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1309,20 +1347,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "liquid.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1333,7 +1372,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/ability.view.lkml" ] } }, @@ -1449,7 +1488,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1466,7 +1506,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1483,7 +1524,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1495,20 +1537,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "ability.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1519,7 +1562,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/owners.view.lkml" ] } }, @@ -1648,7 +1691,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1665,7 +1709,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1682,7 +1727,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1694,20 +1740,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "owners.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1718,7 +1765,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/native_derived_table.view.lkml" ] } }, @@ -1887,7 +1934,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1904,7 +1952,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1921,7 +1970,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1933,20 +1983,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "native_derived_table.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1957,7 +2008,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/flights.view.lkml" ] } }, @@ -2054,7 +2105,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2071,7 +2123,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2088,7 +2141,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2100,20 +2154,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "flights.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2132,7 +2187,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2151,7 +2207,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2170,7 +2227,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2185,7 +2243,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2200,7 +2259,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2215,7 +2275,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json index 5826c4316b539..7b34c566018fe 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo.view.lkml" ] } }, @@ -177,7 +177,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -194,7 +195,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -211,7 +213,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -223,20 +226,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -247,7 +251,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/bar.view.lkml" ] } }, @@ -396,7 +400,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -413,7 +418,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -430,7 +436,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -442,20 +449,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "bar.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -466,7 +474,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/included_view_file.view.lkml" ] } }, @@ -504,7 +512,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -521,7 +530,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -538,7 +548,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -550,20 +561,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "included_view_file.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -574,7 +586,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -612,7 +624,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -629,7 +642,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +660,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -658,20 +673,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -682,7 +698,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -765,7 +781,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -782,7 +799,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -799,7 +817,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -811,20 +830,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -835,7 +855,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -873,7 +893,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -890,7 +911,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -907,7 +929,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -919,20 +942,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -943,7 +967,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/view_declarations.view.lkml" ] } }, @@ -981,7 +1005,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -998,7 +1023,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1015,7 +1041,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1027,20 +1054,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "view_declarations.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1051,7 +1079,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/nested/fragment_derived.view.lkml" ] } }, @@ -1164,7 +1192,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1181,7 +1210,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1198,7 +1228,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1210,20 +1241,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "nested" + }, + { + "id": "fragment_derived.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1234,7 +1269,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/liquid.view.lkml" ] } }, @@ -1272,7 +1307,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1289,7 +1325,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1306,7 +1343,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1318,20 +1356,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "liquid.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1342,7 +1381,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/ability.view.lkml" ] } }, @@ -1459,7 +1498,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1476,7 +1516,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1493,7 +1534,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1505,20 +1547,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "ability.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1529,7 +1572,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/owners.view.lkml" ] } }, @@ -1659,7 +1702,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1676,7 +1720,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1693,7 +1738,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1705,20 +1751,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "owners.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1729,7 +1776,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/native_derived_table.view.lkml" ] } }, @@ -1899,7 +1946,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1916,7 +1964,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1933,7 +1982,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1945,20 +1995,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "native_derived_table.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1969,7 +2020,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/flights.view.lkml" ] } }, @@ -2067,7 +2118,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2084,7 +2136,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2101,7 +2154,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2113,20 +2167,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "flights.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2145,7 +2200,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2164,7 +2220,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2183,7 +2240,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2198,7 +2256,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2213,7 +2272,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2228,7 +2288,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json index 53d1ec0229de1..bb55835438de7 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo.view.lkml" ] } }, @@ -176,7 +176,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -193,7 +194,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -210,7 +212,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -222,20 +225,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -246,7 +250,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/owners.view.lkml" ] } }, @@ -375,7 +379,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -392,7 +397,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -409,7 +415,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -421,20 +428,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "owners.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -445,7 +453,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_samples/views" + "/Develop/lkml_samples/foo2.view.lkml" ] } }, @@ -614,7 +622,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -631,7 +640,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -648,7 +658,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -660,20 +671,21 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_samples" }, { - "id": "views" + "id": "foo2.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -692,7 +704,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -711,7 +724,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -730,7 +744,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -745,7 +760,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -760,7 +776,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -775,7 +792,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json b/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json index 49831ee554ab1..ae944c07e989c 100644 --- a/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json +++ b/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json @@ -7,7 +7,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_refinement_sample1/views" + "/Develop/lkml_refinement_sample1/views/book.view.lkml" ] } }, @@ -219,7 +219,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -236,7 +237,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -253,7 +255,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -265,20 +268,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_refinement_sample1" }, { "id": "views" + }, + { + "id": "book.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -289,7 +296,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_refinement_sample1/views" + "/Develop/lkml_refinement_sample1/views/extend_book.view.lkml" ] } }, @@ -326,7 +333,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -343,7 +351,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -360,7 +369,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -372,20 +382,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_refinement_sample1" }, { "id": "views" + }, + { + "id": "extend_book.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -396,7 +410,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_refinement_sample1/views" + "/Develop/lkml_refinement_sample1/views/order.view.lkml" ] } }, @@ -523,7 +537,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -540,7 +555,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -557,7 +573,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -569,20 +586,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_refinement_sample1" }, { "id": "views" + }, + { + "id": "order.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -593,7 +614,7 @@ { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ - "/prod/looker/lkml_refinement_sample1/views" + "/Develop/lkml_refinement_sample1/views/issue_history.view.lkml" ] } }, @@ -741,7 +762,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -758,7 +780,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -775,7 +798,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -787,20 +811,24 @@ "json": { "path": [ { - "id": "looker" + "id": "Develop" }, { "id": "lkml_refinement_sample1" }, { "id": "views" + }, + { + "id": "issue_history.view.lkml" } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -815,7 +843,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +859,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "lookml-test" + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 7d1e8d053a381..b0d973a060390 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -578,7 +578,7 @@ def test_lookml_git_info(pytestconfig, tmp_path, mock_time): "parse_table_names_from_sql": True, "project_name": "lkml_samples", "model_pattern": {"deny": ["data2"]}, - "github_info": {"repo": "datahub/looker-demo", "branch": "master"}, + "git_info": {"repo": "datahub/looker-demo", "branch": "master"}, "emit_reachable_views_only": False, "process_refinements": False, }, @@ -693,7 +693,7 @@ def test_hive_platform_drops_ids(pytestconfig, tmp_path, mock_time): "parse_table_names_from_sql": True, "project_name": "lkml_samples", "model_pattern": {"deny": ["data2"]}, - "github_info": {"repo": "datahub/looker-demo", "branch": "master"}, + "git_info": {"repo": "datahub/looker-demo", "branch": "master"}, "emit_reachable_views_only": False, "process_refinements": False, }, @@ -791,7 +791,7 @@ def test_lookml_base_folder(): LookMLSourceConfig.parse_obj( { - "github_info": { + "git_info": { "repo": "acryldata/long-tail-companions-looker", "deploy_key": "this-is-fake", }, From 32a2de4dfcacad454288a13d865bc14726bc044e Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 28 Mar 2024 12:43:37 -0700 Subject: [PATCH 4/5] feat(ingest/airflow): allow plugin to load on listener exception (#10152) --- .github/workflows/airflow-plugin.yml | 2 +- .../datahub_airflow_plugin/datahub_plugin.py | 35 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index 68074e626b02f..21fd547114872 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -67,7 +67,7 @@ jobs: - name: Install dependencies run: ./metadata-ingestion/scripts/install_deps.sh - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }}) - run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick + run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build - name: pip freeze show list installed if: always() run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py index 2b0b751bd787b..437c42713ea01 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py @@ -50,21 +50,28 @@ class DatahubPlugin(AirflowPlugin): name = "datahub_plugin" if _USE_AIRFLOW_LISTENER_INTERFACE: - if not NEEDS_AIRFLOW_LISTENER_MODULE: - from datahub_airflow_plugin.datahub_listener import ( # type: ignore[misc] - get_airflow_plugin_listener, + try: + if not NEEDS_AIRFLOW_LISTENER_MODULE: + from datahub_airflow_plugin.datahub_listener import ( # type: ignore[misc] + get_airflow_plugin_listener, + ) + + listeners: list = list(filter(None, [get_airflow_plugin_listener()])) + + else: + # On Airflow < 2.5, we need the listener to be a module. + # This is just a quick shim layer to make that work. + # + # Related Airflow change: https://github.com/apache/airflow/pull/27113. + import datahub_airflow_plugin._datahub_listener_module as _listener_module # type: ignore[misc] + + listeners = [_listener_module] + except Exception as e: + logger.warning( + f"Failed to load the DataHub plugin's event listener: {e}", + exc_info=True, ) - - listeners: list = list(filter(None, [get_airflow_plugin_listener()])) - - else: - # On Airflow < 2.5, we need the listener to be a module. - # This is just a quick shim layer to make that work. - # - # Related Airflow change: https://github.com/apache/airflow/pull/27113. - import datahub_airflow_plugin._datahub_listener_module as _listener_module # type: ignore[misc] - - listeners = [_listener_module] + listeners = [] if not _USE_AIRFLOW_LISTENER_INTERFACE: From 9f2c5d36f360294412a6f657f153d7238e5ffed0 Mon Sep 17 00:00:00 2001 From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com> Date: Fri, 29 Mar 2024 04:20:25 +0530 Subject: [PATCH 5/5] feat(ingestion/bigquery): BigQuery Owner Label to Datahub Ownership (#10047) --- .../docs/transformer/dataset_transformer.md | 60 +++++++++-- .../src/datahub/emitter/mce_builder.py | 4 + .../extract_ownership_from_tags.py | 81 +++++++++----- .../tests/unit/test_bigquery_source.py | 101 ++++++++++++++++++ .../tests/unit/test_transform_dataset.py | 22 ++++ 5 files changed, 236 insertions(+), 32 deletions(-) diff --git a/metadata-ingestion/docs/transformer/dataset_transformer.md b/metadata-ingestion/docs/transformer/dataset_transformer.md index 66cf47c19de70..0acc134d4ef00 100644 --- a/metadata-ingestion/docs/transformer/dataset_transformer.md +++ b/metadata-ingestion/docs/transformer/dataset_transformer.md @@ -20,24 +20,70 @@ The below table shows transformer which can transform aspects of entity [Dataset ### Config Details | Field | Required | Type | Default | Description | |-----------------------------|----------|---------|---------------|---------------------------------------------| -| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | -| `tag_prefix` | | str | | Regex to use for tags to match against. Supports Regex to match a prefix which is used to remove content. Rest of string is considered owner ID for creating owner URN. | -| `is_user` | | bool | `true` | Whether should be consider a user or not. If `false` then considered a group. | +| `tag_pattern` | | str | | Regex to use for tags to match against. Supports Regex to match a pattern which is used to remove content. Rest of string is considered owner ID for creating owner URN. | +| `is_user` | | bool | `true` | Whether should be consider a user or not. If `false` then considered a group. | +| `owner_character_mapping` | | dict[str, str] | | A mapping of extracted owner character to datahub owner character. | | `email_domain` | | str | | If set then this is appended to create owner URN. | +| `extract_owner_type_from_tag_pattern` | | str | `false` | Whether to extract an owner type from provided tag pattern first group. If `true`, no need to provide owner_type and owner_type_urn config. For example: if provided tag pattern is `(.*)_owner_email:` and actual tag is `developer_owner_email`, then extracted owner type will be `developer`.| | `owner_type` | | str | `TECHNICAL_OWNER` | Ownership type. | | `owner_type_urn` | | str | `None` | Set to a custom ownership type's URN if using custom ownership. | -Matches against a tag prefix and considers string in tags after that prefix as owner to create ownership. +Let’s suppose we’d like to add a dataset ownerships based on part of dataset tags. To do so, we can use the `extract_ownership_from_tags` transformer that’s included in the ingestion framework. + +The config, which we’d append to our ingestion recipe YAML, would look like this: ```yaml transformers: - type: "extract_ownership_from_tags" config: - tag_prefix: "dbt:techno-genie:" - is_user: true - email_domain: "coolcompany.com" + tag_pattern: "owner_email:" ``` +So if we have input dataset tag like +- `urn:li:tag:dataset_owner_email:abc@email.com` +- `urn:li:tag:dataset_owner_email:xyz@email.com` + +The portion of the tag after the matched tag pattern will be converted into an owner. Hence users `abc@email.com` and `xyz@email.com` will be added as owners. + +### Examples + +- Add owners, however owner should be considered as group and also email domain not provided in tag string. For example: from tag urn `urn:li:tag:dataset_owner:abc` extracted owner urn should be `urn:li:corpGroup:abc@email.com` then config would look like this: + ```yaml + transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "owner:" + is_user: false + email_domain: "email.com" + ``` +- Add owners, however owner type and owner type urn wanted to provide externally. For example: from tag urn `urn:li:tag:dataset_owner_email:abc@email.com` owner type should be `CUSTOM` and owner type urn as `"urn:li:ownershipType:data_product"` then config would look like this: + ```yaml + transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "owner_email:" + owner_type: "CUSTOM" + owner_type_urn: "urn:li:ownershipType:data_product" + ``` +- Add owners, however some owner characters needs to replace with some other characters before ingestion. For example: from tag urn `urn:li:tag:dataset_owner_email:abc_xyz-email_com` extracted owner urn should be `urn:li:corpGroup:abc.xyz@email.com` then config would look like this: + ```yaml + transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "owner_email:" + owner_character_mapping: + "_": ".", + "-": "@", + ``` +- Add owners, however owner type also need to extracted from tag pattern. For example: from tag urn `urn:li:tag:data_producer_owner_email:abc@email.com` extracted owner type should be `data_producer` then config would look like this: + ```yaml + transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "(.*)_owner_email:" + extract_owner_type_from_tag_pattern: true + ``` + ## Clean suffix prefix from Ownership ### Config Details | Field | Required | Type | Default | Description | diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index d9933db67f66a..1c29d38273d82 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -246,6 +246,10 @@ def make_owner_urn(owner: str, owner_type: OwnerType) -> str: return f"urn:li:{owner_type.value}:{owner}" +def make_ownership_type_urn(type: str) -> str: + return f"urn:li:ownershipType:{type}" + + def make_term_urn(term: str) -> str: """ Makes a term urn if the input is not a term urn already diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py index c4eba0e011de3..e509b4b719166 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py @@ -1,10 +1,11 @@ import logging import re from functools import lru_cache -from typing import List, Optional, Sequence, Union, cast +from typing import Dict, List, Optional, Sequence, Union, cast -from datahub.configuration.common import TransformerSemanticsConfigModel -from datahub.emitter.mce_builder import Aspect +from datahub.configuration.common import ConfigModel +from datahub.configuration.validate_field_rename import pydantic_renamed_field +from datahub.emitter.mce_builder import Aspect, make_ownership_type_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.transformer.dataset_transformer import DatasetTagsTransformer @@ -22,13 +23,19 @@ logger = logging.getLogger(__name__) -class ExtractOwnersFromTagsConfig(TransformerSemanticsConfigModel): - tag_prefix: str +class ExtractOwnersFromTagsConfig(ConfigModel): + tag_pattern: str = "" is_user: bool = True + owner_character_mapping: Optional[Dict[str, str]] = None email_domain: Optional[str] = None + extract_owner_type_from_tag_pattern: bool = False owner_type: str = "TECHNICAL_OWNER" owner_type_urn: Optional[str] = None + _rename_tag_prefix_to_tag_pattern = pydantic_renamed_field( + "tag_prefix", "tag_pattern" + ) + @lru_cache(maxsize=10) def get_owner_type(owner_type_str: str) -> str: @@ -63,6 +70,19 @@ def get_owner_urn(self, owner_str: str) -> str: return owner_str + "@" + self.config.email_domain return owner_str + def convert_owner_as_per_mapping(self, owner: str) -> str: + if self.config.owner_character_mapping: + # Sort the provided mapping by its length. + # Eg: Suppose we have {"_":".", "__":"#"} character mapping. + # In this case "__" character should get replace first compare to "_" character. + for key in sorted( + self.config.owner_character_mapping.keys(), + key=len, + reverse=True, + ): + owner = owner.replace(key, self.config.owner_character_mapping[key]) + return owner + def handle_end_of_stream( self, ) -> Sequence[Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]]: @@ -79,29 +99,41 @@ def transform_aspect( owners: List[OwnerClass] = [] for tag_class in tags: - tag_urn = TagUrn.from_string(tag_class.tag) - tag_str = tag_urn.entity_ids[0] - re_match = re.search(self.config.tag_prefix, tag_str) + tag_str = TagUrn.from_string(tag_class.tag).name + re_match = re.search(self.config.tag_pattern, tag_str) if re_match: owner_str = tag_str[re_match.end() :].strip() + owner_str = self.convert_owner_as_per_mapping(owner_str) owner_urn_str = self.get_owner_urn(owner_str) - if self.config.is_user: - owner_urn = str(CorpuserUrn(owner_urn_str)) + owner_urn = ( + str(CorpuserUrn(owner_urn_str)) + if self.config.is_user + else str(CorpGroupUrn(owner_urn_str)) + ) + + if self.config.extract_owner_type_from_tag_pattern: + if re_match.groups(): + owners.append( + OwnerClass( + owner=owner_urn, + type=OwnershipTypeClass.CUSTOM, + typeUrn=make_ownership_type_urn(re_match.group(1)), + ) + ) else: - owner_urn = str(CorpGroupUrn(owner_urn_str)) - owner_type = get_owner_type(self.config.owner_type) - if owner_type == OwnershipTypeClass.CUSTOM: - assert ( - self.config.owner_type_urn is not None - ), "owner_type_urn must be set if owner_type is CUSTOM" - - owners.append( - OwnerClass( - owner=owner_urn, - type=owner_type, - typeUrn=self.config.owner_type_urn, + owner_type = get_owner_type(self.config.owner_type) + if owner_type == OwnershipTypeClass.CUSTOM: + assert ( + self.config.owner_type_urn is not None + ), "owner_type_urn must be set if owner_type is CUSTOM" + + owners.append( + OwnerClass( + owner=owner_urn, + type=owner_type, + typeUrn=self.config.owner_type_urn, + ) ) - ) self.owner_mcps.append( MetadataChangeProposalWrapper( @@ -111,5 +143,4 @@ def transform_aspect( ), ) ) - - return None + return aspect diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py index 42d65fdf02683..426d4dc12f208 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_source.py @@ -28,6 +28,7 @@ BigqueryDataset, BigqueryProject, BigQuerySchemaApi, + BigqueryTable, BigqueryTableSnapshot, BigqueryView, ) @@ -35,10 +36,19 @@ LineageEdge, LineageEdgeColumnMapping, ) +from datahub.ingestion.source.common.subtypes import DatasetSubTypes from datahub.metadata.com.linkedin.pegasus2avro.dataset import ViewProperties from datahub.metadata.schema_classes import ( + ContainerClass, + DataPlatformInstanceClass, DatasetPropertiesClass, + GlobalTagsClass, MetadataChangeProposalClass, + SchemaMetadataClass, + StatusClass, + SubTypesClass, + TagAssociationClass, + TimeStampClass, ) @@ -352,6 +362,97 @@ def test_get_projects_list_fully_filtered(get_projects_mock, get_bq_client_mock) assert projects == [] +@pytest.fixture +def bigquery_table() -> BigqueryTable: + now = datetime.now(tz=timezone.utc) + return BigqueryTable( + name="table1", + comment="comment1", + created=now, + last_altered=now, + size_in_bytes=2400, + rows_count=2, + expires=now - timedelta(days=10), + labels={"data_producer_owner_email": "games_team-nytimes_com"}, + num_partitions=1, + max_partition_id="1", + max_shard_id="1", + active_billable_bytes=2400, + long_term_billable_bytes=2400, + ) + + +@patch.object(BigQueryV2Config, "get_bigquery_client") +def test_gen_table_dataset_workunits(get_bq_client_mock, bigquery_table): + project_id = "test-project" + dataset_name = "test-dataset" + config = BigQueryV2Config.parse_obj( + { + "project_id": project_id, + "capture_table_label_as_tag": True, + } + ) + source: BigqueryV2Source = BigqueryV2Source( + config=config, ctx=PipelineContext(run_id="test") + ) + + gen = source.gen_table_dataset_workunits( + bigquery_table, [], project_id, dataset_name + ) + mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) + assert mcp.aspect == StatusClass(removed=False) + + mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) + assert isinstance(mcp.aspect, SchemaMetadataClass) + assert mcp.aspect.schemaName == f"{project_id}.{dataset_name}.{bigquery_table.name}" + assert mcp.aspect.fields == [] + + mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) + assert isinstance(mcp.aspect, DatasetPropertiesClass) + assert mcp.aspect.name == bigquery_table.name + assert ( + mcp.aspect.qualifiedName == f"{project_id}.{dataset_name}.{bigquery_table.name}" + ) + assert mcp.aspect.description == bigquery_table.comment + assert mcp.aspect.created == TimeStampClass( + time=int(bigquery_table.created.timestamp() * 1000) + ) + assert mcp.aspect.lastModified == TimeStampClass( + time=int(bigquery_table.last_altered.timestamp() * 1000) + ) + assert mcp.aspect.tags == [] + + assert mcp.aspect.customProperties == { + "expiration_date": str(bigquery_table.expires), + "size_in_bytes": str(bigquery_table.size_in_bytes), + "billable_bytes_active": str(bigquery_table.active_billable_bytes), + "billable_bytes_long_term": str(bigquery_table.long_term_billable_bytes), + "number_of_partitions": str(bigquery_table.num_partitions), + "max_partition_id": str(bigquery_table.max_partition_id), + "is_partitioned": "True", + "max_shard_id": str(bigquery_table.max_shard_id), + "is_sharded": "True", + } + + mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) + assert isinstance(mcp.aspect, GlobalTagsClass) + assert mcp.aspect.tags == [ + TagAssociationClass( + "urn:li:tag:data_producer_owner_email:games_team-nytimes_com" + ) + ] + + mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) + assert isinstance(mcp.aspect, ContainerClass) + + mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) + assert isinstance(mcp.aspect, DataPlatformInstanceClass) + + mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) + assert isinstance(mcp.aspect, SubTypesClass) + assert mcp.aspect.typeNames[1] == DatasetSubTypes.TABLE + + @patch.object(BigQueryV2Config, "get_bigquery_client") def test_simple_upstream_table_generation(get_bq_client_mock): a: BigQueryTableRef = BigQueryTableRef( diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index 6828c741dda2b..c31ec12abfbd7 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -643,6 +643,7 @@ def _test_owner( config: Dict, expected_owner: str, expected_owner_type: Optional[str] = None, + expected_owner_type_urn: Optional[str] = None, ) -> None: dataset = make_generic_dataset( aspects=[ @@ -682,6 +683,8 @@ def _test_owner( assert owner.owner == expected_owner + assert owner.typeUrn == expected_owner_type_urn + _test_owner( tag="owner:foo", config={ @@ -736,6 +739,25 @@ def _test_owner( }, expected_owner="urn:li:corpuser:foo@example.com", expected_owner_type=OwnershipTypeClass.CUSTOM, + expected_owner_type_urn="urn:li:ownershipType:ad8557d6-dcb9-4d2a-83fc-b7d0d54f3e0f", + ) + _test_owner( + tag="data_producer_owner_email:abc_xyz-email_com", + config={ + "tag_pattern": "(.*)_owner_email:", + "owner_character_mapping": { + "_": ".", + "-": "@", + "__": "_", + "--": "-", + "_-": "#", + "-_": " ", + }, + "extract_owner_type_from_tag_pattern": True, + }, + expected_owner="urn:li:corpuser:abc.xyz@email.com", + expected_owner_type=OwnershipTypeClass.CUSTOM, + expected_owner_type_urn="urn:li:ownershipType:data_producer", )