Skip to content

Commit

Permalink
Merge branch 'datahub-project:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
anshbansal authored Jan 11, 2024
2 parents 2537017 + 5476b52 commit 88397db
Show file tree
Hide file tree
Showing 3 changed files with 117 additions and 44 deletions.
37 changes: 31 additions & 6 deletions metadata-ingestion/src/datahub/ingestion/source/tableau.py
Original file line number Diff line number Diff line change
Expand Up @@ -749,11 +749,15 @@ def get_connection_object_page(
count: int = 0,
offset: int = 0,
retry_on_auth_error: bool = True,
retries_remaining: Optional[int] = None,
) -> Tuple[dict, int, int]:
retries_remaining = retries_remaining or self.config.max_retries

logger.debug(
f"Query {connection_type} to get {count} objects with offset {offset}"
)
try:
assert self.server is not None
query_data = query_metadata(
self.server, query, connection_type, count, offset, query_filter
)
Expand All @@ -766,7 +770,31 @@ def get_connection_object_page(
# will be thrown and we need to re-authenticate and retry.
self._authenticate()
return self.get_connection_object_page(
query, connection_type, query_filter, count, offset, False
query,
connection_type,
query_filter,
count,
offset,
retry_on_auth_error=False,
retries_remaining=retries_remaining,
)
except OSError:
# In tableauseverclient 0.26 (which was yanked and released in 0.28 on 2023-10-04),
# the request logic was changed to use threads.
# https://github.com/tableau/server-client-python/commit/307d8a20a30f32c1ce615cca7c6a78b9b9bff081
# I'm not exactly sure why, but since then, we now occasionally see
# `OSError: Response is not a http response?` for some requests. This
# retry logic is basically a bandaid for that.
if retries_remaining <= 0:
raise
return self.get_connection_object_page(
query,
connection_type,
query_filter,
count,
offset,
retry_on_auth_error=False,
retries_remaining=retries_remaining - 1,
)

if c.ERRORS in query_data:
Expand All @@ -781,11 +809,7 @@ def get_connection_object_page(
else:
raise RuntimeError(f"Query {connection_type} error: {errors}")

connection_object = (
query_data.get(c.DATA).get(connection_type, {})
if query_data.get(c.DATA)
else {}
)
connection_object = query_data.get(c.DATA, {}).get(connection_type, {})

total_count = connection_object.get(c.TOTAL_COUNT, 0)
has_next_page = connection_object.get(c.PAGE_INFO, {}).get(
Expand Down Expand Up @@ -2520,3 +2544,4 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:

def get_report(self) -> TableauSourceReport:
return self.report
return self.report
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import Dict, List, Optional, Tuple

from pydantic.fields import Field
from tableauserverclient import Server

import datahub.emitter.mce_builder as builder
from datahub.configuration.common import ConfigModel
Expand Down Expand Up @@ -699,7 +700,6 @@ def get_overridden_info(
platform_instance_map: Optional[Dict[str, str]],
lineage_overrides: Optional[TableauLineageOverrides] = None,
) -> Tuple[Optional[str], Optional[str], str, str]:

original_platform = platform = get_platform(connection_type)
if (
lineage_overrides is not None
Expand Down Expand Up @@ -824,7 +824,14 @@ def clean_query(query: str) -> str:
return query


def query_metadata(server, main_query, connection_name, first, offset, qry_filter=""):
def query_metadata(
server: Server,
main_query: str,
connection_name: str,
first: int,
offset: int,
qry_filter: str = "",
) -> dict:
query = """{{
{connection_name} (first:{first}, offset:{offset}, filter:{{{filter}}})
{{
Expand Down
113 changes: 77 additions & 36 deletions smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js
Original file line number Diff line number Diff line change
@@ -1,53 +1,94 @@
const domainName = "CypressNestedDomain";
const domainDescription = "CypressNestedDomainDescription";

describe("nested domains test", () => {
const createDomain = () => {
cy.clickOptionWithTestId("domains-new-domain-button");
cy.get('[data-testid="create-domain-name"]').click().type(domainName);
cy.get('[data-testid="create-domain-description"]').click().type(domainDescription);
cy.clickOptionWithTestId("create-domain-button");
cy.waitTextVisible("Created domain!");
}

it("create a domain, move under parent, remove domain", () => {
// Create a new domain without a parent
cy.loginWithCredentials();
cy.goToDomainList();
cy.clickOptionWithTestId("domains-new-domain-button");
cy.get('[data-testid="create-domain-name"]').click().type(domainName);
cy.get('[data-testid="create-domain-description"]').click().type(domainDescription);
cy.clickOptionWithTestId("create-domain-button");
cy.waitTextVisible(domainName);
const moveDomaintoRootLevel = () => {
cy.clickOptionWithText(domainName);
cy.openThreeDotDropdown();
cy.clickOptionWithTestId("entity-menu-move-button");
cy.get('[data-testid="move-domain-modal"]').contains("Marketing").click({force: true});
cy.waitTextVisible('Marketing')
cy.clickOptionWithTestId("move-domain-modal-move-button")
}

// Ensure the new domain has no parent in the navigation sidebar
cy.waitTextVisible(domainDescription);
const moveDomaintoParent = () => {
cy.get('[data-testid="domain-list-item"]').contains("Marketing").prev().click();
cy.clickOptionWithText(domainName);
cy.waitTextVisible(domainName)
cy.openThreeDotDropdown();
cy.clickOptionWithTestId("entity-menu-move-button");
cy.clickOptionWithTestId("move-domain-modal-move-button")
}

// Move a domain from the root level to be under a parent domain
cy.clickOptionWithText(domainName);
cy.openThreeDotDropdown();
cy.clickOptionWithTestId("entity-menu-move-button");
cy.get('[data-testid="move-domain-modal"]').contains("Marketing").click({force: true});
cy.get('[data-testid="move-domain-modal"]').contains("Marketing").should("be.visible");
cy.clickOptionWithTestId("move-domain-modal-move-button").wait(5000);
const deleteFromDomainDropdown = () => {
cy.clickOptionWithText('Filters')
cy.openThreeDotDropdown();
cy.clickOptionWithTestId("entity-menu-delete-button");
cy.waitTextVisible("Are you sure you want to remove this Domain?");
cy.clickOptionWithText("Yes");
}

// Wnsure domain is no longer on the sidebar navigator at the top level but shows up under the parent
const deleteDomain = () => {
cy.clickOptionWithText(domainName).waitTextVisible('Domains');
deleteFromDomainDropdown()
}

//Delete Unecessary Existing Domains
const deleteExisitingDomain = () => {
cy.get('a[href*="urn:li"] span[class^="ant-typography"]')
.should('be.visible')
.its('length')
.then((length) => {
for (let i = 0; i < length - 1; i++) {
cy.get('a[href*="urn:li"] span[class^="ant-typography"]')
.should('be.visible')
.first()
.click({ force: true });
deleteFromDomainDropdown();
}
});
cy.waitTextVisible('My custom domain');
}

describe("nested domains test", () => {
beforeEach (() => {
cy.loginWithCredentials();
cy.goToDomainList();
});

it("Create a new domain", () => {
deleteExisitingDomain()
createDomain();
cy.waitTextVisible("Domains");
});

it("Move domain root level to parent level", () => {
cy.waitTextVisible(domainName)
moveDomaintoRootLevel();
cy.waitTextVisible("Moved Domain!")
cy.goToDomainList();
cy.ensureTextNotPresent(domainName);
cy.ensureTextNotPresent(domainDescription);
cy.waitTextVisible("1 sub-domain");
});

// Move a domain from under a parent domain to the root level
cy.get('[data-testid="domain-list-item"]').contains("Marketing").prev().click();
cy.clickOptionWithText(domainName);
cy.openThreeDotDropdown();
cy.clickOptionWithTestId("entity-menu-move-button");
cy.clickOptionWithTestId("move-domain-modal-move-button").wait(5000);
it("Move domain parent level to root level", () => {
moveDomaintoParent();
cy.waitTextVisible("Moved Domain!")
cy.goToDomainList();
cy.waitTextVisible(domainName);
cy.waitTextVisible(domainDescription);
});

// Delete a domain
cy.clickOptionWithText(domainName).wait(3000);
cy.openThreeDotDropdown();
cy.clickOptionWithTestId("entity-menu-delete-button");
cy.waitTextVisible("Are you sure you want to remove this Domain?");
cy.clickOptionWithText("Yes");
cy.waitTextVisible("Deleted Domain!");
it("Remove the domain", () => {
deleteDomain();
cy.goToDomainList();
cy.ensureTextNotPresent(domainName);
cy.ensureTextNotPresent(domainDescription);
});
});
});

0 comments on commit 88397db

Please sign in to comment.