diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index f369d1e5631d8..3c86c7e48c34a 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -129,7 +129,8 @@ jobs: if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }} run: | python ./.github/scripts/check_python_package.py - ./gradlew :smoke-test:lint + ./gradlew :smoke-test:pythonLint + ./gradlew :smoke-test:cypressLint gms_build: name: Build and Push DataHub GMS Docker Image diff --git a/build.gradle b/build.gradle index b61140999a82f..f4fb7b42d8560 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ buildscript { ext.hazelcastVersion = '5.3.6' ext.ebeanVersion = '12.16.1' ext.googleJavaFormatVersion = '1.18.1' - ext.openLineageVersion = '1.5.0' + ext.openLineageVersion = '1.13.1' ext.logbackClassicJava8 = '1.2.12' ext.docker_registry = 'acryldata' diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java index 22f7ca20ab8b8..510804ba17f1a 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java @@ -211,6 +211,8 @@ private Result handleOidcCallback( "Failed to perform post authentication steps. Error message: %s", e.getMessage())); } + log.info("OIDC callback authentication successful for user: {}", userName); + // Successfully logged in - Generate GMS login token final String accessToken = authClient.generateSessionTokenForUser(corpUserUrn.getId()); return result diff --git a/datahub-frontend/app/client/AuthServiceClient.java b/datahub-frontend/app/client/AuthServiceClient.java index baa992994d8ba..30f841d10b4bf 100644 --- a/datahub-frontend/app/client/AuthServiceClient.java +++ b/datahub-frontend/app/client/AuthServiceClient.java @@ -75,7 +75,6 @@ public String generateSessionTokenForUser(@Nonnull final String userId) { CloseableHttpResponse response = null; try { - final String protocol = this.metadataServiceUseSsl ? "https" : "http"; final HttpPost request = new HttpPost( @@ -86,6 +85,8 @@ public String generateSessionTokenForUser(@Nonnull final String userId) { this.metadataServicePort, GENERATE_SESSION_TOKEN_ENDPOINT)); + log.info("Requesting session token for user: {}", userId); + // Build JSON request to generate a token on behalf of a user. final ObjectMapper objectMapper = new ObjectMapper(); final ObjectNode objectNode = objectMapper.createObjectNode(); @@ -100,7 +101,7 @@ public String generateSessionTokenForUser(@Nonnull final String userId) { response = httpClient.execute(request); final HttpEntity entity = response.getEntity(); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && entity != null) { - // Successfully generated a token for the User + log.info("Successfully received session token for user: {}", userId); final String jsonStr = EntityUtils.toString(entity); return getAccessTokenFromJson(jsonStr); } else { @@ -110,6 +111,7 @@ public String generateSessionTokenForUser(@Nonnull final String userId) { response.getStatusLine().toString(), response.getEntity().toString())); } } catch (Exception e) { + log.error("Failed to generate session token for user: {}", userId, e); throw new RuntimeException("Failed to generate session token for user", e); } finally { try { diff --git a/datahub-frontend/app/controllers/AuthenticationController.java b/datahub-frontend/app/controllers/AuthenticationController.java index d9568c25f6e8c..87c4b5ba06793 100644 --- a/datahub-frontend/app/controllers/AuthenticationController.java +++ b/datahub-frontend/app/controllers/AuthenticationController.java @@ -42,7 +42,6 @@ import play.mvc.Results; import security.AuthenticationManager; -// TODO add logging. public class AuthenticationController extends Controller { public static final String AUTH_VERBOSE_LOGGING = "auth.verbose.logging"; private static final String AUTH_REDIRECT_URI_PARAM = "redirect_uri"; @@ -183,10 +182,12 @@ public Result logIn(Http.Request request) { boolean loginSucceeded = tryLogin(username, password); if (!loginSucceeded) { + _logger.info("Login failed for user: {}", username); return Results.badRequest(invalidCredsJson); } final Urn actorUrn = new CorpuserUrn(username); + _logger.info("Login successful for user: {}, urn: {}", username, actorUrn); final String accessToken = _authClient.generateSessionTokenForUser(actorUrn.getId()); return createSession(actorUrn.toString(), accessToken); } @@ -250,6 +251,7 @@ public Result signUp(Http.Request request) { final Urn userUrn = new CorpuserUrn(email); final String userUrnString = userUrn.toString(); _authClient.signUp(userUrnString, fullName, email, title, password, inviteToken); + _logger.info("Signed up user {} using invite tokens", userUrnString); final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId()); return createSession(userUrnString, accessToken); } @@ -351,15 +353,15 @@ private boolean tryLogin(String username, String password) { // First try jaas login, if enabled if (_jaasConfigs.isJAASEnabled()) { try { - _logger.debug("Attempting jaas authentication"); + _logger.debug("Attempting JAAS authentication for user: {}", username); AuthenticationManager.authenticateJaasUser(username, password); - _logger.debug("Jaas authentication successful. Login succeeded"); + _logger.debug("JAAS authentication successful. Login succeeded"); loginSucceeded = true; } catch (Exception e) { if (_verbose) { - _logger.debug("Jaas authentication error. Login failed", e); + _logger.debug("JAAS authentication error. Login failed", e); } else { - _logger.debug("Jaas authentication error. Login failed"); + _logger.debug("JAAS authentication error. Login failed"); } } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDataJobViaNodesCLLConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDataJobViaNodesCLLConfig.java index e7311d23a6d2a..4956254062ff9 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDataJobViaNodesCLLConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDataJobViaNodesCLLConfig.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.upgrade.system.vianodes.ReindexDataJobViaNodesCLL; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; @@ -15,6 +16,7 @@ public class ReindexDataJobViaNodesCLLConfig { @Bean public NonBlockingSystemUpgrade reindexDataJobViaNodesCLL( + final OperationContext opContext, final EntityService entityService, final AspectDao aspectDao, @Value("${systemUpdate.dataJobNodeCLL.enabled}") final boolean enabled, @@ -22,6 +24,6 @@ public NonBlockingSystemUpgrade reindexDataJobViaNodesCLL( @Value("${systemUpdate.dataJobNodeCLL.delayMs}") final Integer delayMs, @Value("${systemUpdate.dataJobNodeCLL.limit}") final Integer limit) { return new ReindexDataJobViaNodesCLL( - entityService, aspectDao, enabled, batchSize, delayMs, limit); + opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java new file mode 100644 index 0000000000000..d28b741fedd2a --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java @@ -0,0 +1,154 @@ +package com.linkedin.datahub.upgrade.system; + +import static com.linkedin.metadata.Constants.DATA_HUB_UPGRADE_RESULT_ASPECT_NAME; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityUtils; +import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; +import com.linkedin.metadata.utils.AuditStampUtils; +import com.linkedin.util.Pair; +import io.datahubproject.metadata.context.OperationContext; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +/** + * Generic upgrade step class for generating MCLs for a given aspect in order to update ES documents + */ +@Slf4j +public abstract class AbstractMCLStep implements UpgradeStep { + private final OperationContext opContext; + private final EntityService entityService; + private final AspectDao aspectDao; + + private final int batchSize; + private final int batchDelayMs; + private final int limit; + + public AbstractMCLStep( + OperationContext opContext, + EntityService entityService, + AspectDao aspectDao, + Integer batchSize, + Integer batchDelayMs, + Integer limit) { + this.opContext = opContext; + this.entityService = entityService; + this.aspectDao = aspectDao; + this.batchSize = batchSize; + this.batchDelayMs = batchDelayMs; + this.limit = limit; + } + + @Nonnull + protected abstract String getAspectName(); + + protected Urn getUpgradeIdUrn() { + return BootstrapStep.getUpgradeUrn(id()); + } + + /** Optionally apply an urn-like sql filter, otherwise all urns */ + @Nullable + protected abstract String getUrnLike(); + + @Override + public Function executable() { + return (context) -> { + + // re-using for configuring the sql scan + RestoreIndicesArgs args = + new RestoreIndicesArgs().aspectName(getAspectName()).batchSize(batchSize).limit(limit); + + if (getUrnLike() != null) { + args = args.urnLike(getUrnLike()); + } + + aspectDao + .streamAspectBatches(args) + .forEach( + batch -> { + log.info("Processing batch({}) of size {}.", getAspectName(), batchSize); + + List, Boolean>> futures = + EntityUtils.toSystemAspectFromEbeanAspects( + opContext.getRetrieverContext().get(), + batch.collect(Collectors.toList())) + .stream() + .map( + systemAspect -> + entityService.alwaysProduceMCLAsync( + opContext, + systemAspect.getUrn(), + systemAspect.getUrn().getEntityType(), + getAspectName(), + systemAspect.getAspectSpec(), + null, + systemAspect.getRecordTemplate(), + null, + systemAspect + .getSystemMetadata() + .setRunId(id()) + .setLastObserved(System.currentTimeMillis()), + AuditStampUtils.createDefaultAuditStamp(), + ChangeType.UPSERT)) + .collect(Collectors.toList()); + + futures.forEach( + f -> { + try { + f.getFirst().get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + + if (batchDelayMs > 0) { + log.info("Sleeping for {} ms", batchDelayMs); + try { + Thread.sleep(batchDelayMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + + entityService + .streamRestoreIndices(opContext, args, x -> context.report().addLine((String) x)) + .forEach( + result -> { + context.report().addLine("Rows migrated: " + result.rowsMigrated); + context.report().addLine("Rows ignored: " + result.ignored); + }); + + BootstrapStep.setUpgradeResult(opContext, getUpgradeIdUrn(), entityService); + context.report().addLine("State updated: " + getUpgradeIdUrn()); + + return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + }; + } + + @Override + /** Returns whether the upgrade should be skipped. */ + public boolean skip(UpgradeContext context) { + boolean previouslyRun = + entityService.exists( + opContext, getUpgradeIdUrn(), DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, true); + if (previouslyRun) { + log.info("{} was already run. Skipping.", id()); + } + return previouslyRun; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLL.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLL.java index 9ad673c599758..fc0b44f57ab49 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLL.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLL.java @@ -5,7 +5,9 @@ import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; import java.util.List; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; /** @@ -18,6 +20,7 @@ public class ReindexDataJobViaNodesCLL implements NonBlockingSystemUpgrade { private final List _steps; public ReindexDataJobViaNodesCLL( + @Nonnull OperationContext opContext, EntityService entityService, AspectDao aspectDao, boolean enabled, @@ -28,7 +31,7 @@ public ReindexDataJobViaNodesCLL( _steps = ImmutableList.of( new ReindexDataJobViaNodesCLLStep( - entityService, aspectDao, batchSize, batchDelayMs, limit)); + opContext, entityService, aspectDao, batchSize, batchDelayMs, limit)); } else { _steps = ImmutableList.of(); } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLLStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLLStep.java index 5e135124524d9..cf580670ee3a9 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLLStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLLStep.java @@ -2,148 +2,43 @@ import static com.linkedin.metadata.Constants.*; -import com.linkedin.common.urn.Urn; import com.linkedin.datahub.upgrade.UpgradeContext; -import com.linkedin.datahub.upgrade.UpgradeStep; -import com.linkedin.datahub.upgrade.UpgradeStepResult; -import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.datahub.upgrade.system.AbstractMCLStep; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.entity.EntityUtils; -import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; -import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.utils.AuditStampUtils; -import com.linkedin.util.Pair; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; +import io.datahubproject.metadata.context.OperationContext; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; @Slf4j -public class ReindexDataJobViaNodesCLLStep implements UpgradeStep { - - public static final String UPGRADE_ID = "via-node-cll-reindex-datajob-v3"; - private static final Urn UPGRADE_ID_URN = BootstrapStep.getUpgradeUrn(UPGRADE_ID); - - private final EntityService entityService; - private final AspectDao aspectDao; - private final int batchSize; - private final int batchDelayMs; - private final int limit; +public class ReindexDataJobViaNodesCLLStep extends AbstractMCLStep { public ReindexDataJobViaNodesCLLStep( + OperationContext opContext, EntityService entityService, AspectDao aspectDao, Integer batchSize, Integer batchDelayMs, Integer limit) { - this.entityService = entityService; - this.aspectDao = aspectDao; - this.batchSize = batchSize != null ? batchSize : 200; - this.batchDelayMs = batchDelayMs; - this.limit = limit; + super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit); } @Override - public Function executable() { - return (context) -> { - - // re-using for configuring the sql scan - RestoreIndicesArgs args = - new RestoreIndicesArgs() - .aspectName(DATA_JOB_INPUT_OUTPUT_ASPECT_NAME) - .urnLike("urn:li:" + DATA_JOB_ENTITY_NAME + ":%") - .batchSize(batchSize) - .limit(limit); - - final AspectSpec aspectSpec = - context - .opContext() - .getEntityRegistry() - .getAspectSpecs() - .get(DATA_JOB_INPUT_OUTPUT_ASPECT_NAME); - - aspectDao - .streamAspectBatches(args) - .forEach( - batch -> { - log.info("Processing batch of size {}.", batchSize); - - List, Boolean>> futures = - EntityUtils.toSystemAspectFromEbeanAspects( - context.opContext().getRetrieverContext().get(), - batch.collect(Collectors.toList())) - .stream() - .map( - systemAspect -> - entityService.alwaysProduceMCLAsync( - context.opContext(), - systemAspect.getUrn(), - systemAspect.getUrn().getEntityType(), - DATA_JOB_INPUT_OUTPUT_ASPECT_NAME, - aspectSpec, - null, - systemAspect.getRecordTemplate(), - null, - systemAspect - .getSystemMetadata() - .setRunId(UPGRADE_ID) - .setLastObserved(System.currentTimeMillis()), - AuditStampUtils.createDefaultAuditStamp(), - ChangeType.UPSERT)) - .collect(Collectors.toList()); - - futures.forEach( - f -> { - try { - f.getFirst().get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - }); - - if (batchDelayMs > 0) { - log.info("Sleeping for {} ms", batchDelayMs); - try { - Thread.sleep(batchDelayMs); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - }); - - entityService - .streamRestoreIndices( - context.opContext(), args, x -> context.report().addLine((String) x)) - .forEach( - result -> { - context.report().addLine("Rows migrated: " + result.rowsMigrated); - context.report().addLine("Rows ignored: " + result.ignored); - }); - - BootstrapStep.setUpgradeResult(context.opContext(), UPGRADE_ID_URN, entityService); - context.report().addLine("State updated: " + UPGRADE_ID_URN); - - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); - }; + public String id() { + return "via-node-cll-reindex-datajob-v3"; } + @Nonnull @Override - public String id() { - return UPGRADE_ID; + protected String getAspectName() { + return DATA_JOB_INPUT_OUTPUT_ASPECT_NAME; } - /** - * Returns whether the upgrade should proceed if the step fails after exceeding the maximum - * retries. - */ + @Nullable @Override - public boolean isOptional() { - return false; + protected String getUrnLike() { + return "urn:li:" + DATA_JOB_ENTITY_NAME + ":%"; } @Override @@ -152,17 +47,11 @@ public boolean isOptional() { * variable SKIP_REINDEX_DATA_JOB_INPUT_OUTPUT to determine whether to skip. */ public boolean skip(UpgradeContext context) { - boolean previouslyRun = - entityService.exists( - context.opContext(), UPGRADE_ID_URN, DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, true); boolean envFlagRecommendsSkip = Boolean.parseBoolean(System.getenv("SKIP_REINDEX_DATA_JOB_INPUT_OUTPUT")); - if (previouslyRun) { - log.info("{} was already run. Skipping.", id()); - } if (envFlagRecommendsSkip) { log.info("Environment variable SKIP_REINDEX_DATA_JOB_INPUT_OUTPUT is set to true. Skipping."); } - return (previouslyRun || envFlagRecommendsSkip); + return (super.skip(context) || envFlagRecommendsSkip); } } diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java index c28ff7fd29dfb..154b1de71f46c 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java @@ -19,6 +19,7 @@ import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.mxe.Topics; +import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; import java.util.List; import javax.inject.Named; @@ -58,6 +59,8 @@ public class DatahubUpgradeNonBlockingTest extends AbstractTestNGSpringContextTe @Autowired private EntityServiceImpl entityService; + @Autowired private OperationContext opContext; + @Test public void testSystemUpdateNonBlockingInit() { assertNotNull(systemUpdateNonBlocking); @@ -76,7 +79,7 @@ public void testReindexDataJobViaNodesCLLPaging() { AspectDao mockAspectDao = mock(AspectDao.class); ReindexDataJobViaNodesCLL cllUpgrade = - new ReindexDataJobViaNodesCLL(mockService, mockAspectDao, true, 10, 0, 0); + new ReindexDataJobViaNodesCLL(opContext, mockService, mockAspectDao, true, 10, 0, 0); SystemUpdateNonBlocking upgrade = new SystemUpdateNonBlocking(List.of(), List.of(cllUpgrade), null); DefaultUpgradeManager manager = new DefaultUpgradeManager(); diff --git a/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx b/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx index 9829cac9befe4..88b3c25162ec5 100644 --- a/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx +++ b/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx @@ -15,6 +15,7 @@ import FormRequestedBy from './FormSelectionModal/FormRequestedBy'; import useHasComponentRendered from '../../../shared/useHasComponentRendered'; import Loading from '../../../shared/Loading'; import { DeferredRenderComponent } from '../../../shared/DeferredRenderComponent'; +import { Editor } from '../tabs/Documentation/components/editor/Editor'; const TabWrapper = styled.div` background-color: ${ANTD_GRAY_V2[1]}; @@ -70,7 +71,9 @@ function Form({ formUrn }: Props) { )} {description ? ( - {description} + + + ) : ( Please fill out the following information for this {entityRegistry.getEntityName(entityType)} so diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx index 5a02067deb33d..fe2a8c51f9377 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx @@ -42,10 +42,11 @@ type EditorProps = { doNotFocus?: boolean; dataTestId?: string; onKeyDown?: (event: React.KeyboardEvent) => void; + editorStyle?: string; }; export const Editor = forwardRef((props: EditorProps, ref) => { - const { content, readOnly, onChange, className, dataTestId, onKeyDown } = props; + const { content, readOnly, onChange, className, dataTestId, onKeyDown, editorStyle } = props; const { manager, state, getContext } = useRemirror({ extensions: () => [ new BlockquoteExtension(), @@ -100,7 +101,7 @@ export const Editor = forwardRef((props: EditorProps, ref) => { }, [readOnly, content]); return ( - + {!readOnly && ( diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/EditorTheme.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/EditorTheme.tsx index c37e50d382435..ec094fb84e59a 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/EditorTheme.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/EditorTheme.tsx @@ -47,7 +47,7 @@ export const EditorTheme: RemirrorThemeType = { }, }; -export const EditorContainer = styled.div` +export const EditorContainer = styled.div<{ editorStyle?: string }>` ${extensionBlockquoteStyledCss} ${extensionCalloutStyledCss} ${extensionCodeBlockStyledCss} @@ -81,6 +81,7 @@ export const EditorContainer = styled.div` line-height: 1.5; white-space: pre-wrap; margin: 0; + ${props => props.editorStyle} a { font-weight: 500; diff --git a/docker/profiles/docker-compose.actions.yml b/docker/profiles/docker-compose.actions.yml index 64ad5d9211ed8..c0a0fd5932871 100644 --- a/docker/profiles/docker-compose.actions.yml +++ b/docker/profiles/docker-compose.actions.yml @@ -5,7 +5,7 @@ x-datahub-actions-service: &datahub-actions-service env_file: - datahub-actions/env/docker.env - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env} - - ${DATAHUB_LOCAL_ACTIONS_ENV:-empty.env} + - ${DATAHUB_LOCAL_ACTIONS_ENV:-empty2.env} environment: ACTIONS_EXTRA_PACKAGES: ${ACTIONS_EXTRA_PACKAGES:-} ACTIONS_CONFIG: ${ACTIONS_CONFIG:-} diff --git a/docker/profiles/docker-compose.frontend.yml b/docker/profiles/docker-compose.frontend.yml index a0239402cad86..b43db8297cb1e 100644 --- a/docker/profiles/docker-compose.frontend.yml +++ b/docker/profiles/docker-compose.frontend.yml @@ -7,7 +7,7 @@ x-datahub-frontend-service: &datahub-frontend-service env_file: - datahub-frontend/env/docker.env - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env} - - ${DATAHUB_LOCAL_FRONTEND_ENV:-empty.env} + - ${DATAHUB_LOCAL_FRONTEND_ENV:-empty2.env} environment: &datahub-frontend-service-env KAFKA_BOOTSTRAP_SERVER: broker:29092 volumes: diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml index 7950c12cfbb31..208ed763dd6a3 100644 --- a/docker/profiles/docker-compose.gms.yml +++ b/docker/profiles/docker-compose.gms.yml @@ -61,7 +61,7 @@ x-datahub-system-update-service: &datahub-system-update-service env_file: - datahub-upgrade/env/docker.env - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env} - - ${DATAHUB_LOCAL_SYS_UPDATE_ENV:-empty.env} + - ${DATAHUB_LOCAL_SYS_UPDATE_ENV:-empty2.env} environment: &datahub-system-update-env <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *kafka-env] SCHEMA_REGISTRY_SYSTEM_UPDATE: ${SCHEMA_REGISTRY_SYSTEM_UPDATE:-true} @@ -96,9 +96,11 @@ x-datahub-gms-service: &datahub-gms-service env_file: - datahub-gms/env/docker.env - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env} - - ${DATAHUB_LOCAL_GMS_ENV:-empty.env} + - ${DATAHUB_LOCAL_GMS_ENV:-empty2.env} environment: &datahub-gms-env <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env] + ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED: true + ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: '/etc/datahub/search/search_config.yaml' healthcheck: test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health start_period: 90s @@ -107,6 +109,7 @@ x-datahub-gms-service: &datahub-gms-service timeout: 5s volumes: - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/search:/etc/datahub/search labels: io.datahubproject.datahub.component: "gms" @@ -131,6 +134,7 @@ x-datahub-gms-service-dev: &datahub-gms-service-dev - ../../metadata-models/src/main/resources/:/datahub/datahub-gms/resources - ../../metadata-service/war/build/libs/:/datahub/datahub-gms/bin - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/search:/etc/datahub/search ################################# # MAE Consumer @@ -143,7 +147,7 @@ x-datahub-mae-consumer-service: &datahub-mae-consumer-service env_file: - datahub-mae-consumer/env/docker.env - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env} - - ${DATAHUB_LOCAL_MAE_ENV:-empty.env} + - ${DATAHUB_LOCAL_MAE_ENV:-empty2.env} environment: &datahub-mae-consumer-env <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *kafka-env] @@ -169,7 +173,7 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service env_file: - datahub-mce-consumer/env/docker.env - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env} - - ${DATAHUB_LOCAL_MCE_ENV:-empty.env} + - ${DATAHUB_LOCAL_MCE_ENV:-empty2.env} environment: &datahub-mce-consumer-env <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env] diff --git a/docker/profiles/docker-compose.yml b/docker/profiles/docker-compose.yml index 534ca9702e2d7..4d12061196fbf 100644 --- a/docker/profiles/docker-compose.yml +++ b/docker/profiles/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: '3.9' name: datahub include: diff --git a/docker/profiles/empty.env b/docker/profiles/empty.env index 6a3aaf83f8378..f07970153f455 100644 --- a/docker/profiles/empty.env +++ b/docker/profiles/empty.env @@ -1,4 +1,5 @@ -# Docker compose requires that all env_file entries exist. +# Docker compose requires that all env_file entries exist and +# are unique. # Because we have some optional env_file entries that can be set # as environment variables, we need a default file to point at # when those are not set. diff --git a/docker/profiles/empty2.env b/docker/profiles/empty2.env new file mode 100644 index 0000000000000..acd68426f0985 --- /dev/null +++ b/docker/profiles/empty2.env @@ -0,0 +1 @@ +# See empty.env. diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index c6e298728e07f..30bab24267d70 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -110,7 +110,7 @@ module.exports = { label: "Demo", }, { - href: "https://www.acryldata.io/blog", + href: "https://blog.datahubproject.io/", label: "Blog", }, { diff --git a/docs-website/filterTagIndexes.json b/docs-website/filterTagIndexes.json index 64cb734e6d984..0c1f541cf53d3 100644 --- a/docs-website/filterTagIndexes.json +++ b/docs-website/filterTagIndexes.json @@ -77,6 +77,17 @@ "Features": "" } }, + { + "Path": "docs/lineage/dagster", + "imgPath": "img/logos/platforms/dagster.svg", + "Title": "Dagster", + "Description": "Dagster is a next-generation open source orchestration platform for the development, production, and observation of data assets..", + "tags": { + "Platform Type": "Orchestrator", + "Connection Type": "Pull", + "Features": "Stateful Ingestion, UI Ingestion, Status Aspect" + } + }, { "Path": "docs/generated/ingestion/sources/databricks", "imgPath": "img/logos/platforms/databricks.png", @@ -433,7 +444,7 @@ "Path": "docs/generated/ingestion/sources/hive-metastore", "imgPath": "img/logos/platforms/presto.svg", "Title": "Hive Metastore", - "Description": "Presto on Hive is a data tool that allows users to query and analyze large datasets stored in Hive using SQL-like syntax.", + "Description": "Hive Metastore (HMS) is a service that stores metadata that is related to Hive, Presto, Trino and other services in a backend Relational Database Management System (RDBMS) ", "tags": { "Platform Type": "Datastore", "Connection Type": "Pull", @@ -551,7 +562,7 @@ } }, { - "Path": "docs/metadata-integration/java/spark-lineage", + "Path": "docs/metadata-integration/java/spark-lineage-beta", "imgPath": "img/logos/platforms/spark.svg", "Title": "Spark", "Description": "Spark is a data processing tool that enables fast and efficient processing of large-scale data sets using distributed computing.", diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 865b37c961a71..f26853a488b62 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -317,6 +317,11 @@ module.exports = { id: "docs/lineage/dagster", label: "Dagster", }, + { + type: "doc", + id: "docs/lineage/openlineage", + label: "OpenLineage", + }, { type: "doc", id: "metadata-integration/java/spark-lineage/README", diff --git a/docs-website/static/img/logos/platforms/dagster.svg b/docs-website/static/img/logos/platforms/dagster.svg new file mode 100644 index 0000000000000..d2ae628553a7d --- /dev/null +++ b/docs-website/static/img/logos/platforms/dagster.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/docs/how/search.md b/docs/how/search.md index 0b718fc9dc77f..3b50a0da7fec4 100644 --- a/docs/how/search.md +++ b/docs/how/search.md @@ -291,11 +291,11 @@ If enabled in #2 above, those queries will appear in the `should` section of the `boolean query`[[4](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html)]. 4. `functionScore` - The Elasticsearch `function score`[[5](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-function-score-query.html#score-functions)] section of the overall query. -### Examples +#### Examples These examples assume a match-all `queryRegex` of `.*` so that it would impact any search query for simplicity. -#### Example 1: Ranking By Tags/Terms +##### Example 1: Ranking By Tags/Terms Boost entities with tags of `primary` or `gold` and an example glossary term's uuid. @@ -327,7 +327,7 @@ queryConfigurations: boost_mode: multiply ``` -#### Example 2: Preferred Data Platform +##### Example 2: Preferred Data Platform Boost the `urn:li:dataPlatform:hive` platform. @@ -350,7 +350,7 @@ queryConfigurations: boost_mode: multiply ``` -#### Example 3: Exclusion & Bury +##### Example 3: Exclusion & Bury This configuration extends the 3 built-in queries with a rule to exclude `deprecated` entities from search results because they are not generally relevant as well as reduces the score of `materialized`. @@ -380,6 +380,73 @@ queryConfigurations: boost_mode: multiply ``` +### Search Autocomplete Configuration + +Similar to the options provided in the previous section for search configuration, there are autocomplete specific options +which can be configured. + +Note: The scoring functions defined in the previous section are inherited for autocomplete by default, unless +overrides are provided in the autocomplete section. + +For the most part the configuration options are identical to the search customization options in the previous +section, however they are located under `autocompleteConfigurations` in the yaml configuration file. + +1. `queryRegex` - Responsible for selecting the search customization based on the [regex matching](https://www.w3schools.com/java/java_regex.asp) the search query string. + *The first match is applied.* +2. The following boolean enables/disables the function score inheritance from the normal search configuration: [`inheritFunctionScore`] + This flag will automatically be set to `false` when the `functionScore` section is provided. If set to `false` with no + `functionScore` provided, the default Elasticsearch `_score` is used. +3. Built-in query booleans - There is 1 built-in query which can be enabled/disabled. These include + the `default autocomplete query` query, + enabled with the following booleans + respectively [`defaultQuery`] +4. `boolQuery` - The base Elasticsearch `boolean query`[[4](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html)]. + If enabled in #2 above, those queries will + appear in the `should` section of the `boolean query`[[4](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html)]. +5. `functionScore` - The Elasticsearch `function score`[[5](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-function-score-query.html#score-functions)] section of the overall query. + +#### Examples + +These examples assume a match-all `queryRegex` of `.*` so that it would impact any search query for simplicity. Also +note that the `queryRegex` is applied individually for `searchConfigurations` and `autocompleteConfigurations` and they +do not have to be identical. + +##### Example 1: Exclude `deprecated` entities from autocomplete + +```yaml +autocompleteConfigurations: + - queryRegex: .* + defaultQuery: true + + boolQuery: + must: + - term: + deprecated: 'false' +``` + +#### Example 2: Override scoring for autocomplete + +```yaml +autocompleteConfigurations: + - queryRegex: .* + defaultQuery: true + + functionScore: + functions: + - filter: + term: + materialized: + value: true + weight: 1.1 + - filter: + term: + deprecated: + value: false + weight: 0.5 + score_mode: avg + boost_mode: multiply +``` + ## FAQ and Troubleshooting **How are the results ordered?** diff --git a/docs/lineage/openlineage.md b/docs/lineage/openlineage.md new file mode 100644 index 0000000000000..0b9423bf2c4da --- /dev/null +++ b/docs/lineage/openlineage.md @@ -0,0 +1,92 @@ +# OpenLineage + +DataHub, now supports [OpenLineage](https://openlineage.io/) integration. With this support, DataHub can ingest and display lineage information from various data processing frameworks, providing users with a comprehensive understanding of their data pipelines. + +## Features + +- **REST Endpoint Support**: DataHub now includes a REST endpoint that can understand OpenLineage events. This allows users to send lineage information directly to DataHub, enabling easy integration with various data processing frameworks. + +- **[Spark Event Listener Plugin](https://datahubproject.io/docs/metadata-integration/java/spark-lineage-beta)**: DataHub provides a Spark Event Listener plugin that seamlessly integrates OpenLineage's Spark plugin. This plugin enhances DataHub's OpenLineage support by offering additional features such as PathSpec support, column-level lineage, patch support and more. + +## OpenLineage Support with DataHub + +### 1. REST Endpoint Support + +DataHub's REST endpoint allows users to send OpenLineage events directly to DataHub. This enables easy integration with various data processing frameworks, providing users with a centralized location for viewing and managing data lineage information. + +With Spark and Airflow we recommend using the Spark Lineage or DataHub's Airflow plugin for tighter integration with DataHub. + +#### How to Use + +To send OpenLineage messages to DataHub using the REST endpoint, simply make a POST request to the following endpoint: + +``` +POST GMS_SERVER_HOST:GMS_PORT/api/v2/lineage +``` + +Include the OpenLineage message in the request body in JSON format. + +Example: + +```json +{ + "eventType": "START", + "eventTime": "2020-12-28T19:52:00.001+10:00", + "run": { + "runId": "d46e465b-d358-4d32-83d4-df660ff614dd" + }, + "job": { + "namespace": "workshop", + "name": "process_taxes" + }, + "inputs": [ + { + "namespace": "postgres://workshop-db:None", + "name": "workshop.public.taxes", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.10.0/integration/airflow", + "_schemaURL": "https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/DataSourceDatasetFacet", + "name": "postgres://workshop-db:None", + "uri": "workshop-db" + } + } + } + ], + "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client" +} +``` +##### How to set up Airflow +Follow the Airflow guide to setup the Airflow DAGs to send lineage information to DataHub. The guide can be found [here](https://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/guides/user.html +The transport should look like this: +```json +{"type": "http", + "url": "https://GMS_SERVER_HOST:GMS_PORT/openapi/openlineage/", + "endpoint": "api/v1/lineage", + "auth": { + "type": "api_key", + "api_key": "your-datahub-api-key" + } +} +``` + +#### Known Limitations +With Spark and Airflow we recommend using the Spark Lineage or DataHub's Airflow plugin for tighter integration with DataHub. + +- **[PathSpec](https://datahubproject.io/docs/metadata-integration/java/spark-lineage-beta/#configuring-hdfs-based-dataset-urns) Support**: While the REST endpoint supports OpenLineage messages, full [PathSpec](https://datahubproject.io/docs/metadata-integration/java/spark-lineage-beta/#configuring-hdfs-based-dataset-urns)) support is not yet available. + +- **Column-level Lineage**: DataHub's current OpenLineage support does not provide full column-level lineage tracking. +- etc... +### 2. Spark Event Listener Plugin + +DataHub's Spark Event Listener plugin enhances OpenLineage support by providing additional features such as PathSpec support, column-level lineage, and more. + +#### How to Use + +Follow the guides of the Spark Lineage plugin page for more information on how to set up the Spark Lineage plugin. The guide can be found [here](https://datahubproject.io/docs/metadata-integration/java/spark-lineage-beta) + +## References + +- [OpenLineage](https://openlineage.io/) +- [DataHub OpenAPI Guide](../api/openapi/openapi-usage-guide.md) +- [DataHub Spark Lineage Plugin](https://datahubproject.io/docs/metadata-integration/java/spark-lineage-beta) diff --git a/docs/managed-datahub/datahub-api/entity-events-api.md b/docs/managed-datahub/datahub-api/entity-events-api.md index ebc3bb97f9554..e59f1650c7d76 100644 --- a/docs/managed-datahub/datahub-api/entity-events-api.md +++ b/docs/managed-datahub/datahub-api/entity-events-api.md @@ -352,7 +352,7 @@ This event is emitted when a description has been added to an entity on DataHub. #### Header -
CategoryOperationEntity Types
DOCUMENTATIONADDdataset, dashboard, chart, dataJob, dataFlow , container, glossaryTerm, domain, tag
+
CategoryOperationEntity Types
DOCUMENTATIONADDdataset, dashboard, chart, dataJob, dataFlow , container, glossaryTerm, domain, tag, schemaField
#### Parameters @@ -384,7 +384,7 @@ This event is emitted when an existing description has been removed from an enti #### Header -
CategoryOperationEntity Types
DOCUMENTATIONREMOVEdataset, dashboard, chart, dataJob, container ,dataFlow , glossaryTerm, domain, tag
+
CategoryOperationEntity Types
DOCUMENTATIONREMOVEdataset, dashboard, chart, dataJob, container ,dataFlow , glossaryTerm, domain, tag, schemaField
#### Parameters diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py index ac5dc00e0e639..737ea99e87824 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py @@ -3,6 +3,7 @@ import logging import os import threading +import time from typing import TYPE_CHECKING, Callable, Dict, List, Optional, TypeVar, cast import airflow @@ -16,6 +17,8 @@ FineGrainedLineageClass, FineGrainedLineageDownstreamTypeClass, FineGrainedLineageUpstreamTypeClass, + OperationClass, + OperationTypeClass, StatusClass, ) from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult @@ -414,6 +417,23 @@ def on_task_instance_running( f"DataHub listener finished processing notification about task instance start for {task_instance.task_id}" ) + if self.config.materialize_iolets: + for outlet in datajob.outlets: + reported_time: int = int(time.time() * 1000) + operation = OperationClass( + timestampMillis=reported_time, + operationType=OperationTypeClass.CREATE, + lastUpdatedTimestamp=reported_time, + actor=builder.make_user_urn("airflow"), + ) + + operation_mcp = MetadataChangeProposalWrapper( + entityUrn=str(outlet), aspect=operation + ) + + self.emitter.emit(operation_mcp) + logger.debug(f"Emitted Dataset Operation: {outlet}") + def on_task_instance_finish( self, task_instance: "TaskInstance", status: InstanceRunResult ) -> None: diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json index 7c52cbcddc13c..8b1bad5b55874 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json @@ -368,6 +368,42 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714671978982, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714671978982 + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714671978991, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714671978991 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", @@ -503,6 +539,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -528,7 +565,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223417702, + "timestampMillis": 1714671979032, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json index 150f95d5171c7..589cd32ae3eb7 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json @@ -368,6 +368,42 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714676628119, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714676628119 + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714676628127, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714676628127 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json index 0248ab0473c9e..653d8f7e30530 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json @@ -297,6 +297,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714671938600, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714671938600 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json index 7860251fc22dc..da08d2addf7c9 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json @@ -297,6 +297,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714676586630, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714676586630 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json index 1bf0820c7cb41..331ecd353ba26 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json @@ -328,6 +328,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,datahub_test_database.datahub_test_schema.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714672017187, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714672017187 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,snowflake_operator,prod),transform_cost_table)", diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json index 313abad9c5546..e85a07b194e4f 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json @@ -271,6 +271,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714672059338, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714672059338 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -635,6 +653,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714672062927, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714672062927 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", @@ -1022,6 +1058,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714672066747, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714672066747 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json index 60beff71c46c6..47f7cdca68d49 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json @@ -271,6 +271,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714676666839, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714676666839 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -692,6 +710,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714676669640, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714676669640 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", @@ -1136,6 +1172,24 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1714676672665, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1714676672665 + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", diff --git a/metadata-ingestion/sink_docs/datahub.md b/metadata-ingestion/sink_docs/datahub.md index 53b76ddd7288a..85bd8cc2f8531 100644 --- a/metadata-ingestion/sink_docs/datahub.md +++ b/metadata-ingestion/sink_docs/datahub.md @@ -28,6 +28,17 @@ sink: server: "http://localhost:8080" ``` +If you are connecting to a hosted Acryl instance, your sink will look like +```yml +source: + # source configs +sink: + type: "datahub-rest" + config: + server: "https://.acryl.io/gms" + token: +``` + If you are running the ingestion in a container in docker and your [GMS is also running in docker](../../docker/README.md) then you should use the internal docker hostname of the GMS pod. Usually it would look something like ```yml diff --git a/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py b/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py index 285ad9c088447..3680546d307d9 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py +++ b/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py @@ -26,9 +26,7 @@ class IngestionCheckpointingProviderBase(StatefulCommittable[CheckpointJobStates The base class for all checkpointing state provider implementations. """ - def __init__( - self, name: str, commit_policy: CommitPolicy = CommitPolicy.ON_NO_ERRORS - ): + def __init__(self, name: str, commit_policy: CommitPolicy = CommitPolicy.ALWAYS): # Set the initial state to an empty dict. super().__init__(name, commit_policy, {}) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index d918782691c77..ca09496eda341 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -275,7 +275,7 @@ def _make_bigquery_table( table.get("last_altered") / 1000, tz=timezone.utc ) if table.get("last_altered") is not None - else table.created, + else None, size_in_bytes=table.get("bytes"), rows_count=table.get("row_count"), comment=table.comment, @@ -339,7 +339,7 @@ def _make_bigquery_view(view: bigquery.Row) -> BigqueryView: view.get("last_altered") / 1000, tz=timezone.utc ) if view.get("last_altered") is not None - else view.created, + else None, comment=view.comment, view_definition=view.view_definition, materialized=view.table_type == BigqueryTableType.MATERIALIZED_VIEW, @@ -487,7 +487,7 @@ def _make_bigquery_table_snapshot(snapshot: bigquery.Row) -> BigqueryTableSnapsh snapshot.get("last_altered") / 1000, tz=timezone.utc ) if snapshot.get("last_altered") is not None - else snapshot.created, + else None, comment=snapshot.comment, ddl=snapshot.ddl, snapshot_time=snapshot.snapshot_time, diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py index 9a37f779bbcd5..4a73d26e11eaf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py @@ -102,8 +102,15 @@ def get_batch_kwargs( # We are using fraction-based sampling here, instead of fixed-size sampling because # Fixed-size sampling can be slower than equivalent fraction-based sampling # as per https://docs.snowflake.com/en/sql-reference/constructs/sample#performance-considerations + + sample_method = "BERNOULLI" + if table.rows_count > self.config.profiling.sample_size * 1000: + # If the table is significantly larger than the sample size, we use BLOCK + # sampling for better performance. + sample_method = "BLOCK" + sample_pc = 100 * self.config.profiling.sample_size / table.rows_count - custom_sql = f'select * from "{db_name}"."{schema_name}"."{table.name}" TABLESAMPLE ({sample_pc:.8f})' + custom_sql = f'select * from "{db_name}"."{schema_name}"."{table.name}" TABLESAMPLE {sample_method} ({sample_pc:.8f})' return { **super().get_batch_kwargs(table, schema_name, db_name), # Lowercase/Mixedcase table names in Snowflake do not work by default. diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py index b80067aa0892c..0145c922696e8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py @@ -164,6 +164,9 @@ def set_job_id(self, unique_id): def is_checkpointing_enabled(self) -> bool: return self.checkpointing_enabled + def _get_state_obj(self): + return self.state_type_class() + def create_checkpoint(self) -> Optional[Checkpoint]: if self.is_checkpointing_enabled() and not self._ignore_new_state(): assert self.stateful_ingestion_config is not None @@ -172,7 +175,7 @@ def create_checkpoint(self) -> Optional[Checkpoint]: job_name=self.job_id, pipeline_name=self.pipeline_name, run_id=self.run_id, - state=self.state_type_class(), + state=self._get_state_obj(), ) return None @@ -255,9 +258,13 @@ def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: # If the source already had a failure, skip soft-deletion. # TODO: Eventually, switch this to check if anything in the pipeline had a failure so far. if self.source.get_report().failures: + for urn in last_checkpoint_state.get_urns_not_in( + type="*", other_checkpoint_state=cur_checkpoint_state + ): + self.add_entity_to_state("", urn) self.source.get_report().report_warning( "stale-entity-removal", - "Skipping stale entity soft-deletion since source already had failures.", + "Skipping stale entity soft-deletion and coping urns from last state since source already had failures.", ) return diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index 23a75745698d9..d6f63ab385f52 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -1693,7 +1693,7 @@ def parse_custom_sql( ) -> Optional["SqlParsingResult"]: database_info = datasource.get(c.DATABASE) or { c.NAME: c.UNKNOWN.lower(), - c.CONNECTION_TYPE: "databricks", + c.CONNECTION_TYPE: datasource.get(c.CONNECTION_TYPE), } if ( @@ -1703,7 +1703,10 @@ def parse_custom_sql( logger.debug(f"datasource {datasource_urn} is not created from custom sql") return None - if c.NAME not in database_info or c.CONNECTION_TYPE not in database_info: + if ( + database_info.get(c.NAME) is None + or database_info.get(c.CONNECTION_TYPE) is None + ): logger.debug( f"database information is missing from datasource {datasource_urn}" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py index 98536472c5f61..fcfa434e00fee 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py @@ -324,6 +324,7 @@ class MetadataQueryException(Exception): totalCount } } + connectionType database{ name connectionType @@ -827,6 +828,7 @@ def get_unique_custom_sql(custom_sql_list: List[dict]) -> List[dict]: # are missing from api result. "isUnsupportedCustomSql": True if not custom_sql.get("tables") else False, "query": custom_sql.get("query"), + "connectionType": custom_sql.get("connectionType"), "columns": custom_sql.get("columns"), "tables": custom_sql.get("tables"), "database": custom_sql.get("database"), diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index de648ec29b233..b8dce11616981 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -406,10 +406,11 @@ def _schema_aware_fuzzy_column_resolve( return default_col_name # Optimize the statement + qualify column references. - logger.debug( - "Prior to column qualification sql %s", - statement.sql(pretty=True, dialect=dialect), - ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Prior to column qualification sql %s", + statement.sql(pretty=True, dialect=dialect), + ) try: # Second time running qualify, this time with: # - the select instead of the full outer statement @@ -434,7 +435,8 @@ def _schema_aware_fuzzy_column_resolve( raise SqlUnderstandingError( f"sqlglot failed to map columns to their source tables; likely missing/outdated table schema info: {e}" ) from e - logger.debug("Qualified sql %s", statement.sql(pretty=True, dialect=dialect)) + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Qualified sql %s", statement.sql(pretty=True, dialect=dialect)) # Handle the create DDL case. if is_create_ddl: @@ -805,7 +807,7 @@ def _sqlglot_lineage_inner( logger.debug("Parsing lineage from sql statement: %s", sql) statement = parse_statement(sql, dialect=dialect) - original_statement = statement.copy() + original_statement, statement = statement, statement.copy() # logger.debug( # "Formatted sql statement: %s", # original_statement.sql(pretty=True, dialect=dialect), diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py index c7cf975a3a953..25988f9905ec1 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py @@ -1,3 +1,4 @@ +import functools import hashlib import logging from typing import Dict, Iterable, Optional, Tuple, Union @@ -7,6 +8,7 @@ logger = logging.getLogger(__name__) DialectOrStr = Union[sqlglot.Dialect, str] +SQL_PARSE_CACHE_SIZE = 1000 def _get_dialect_str(platform: str) -> str: @@ -55,7 +57,8 @@ def is_dialect_instance( return False -def parse_statement( +@functools.lru_cache(maxsize=SQL_PARSE_CACHE_SIZE) +def _parse_statement( sql: sqlglot.exp.ExpOrStr, dialect: sqlglot.Dialect ) -> sqlglot.Expression: statement: sqlglot.Expression = sqlglot.maybe_parse( @@ -64,6 +67,16 @@ def parse_statement( return statement +def parse_statement( + sql: sqlglot.exp.ExpOrStr, dialect: sqlglot.Dialect +) -> sqlglot.Expression: + # Parsing is significantly more expensive than copying the expression. + # Because the expressions are mutable, we don't want to allow the caller + # to modify the parsed expression that sits in the cache. We keep + # the cached versions pristine by returning a copy on each call. + return _parse_statement(sql, dialect).copy() + + def parse_statements_and_pick(sql: str, platform: DialectOrStr) -> sqlglot.Expression: dialect = get_dialect(platform) statements = [ @@ -277,4 +290,5 @@ def replace_cte_refs(node: sqlglot.exp.Expression) -> sqlglot.exp.Expression: else: return node + statement = statement.copy() return statement.transform(replace_cte_refs, copy=False) diff --git a/metadata-ingestion/tests/integration/mysql/docker-compose.yml b/metadata-ingestion/tests/integration/mysql/docker-compose.yml index 6a46a1b8dccd4..db7ea3d4414bc 100644 --- a/metadata-ingestion/tests/integration/mysql/docker-compose.yml +++ b/metadata-ingestion/tests/integration/mysql/docker-compose.yml @@ -4,7 +4,6 @@ services: image: mysql:8 # image: mariadb:10.5.8 # Uncomment to run on m1 container_name: "testmysql" - command: --default-authentication-plugin=mysql_native_password environment: MYSQL_ROOT_PASSWORD: example ports: diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index cccf79ccbd8e0..2b122897a333f 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -16,6 +16,7 @@ ) from datahub.configuration.source_common import DEFAULT_ENV +from datahub.emitter.mce_builder import make_schema_field_urn from datahub.ingestion.run.pipeline import Pipeline, PipelineContext from datahub.ingestion.source.tableau import TableauConfig, TableauSource from datahub.ingestion.source.tableau_common import ( @@ -24,10 +25,12 @@ ) from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( DatasetLineageType, + FineGrainedLineage, + FineGrainedLineageDownstreamType, + FineGrainedLineageUpstreamType, UpstreamLineage, ) from datahub.metadata.schema_classes import MetadataChangeProposalClass, UpstreamClass -from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult from tests.test_helpers import mce_helpers, test_connection_helpers from tests.test_helpers.state_helpers import ( get_current_checkpoint_from_pipeline, @@ -805,55 +808,90 @@ def test_tableau_signout_timeout(pytestconfig, tmp_path, mock_datahub_graph): ) -def test_tableau_unsupported_csql(mock_datahub_graph): +def test_tableau_unsupported_csql(): context = PipelineContext(run_id="0", pipeline_name="test_tableau") - context.graph = mock_datahub_graph - config = TableauConfig.parse_obj(config_source_default.copy()) + config_dict = config_source_default.copy() + del config_dict["stateful_ingestion"] + config = TableauConfig.parse_obj(config_dict) config.extract_lineage_from_unsupported_custom_sql_queries = True config.lineage_overrides = TableauLineageOverrides( database_override_map={"production database": "prod"} ) - with mock.patch( - "datahub.ingestion.source.tableau.create_lineage_sql_parsed_result", - return_value=SqlParsingResult( - in_tables=[ - "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_bigquery_project.invent_dw.userdetail,PROD)" - ], - out_tables=[], - column_lineage=None, - ), + def test_lineage_metadata( + lineage, expected_entity_urn, expected_upstream_table, expected_cll ): - source = TableauSource(config=config, ctx=context) - - lineage = source._create_lineage_from_unsupported_csql( - csql_urn="urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)", - csql={ - "query": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (partition BY user_id ORDER BY __partition_day DESC) AS rank_ FROM invent_dw.UserDetail ) source_user WHERE rank_ = 1", - "isUnsupportedCustomSql": "true", - "database": { - "name": "my-bigquery-project", - "connectionType": "bigquery", - }, - }, - out_columns=[], - ) - mcp = cast(MetadataChangeProposalClass, next(iter(lineage)).metadata) - assert mcp.aspect == UpstreamLineage( upstreams=[ UpstreamClass( - dataset="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_bigquery_project.invent_dw.userdetail,PROD)", + dataset=expected_upstream_table, type=DatasetLineageType.TRANSFORMED, ) ], - fineGrainedLineages=[], - ) - assert ( - mcp.entityUrn - == "urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)" + fineGrainedLineages=[ + FineGrainedLineage( + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + upstreams=[ + make_schema_field_urn(expected_upstream_table, upstream_column) + ], + downstreamType=FineGrainedLineageDownstreamType.FIELD, + downstreams=[ + make_schema_field_urn(expected_entity_urn, downstream_column) + ], + ) + for upstream_column, downstream_column in expected_cll.items() + ], ) + assert mcp.entityUrn == expected_entity_urn + + csql_urn = "urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)" + expected_upstream_table = "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_bigquery_project.invent_dw.UserDetail,PROD)" + expected_cll = { + "user_id": "user_id", + "source": "source", + "user_source": "user_source", + } + + source = TableauSource(config=config, ctx=context) + + lineage = source._create_lineage_from_unsupported_csql( + csql_urn=csql_urn, + csql={ + "query": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (partition BY user_id ORDER BY __partition_day DESC) AS rank_ FROM invent_dw.UserDetail ) source_user WHERE rank_ = 1", + "isUnsupportedCustomSql": "true", + "connectionType": "bigquery", + "database": { + "name": "my_bigquery_project", + "connectionType": "bigquery", + }, + }, + out_columns=[], + ) + test_lineage_metadata( + lineage=lineage, + expected_entity_urn=csql_urn, + expected_upstream_table=expected_upstream_table, + expected_cll=expected_cll, + ) + + # With database as None + lineage = source._create_lineage_from_unsupported_csql( + csql_urn=csql_urn, + csql={ + "query": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (partition BY user_id ORDER BY __partition_day DESC) AS rank_ FROM my_bigquery_project.invent_dw.UserDetail ) source_user WHERE rank_ = 1", + "isUnsupportedCustomSql": "true", + "connectionType": "bigquery", + "database": None, + }, + out_columns=[], + ) + test_lineage_metadata( + lineage=lineage, + expected_entity_urn=csql_urn, + expected_upstream_table=expected_upstream_table, + expected_cll=expected_cll, + ) @freeze_time(FROZEN_TIME) diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json index 4e62492918bfb..fcf73d9614f24 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json @@ -16,8 +16,8 @@ "config": "", "state": { "formatVersion": "1.0", - "serde": "base85-bz2-json", - "payload": "LRx4!F+o`-Q(1w>5G4QrYoCBnWH=B60MH7jr`{?c0BA?5L)2-AGyu>6y;V<9hz%Mv0Bt1*)lOMzr>a0|Iq-4VtTsYONQsFPLn1EpdQS;HIy|&CvSAlRvAJwmtCEM+Rx(v_)~sVvkx3V@WX4O`=losC6yZWb2OL0@" + "serde": "utf-8", + "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)\"]}" }, "runId": "dummy-test-stateful-ingestion" } diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json index 6ecd43483d948..5477af72a1939 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json @@ -16,8 +16,8 @@ "config": "", "state": { "formatVersion": "1.0", - "serde": "base85-bz2-json", - "payload": "LRx4!F+o`-Q(317h`0a%NgsevWH1l}0MH7jr`{?c0B9vdZ9%mLfYG4P6;f$2G%+v`9z&~6n|e(JEPC2_Iix~CA_im)jR-zsjEK*yo|HQz#IUUHtf@DYVEme-lUW9{Xmmt~y^2jCdyY95az!{$kf#WUxB" + "serde": "utf-8", + "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)\"]}" }, "runId": "dummy-test-stateful-ingestion" } diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted_failure.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted_failure.json new file mode 100644 index 0000000000000..fcf73d9614f24 --- /dev/null +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted_failure.json @@ -0,0 +1,26 @@ +[ +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(file,dummy_stateful,prod),default_stale_entity_removal)", + "changeType": "UPSERT", + "aspectName": "datahubIngestionCheckpoint", + "aspect": { + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "pipelineName": "dummy_stateful", + "platformInstanceId": "", + "config": "", + "state": { + "formatVersion": "1.0", + "serde": "utf-8", + "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)\"]}" + }, + "runId": "dummy-test-stateful-ingestion" + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_failure.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_failure.json new file mode 100644 index 0000000000000..fcf73d9614f24 --- /dev/null +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_failure.json @@ -0,0 +1,26 @@ +[ +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(file,dummy_stateful,prod),default_stale_entity_removal)", + "changeType": "UPSERT", + "aspectName": "datahubIngestionCheckpoint", + "aspect": { + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "pipelineName": "dummy_stateful", + "platformInstanceId": "", + "config": "", + "state": { + "formatVersion": "1.0", + "serde": "utf-8", + "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)\"]}" + }, + "runId": "dummy-test-stateful-ingestion" + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted_failure.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted_failure.json new file mode 100644 index 0000000000000..a1f5132cac0a3 --- /dev/null +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted_failure.json @@ -0,0 +1,34 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_failure.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_failure.json new file mode 100644 index 0000000000000..4a77651c93066 --- /dev/null +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_failure.json @@ -0,0 +1,50 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py index 2b811d5e5e3a3..783b0fe18b29a 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py @@ -1,7 +1,9 @@ from dataclasses import dataclass, field as dataclass_field from typing import Any, Dict, Iterable, List, Optional, cast +from unittest import mock import pydantic +import pytest from freezegun import freeze_time from pydantic import Field @@ -56,6 +58,10 @@ class DummySourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field( default=None, description="Dummy source Ingestion Config." ) + report_failure: bool = Field( + default=False, + description="Should this dummy source report a failure.", + ) class DummySource(StatefulIngestionSourceBase): @@ -103,10 +109,23 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: aspect=StatusClass(removed=False), ).as_workunit() + if self.source_config.report_failure: + self.reporter.report_failure("Dummy error", "Error") + def get_report(self) -> SourceReport: return self.reporter +@pytest.fixture(scope="module") +def mock_generic_checkpoint_state(): + with mock.patch( + "datahub.ingestion.source.state.entity_removal_state.GenericCheckpointState" + ) as mock_checkpoint_state: + checkpoint_state = mock_checkpoint_state.return_value + checkpoint_state.serde.return_value = "utf-8" + yield mock_checkpoint_state + + @freeze_time(FROZEN_TIME) def test_stateful_ingestion(pytestconfig, tmp_path, mock_time): # test stateful ingestion using dummy source @@ -148,80 +167,209 @@ def test_stateful_ingestion(pytestconfig, tmp_path, mock_time): }, } - pipeline_run1 = None - pipeline_run1_config: Dict[str, Dict[str, Dict[str, Any]]] = dict( # type: ignore - base_pipeline_config # type: ignore + with mock.patch( + "datahub.ingestion.source.state.stale_entity_removal_handler.StaleEntityRemovalHandler._get_state_obj" + ) as mock_state: + mock_state.return_value = GenericCheckpointState(serde="utf-8") + pipeline_run1 = None + pipeline_run1_config: Dict[str, Dict[str, Dict[str, Any]]] = dict( # type: ignore + base_pipeline_config # type: ignore + ) + pipeline_run1_config["sink"]["config"][ + "filename" + ] = f"{tmp_path}/{output_file_name}" + pipeline_run1 = Pipeline.create(pipeline_run1_config) + pipeline_run1.run() + pipeline_run1.raise_from_status() + pipeline_run1.pretty_print_summary() + + # validate both dummy source mces and checkpoint state mces files + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / output_file_name, + golden_path=f"{test_resources_dir}/{golden_file_name}", + ) + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / state_file_name, + golden_path=f"{test_resources_dir}/{golden_state_file_name}", + ) + checkpoint1 = get_current_checkpoint_from_pipeline(pipeline_run1) + assert checkpoint1 + assert checkpoint1.state + + with mock.patch( + "datahub.ingestion.source.state.stale_entity_removal_handler.StaleEntityRemovalHandler._get_state_obj" + ) as mock_state: + mock_state.return_value = GenericCheckpointState(serde="utf-8") + pipeline_run2 = None + pipeline_run2_config: Dict[str, Dict[str, Dict[str, Any]]] = dict(base_pipeline_config) # type: ignore + pipeline_run2_config["source"]["config"]["dataset_patterns"] = { + "allow": ["dummy_dataset1", "dummy_dataset2"], + } + pipeline_run2_config["sink"]["config"][ + "filename" + ] = f"{tmp_path}/{output_file_name_after_deleted}" + pipeline_run2 = Pipeline.create(pipeline_run2_config) + pipeline_run2.run() + pipeline_run2.raise_from_status() + pipeline_run2.pretty_print_summary() + + # validate both updated dummy source mces and checkpoint state mces files after deleting dataset + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / output_file_name_after_deleted, + golden_path=f"{test_resources_dir}/{golden_file_name_after_deleted}", + ) + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / state_file_name, + golden_path=f"{test_resources_dir}/{golden_state_file_name_after_deleted}", + ) + checkpoint2 = get_current_checkpoint_from_pipeline(pipeline_run2) + assert checkpoint2 + assert checkpoint2.state + + # Validate that all providers have committed successfully. + validate_all_providers_have_committed_successfully( + pipeline=pipeline_run1, expected_providers=1 + ) + validate_all_providers_have_committed_successfully( + pipeline=pipeline_run2, expected_providers=1 + ) + + # Perform all assertions on the states. The deleted table should not be + # part of the second state + state1 = cast(GenericCheckpointState, checkpoint1.state) + state2 = cast(GenericCheckpointState, checkpoint2.state) + + difference_dataset_urns = list( + state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) + ) + # the difference in dataset urns is the dataset which is not allowed to ingest + assert len(difference_dataset_urns) == 1 + deleted_dataset_urns: List[str] = [ + "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)", + ] + assert sorted(deleted_dataset_urns) == sorted(difference_dataset_urns) + + +@freeze_time(FROZEN_TIME) +def test_stateful_ingestion_failure(pytestconfig, tmp_path, mock_time): + # test stateful ingestion using dummy source with pipeline execution failed in second ingestion + state_file_name: str = "checkpoint_state_mces_failure.json" + golden_state_file_name: str = "golden_test_checkpoint_state_failure.json" + golden_state_file_name_after_deleted: str = ( + "golden_test_checkpoint_state_after_deleted_failure.json" ) - pipeline_run1_config["sink"]["config"][ - "filename" - ] = f"{tmp_path}/{output_file_name}" - pipeline_run1 = Pipeline.create(pipeline_run1_config) - pipeline_run1.run() - pipeline_run1.raise_from_status() - pipeline_run1.pretty_print_summary() - - # validate both dummy source mces and checkpoint state mces files - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / output_file_name, - golden_path=f"{test_resources_dir}/{golden_file_name}", + output_file_name: str = "dummy_mces_failure.json" + golden_file_name: str = "golden_test_stateful_ingestion_failure.json" + output_file_name_after_deleted: str = ( + "dummy_mces_stateful_after_deleted_failure.json" ) - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / state_file_name, - golden_path=f"{test_resources_dir}/{golden_state_file_name}", + golden_file_name_after_deleted: str = ( + "golden_test_stateful_ingestion_after_deleted_failure.json" ) - checkpoint1 = get_current_checkpoint_from_pipeline(pipeline_run1) - assert checkpoint1 - assert checkpoint1.state - - pipeline_run2 = None - pipeline_run2_config: Dict[str, Dict[str, Dict[str, Any]]] = dict(base_pipeline_config) # type: ignore - pipeline_run2_config["source"]["config"]["dataset_patterns"] = { - "allow": ["dummy_dataset1", "dummy_dataset2"], + + test_resources_dir = pytestconfig.rootpath / "tests/unit/stateful_ingestion/state" + + base_pipeline_config = { + "run_id": "dummy-test-stateful-ingestion", + "pipeline_name": "dummy_stateful", + "source": { + "type": "tests.unit.stateful_ingestion.state.test_stateful_ingestion.DummySource", + "config": { + "stateful_ingestion": { + "enabled": True, + "remove_stale_metadata": True, + "state_provider": { + "type": "file", + "config": { + "filename": f"{tmp_path}/{state_file_name}", + }, + }, + }, + }, + }, + "sink": { + "type": "file", + "config": {}, + }, } - pipeline_run2_config["sink"]["config"][ - "filename" - ] = f"{tmp_path}/{output_file_name_after_deleted}" - pipeline_run2 = Pipeline.create(pipeline_run2_config) - pipeline_run2.run() - pipeline_run2.raise_from_status() - pipeline_run2.pretty_print_summary() - - # validate both updated dummy source mces and checkpoint state mces files after deleting dataset - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / output_file_name_after_deleted, - golden_path=f"{test_resources_dir}/{golden_file_name_after_deleted}", - ) - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / state_file_name, - golden_path=f"{test_resources_dir}/{golden_state_file_name_after_deleted}", - ) - checkpoint2 = get_current_checkpoint_from_pipeline(pipeline_run2) - assert checkpoint2 - assert checkpoint2.state - # Validate that all providers have committed successfully. - validate_all_providers_have_committed_successfully( - pipeline=pipeline_run1, expected_providers=1 - ) - validate_all_providers_have_committed_successfully( - pipeline=pipeline_run2, expected_providers=1 - ) + with mock.patch( + "datahub.ingestion.source.state.stale_entity_removal_handler.StaleEntityRemovalHandler._get_state_obj" + ) as mock_state: + mock_state.return_value = GenericCheckpointState(serde="utf-8") + pipeline_run1 = None + pipeline_run1_config: Dict[str, Dict[str, Dict[str, Any]]] = dict( # type: ignore + base_pipeline_config # type: ignore + ) + pipeline_run1_config["sink"]["config"][ + "filename" + ] = f"{tmp_path}/{output_file_name}" + pipeline_run1 = Pipeline.create(pipeline_run1_config) + pipeline_run1.run() + pipeline_run1.raise_from_status() + pipeline_run1.pretty_print_summary() + + # validate both dummy source mces and checkpoint state mces files + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / output_file_name, + golden_path=f"{test_resources_dir}/{golden_file_name}", + ) + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / state_file_name, + golden_path=f"{test_resources_dir}/{golden_state_file_name}", + ) + checkpoint1 = get_current_checkpoint_from_pipeline(pipeline_run1) + assert checkpoint1 + assert checkpoint1.state - # Perform all assertions on the states. The deleted table should not be - # part of the second state - state1 = cast(GenericCheckpointState, checkpoint1.state) - state2 = cast(GenericCheckpointState, checkpoint2.state) + with mock.patch( + "datahub.ingestion.source.state.stale_entity_removal_handler.StaleEntityRemovalHandler._get_state_obj" + ) as mock_state: + mock_state.return_value = GenericCheckpointState(serde="utf-8") + pipeline_run2 = None + pipeline_run2_config: Dict[str, Dict[str, Dict[str, Any]]] = dict(base_pipeline_config) # type: ignore + pipeline_run2_config["source"]["config"]["dataset_patterns"] = { + "allow": ["dummy_dataset1", "dummy_dataset2"], + } + pipeline_run2_config["source"]["config"]["report_failure"] = True + pipeline_run2_config["sink"]["config"][ + "filename" + ] = f"{tmp_path}/{output_file_name_after_deleted}" + pipeline_run2 = Pipeline.create(pipeline_run2_config) + pipeline_run2.run() + pipeline_run2.pretty_print_summary() - difference_dataset_urns = list( - state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) - ) - # the difference in dataset urns is the dataset which is not allowed to ingest - assert len(difference_dataset_urns) == 1 - deleted_dataset_urns: List[str] = [ - "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)", - ] - assert sorted(deleted_dataset_urns) == sorted(difference_dataset_urns) + # validate both updated dummy source mces and checkpoint state mces files after deleting dataset + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / output_file_name_after_deleted, + golden_path=f"{test_resources_dir}/{golden_file_name_after_deleted}", + ) + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / state_file_name, + golden_path=f"{test_resources_dir}/{golden_state_file_name_after_deleted}", + ) + checkpoint2 = get_current_checkpoint_from_pipeline(pipeline_run2) + assert checkpoint2 + assert checkpoint2.state + + # Validate that all providers have committed successfully. + validate_all_providers_have_committed_successfully( + pipeline=pipeline_run1, expected_providers=1 + ) + validate_all_providers_have_committed_successfully( + pipeline=pipeline_run2, expected_providers=1 + ) + + # Perform assertions on the states. The deleted table should be + # still part of the second state as pipeline run failed + state1 = cast(GenericCheckpointState, checkpoint1.state) + state2 = cast(GenericCheckpointState, checkpoint2.state) + assert state1 == state2 diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py index 426d4dc12f208..c501593fbed01 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_source.py @@ -829,7 +829,7 @@ def bigquery_view_2() -> BigqueryView: return BigqueryView( name="table2", created=now, - last_altered=now, + last_altered=None, comment="comment2", view_definition="CREATE VIEW 2", materialized=True, diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index eee84b1f8c827..2328697632434 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -27,7 +27,7 @@ dependencies { } } - compileOnly externalDependency.httpAsyncClient + compileOnly externalDependency.httpClient implementation externalDependency.jacksonDataBind runtimeOnly externalDependency.jna @@ -41,7 +41,7 @@ dependencies { testImplementation externalDependency.mockServer testImplementation externalDependency.mockServerClient testImplementation externalDependency.testContainers - testImplementation externalDependency.httpAsyncClient + testImplementation externalDependency.httpClient testRuntimeOnly externalDependency.logbackClassic } diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/MetadataResponseFuture.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/MetadataResponseFuture.java index 89db9738efda6..11be10186f1ef 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/MetadataResponseFuture.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/MetadataResponseFuture.java @@ -7,16 +7,16 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import lombok.SneakyThrows; -import org.apache.http.HttpResponse; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; public class MetadataResponseFuture implements Future { - private final Future requestFuture; + private final Future requestFuture; private final AtomicReference responseReference; private final CountDownLatch responseLatch; private final ResponseMapper mapper; public MetadataResponseFuture( - Future underlyingFuture, + Future underlyingFuture, AtomicReference responseAtomicReference, CountDownLatch responseLatch) { this.requestFuture = underlyingFuture; @@ -25,7 +25,8 @@ public MetadataResponseFuture( this.mapper = null; } - public MetadataResponseFuture(Future underlyingFuture, ResponseMapper mapper) { + public MetadataResponseFuture( + Future underlyingFuture, ResponseMapper mapper) { this.requestFuture = underlyingFuture; this.responseReference = null; this.responseLatch = null; @@ -50,7 +51,7 @@ public boolean isDone() { @SneakyThrows @Override public MetadataWriteResponse get() throws InterruptedException, ExecutionException { - HttpResponse response = requestFuture.get(); + SimpleHttpResponse response = requestFuture.get(); if (mapper != null) { return mapper.map(response); } else { @@ -63,7 +64,7 @@ public MetadataWriteResponse get() throws InterruptedException, ExecutionExcepti @Override public MetadataWriteResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - HttpResponse response = requestFuture.get(timeout, unit); + SimpleHttpResponse response = requestFuture.get(timeout, unit); if (mapper != null) { return mapper.map(response); } else { @@ -75,6 +76,6 @@ public MetadataWriteResponse get(long timeout, TimeUnit unit) @FunctionalInterface public interface ResponseMapper { - MetadataWriteResponse map(HttpResponse httpResponse); + MetadataWriteResponse map(SimpleHttpResponse httpResponse); } } diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java new file mode 100644 index 0000000000000..71a4b93baf48f --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java @@ -0,0 +1,54 @@ +package datahub.client.rest; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.UnknownHostException; +import java.util.Arrays; +import javax.net.ssl.SSLException; +import lombok.extern.slf4j.Slf4j; +import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; +import org.apache.hc.core5.http.ConnectionClosedException; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.apache.hc.core5.util.TimeValue; + +@Slf4j +public class DatahubHttpRequestRetryStrategy extends DefaultHttpRequestRetryStrategy { + public DatahubHttpRequestRetryStrategy() { + this(1, TimeValue.ofSeconds(10)); + } + + public DatahubHttpRequestRetryStrategy(int maxRetries, TimeValue retryInterval) { + super( + maxRetries, + retryInterval, + Arrays.asList( + InterruptedIOException.class, + UnknownHostException.class, + ConnectException.class, + ConnectionClosedException.class, + NoRouteToHostException.class, + SSLException.class), + Arrays.asList( + HttpStatus.SC_TOO_MANY_REQUESTS, + HttpStatus.SC_SERVICE_UNAVAILABLE, + HttpStatus.SC_INTERNAL_SERVER_ERROR)); + } + + @Override + public boolean retryRequest( + HttpRequest request, IOException exception, int execCount, HttpContext context) { + log.warn("Checking if retry is needed: {}", execCount); + return super.retryRequest(request, exception, execCount, context); + } + + @Override + public boolean retryRequest(HttpResponse response, int execCount, HttpContext context) { + log.warn("Retrying request due to error: {}", response); + return super.retryRequest(response, execCount, context); + } +} diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java index a2692c432513e..ed4cee060bd69 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java @@ -18,31 +18,35 @@ import datahub.event.UpsertAspectRequest; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.util.List; +import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import javax.annotation.concurrent.ThreadSafe; +import javax.net.ssl.SSLContext; import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.concurrent.FutureCallback; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.TrustAllStrategy; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; -import org.apache.http.nio.client.HttpAsyncClient; -import org.apache.http.ssl.SSLContextBuilder; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.client5.http.ssl.TrustAllStrategy; +import org.apache.hc.core5.concurrent.FutureCallback; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.ssl.SSLContexts; +import org.apache.hc.core5.util.TimeValue; @ThreadSafe @Slf4j @@ -89,28 +93,43 @@ public RestEmitter(RestEmitterConfig config) { dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory()); this.config = config; + HttpAsyncClientBuilder httpClientBuilder = this.config.getAsyncHttpClientBuilder(); + httpClientBuilder.setRetryStrategy(new DatahubHttpRequestRetryStrategy()); + // Override httpClient settings with RestEmitter configs if present if (config.getTimeoutSec() != null) { - HttpAsyncClientBuilder httpClientBuilder = this.config.getAsyncHttpClientBuilder(); httpClientBuilder.setDefaultRequestConfig( RequestConfig.custom() - .setConnectTimeout(config.getTimeoutSec() * 1000) - .setSocketTimeout(config.getTimeoutSec() * 1000) + .setConnectionRequestTimeout( + config.getTimeoutSec() * 1000, java.util.concurrent.TimeUnit.MILLISECONDS) + .setResponseTimeout( + config.getTimeoutSec() * 1000, java.util.concurrent.TimeUnit.MILLISECONDS) .build()); } if (config.isDisableSslVerification()) { - HttpAsyncClientBuilder httpClientBuilder = this.config.getAsyncHttpClientBuilder(); try { - httpClientBuilder - .setSSLContext( - new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build()) - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + SSLContext sslcontext = + SSLContexts.custom().loadTrustMaterial(TrustAllStrategy.INSTANCE).build(); + TlsStrategy tlsStrategy = + ClientTlsStrategyBuilder.create() + .setSslContext(sslcontext) + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build(); + + httpClientBuilder.setConnectionManager( + PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build()); } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { throw new RuntimeException("Error while creating insecure http client", e); } } - this.httpClient = this.config.getAsyncHttpClientBuilder().build(); + httpClientBuilder.setRetryStrategy( + new DatahubHttpRequestRetryStrategy( + config.getMaxRetries(), TimeValue.ofSeconds(config.getRetryIntervalSec()))); + + this.httpClient = httpClientBuilder.build(); this.httpClient.start(); this.ingestProposalUrl = this.config.getServer() + "/aspects?action=ingestProposal"; this.ingestOpenApiUrl = config.getServer() + "/openapi/entities/v1/"; @@ -118,13 +137,11 @@ public RestEmitter(RestEmitterConfig config) { this.eventFormatter = this.config.getEventFormatter(); } - private static MetadataWriteResponse mapResponse(HttpResponse response) { + private static MetadataWriteResponse mapResponse(SimpleHttpResponse response) { MetadataWriteResponse.MetadataWriteResponseBuilder builder = MetadataWriteResponse.builder().underlyingResponse(response); - if ((response != null) - && (response.getStatusLine() != null) - && (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK - || response.getStatusLine().getStatusCode() == HttpStatus.SC_CREATED)) { + if ((response != null) && (response.getCode()) == HttpStatus.SC_OK + || Objects.requireNonNull(response).getCode() == HttpStatus.SC_CREATED) { builder.success(true); } else { builder.success(false); @@ -132,14 +149,7 @@ private static MetadataWriteResponse mapResponse(HttpResponse response) { // Read response content try { ByteArrayOutputStream result = new ByteArrayOutputStream(); - InputStream contentStream = response.getEntity().getContent(); - byte[] buffer = new byte[1024]; - int length = contentStream.read(buffer); - while (length > 0) { - result.write(buffer, 0, length); - length = contentStream.read(buffer); - } - builder.responseContent(result.toString("UTF-8")); + builder.responseContent(response.getBody().getBodyText()); } catch (Exception e) { // Catch all exceptions and still return a valid response object log.warn("Wasn't able to convert response into a string", e); @@ -198,21 +208,22 @@ public Future emit(MetadataChangeProposal mcp, Callback c private Future postGeneric( String urlStr, String payloadJson, Object originalRequest, Callback callback) throws IOException { - HttpPost httpPost = new HttpPost(urlStr); - httpPost.setHeader("Content-Type", "application/json"); - httpPost.setHeader("X-RestLi-Protocol-Version", "2.0.0"); - httpPost.setHeader("Accept", "application/json"); - this.config.getExtraHeaders().forEach((k, v) -> httpPost.setHeader(k, v)); + SimpleRequestBuilder simpleRequestBuilder = SimpleRequestBuilder.post(urlStr); + simpleRequestBuilder.setHeader("Content-Type", "application/json"); + simpleRequestBuilder.setHeader("X-RestLi-Protocol-Version", "2.0.0"); + simpleRequestBuilder.setHeader("Accept", "application/json"); + this.config.getExtraHeaders().forEach(simpleRequestBuilder::setHeader); if (this.config.getToken() != null) { - httpPost.setHeader("Authorization", "Bearer " + this.config.getToken()); + simpleRequestBuilder.setHeader("Authorization", "Bearer " + this.config.getToken()); } - httpPost.setEntity(new StringEntity(payloadJson)); + + simpleRequestBuilder.setBody(payloadJson, ContentType.APPLICATION_JSON); AtomicReference responseAtomicReference = new AtomicReference<>(); CountDownLatch responseLatch = new CountDownLatch(1); - FutureCallback httpCallback = - new FutureCallback() { + FutureCallback httpCallback = + new FutureCallback() { @Override - public void completed(HttpResponse response) { + public void completed(SimpleHttpResponse response) { MetadataWriteResponse writeResponse = null; try { writeResponse = mapResponse(response); @@ -252,16 +263,20 @@ public void cancelled() { } } }; - Future requestFuture = httpClient.execute(httpPost, httpCallback); + Future requestFuture = + httpClient.execute(simpleRequestBuilder.build(), httpCallback); return new MetadataResponseFuture(requestFuture, responseAtomicReference, responseLatch); } private Future getGeneric(String urlStr) throws IOException { - HttpGet httpGet = new HttpGet(urlStr); - httpGet.setHeader("Content-Type", "application/json"); - httpGet.setHeader("X-RestLi-Protocol-Version", "2.0.0"); - httpGet.setHeader("Accept", "application/json"); - Future response = this.httpClient.execute(httpGet, null); + SimpleHttpRequest simpleHttpRequest = + SimpleRequestBuilder.get(urlStr) + .addHeader("Content-Type", "application/json") + .addHeader("X-RestLi-Protocol-Version", "2.0.0") + .addHeader("Accept", "application/json") + .build(); + + Future response = this.httpClient.execute(simpleHttpRequest, null); return new MetadataResponseFuture(response, RestEmitter::mapResponse); } @@ -284,20 +299,25 @@ public Future emit(List request, Cal private Future postOpenAPI( List payload, Callback callback) throws IOException { - HttpPost httpPost = new HttpPost(ingestOpenApiUrl); - httpPost.setHeader("Content-Type", "application/json"); - httpPost.setHeader("Accept", "application/json"); - this.config.getExtraHeaders().forEach((k, v) -> httpPost.setHeader(k, v)); + SimpleRequestBuilder simpleRequestBuilder = + SimpleRequestBuilder.post(ingestOpenApiUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Accept", "application/json") + .addHeader("X-RestLi-Protocol-Version", "2.0.0"); + + this.config.getExtraHeaders().forEach(simpleRequestBuilder::addHeader); + if (this.config.getToken() != null) { - httpPost.setHeader("Authorization", "Bearer " + this.config.getToken()); + simpleRequestBuilder.addHeader("Authorization", "Bearer " + this.config.getToken()); } - httpPost.setEntity(new StringEntity(objectMapper.writeValueAsString(payload))); + simpleRequestBuilder.setBody( + objectMapper.writeValueAsString(payload), ContentType.APPLICATION_JSON); AtomicReference responseAtomicReference = new AtomicReference<>(); CountDownLatch responseLatch = new CountDownLatch(1); - FutureCallback httpCallback = - new FutureCallback() { + FutureCallback httpCallback = + new FutureCallback() { @Override - public void completed(HttpResponse response) { + public void completed(SimpleHttpResponse response) { MetadataWriteResponse writeResponse = null; try { writeResponse = mapResponse(response); @@ -337,12 +357,13 @@ public void cancelled() { } } }; - Future requestFuture = httpClient.execute(httpPost, httpCallback); + Future requestFuture = + httpClient.execute(simpleRequestBuilder.build(), httpCallback); return new MetadataResponseFuture(requestFuture, responseAtomicReference, responseLatch); } @VisibleForTesting - HttpAsyncClient getHttpClient() { + CloseableHttpAsyncClient getHttpClient() { return this.httpClient; } } diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java index 7e24429213246..e28ad4ed660f0 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java @@ -10,8 +10,10 @@ import lombok.NonNull; import lombok.Value; import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.impl.async.HttpAsyncClients; +import org.apache.hc.core5.util.TimeValue; @Value @Builder @@ -23,20 +25,23 @@ public class RestEmitterConfig { public static final String DEFAULT_AUTH_TOKEN = null; public static final String CLIENT_VERSION_PROPERTY = "clientVersion"; - @Builder.Default private final String server = "http://localhost:8080"; + @Builder.Default String server = "http://localhost:8080"; - private final Integer timeoutSec; - @Builder.Default private final boolean disableSslVerification = false; + Integer timeoutSec; + @Builder.Default boolean disableSslVerification = false; - @Builder.Default private final String token = DEFAULT_AUTH_TOKEN; + @Builder.Default int maxRetries = 0; - @Builder.Default @NonNull private final Map extraHeaders = Collections.EMPTY_MAP; + @Builder.Default int retryIntervalSec = 10; - private final HttpAsyncClientBuilder asyncHttpClientBuilder; + @Builder.Default String token = DEFAULT_AUTH_TOKEN; + + @Builder.Default @NonNull Map extraHeaders = Collections.EMPTY_MAP; @Builder.Default - private final EventFormatter eventFormatter = - new EventFormatter(EventFormatter.Format.PEGASUS_JSON); + EventFormatter eventFormatter = new EventFormatter(EventFormatter.Format.PEGASUS_JSON); + + HttpAsyncClientBuilder asyncHttpClientBuilder; public static class RestEmitterConfigBuilder { @@ -53,13 +58,19 @@ private String getVersion() { } private HttpAsyncClientBuilder asyncHttpClientBuilder = - HttpAsyncClientBuilder.create() + HttpAsyncClients.custom() + .setUserAgent("DataHub-RestClient/" + getVersion()) .setDefaultRequestConfig( RequestConfig.custom() - .setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_SEC * 1000) - .setSocketTimeout(DEFAULT_READ_TIMEOUT_SEC * 1000) + .setConnectionRequestTimeout( + DEFAULT_CONNECT_TIMEOUT_SEC * 1000, + java.util.concurrent.TimeUnit.MILLISECONDS) + .setResponseTimeout( + DEFAULT_READ_TIMEOUT_SEC * 1000, java.util.concurrent.TimeUnit.MILLISECONDS) .build()) - .setUserAgent("DataHub-RestClient/" + getVersion()); + .setRetryStrategy( + new DatahubHttpRequestRetryStrategy( + maxRetries$value, TimeValue.ofSeconds(retryIntervalSec$value))); public RestEmitterConfigBuilder with(Consumer builderFunction) { builderFunction.accept(this); diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/rest/RestEmitterTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/rest/RestEmitterTest.java index 657669d19439c..a22b2736e750d 100644 --- a/metadata-integration/java/datahub-client/src/test/java/datahub/client/rest/RestEmitterTest.java +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/rest/RestEmitterTest.java @@ -11,8 +11,8 @@ import datahub.event.MetadataChangeProposalWrapper; import datahub.server.TestDataHubServer; import java.io.IOException; -import java.io.InputStream; import java.net.SocketTimeoutException; +import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -32,117 +32,148 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import javax.net.ssl.SSLHandshakeException; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.concurrent.FutureCallback; -import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.core5.http.Method; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.mockserver.matchers.Times; +import org.mockserver.model.HttpError; import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; +import org.mockserver.model.HttpStatusCode; import org.mockserver.model.RequestDefinition; +import org.mockserver.verify.VerificationTimes; @RunWith(MockitoJUnitRunner.class) public class RestEmitterTest { - @Mock HttpAsyncClientBuilder mockHttpClientFactory; - - @Mock CloseableHttpAsyncClient mockClient; - - @Captor ArgumentCaptor postArgumentCaptor; - - @Captor ArgumentCaptor callbackCaptor; + @Test + public void testPost() + throws URISyntaxException, IOException, ExecutionException, InterruptedException { + TestDataHubServer testDataHubServer = new TestDataHubServer(); + Integer port = testDataHubServer.getMockServer().getPort(); + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:" + port)); - @Before - public void setupMocks() { - Mockito.when(mockHttpClientFactory.build()).thenReturn(mockClient); + MetadataChangeProposalWrapper mcp = + getMetadataChangeProposalWrapper( + "Test Dataset", "urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)"); + Future future = emitter.emit(mcp, null); + MetadataWriteResponse response = future.get(); + String expectedContent = + "{\"proposal\":{\"aspectName\":\"datasetProperties\"," + + "\"entityUrn\":\"urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)\"," + + "\"entityType\":\"dataset\",\"changeType\":\"UPSERT\",\"aspect\":{\"contentType\":\"application/json\"" + + ",\"value\":\"{\\\"description\\\":\\\"Test Dataset\\\"}\"}}}"; + testDataHubServer + .getMockServer() + .verify( + request().withHeader("X-RestLi-Protocol-Version", "2.0.0").withBody(expectedContent)); } @Test - public void testPost() throws URISyntaxException, IOException { + public void testPostWithRetry() + throws URISyntaxException, IOException, ExecutionException, InterruptedException { + TestDataHubServer testDataHubServer = new TestDataHubServer(); + Integer port = testDataHubServer.getMockServer().getPort(); + RestEmitterConfig config = + RestEmitterConfig.builder() + .server("http://localhost:" + port) + .maxRetries(3) + .retryIntervalSec(1) + .build(); + RestEmitter emitter = new RestEmitter(config); - RestEmitter emitter = RestEmitter.create(b -> b.asyncHttpClientBuilder(mockHttpClientFactory)); MetadataChangeProposalWrapper mcp = getMetadataChangeProposalWrapper( "Test Dataset", "urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)"); - emitter.emit(mcp, null); - Mockito.verify(mockClient).execute(postArgumentCaptor.capture(), callbackCaptor.capture()); - FutureCallback callback = callbackCaptor.getValue(); - Assert.assertNotNull(callback); - HttpPost testPost = postArgumentCaptor.getValue(); - Assert.assertEquals("2.0.0", testPost.getFirstHeader("X-RestLi-Protocol-Version").getValue()); - InputStream is = testPost.getEntity().getContent(); - byte[] contentBytes = new byte[(int) testPost.getEntity().getContentLength()]; - is.read(contentBytes); - String contentString = new String(contentBytes, StandardCharsets.UTF_8); + Future future = emitter.emit(mcp, null); + MetadataWriteResponse response = future.get(); String expectedContent = "{\"proposal\":{\"aspectName\":\"datasetProperties\"," + "\"entityUrn\":\"urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)\"," + "\"entityType\":\"dataset\",\"changeType\":\"UPSERT\",\"aspect\":{\"contentType\":\"application/json\"" + ",\"value\":\"{\\\"description\\\":\\\"Test Dataset\\\"}\"}}}"; - Assert.assertEquals(expectedContent, contentString); + testDataHubServer + .getMockServer() + .verify( + request().withHeader("X-RestLi-Protocol-Version", "2.0.0").withBody(expectedContent), + VerificationTimes.exactly(1)) + .when( + request() + .withPath("/aspect") + .withHeader("X-RestLi-Protocol-Version", "2.0.0") + .withBody(expectedContent), + Times.exactly(4)) + .respond(HttpResponse.response().withStatusCode(500).withBody("exception")); } @Test public void testExceptions() throws URISyntaxException, IOException, ExecutionException, InterruptedException { - - RestEmitter emitter = RestEmitter.create($ -> $.asyncHttpClientBuilder(mockHttpClientFactory)); + TestDataHubServer testDataHubServer = new TestDataHubServer(); + Integer port = testDataHubServer.getMockServer().getPort(); + RestEmitter emitter = + RestEmitter.create( + b -> + b.server("http://localhost:" + port) + .extraHeaders(Collections.singletonMap("Test-Header", "Test-Value"))); MetadataChangeProposalWrapper mcp = - MetadataChangeProposalWrapper.create( - b -> - b.entityType("dataset") - .entityUrn("urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)") - .upsert() - .aspect(new DatasetProperties().setDescription("Test Dataset"))); - - Future mockFuture = Mockito.mock(Future.class); - Mockito.when(mockClient.execute(Mockito.any(), Mockito.any())).thenReturn(mockFuture); - Mockito.when(mockFuture.get()) - .thenThrow(new ExecutionException("Test execution exception", null)); + getMetadataChangeProposalWrapper( + "Test Dataset", "urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)"); + Future future = emitter.emit(mcp, null); + MetadataWriteResponse response = future.get(); + String expectedContent = + "{\"proposal\":{\"aspectName\":\"datasetProperties\"," + + "\"entityUrn\":\"urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)\"," + + "\"entityType\":\"dataset\",\"changeType\":\"UPSERT\",\"aspect\":{\"contentType\":\"application/json\"" + + ",\"value\":\"{\\\"description\\\":\\\"Test Dataset\\\"}\"}}}"; + testDataHubServer + .getMockServer() + .when(request(), Times.once()) + .error(HttpError.error().withDropConnection(true)); + try { emitter.emit(mcp, null).get(); Assert.fail("should not be here"); } catch (ExecutionException e) { - Assert.assertEquals(e.getMessage(), "Test execution exception"); + Assert.assertEquals( + e.getMessage(), + "org.apache.hc.core5.http.ConnectionClosedException: Connection closed by peer"); } } @Test - public void testExtraHeaders() throws Exception { + public void testExtraHeaders() + throws URISyntaxException, IOException, ExecutionException, InterruptedException { + TestDataHubServer testDataHubServer = new TestDataHubServer(); + Integer port = testDataHubServer.getMockServer().getPort(); RestEmitter emitter = RestEmitter.create( b -> - b.asyncHttpClientBuilder(mockHttpClientFactory) + b.server("http://localhost:" + port) .extraHeaders(Collections.singletonMap("Test-Header", "Test-Value"))); - MetadataChangeProposalWrapper mcpw = - MetadataChangeProposalWrapper.create( - b -> - b.entityType("dataset") - .entityUrn("urn:li:dataset:foo") - .upsert() - .aspect(new DatasetProperties())); - Future mockFuture = Mockito.mock(Future.class); - Mockito.when(mockClient.execute(Mockito.any(), Mockito.any())).thenReturn(mockFuture); - emitter.emit(mcpw, null); - Mockito.verify(mockClient).execute(postArgumentCaptor.capture(), callbackCaptor.capture()); - FutureCallback callback = callbackCaptor.getValue(); - Assert.assertNotNull(callback); - HttpPost testPost = postArgumentCaptor.getValue(); - // old headers are not modified - Assert.assertEquals("2.0.0", testPost.getFirstHeader("X-RestLi-Protocol-Version").getValue()); - // new headers are added - Assert.assertEquals("Test-Value", testPost.getFirstHeader("Test-Header").getValue()); + + MetadataChangeProposalWrapper mcp = + getMetadataChangeProposalWrapper( + "Test Dataset", "urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)"); + Future future = emitter.emit(mcp, null); + MetadataWriteResponse response = future.get(); + String expectedContent = + "{\"proposal\":{\"aspectName\":\"datasetProperties\"," + + "\"entityUrn\":\"urn:li:dataset:(urn:li:dataPlatform:hive,foo.bar,PROD)\"," + + "\"entityType\":\"dataset\",\"changeType\":\"UPSERT\",\"aspect\":{\"contentType\":\"application/json\"" + + ",\"value\":\"{\\\"description\\\":\\\"Test Dataset\\\"}\"}}}"; + testDataHubServer + .getMockServer() + .verify( + request() + .withHeader("Test-Header", "Test-Value") + .withHeader("X-RestLi-Protocol-Version", "2.0.0") + .withBody(expectedContent)); } @Test @@ -168,7 +199,7 @@ public void multithreadedTestExecutors() throws Exception { .withQueryStringParameter("action", "ingestProposal") .withHeader("Content-type", "application/json"), Times.unlimited()) - .respond(org.mockserver.model.HttpResponse.response().withStatusCode(200)); + .respond(HttpResponse.response().withStatusCode(200)); ExecutorService executor = Executors.newFixedThreadPool(10); ArrayList results = new ArrayList(); Random random = new Random(); @@ -476,26 +507,27 @@ public void testUserAgentHeader() throws IOException, ExecutionException, Interr @Test public void testDisableSslVerification() - throws IOException, InterruptedException, ExecutionException { + throws IOException, InterruptedException, ExecutionException, URISyntaxException { RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().disableSslVerification(true).build()); final String hostWithSsl = "https://self-signed.badssl.com"; - final HttpGet request = new HttpGet(hostWithSsl); + final SimpleHttpRequest request = SimpleHttpRequest.create(Method.GET, new URI(hostWithSsl)); - final HttpResponse response = restEmitter.getHttpClient().execute(request, null).get(); + final SimpleHttpResponse response = restEmitter.getHttpClient().execute(request, null).get(); restEmitter.close(); - Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + Assert.assertEquals(HttpStatusCode.OK_200.code(), response.getCode()); } @Test public void testSslVerificationException() - throws IOException, InterruptedException, ExecutionException { + throws IOException, InterruptedException, ExecutionException, URISyntaxException { RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().disableSslVerification(false).build()); final String hostWithSsl = "https://self-signed.badssl.com"; - final HttpGet request = new HttpGet(hostWithSsl); + final SimpleHttpRequest request = SimpleHttpRequest.create(Method.GET, new URI(hostWithSsl)); + try { - HttpResponse response = restEmitter.getHttpClient().execute(request, null).get(); + SimpleHttpResponse response = restEmitter.getHttpClient().execute(request, null).get(); Assert.fail(); } catch (Exception e) { Assert.assertTrue(e instanceof ExecutionException); diff --git a/metadata-integration/java/datahub-event/build.gradle b/metadata-integration/java/datahub-event/build.gradle index a516b9d43da4b..395065404d1db 100644 --- a/metadata-integration/java/datahub-event/build.gradle +++ b/metadata-integration/java/datahub-event/build.gradle @@ -25,7 +25,7 @@ dependencies { testImplementation externalDependency.testng testImplementation externalDependency.mockito testImplementation externalDependency.testContainers - testImplementation externalDependency.httpAsyncClient + testImplementation externalDependency.httpClient testRuntimeOnly externalDependency.logbackClassicJava8 } diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java index 038e8d33a97c4..59cac8719c303 100644 --- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java +++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java @@ -621,6 +621,11 @@ private static void processParentJob( private static void processJobInputs( DatahubJob datahubJob, OpenLineage.RunEvent event, DatahubOpenlineageConfig datahubConf) { + + if (event.getInputs() == null) { + return; + } + for (OpenLineage.InputDataset input : event.getInputs().stream() .filter(input -> input.getFacets() != null) @@ -646,6 +651,11 @@ private static void processJobInputs( private static void processJobOutputs( DatahubJob datahubJob, OpenLineage.RunEvent event, DatahubOpenlineageConfig datahubConf) { + + if (event.getOutputs() == null) { + return; + } + for (OpenLineage.OutputDataset output : event.getOutputs().stream() .filter(input -> input.getFacets() != null) diff --git a/metadata-integration/java/spark-lineage-beta/README.md b/metadata-integration/java/spark-lineage-beta/README.md index 6a520071ba797..e09bc3938b686 100644 --- a/metadata-integration/java/spark-lineage-beta/README.md +++ b/metadata-integration/java/spark-lineage-beta/README.md @@ -24,7 +24,7 @@ When running jobs using spark-submit, the agent needs to be configured in the co ```text #Configuring DataHub spark agent jar -spark.jars.packages io.acryl:acryl-spark-lineage:0.2.1 +spark.jars.packages io.acryl:acryl-spark-lineage:0.2.3 spark.extraListeners datahub.spark.DatahubSparkListener spark.datahub.rest.server http://localhost:8080 ``` @@ -32,7 +32,7 @@ spark.datahub.rest.server http://localhost:8080 ## spark-submit command line ```sh -spark-submit --packages io.acryl:acryl-spark-lineage:0.2.1 --conf "spark.extraListeners=datahub.spark.DatahubSparkListener" my_spark_job_to_run.py +spark-submit --packages io.acryl:acryl-spark-lineage:0.2.3 --conf "spark.extraListeners=datahub.spark.DatahubSparkListener" my_spark_job_to_run.py ``` ### Configuration Instructions: Amazon EMR @@ -41,7 +41,7 @@ Set the following spark-defaults configuration properties as it stated [here](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark-configure.html) ```text -spark.jars.packages io.acryl:acryl-spark-lineage:0.2.1 +spark.jars.packages io.acryl:acryl-spark-lineage:0.2.3 spark.extraListeners datahub.spark.DatahubSparkListener spark.datahub.rest.server https://your_datahub_host/gms #If you have authentication set up then you also need to specify the Datahub access token @@ -56,7 +56,7 @@ When running interactive jobs from a notebook, the listener can be configured wh spark = SparkSession.builder .master("spark://spark-master:7077") .appName("test-application") -.config("spark.jars.packages", "io.acryl:acryl-spark-lineage:0.1.0") +.config("spark.jars.packages", "io.acryl:acryl-spark-lineage:0.2.3") .config("spark.extraListeners", "datahub.spark.DatahubSparkListener") .config("spark.datahub.rest.server", "http://localhost:8080") .enableHiveSupport() @@ -79,7 +79,7 @@ appName("test-application") config("spark.master","spark://spark-master:7077") . -config("spark.jars.packages","io.acryl:acryl-spark-lineage:0.2.1") +config("spark.jars.packages","io.acryl:acryl-spark-lineage:0.2.3") . config("spark.extraListeners","datahub.spark.DatahubSparkListener") @@ -164,6 +164,8 @@ information like tokens. | spark.datahub.rest.server | ✅ | | Datahub server url eg: | | spark.datahub.rest.token | | | Authentication token. | | spark.datahub.rest.disable_ssl_verification | | false | Disable SSL certificate validation. Caution: Only use this if you know what you are doing! | +| spark.datahub.rest.rest.max_retries | | 0 | Number of times a request retried if failed | +| spark.datahub.rest.rest.retry_interval | | 10 | Number of seconds to wait between retries | | spark.datahub.metadata.pipeline.platformInstance | | | Pipeline level platform instance | | spark.datahub.metadata.dataset.platformInstance | | | dataset level platform instance | | spark.datahub.metadata.dataset.env | | PROD | [Supported values](https://datahubproject.io/docs/graphql/enums#fabrictype). In all other cases, will fallback to PROD | @@ -180,7 +182,7 @@ information like tokens. | spark.datahub.tags | | | Comma separated list of tags to attach to the DataFlow | | spark.datahub.domains | | | Comma separated list of domain urns to attach to the DataFlow | | spark.datahub.stage_metadata_coalescing | | | Normally it coalesce and send metadata at the onApplicationEnd event which is never called on Databricsk. You should enable this on Databricks if you want coalesced run . | -| spark.datahub.patch.enabled | | | Set this to true to send lineage as a patch, which appends rather than overwrites existing Dataset lineage edges. By default it is enabled. +| spark.datahub.patch.enabled | | false | Set this to true to send lineage as a patch, which appends rather than overwrites existing Dataset lineage edges. By default it is enabled. | ## What to Expect: The Metadata Model diff --git a/metadata-integration/java/spark-lineage-beta/build.gradle b/metadata-integration/java/spark-lineage-beta/build.gradle index 4cd2ddfec3dfc..d83753028d0b4 100644 --- a/metadata-integration/java/spark-lineage-beta/build.gradle +++ b/metadata-integration/java/spark-lineage-beta/build.gradle @@ -37,7 +37,7 @@ dependencies { provided(externalDependency.sparkSql) provided(externalDependency.sparkHive) implementation 'org.slf4j:slf4j-log4j12:2.0.7' - implementation externalDependency.httpAsyncClient + implementation externalDependency.httpClient implementation externalDependency.logbackClassicJava8 implementation externalDependency.typesafeConfig implementation externalDependency.commonsLang @@ -53,7 +53,7 @@ dependencies { implementation project(path: ':metadata-integration:java:openlineage-converter', configuration: 'shadow') //implementation "io.acryl:datahub-client:0.10.2" - implementation "io.openlineage:openlineage-spark:$openLineageVersion" + implementation "io.openlineage:openlineage-spark_2.12:$openLineageVersion" compileOnly "org.apache.iceberg:iceberg-spark3-runtime:0.12.1" compileOnly "org.apache.spark:spark-sql_2.12:3.1.3" @@ -123,7 +123,7 @@ shadowJar { relocate 'com.fasterxml.jackson', 'datahub.spark2.shaded.jackson' relocate 'org.slf4j', 'datahub.spark2.shaded.org.slf4j' // - relocate 'org.apache.http', 'io.acryl.shaded.http' + relocate 'org.apache.hc', 'io.acryl.shaded.http' relocate 'org.apache.commons.codec', 'datahub.spark2.shaded.o.a.c.codec' relocate 'org.apache.commons.compress', 'datahub.spark2.shaded.o.a.c.compress' relocate 'org.apache.commons.lang3', 'datahub.spark2.shaded.o.a.c.lang3' diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubEventEmitter.java b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubEventEmitter.java index 6b430c5c2ab26..1dc086e4af585 100644 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubEventEmitter.java +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubEventEmitter.java @@ -23,8 +23,8 @@ import io.datahubproject.openlineage.dataset.DatahubJob; import io.openlineage.client.OpenLineage; import io.openlineage.client.OpenLineageClientUtils; -import io.openlineage.spark.agent.ArgumentParser; import io.openlineage.spark.agent.EventEmitter; +import io.openlineage.spark.api.SparkOpenLineageConfig; import java.io.IOException; import java.net.URISyntaxException; import java.time.Instant; @@ -44,7 +44,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; -import org.apache.spark.SparkConf; import org.apache.spark.sql.streaming.StreamingQueryProgress; @Slf4j @@ -55,10 +54,11 @@ public class DatahubEventEmitter extends EventEmitter { private final Map schemaMap = new HashMap<>(); private SparkLineageConf datahubConf; - private EventFormatter eventFormatter = new EventFormatter(); + private final EventFormatter eventFormatter = new EventFormatter(); - public DatahubEventEmitter() throws URISyntaxException { - super(ArgumentParser.parse(new SparkConf())); + public DatahubEventEmitter(SparkOpenLineageConfig config, String applicationJobName) + throws URISyntaxException { + super(config, applicationJobName); } private Optional getEmitter() { @@ -167,7 +167,7 @@ public List generateCoalescedMcps() { List mcps = new ArrayList<>(); if (_datahubJobs.isEmpty()) { - log.warn("No lineage events to emit. Maybe the spark job finished premaraturely?"); + log.warn("No lineage events to emit. Maybe the spark job finished prematurely?"); return mcps; } diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubSparkListener.java b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubSparkListener.java index 060402723d194..38de142c4dd17 100644 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubSparkListener.java +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubSparkListener.java @@ -1,6 +1,7 @@ package datahub.spark; import static datahub.spark.conf.SparkConfigParser.*; +import static io.openlineage.spark.agent.util.ScalaConversionUtils.*; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; @@ -10,16 +11,33 @@ import datahub.spark.conf.SparkAppContext; import datahub.spark.conf.SparkConfigParser; import datahub.spark.conf.SparkLineageConf; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.openlineage.client.OpenLineageConfig; +import io.openlineage.client.circuitBreaker.CircuitBreaker; +import io.openlineage.client.circuitBreaker.CircuitBreakerFactory; +import io.openlineage.client.circuitBreaker.NoOpCircuitBreaker; +import io.openlineage.client.metrics.MicrometerProvider; +import io.openlineage.spark.agent.ArgumentParser; import io.openlineage.spark.agent.OpenLineageSparkListener; +import io.openlineage.spark.agent.Versions; import io.openlineage.spark.agent.lifecycle.ContextFactory; +import io.openlineage.spark.agent.util.ScalaConversionUtils; +import io.openlineage.spark.api.SparkOpenLineageConfig; import java.net.URISyntaxException; import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Properties; +import org.apache.spark.SparkConf; +import org.apache.spark.SparkContext; +import org.apache.spark.SparkContext$; import org.apache.spark.SparkEnv; import org.apache.spark.SparkEnv$; +import org.apache.spark.package$; import org.apache.spark.scheduler.SparkListener; import org.apache.spark.scheduler.SparkListenerApplicationEnd; import org.apache.spark.scheduler.SparkListenerApplicationStart; @@ -30,20 +48,28 @@ import org.apache.spark.sql.streaming.StreamingQueryListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Function0; +import scala.Option; public class DatahubSparkListener extends SparkListener { private static final Logger log = LoggerFactory.getLogger(DatahubSparkListener.class); private final Map batchLastUpdated = new HashMap(); private final OpenLineageSparkListener listener; - private final DatahubEventEmitter emitter; + private DatahubEventEmitter emitter; private Config datahubConf = ConfigFactory.empty(); private SparkAppContext appContext; + private static ContextFactory contextFactory; + private static CircuitBreaker circuitBreaker = new NoOpCircuitBreaker(); + private static final String sparkVersion = package$.MODULE$.SPARK_VERSION(); + + private final Function0> activeSparkContext = + ScalaConversionUtils.toScalaFn(SparkContext$.MODULE$::getActive); + + private static MeterRegistry meterRegistry; + private boolean isDisabled; public DatahubSparkListener() throws URISyntaxException { listener = new OpenLineageSparkListener(); - emitter = new DatahubEventEmitter(); - ContextFactory contextFactory = new ContextFactory(emitter); - OpenLineageSparkListener.init(contextFactory); } private static SparkAppContext getSparkAppContext( @@ -61,13 +87,14 @@ private static SparkAppContext getSparkAppContext( public void onApplicationStart(SparkListenerApplicationStart applicationStart) { long startTime = System.currentTimeMillis(); + initializeContextFactoryIfNotInitialized(); - log.debug("Application start called"); + log.info("Application start called"); this.appContext = getSparkAppContext(applicationStart); listener.onApplicationStart(applicationStart); long elapsedTime = System.currentTimeMillis() - startTime; - log.debug("onApplicationStart completed successfully in {} ms", elapsedTime); + log.info("onApplicationStart completed successfully in {} ms", elapsedTime); } public Optional initializeEmitter(Config sparkConf) { @@ -87,6 +114,17 @@ public Optional initializeEmitter(Config sparkConf) { boolean disableSslVerification = sparkConf.hasPath(SparkConfigParser.DISABLE_SSL_VERIFICATION_KEY) && sparkConf.getBoolean(SparkConfigParser.DISABLE_SSL_VERIFICATION_KEY); + + int retry_interval_in_sec = + sparkConf.hasPath(SparkConfigParser.RETRY_INTERVAL_IN_SEC) + ? sparkConf.getInt(SparkConfigParser.RETRY_INTERVAL_IN_SEC) + : 5; + + int max_retries = + sparkConf.hasPath(SparkConfigParser.MAX_RETRIES) + ? sparkConf.getInt(SparkConfigParser.MAX_RETRIES) + : 0; + log.info( "REST Emitter Configuration: GMS url {}{}", gmsUrl, @@ -94,14 +132,18 @@ public Optional initializeEmitter(Config sparkConf) { if (token != null) { log.info("REST Emitter Configuration: Token {}", "XXXXX"); } + if (disableSslVerification) { log.warn("REST Emitter Configuration: ssl verification will be disabled."); } + RestEmitterConfig restEmitterConf = RestEmitterConfig.builder() .server(gmsUrl) .token(token) .disableSslVerification(disableSslVerification) + .maxRetries(max_retries) + .retryIntervalSec(retry_interval_in_sec) .build(); return Optional.of(new RestDatahubEmitterConfig(restEmitterConf)); } else { @@ -145,7 +187,12 @@ public void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) { if (datahubConf.hasPath(STREAMING_JOB) && (datahubConf.getBoolean(STREAMING_JOB))) { return; } - emitter.emitCoalesced(); + if (emitter != null) { + emitter.emitCoalesced(); + } else { + log.warn("Emitter is not initialized, unable to emit coalesced events"); + } + long elapsedTime = System.currentTimeMillis() - startTime; log.debug("onApplicationEnd completed successfully in {} ms", elapsedTime); } @@ -170,6 +217,8 @@ public void onJobEnd(SparkListenerJobEnd jobEnd) { public void onJobStart(SparkListenerJobStart jobStart) { long startTime = System.currentTimeMillis(); + initializeContextFactoryIfNotInitialized(); + log.debug("Job start called"); loadDatahubConfig(this.appContext, jobStart.properties()); listener.onJobStart(jobStart); @@ -227,4 +276,72 @@ public void onOtherEvent(SparkListenerEvent event) { log.debug("onOtherEvent completed successfully in {} ms", elapsedTime); } } + + private static void initializeMetrics(OpenLineageConfig openLineageConfig) { + meterRegistry = + MicrometerProvider.addMeterRegistryFromConfig(openLineageConfig.getMetricsConfig()); + String disabledFacets; + if (openLineageConfig.getFacetsConfig() != null + && openLineageConfig.getFacetsConfig().getDisabledFacets() != null) { + disabledFacets = String.join(";", openLineageConfig.getFacetsConfig().getDisabledFacets()); + } else { + disabledFacets = ""; + } + meterRegistry + .config() + .commonTags( + Tags.of( + Tag.of("openlineage.spark.integration.version", Versions.getVersion()), + Tag.of("openlineage.spark.version", sparkVersion), + Tag.of("openlineage.spark.disabled.facets", disabledFacets))); + ((CompositeMeterRegistry) meterRegistry) + .getRegistries() + .forEach( + r -> + r.config() + .commonTags( + Tags.of( + Tag.of("openlineage.spark.integration.version", Versions.getVersion()), + Tag.of("openlineage.spark.version", sparkVersion), + Tag.of("openlineage.spark.disabled.facets", disabledFacets)))); + } + + private void initializeContextFactoryIfNotInitialized() { + if (contextFactory != null || isDisabled) { + return; + } + asJavaOptional(activeSparkContext.apply()) + .ifPresent(context -> initializeContextFactoryIfNotInitialized(context.appName())); + } + + private void initializeContextFactoryIfNotInitialized(String appName) { + if (contextFactory != null || isDisabled) { + return; + } + SparkEnv sparkEnv = SparkEnv$.MODULE$.get(); + if (sparkEnv == null) { + log.warn( + "OpenLineage listener instantiated, but no configuration could be found. " + + "Lineage events will not be collected"); + return; + } + initializeContextFactoryIfNotInitialized(sparkEnv.conf(), appName); + } + + private void initializeContextFactoryIfNotInitialized(SparkConf sparkConf, String appName) { + if (contextFactory != null || isDisabled) { + return; + } + try { + SparkOpenLineageConfig config = ArgumentParser.parse(sparkConf); + // Needs to be done before initializing OpenLineageClient + initializeMetrics(config); + emitter = new DatahubEventEmitter(config, appName); + contextFactory = new ContextFactory(emitter, meterRegistry, config); + circuitBreaker = new CircuitBreakerFactory(config.getCircuitBreaker()).build(); + OpenLineageSparkListener.init(contextFactory); + } catch (URISyntaxException e) { + log.error("Unable to parse OpenLineage endpoint. Lineage events will not be collected", e); + } + } } diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/conf/SparkConfigParser.java b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/conf/SparkConfigParser.java index 7e10f51feb38a..f1af56ff888d3 100644 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/conf/SparkConfigParser.java +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/conf/SparkConfigParser.java @@ -29,6 +29,9 @@ public class SparkConfigParser { public static final String GMS_URL_KEY = "rest.server"; public static final String GMS_AUTH_TOKEN = "rest.token"; public static final String DISABLE_SSL_VERIFICATION_KEY = "rest.disable_ssl_verification"; + public static final String MAX_RETRIES = "rest.max_retries"; + public static final String RETRY_INTERVAL_IN_SEC = "rest.retry_interval_in_sec"; + public static final String COALESCE_KEY = "coalesce_jobs"; public static final String PATCH_ENABLED = "patch.enabled"; @@ -304,7 +307,7 @@ public static boolean isCoalesceEnabled(Config datahubConfig) { public static boolean isPatchEnabled(Config datahubConfig) { if (!datahubConfig.hasPath(PATCH_ENABLED)) { - return true; + return false; } return datahubConfig.hasPath(PATCH_ENABLED) && datahubConfig.getBoolean(PATCH_ENABLED); } diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/OpenLineageRunEventBuilder.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/OpenLineageRunEventBuilder.java deleted file mode 100644 index 99643592dc200..0000000000000 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/OpenLineageRunEventBuilder.java +++ /dev/null @@ -1,493 +0,0 @@ -/* -/* Copyright 2018-2023 contributors to the OpenLineage project -/* SPDX-License-Identifier: Apache-2.0 -*/ - -package io.openlineage.spark.agent.lifecycle; - -import static io.openlineage.client.OpenLineageClientUtils.mergeFacets; -import static io.openlineage.spark.agent.util.ScalaConversionUtils.fromSeq; -import static io.openlineage.spark.agent.util.ScalaConversionUtils.toScalaFn; - -import io.openlineage.client.OpenLineage; -import io.openlineage.client.OpenLineage.DatasetFacet; -import io.openlineage.client.OpenLineage.DatasetFacets; -import io.openlineage.client.OpenLineage.InputDataset; -import io.openlineage.client.OpenLineage.InputDatasetFacet; -import io.openlineage.client.OpenLineage.InputDatasetInputFacets; -import io.openlineage.client.OpenLineage.JobBuilder; -import io.openlineage.client.OpenLineage.JobFacet; -import io.openlineage.client.OpenLineage.OutputDataset; -import io.openlineage.client.OpenLineage.OutputDatasetFacet; -import io.openlineage.client.OpenLineage.OutputDatasetOutputFacets; -import io.openlineage.client.OpenLineage.ParentRunFacet; -import io.openlineage.client.OpenLineage.RunEvent; -import io.openlineage.client.OpenLineage.RunEventBuilder; -import io.openlineage.client.OpenLineage.RunFacet; -import io.openlineage.client.OpenLineage.RunFacets; -import io.openlineage.client.OpenLineage.RunFacetsBuilder; -import io.openlineage.spark.agent.hooks.HookUtils; -import io.openlineage.spark.agent.lifecycle.plan.column.ColumnLevelLineageUtils; -import io.openlineage.spark.agent.lifecycle.plan.column.ColumnLevelLineageVisitor; -import io.openlineage.spark.agent.util.FacetUtils; -import io.openlineage.spark.agent.util.PlanUtils; -import io.openlineage.spark.agent.util.ScalaConversionUtils; -import io.openlineage.spark.api.CustomFacetBuilder; -import io.openlineage.spark.api.OpenLineageContext; -import io.openlineage.spark.api.OpenLineageEventHandlerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import lombok.AllArgsConstructor; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.apache.spark.rdd.RDD; -import org.apache.spark.scheduler.ActiveJob; -import org.apache.spark.scheduler.JobFailed; -import org.apache.spark.scheduler.SparkListenerJobEnd; -import org.apache.spark.scheduler.SparkListenerJobStart; -import org.apache.spark.scheduler.SparkListenerStageCompleted; -import org.apache.spark.scheduler.SparkListenerStageSubmitted; -import org.apache.spark.scheduler.Stage; -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; -import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd; -import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart; -import scala.Function1; -import scala.PartialFunction; - -/** - * Event handler that accepts various {@link org.apache.spark.scheduler.SparkListener} events and - * helps build up an {@link RunEvent} by passing event components to partial functions that know how - * to convert those event components into {@link RunEvent} properties. - * - *

The event types that can be consumed to generate @link OpenLineage.RunEvent} properties have - * no common supertype, so the generic argument for the function input is simply {@link Object}. The - * types of arguments that may be found include - * - *

    - *
  • {@link org.apache.spark.scheduler.StageInfo} - *
  • {@link Stage} - *
  • {@link RDD} - *
  • {@link ActiveJob} - *
  • {@link org.apache.spark.sql.execution.QueryExecution} - *
- * - *

These components are extracted from various {@link org.apache.spark.scheduler.SparkListener} - * events, such as {@link SparkListenerStageCompleted}, {@link SparkListenerJobStart}, and {@link - * org.apache.spark.scheduler.SparkListenerTaskEnd}. - * - *

{@link RDD} chains will be _flattened_ so each `RDD` dependency is passed to the builders one - * at a time. This means a builder can directly specify the type of {@link RDD} it handles, such as - * a {@link org.apache.spark.rdd.HadoopRDD} or a {@link - * org.apache.spark.sql.execution.datasources.FileScanRDD}, without having to check the dependencies - * of every {@link org.apache.spark.rdd.MapPartitionsRDD} or {@link - * org.apache.spark.sql.execution.SQLExecutionRDD}. - * - *

Any {@link RunFacet}s and {@link JobFacet}s returned by the {@link CustomFacetBuilder}s are - * appended to the {@link OpenLineage.Run} and {@link OpenLineage.Job}, respectively. - * - *

If any {@link OpenLineage.InputDatasetBuilder}s or {@link - * OpenLineage.OutputDatasetBuilder}s are returned from the partial functions, the {@link - * #inputDatasetBuilders} or {@link #outputDatasetBuilders} will be invoked using the same input - * arguments in order to construct any {@link InputDatasetFacet}s or {@link OutputDatasetFacet}s to - * the returned dataset. {@link InputDatasetFacet}s and {@link OutputDatasetFacet}s will be attached - * to any {@link OpenLineage.InputDatasetBuilder} or {@link OpenLineage.OutputDatasetBuilder} - * found for the event. This is because facets may be constructed from generic information that is - * not specifically tied to a Dataset. For example, {@link - * OpenLineage.OutputStatisticsOutputDatasetFacet}s are created from {@link - * org.apache.spark.executor.TaskMetrics} attached to the last {@link - * org.apache.spark.scheduler.StageInfo} for a given job execution. However, the {@link - * OutputDataset} is constructed by reading the {@link LogicalPlan}. There's no way to tie the - * output metrics in the {@link org.apache.spark.scheduler.StageInfo} to the {@link OutputDataset} - * in the {@link LogicalPlan} except by inference. Similarly, input metrics can be found in the - * {@link org.apache.spark.scheduler.StageInfo} for the stage that reads a dataset and the {@link - * InputDataset} can usually be constructed by walking the {@link RDD} dependency tree for that - * {@link Stage} and finding a {@link org.apache.spark.sql.execution.datasources.FileScanRDD} or - * other concrete implementation. But while there is typically only one {@link InputDataset} read in - * a given stage, there's no guarantee of that and the {@link org.apache.spark.executor.TaskMetrics} - * in the {@link org.apache.spark.scheduler.StageInfo} won't disambiguate. - * - *

If a facet needs to be attached to a specific dataset, the user must take care to construct - * both the Dataset and the Facet in the same builder. - */ -@Slf4j -@AllArgsConstructor -class OpenLineageRunEventBuilder { - - @NonNull private final OpenLineageContext openLineageContext; - - @NonNull - private final Collection>> inputDatasetBuilders; - - @NonNull - private final Collection>> - inputDatasetQueryPlanVisitors; - - @NonNull - private final Collection>> outputDatasetBuilders; - - @NonNull - private final Collection>> - outputDatasetQueryPlanVisitors; - - @NonNull - private final Collection> datasetFacetBuilders; - - @NonNull - private final Collection> - inputDatasetFacetBuilders; - - @NonNull - private final Collection> - outputDatasetFacetBuilders; - - @NonNull private final Collection> runFacetBuilders; - @NonNull private final Collection> jobFacetBuilders; - @NonNull private final Collection columnLineageVisitors; - private final UnknownEntryFacetListener unknownEntryFacetListener = - UnknownEntryFacetListener.getInstance(); - private final Map jobMap = new HashMap<>(); - private final Map stageMap = new HashMap<>(); - - OpenLineageRunEventBuilder(OpenLineageContext context, OpenLineageEventHandlerFactory factory) { - this( - context, - factory.createInputDatasetBuilder(context), - factory.createInputDatasetQueryPlanVisitors(context), - factory.createOutputDatasetBuilder(context), - factory.createOutputDatasetQueryPlanVisitors(context), - factory.createDatasetFacetBuilders(context), - factory.createInputDatasetFacetBuilders(context), - factory.createOutputDatasetFacetBuilders(context), - factory.createRunFacetBuilders(context), - factory.createJobFacetBuilders(context), - factory.createColumnLevelLineageVisitors(context)); - } - - /** - * Add an {@link ActiveJob} and all of its {@link Stage}s to the maps so we can look them up by id - * later. - * - * @param job - */ - void registerJob(ActiveJob job) { - jobMap.put(job.jobId(), job); - stageMap.put(job.finalStage().id(), job.finalStage()); - job.finalStage() - .parents() - .forall( - toScalaFn( - stage -> { - stageMap.put(stage.id(), stage); - return true; - })); - } - - RunEvent buildRun( - Optional parentRunFacet, - RunEventBuilder runEventBuilder, - JobBuilder jobBuilder, - SparkListenerStageSubmitted event) { - Stage stage = stageMap.get(event.stageInfo().stageId()); - RDD rdd = stage.rdd(); - - List nodes = new ArrayList<>(); - nodes.addAll(Arrays.asList(event.stageInfo(), stage)); - - nodes.addAll(Rdds.flattenRDDs(rdd)); - - return populateRun(parentRunFacet, runEventBuilder, jobBuilder, nodes); - } - - RunEvent buildRun( - Optional parentRunFacet, - RunEventBuilder runEventBuilder, - JobBuilder jobBuilder, - SparkListenerStageCompleted event) { - Stage stage = stageMap.get(event.stageInfo().stageId()); - RDD rdd = stage.rdd(); - - List nodes = new ArrayList<>(); - nodes.addAll(Arrays.asList(event.stageInfo(), stage)); - - nodes.addAll(Rdds.flattenRDDs(rdd)); - - return populateRun(parentRunFacet, runEventBuilder, jobBuilder, nodes); - } - - RunEvent buildRun( - Optional parentRunFacet, - RunEventBuilder runEventBuilder, - JobBuilder jobBuilder, - SparkListenerSQLExecutionStart event) { - runEventBuilder.eventType(RunEvent.EventType.START); - return buildRun(parentRunFacet, runEventBuilder, jobBuilder, event, Optional.empty()); - } - - RunEvent buildRun( - Optional parentRunFacet, - RunEventBuilder runEventBuilder, - JobBuilder jobBuilder, - SparkListenerSQLExecutionEnd event) { - runEventBuilder.eventType(RunEvent.EventType.COMPLETE); - return buildRun(parentRunFacet, runEventBuilder, jobBuilder, event, Optional.empty()); - } - - RunEvent buildRun( - Optional parentRunFacet, - RunEventBuilder runEventBuilder, - JobBuilder jobBuilder, - SparkListenerJobStart event) { - runEventBuilder.eventType(RunEvent.EventType.START); - return buildRun( - parentRunFacet, - runEventBuilder, - jobBuilder, - event, - Optional.ofNullable(jobMap.get(event.jobId()))); - } - - RunEvent buildRun( - Optional parentRunFacet, - RunEventBuilder runEventBuilder, - JobBuilder jobBuilder, - SparkListenerJobEnd event) { - runEventBuilder.eventType( - event.jobResult() instanceof JobFailed - ? RunEvent.EventType.FAIL - : RunEvent.EventType.COMPLETE); - return buildRun( - parentRunFacet, - runEventBuilder, - jobBuilder, - event, - Optional.ofNullable(jobMap.get(event.jobId()))); - } - - private RunEvent buildRun( - Optional parentRunFacet, - RunEventBuilder runEventBuilder, - JobBuilder jobBuilder, - Object event, - Optional job) { - List nodes = new ArrayList<>(); - nodes.add(event); - job.ifPresent( - j -> { - nodes.add(j); - nodes.addAll(Rdds.flattenRDDs(j.finalStage().rdd())); - }); - - return populateRun(parentRunFacet, runEventBuilder, jobBuilder, nodes); - } - - private RunEvent populateRun( - Optional parentRunFacet, - RunEventBuilder runEventBuilder, - JobBuilder jobBuilder, - List nodes) { - OpenLineage openLineage = openLineageContext.getOpenLineage(); - - RunFacetsBuilder runFacetsBuilder = openLineage.newRunFacetsBuilder(); - OpenLineage.JobFacetsBuilder jobFacetsBuilder = - openLineageContext.getOpenLineage().newJobFacetsBuilder(); - - parentRunFacet.ifPresent(runFacetsBuilder::parent); - OpenLineage.JobFacets jobFacets = buildJobFacets(nodes, jobFacetBuilders, jobFacetsBuilder); - List inputDatasets = buildInputDatasets(nodes); - List outputDatasets = buildOutputDatasets(nodes); - openLineageContext - .getQueryExecution() - .filter(qe -> !FacetUtils.isFacetDisabled(openLineageContext, "spark_unknown")) - .flatMap(qe -> unknownEntryFacetListener.build(qe.optimizedPlan())) - .ifPresent(facet -> runFacetsBuilder.put("spark_unknown", facet)); - - RunFacets runFacets = buildRunFacets(nodes, runFacetBuilders, runFacetsBuilder); - OpenLineage.RunBuilder runBuilder = - openLineage.newRunBuilder().runId(openLineageContext.getRunUuid()).facets(runFacets); - runEventBuilder - .run(runBuilder.build()) - .job(jobBuilder.facets(jobFacets).build()) - .inputs(inputDatasets) - .outputs(outputDatasets); - - HookUtils.preBuild(openLineageContext, runEventBuilder); - return runEventBuilder.build(); - } - - private List buildInputDatasets(List nodes) { - openLineageContext - .getQueryExecution() - .ifPresent( - qe -> { - if (log.isDebugEnabled()) { - log.debug("Traversing optimized plan {}", qe.optimizedPlan().toJSON()); - log.debug("Physical plan executed {}", qe.executedPlan().toJSON()); - } - }); - log.debug( - "Visiting query plan {} with input dataset builders {}", - openLineageContext.getQueryExecution(), - inputDatasetBuilders); - - Function1> inputVisitor = - visitLogicalPlan(PlanUtils.merge(inputDatasetQueryPlanVisitors)); - - List datasets = - Stream.concat( - buildDatasets(nodes, inputDatasetBuilders), - openLineageContext - .getQueryExecution() - .map( - qe -> - fromSeq(qe.optimizedPlan().map(inputVisitor)).stream() - .flatMap(Collection::stream) - .map(((Class) InputDataset.class)::cast)) - .orElse(Stream.empty())) - .collect(Collectors.toList()); - OpenLineage openLineage = openLineageContext.getOpenLineage(); - if (!datasets.isEmpty()) { - Map inputFacetsMap = new HashMap<>(); - nodes.forEach( - event -> inputDatasetFacetBuilders.forEach(fn -> fn.accept(event, inputFacetsMap::put))); - Map datasetFacetsMap = new HashMap<>(); - nodes.forEach( - event -> inputDatasetFacetBuilders.forEach(fn -> fn.accept(event, inputFacetsMap::put))); - return datasets.stream() - .map( - ds -> - openLineage - .newInputDatasetBuilder() - .name(ds.getName()) - .namespace(ds.getNamespace()) - .inputFacets( - mergeFacets( - inputFacetsMap, ds.getInputFacets(), InputDatasetInputFacets.class)) - .facets(mergeFacets(datasetFacetsMap, ds.getFacets(), DatasetFacets.class)) - .build()) - .collect(Collectors.toList()); - } - return datasets; - } - - /** - * Returns a {@link Function1} that passes the input {@link LogicalPlan} node to the {@link - * #unknownEntryFacetListener} if the inputVisitor is defined for the input node. - * - * @param inputVisitor - * @param - * @return - */ - private Function1> visitLogicalPlan( - PartialFunction> inputVisitor) { - return ScalaConversionUtils.toScalaFn( - node -> - inputVisitor - .andThen( - toScalaFn( - ds -> { - unknownEntryFacetListener.accept(node); - return ds; - })) - .applyOrElse(node, toScalaFn(n -> Collections.emptyList()))); - } - - private List buildOutputDatasets(List nodes) { - log.debug( - "Visiting query plan {} with output dataset builders {}", - openLineageContext.getQueryExecution(), - outputDatasetBuilders); - Function1> visitor = - visitLogicalPlan(PlanUtils.merge(outputDatasetQueryPlanVisitors)); - List datasets = - Stream.concat( - buildDatasets(nodes, outputDatasetBuilders), - openLineageContext - .getQueryExecution() - .map(qe -> visitor.apply(qe.optimizedPlan())) - .map(Collection::stream) - .orElse(Stream.empty())) - .collect(Collectors.toList()); - - OpenLineage openLineage = openLineageContext.getOpenLineage(); - - if (!datasets.isEmpty()) { - Map outputFacetsMap = new HashMap<>(); - nodes.forEach( - event -> - outputDatasetFacetBuilders.forEach(fn -> fn.accept(event, outputFacetsMap::put))); - Map datasetFacetsMap = new HashMap<>(); - nodes.forEach( - event -> datasetFacetBuilders.forEach(fn -> fn.accept(event, datasetFacetsMap::put))); - return datasets.stream() - .map( - ds -> { - Map dsFacetsMap = new HashMap(datasetFacetsMap); - ColumnLevelLineageUtils.buildColumnLineageDatasetFacet( - openLineageContext, ds.getFacets().getSchema()) - .ifPresent(facet -> dsFacetsMap.put("columnLineage", facet)); - return openLineage - .newOutputDatasetBuilder() - .name(ds.getName()) - .namespace(ds.getNamespace()) - .outputFacets( - mergeFacets( - outputFacetsMap, ds.getOutputFacets(), OutputDatasetOutputFacets.class)) - .facets(mergeFacets(dsFacetsMap, ds.getFacets(), DatasetFacets.class)) - .build(); - }) - .collect(Collectors.toList()); - } - return datasets; - } - - private Stream buildDatasets( - List nodes, Collection>> builders) { - return nodes.stream() - .flatMap( - event -> - builders.stream() - .filter(pfn -> PlanUtils.safeIsDefinedAt(pfn, event)) - .map(pfn -> PlanUtils.safeApply(pfn, event)) - .flatMap(Collection::stream)); - } - - /** - * Attach facets to a facet container, such as an {@link InputDatasetInputFacets} or an {@link - * OutputDatasetOutputFacets}. Facets returned by a {@link CustomFacetBuilder} may be attached to - * a field in the container, such as {@link InputDatasetInputFacets#dataQualityMetrics} or may be - * attached as a key/value pair in the {@link InputDatasetInputFacets#additionalProperties} map. - * The serialized JSON does not distinguish between these, but the java class does. The Java class - * also has some fields, such as the {@link InputDatasetInputFacets#producer} URI, which need to - * be included in the serialized JSON. - * - *

This methods will generate a new facet container with properties potentially overridden by - * the values set by the custom facet generators. - * - * @param events - * @param builders - * @return - */ - private OpenLineage.JobFacets buildJobFacets( - List events, - Collection> builders, - OpenLineage.JobFacetsBuilder jobFacetsBuilder) { - events.forEach(event -> builders.forEach(fn -> fn.accept(event, jobFacetsBuilder::put))); - return jobFacetsBuilder.build(); - } - - private RunFacets buildRunFacets( - List events, - Collection> builders, - RunFacetsBuilder runFacetsBuilder) { - events.forEach(event -> builders.forEach(fn -> fn.accept(event, runFacetsBuilder::put))); - return runFacetsBuilder.build(); - } -} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/plan/LogicalRelationDatasetBuilder.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/plan/LogicalRelationDatasetBuilder.java deleted file mode 100644 index dd58b9eaf140b..0000000000000 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/plan/LogicalRelationDatasetBuilder.java +++ /dev/null @@ -1,220 +0,0 @@ -/* -/* Copyright 2018-2023 contributors to the OpenLineage project -/* SPDX-License-Identifier: Apache-2.0 -*/ - -package io.openlineage.spark.agent.lifecycle.plan; - -import io.openlineage.client.OpenLineage; -import io.openlineage.client.OpenLineage.DatasetFacetsBuilder; -import io.openlineage.client.utils.DatasetIdentifier; -import io.openlineage.spark.agent.lifecycle.plan.handlers.JdbcRelationHandler; -import io.openlineage.spark.agent.util.PathUtils; -import io.openlineage.spark.agent.util.PlanUtils; -import io.openlineage.spark.api.AbstractQueryPlanDatasetBuilder; -import io.openlineage.spark.api.DatasetFactory; -import io.openlineage.spark.api.OpenLineageContext; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.spark.scheduler.SparkListenerEvent; -import org.apache.spark.sql.catalyst.catalog.CatalogTable; -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; -import org.apache.spark.sql.execution.datasources.HadoopFsRelation; -import org.apache.spark.sql.execution.datasources.LogicalRelation; -import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions; -import org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation; -import scala.collection.JavaConversions; - -/** - * {@link LogicalPlan} visitor that attempts to extract a {@link OpenLineage.Dataset} from a {@link - * LogicalRelation}. The {@link org.apache.spark.sql.sources.BaseRelation} is tested for known - * types, such as {@link HadoopFsRelation} or {@link JDBCRelation}s, as those are easy to extract - * exact dataset information. - * - *

For {@link HadoopFsRelation}s, it is assumed that a single directory maps to a single {@link - * OpenLineage.Dataset}. Any files referenced are replaced by their parent directory and all files - * in a given directory are assumed to belong to the same {@link OpenLineage.Dataset}. Directory - * partitioning is currently not addressed. - * - *

For {@link JDBCRelation}s, {@link OpenLineage.Dataset} naming expects the namespace to be the - * JDBC connection URL (schema and authority only) and the table name to be the - * <database> - * .<tableName>. - * - *

{@link CatalogTable}s, if present, can be used to describe the {@link OpenLineage.Dataset} if - * its {@link org.apache.spark.sql.sources.BaseRelation} is unknown. - * - *

TODO If a user specifies the {@link JDBCOptions#JDBC_QUERY_STRING()} option, we do not parse - * the sql to determine the specific tables used. Since we return a List of {@link - * OpenLineage.Dataset}s, we can parse the sql and determine each table referenced to return a - * complete list of datasets referenced. - */ -@Slf4j -public class LogicalRelationDatasetBuilder - extends AbstractQueryPlanDatasetBuilder { - - private final DatasetFactory datasetFactory; - - public LogicalRelationDatasetBuilder( - OpenLineageContext context, DatasetFactory datasetFactory, boolean searchDependencies) { - super(context, searchDependencies); - this.datasetFactory = datasetFactory; - } - - @Override - public boolean isDefinedAtLogicalPlan(LogicalPlan x) { - // if a LogicalPlan is a single node plan like `select * from temp`, - // then it's leaf node and should not be considered output node - if (x instanceof LogicalRelation && isSingleNodeLogicalPlan(x) && !searchDependencies) { - return false; - } - - return x instanceof LogicalRelation - && (((LogicalRelation) x).relation() instanceof HadoopFsRelation - || ((LogicalRelation) x).relation() instanceof JDBCRelation - || ((LogicalRelation) x).catalogTable().isDefined()); - } - - private boolean isSingleNodeLogicalPlan(LogicalPlan x) { - return context - .getQueryExecution() - .map(qe -> qe.optimizedPlan()) - .filter(p -> p.equals(x)) - .isPresent() - && (x.children() == null || x.children().isEmpty()); - } - - @Override - public List apply(LogicalRelation logRel) { - if (logRel.catalogTable() != null && logRel.catalogTable().isDefined()) { - return handleCatalogTable(logRel); - } else if (logRel.relation() instanceof HadoopFsRelation) { - return handleHadoopFsRelation(logRel); - } else if (logRel.relation() instanceof JDBCRelation) { - return new JdbcRelationHandler<>(datasetFactory).handleRelation(logRel); - } - throw new IllegalArgumentException( - "Expected logical plan to be either HadoopFsRelation, JDBCRelation, " - + "or CatalogTable but was " - + logRel); - } - - private List handleCatalogTable(LogicalRelation logRel) { - CatalogTable catalogTable = logRel.catalogTable().get(); - - DatasetIdentifier di = PathUtils.fromCatalogTable(catalogTable); - - OpenLineage.DatasetFacetsBuilder datasetFacetsBuilder = - context.getOpenLineage().newDatasetFacetsBuilder(); - datasetFacetsBuilder.schema(PlanUtils.schemaFacet(context.getOpenLineage(), logRel.schema())); - datasetFacetsBuilder.dataSource( - PlanUtils.datasourceFacet(context.getOpenLineage(), di.getNamespace())); - - getDatasetVersion(logRel) - .map( - version -> - datasetFacetsBuilder.version( - context.getOpenLineage().newDatasetVersionDatasetFacet(version))); - - return Collections.singletonList(datasetFactory.getDataset(di, datasetFacetsBuilder)); - } - - private List handleHadoopFsRelation(LogicalRelation x) { - HadoopFsRelation relation = (HadoopFsRelation) x.relation(); - try { - return context - .getSparkSession() - .map( - session -> { - Configuration hadoopConfig = - session.sessionState().newHadoopConfWithOptions(relation.options()); - - DatasetFacetsBuilder datasetFacetsBuilder = - context.getOpenLineage().newDatasetFacetsBuilder(); - getDatasetVersion(x) - .map( - version -> - datasetFacetsBuilder.version( - context.getOpenLineage().newDatasetVersionDatasetFacet(version))); - - Collection rootPaths = - JavaConversions.asJavaCollection(relation.location().rootPaths()); - - if (isSingleFileRelation(rootPaths, hadoopConfig)) { - return Collections.singletonList( - datasetFactory.getDataset( - rootPaths.stream().findFirst().get().toUri(), - relation.schema(), - datasetFacetsBuilder)); - } else { - return rootPaths.stream() - .map(p -> PlanUtils.getDirectoryPath(p, hadoopConfig)) - .distinct() - .map( - p -> { - // TODO- refactor this to return a single partitioned dataset based on - // static - // static partitions in the relation - return datasetFactory.getDataset( - p.toUri(), relation.schema(), datasetFacetsBuilder); - }) - .collect(Collectors.toList()); - } - }) - .orElse(Collections.emptyList()); - } catch (Exception e) { - if ("com.databricks.backend.daemon.data.client.adl.AzureCredentialNotFoundExcepgittion" - .equals(e.getClass().getName())) { - // This is a fallback that can occur when hadoop configurations cannot be - // reached. This occurs in Azure Databricks when credential passthrough - // is enabled and you're attempting to get the data lake credentials. - // The Spark Listener context cannot use the user credentials - // thus we need a fallback. - // This is similar to the InsertIntoHadoopRelationVisitor's process for getting - // Datasets - List inputDatasets = new ArrayList(); - List paths = - new ArrayList<>(JavaConversions.asJavaCollection(relation.location().rootPaths())); - for (Path p : paths) { - inputDatasets.add(datasetFactory.getDataset(p.toUri(), relation.schema())); - } - if (inputDatasets.isEmpty()) { - return Collections.emptyList(); - } else { - return inputDatasets; - } - } else { - throw e; - } - } - } - - private boolean isSingleFileRelation(Collection paths, Configuration hadoopConfig) { - if (paths.size() != 1) { - return false; - } - - try { - Path path = paths.stream().findFirst().get(); - return path.getFileSystem(hadoopConfig).isFile(path); - /* - Unfortunately it seems like on DataBricks this can throw an SparkException as well if credentials are missing. - Like org.apache.spark.SparkException: There is no Credential Scope. - */ - } catch (Exception e) { - return false; - } - } - - protected Optional getDatasetVersion(LogicalRelation x) { - // not implemented - return Optional.empty(); - } -} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/PathUtils.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/PathUtils.java deleted file mode 100644 index b72d28ce72dd9..0000000000000 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/PathUtils.java +++ /dev/null @@ -1,207 +0,0 @@ -/* -/* Copyright 2018-2023 contributors to the OpenLineage project -/* SPDX-License-Identifier: Apache-2.0 -*/ - -package io.openlineage.spark.agent.util; - -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import datahub.spark.conf.SparkAppContext; -import datahub.spark.conf.SparkConfigParser; -import io.datahubproject.openlineage.config.DatahubOpenlineageConfig; -import io.datahubproject.openlineage.dataset.HdfsPathDataset; -import io.openlineage.client.utils.DatasetIdentifier; -import io.openlineage.client.utils.DatasetIdentifierUtils; -import java.io.File; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.Optional; -import java.util.stream.Collectors; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.fs.Path; -import org.apache.spark.SparkConf; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.catalyst.TableIdentifier; -import org.apache.spark.sql.catalyst.catalog.CatalogTable; -import org.apache.spark.sql.internal.StaticSQLConf; - -@Slf4j -@SuppressWarnings("checkstyle:HideUtilityClassConstructor") -public class PathUtils { - - private static final String DEFAULT_SCHEME = "file"; - public static final String SPARK_OPENLINEAGE_DATASET_REMOVE_PATH_PATTERN = - "spark.openlineage.dataset.removePath.pattern"; - public static final String REMOVE_PATTERN_GROUP = "remove"; - - private static Optional sparkConf = Optional.empty(); - - public static DatasetIdentifier fromPath(Path path) { - return fromPath(path, DEFAULT_SCHEME); - } - - public static DatasetIdentifier fromPath(Path path, String defaultScheme) { - return fromURI(path.toUri(), defaultScheme); - } - - public static DatasetIdentifier fromURI(URI location) { - return fromURI(location, DEFAULT_SCHEME); - } - - public static DatasetIdentifier fromURI(URI location, String defaultScheme) { - DatasetIdentifier di = DatasetIdentifierUtils.fromURI(location, defaultScheme); - return new DatasetIdentifier(removePathPattern(di.getName()), di.getNamespace()); - } - - public static DatasetIdentifier fromCatalogTable(CatalogTable catalogTable) { - return fromCatalogTable(catalogTable, loadSparkConf()); - } - - /** - * Create DatasetIdentifier from CatalogTable, using storage's locationURI if it exists. In other - * way, use defaultTablePath. - */ - @SneakyThrows - public static DatasetIdentifier fromCatalogTable( - CatalogTable catalogTable, Optional sparkConf) { - - DatasetIdentifier di; - if (catalogTable.storage() != null && catalogTable.storage().locationUri().isDefined()) { - di = PathUtils.fromURI(catalogTable.storage().locationUri().get(), DEFAULT_SCHEME); - } else { - // try to obtain location - try { - di = prepareDatasetIdentifierFromDefaultTablePath(catalogTable); - } catch (IllegalStateException e) { - // session inactive - no way to find DatasetProvider - throw new IllegalArgumentException( - "Unable to extract DatasetIdentifier from a CatalogTable", e); - } - } - - Optional metastoreUri = extractMetastoreUri(sparkConf); - // TODO: Is the call to "metastoreUri.get()" really needed? - // Java's Optional should prevent the null in the first place. - if (metastoreUri.isPresent() && metastoreUri.get() != null) { - // dealing with Hive tables - DatasetIdentifier symlink = prepareHiveDatasetIdentifier(catalogTable, metastoreUri.get()); - return di.withSymlink( - symlink.getName(), symlink.getNamespace(), DatasetIdentifier.SymlinkType.TABLE); - } else { - return di.withSymlink( - nameFromTableIdentifier(catalogTable.identifier()), - StringUtils.substringBeforeLast(di.getName(), File.separator), - DatasetIdentifier.SymlinkType.TABLE); - } - } - - @SneakyThrows - private static DatasetIdentifier prepareDatasetIdentifierFromDefaultTablePath( - CatalogTable catalogTable) { - URI uri = - SparkSession.active().sessionState().catalog().defaultTablePath(catalogTable.identifier()); - - return PathUtils.fromURI(uri); - } - - @SneakyThrows - private static DatasetIdentifier prepareHiveDatasetIdentifier( - CatalogTable catalogTable, URI metastoreUri) { - String qualifiedName = nameFromTableIdentifier(catalogTable.identifier()); - if (!qualifiedName.startsWith("/")) { - qualifiedName = String.format("/%s", qualifiedName); - } - return PathUtils.fromPath( - new Path(enrichHiveMetastoreURIWithTableName(metastoreUri, qualifiedName))); - } - - @SneakyThrows - public static URI enrichHiveMetastoreURIWithTableName(URI metastoreUri, String qualifiedName) { - return new URI( - "hive", null, metastoreUri.getHost(), metastoreUri.getPort(), qualifiedName, null, null); - } - - /** - * SparkConf does not change through job lifetime but it can get lost once session is closed. It's - * good to have it set in case of SPARK-29046 - */ - private static Optional loadSparkConf() { - if (!sparkConf.isPresent() && SparkSession.getDefaultSession().isDefined()) { - sparkConf = Optional.of(SparkSession.getDefaultSession().get().sparkContext().getConf()); - } - return sparkConf; - } - - private static Optional extractMetastoreUri(Optional sparkConf) { - // make sure SparkConf is present - if (!sparkConf.isPresent()) { - return Optional.empty(); - } - - // make sure enableHiveSupport is called - Optional setting = - SparkConfUtils.findSparkConfigKey( - sparkConf.get(), StaticSQLConf.CATALOG_IMPLEMENTATION().key()); - if (!setting.isPresent() || !"hive".equals(setting.get())) { - return Optional.empty(); - } - - return SparkConfUtils.getMetastoreUri(sparkConf.get()); - } - - private static String removeFirstSlashIfSingleSlashInString(String name) { - if (name.chars().filter(x -> x == '/').count() == 1 && name.startsWith("/")) { - return name.substring(1); - } - return name; - } - - private static String removePathPattern(String datasetName) { - // TODO: The reliance on global-mutable state here should be changed - // this led to problems in the PathUtilsTest class, where some tests interfered with others - log.info("Removing path pattern from dataset name {}", datasetName); - Optional conf = loadSparkConf(); - if (!conf.isPresent()) { - return datasetName; - } - try { - String propertiesString = - Arrays.stream(conf.get().getAllWithPrefix("spark.datahub.")) - .map(tup -> tup._1 + "= \"" + tup._2 + "\"") - .collect(Collectors.joining("\n")); - Config datahubConfig = ConfigFactory.parseString(propertiesString); - DatahubOpenlineageConfig datahubOpenlineageConfig = - SparkConfigParser.sparkConfigToDatahubOpenlineageConf( - datahubConfig, new SparkAppContext()); - HdfsPathDataset hdfsPath = - HdfsPathDataset.create(new URI(datasetName), datahubOpenlineageConfig); - log.debug("Transformed path is {}", hdfsPath.getDatasetPath()); - return hdfsPath.getDatasetPath(); - } catch (InstantiationException e) { - log.warn( - "Unable to convert dataset {} to path the exception was {}", datasetName, e.getMessage()); - return datasetName; - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - - private static String nameFromTableIdentifier(TableIdentifier identifier) { - // we create name instead of calling `unquotedString` method which includes spark_catalog - // for Spark 3.4 - String name; - if (identifier.database().isDefined()) { - // include database in name - name = String.format("%s.%s", identifier.database().get(), identifier.table()); - } else { - // just table name - name = identifier.table(); - } - - return name; - } -} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java index 8d93b0288b515..d46d741d155b8 100644 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java @@ -1,16 +1,19 @@ /* -/* Copyright 2018-2023 contributors to the OpenLineage project +/* Copyright 2018-2024 contributors to the OpenLineage project /* SPDX-License-Identifier: Apache-2.0 */ package io.openlineage.spark.agent.util; +import static io.openlineage.spark.agent.lifecycle.ExecutionContext.CAMEL_TO_SNAKE_CASE; + import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import datahub.spark.conf.SparkLineageConf; import io.datahubproject.openlineage.dataset.HdfsPathDataset; import io.openlineage.client.OpenLineage; import io.openlineage.spark.agent.Versions; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -18,71 +21,44 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.FileInputFormat; import org.apache.spark.SparkConf; import org.apache.spark.SparkEnv; -import org.apache.spark.package$; -import org.apache.spark.rdd.HadoopRDD; import org.apache.spark.rdd.RDD; import org.apache.spark.sql.catalyst.expressions.Attribute; -import org.apache.spark.sql.execution.datasources.FileScanRDD; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import scala.PartialFunction; -import scala.runtime.AbstractPartialFunction; +import scala.PartialFunction$; /** * Utility functions for traversing a {@link * org.apache.spark.sql.catalyst.plans.logical.LogicalPlan}. */ @Slf4j -@SuppressWarnings("checkstyle:HideUtilityClassConstructor") public class PlanUtils { - - public static final String SLASH_DELIMITER_USER_PASSWORD_REGEX = - "[A-Za-z0-9_%]+//?[A-Za-z0-9_%]*@"; - public static final String COLON_DELIMITER_USER_PASSWORD_REGEX = - "([/|,])[A-Za-z0-9_%]+:?[A-Za-z0-9_%]*@"; - - /** - * Merge a list of {@link PartialFunction}s and return the first value where the function is - * defined or empty list if no function matches the input. - * - * @param fns - * @param arg - * @param - * @param - * @return - */ - public static Collection applyAll( - List>> fns, T arg) { - PartialFunction> fn = merge(fns); - if (fn.isDefinedAt(arg)) { - return fn.apply(arg); - } - return Collections.emptyList(); - } - /** * Given a list of {@link PartialFunction}s merge to produce a single function that will test the - * input against each function one by one until a match is found or empty() is returned. + * input against each function one by one until a match is found or {@link + * PartialFunction$#empty()} is returned. * * @param fns * @param * @param * @return */ - public static PartialFunction> merge( + public static OpenLineageAbstractPartialFunction> merge( Collection>> fns) { - return new AbstractPartialFunction>() { + return new OpenLineageAbstractPartialFunction>() { + String appliedClassName; + @Override public boolean isDefinedAt(T x) { return fns.stream() @@ -110,6 +86,7 @@ public Collection apply(T x) { x.getClass().getCanonicalName(), collection); } + appliedClassName = x.getClass().getName(); return collection; } catch (RuntimeException | NoClassDefFoundError | NoSuchMethodError e) { log.error("Apply failed:", e); @@ -120,6 +97,11 @@ public Collection apply(T x) { .flatMap(Collection::stream) .collect(Collectors.toList()); } + + @Override + String appliedName() { + return appliedClassName; + } }; } @@ -204,12 +186,26 @@ public static OpenLineage.ParentRunFacet parentRunFacet( .run(new OpenLineage.ParentRunFacetRunBuilder().runId(parentRunId).build()) .job( new OpenLineage.ParentRunFacetJobBuilder() - .name(parentJob) + .name(parentJob.replaceAll(CAMEL_TO_SNAKE_CASE, "_$1").toLowerCase(Locale.ROOT)) .namespace(parentJobNamespace) .build()) .build(); } + public static Path getDirectoryPathOl(Path p, Configuration hadoopConf) { + try { + if (p.getFileSystem(hadoopConf).getFileStatus(p).isFile()) { + return p.getParent(); + } else { + return p; + } + } catch (IOException e) { + log.warn("Unable to get file system for path ", e); + return p; + } + } + + // This method was replaced to support Datahub PathSpecs public static Path getDirectoryPath(Path p, Configuration hadoopConf) { SparkConf conf = SparkEnv.get().conf(); String propertiesString = @@ -229,17 +225,6 @@ public static Path getDirectoryPath(Path p, Configuration hadoopConf) { log.warn("Unable to convert path to hdfs path {} the exception was {}", p, e.getMessage()); return p; } - - // try { - // if (p.getFileSystem(hadoopConf).getFileStatus(p).isFile()) { - // return p.getParent(); - // } else { - // return p; - // } - // } catch (IOException e) { - // log.warn("Unable to get file system for path ", e); - // return p; - // } } /** @@ -251,36 +236,7 @@ public static Path getDirectoryPath(Path p, Configuration hadoopConf) { */ public static List findRDDPaths(List> fileRdds) { return fileRdds.stream() - .flatMap( - rdd -> { - if (rdd instanceof HadoopRDD) { - HadoopRDD hadoopRDD = (HadoopRDD) rdd; - Path[] inputPaths = FileInputFormat.getInputPaths(hadoopRDD.getJobConf()); - Configuration hadoopConf = hadoopRDD.getConf(); - return Arrays.stream(inputPaths) - .map(p -> PlanUtils.getDirectoryPath(p, hadoopConf)); - } else if (rdd instanceof FileScanRDD) { - FileScanRDD fileScanRDD = (FileScanRDD) rdd; - return ScalaConversionUtils.fromSeq(fileScanRDD.filePartitions()).stream() - .flatMap(fp -> Arrays.stream(fp.files())) - .map( - f -> { - if (package$.MODULE$.SPARK_VERSION().compareTo("3.4") > 0) { - // filePath returns SparkPath for Spark 3.4 - return ReflectionUtils.tryExecuteMethod(f, "filePath") - .map(o -> ReflectionUtils.tryExecuteMethod(o, "toPath")) - .map(o -> (Path) o.get()) - .get() - .getParent(); - } else { - return new Path(f.filePath()).getParent(); - } - }); - } else { - log.warn("Unknown RDD class {}", rdd.getClass().getCanonicalName()); - return Stream.empty(); - } - }) + .flatMap(RddPathUtils::findRDDPaths) .distinct() .collect(Collectors.toList()); } @@ -316,11 +272,11 @@ public static boolean safeIsDefinedAt(PartialFunction pfn, Object x) { return false; } catch (Exception e) { if (e != null) { - log.debug("isDefinedAt method failed on {}", e); + log.info("isDefinedAt method failed on {}", e); } return false; } catch (NoClassDefFoundError e) { - log.debug("isDefinedAt method failed on {}", e.getMessage()); + log.info("isDefinedAt method failed on {}", e.getMessage()); return false; } } @@ -331,6 +287,8 @@ public static boolean safeIsDefinedAt(PartialFunction pfn, Object x) { * @param pfn * @param x * @return + * @param + * @param */ public static List safeApply(PartialFunction> pfn, D x) { try { diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java new file mode 100644 index 0000000000000..a606a44ddd516 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java @@ -0,0 +1,182 @@ +/* +/* Copyright 2018-2024 contributors to the OpenLineage project +/* SPDX-License-Identifier: Apache-2.0 +*/ + +package io.openlineage.spark.agent.util; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import datahub.spark.conf.SparkAppContext; +import datahub.spark.conf.SparkConfigParser; +import io.datahubproject.openlineage.config.DatahubOpenlineageConfig; +import io.datahubproject.openlineage.dataset.HdfsPathDataset; +import io.openlineage.client.OpenLineage.InputDataset; +import io.openlineage.client.OpenLineage.OutputDataset; +import io.openlineage.spark.api.OpenLineageContext; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; + +/** + * Utility class to handle removing path patterns in dataset names. Given a configured regex pattern + * with "remove" group defined, class methods run regex replacements on all the datasets available + * within the event + */ +@Slf4j +public class RemovePathPatternUtils { + public static final String REMOVE_PATTERN_GROUP = "remove"; + public static final String SPARK_OPENLINEAGE_DATASET_REMOVE_PATH_PATTERN = + "spark.openlineage.dataset.removePath.pattern"; + + private static Optional sparkConf = Optional.empty(); + + public static List removeOutputsPathPattern_ol( + OpenLineageContext context, List outputs) { + return getPattern(context) + .map( + pattern -> + outputs.stream() + .map( + dataset -> { + String newName = removePath(pattern, dataset.getName()); + if (newName != dataset.getName()) { + return context + .getOpenLineage() + .newOutputDatasetBuilder() + .name(removePath(pattern, dataset.getName())) + .namespace(dataset.getNamespace()) + .facets(dataset.getFacets()) + .outputFacets(dataset.getOutputFacets()) + .build(); + } else { + return dataset; + } + }) + .collect(Collectors.toList())) + .orElse(outputs); + } + + // This method was replaced to support Datahub PathSpecs + public static List removeOutputsPathPattern( + OpenLineageContext context, List outputs) { + return outputs.stream() + .map( + dataset -> { + String newName = removePathPattern(dataset.getName()); + if (newName != dataset.getName()) { + return context + .getOpenLineage() + .newOutputDatasetBuilder() + .name(newName) + .namespace(dataset.getNamespace()) + .facets(dataset.getFacets()) + .outputFacets(dataset.getOutputFacets()) + .build(); + } else { + return dataset; + } + }) + .collect(Collectors.toList()); + } + + // This method was replaced to support Datahub PathSpecs + public static List removeInputsPathPattern( + OpenLineageContext context, List inputs) { + return inputs.stream() + .map( + dataset -> { + String newName = removePathPattern(dataset.getName()); + if (newName != dataset.getName()) { + return context + .getOpenLineage() + .newInputDatasetBuilder() + .name(newName) + .namespace(dataset.getNamespace()) + .facets(dataset.getFacets()) + .inputFacets(dataset.getInputFacets()) + .build(); + } else { + return dataset; + } + }) + .collect(Collectors.toList()); + } + + private static Optional getPattern(OpenLineageContext context) { + return Optional.ofNullable(context.getSparkContext()) + .map(sparkContext -> sparkContext.conf()) + .filter(conf -> conf.contains(SPARK_OPENLINEAGE_DATASET_REMOVE_PATH_PATTERN)) + .map(conf -> conf.get(SPARK_OPENLINEAGE_DATASET_REMOVE_PATH_PATTERN)) + .map(pattern -> Pattern.compile(pattern)); + } + + private static String removePath(Pattern pattern, String name) { + return Optional.ofNullable(pattern.matcher(name)) + .filter(matcher -> matcher.find()) + .filter( + matcher -> { + try { + matcher.group(REMOVE_PATTERN_GROUP); + return true; + } catch (IllegalStateException | IllegalArgumentException e) { + return false; + } + }) + .filter(matcher -> StringUtils.isNotEmpty(matcher.group(REMOVE_PATTERN_GROUP))) + .map( + matcher -> + name.substring(0, matcher.start(REMOVE_PATTERN_GROUP)) + + name.substring(matcher.end(REMOVE_PATTERN_GROUP), name.length())) + .orElse(name); + } + + /** + * SparkConf does not change through job lifetime but it can get lost once session is closed. It's + * good to have it set in case of SPARK-29046 + */ + private static Optional loadSparkConf() { + if (!sparkConf.isPresent() && SparkSession.getDefaultSession().isDefined()) { + sparkConf = Optional.of(SparkSession.getDefaultSession().get().sparkContext().getConf()); + } + return sparkConf; + } + + private static String removePathPattern(String datasetName) { + // TODO: The reliance on global-mutable state here should be changed + // this led to problems in the PathUtilsTest class, where some tests interfered with others + log.info("Removing path pattern from dataset name {}", datasetName); + Optional conf = loadSparkConf(); + if (!conf.isPresent()) { + return datasetName; + } + try { + String propertiesString = + Arrays.stream(conf.get().getAllWithPrefix("spark.datahub.")) + .map(tup -> tup._1 + "= \"" + tup._2 + "\"") + .collect(Collectors.joining("\n")); + Config datahubConfig = ConfigFactory.parseString(propertiesString); + DatahubOpenlineageConfig datahubOpenlineageConfig = + SparkConfigParser.sparkConfigToDatahubOpenlineageConf( + datahubConfig, new SparkAppContext()); + HdfsPathDataset hdfsPath = + HdfsPathDataset.create(new URI(datasetName), datahubOpenlineageConfig); + log.debug("Transformed path is {}", hdfsPath.getDatasetPath()); + return hdfsPath.getDatasetPath(); + } catch (InstantiationException e) { + log.warn( + "Unable to convert dataset {} to path the exception was {}", datasetName, e.getMessage()); + return datasetName; + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/sample_spark.json b/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/sample_spark.json index 77a6ebc4044bd..acb7f585e98c9 100644 --- a/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/sample_spark.json +++ b/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/sample_spark.json @@ -40,7 +40,7 @@ "inputs": [ { "namespace": "file", - "name": "/Users/treff7es/shadow/spark-test/people.json", + "name": "/my_folder/spark-test/people.json", "facets": { "dataSource": { "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.2.2/integration/spark", @@ -69,7 +69,7 @@ "outputs": [ { "namespace": "file", - "name": "/Users/treff7es/shadow/spark-test/result", + "name": "/my_folder/shadow/spark-test/result", "facets": { "dataSource": { "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.2.2/integration/spark", @@ -95,7 +95,7 @@ "inputFields": [ { "namespace": "file", - "name": "/Users/treff7es/shadow/spark-test/people.json", + "name": "/my_folder/spark-test/people.json", "field": "name" } ] diff --git a/metadata-integration/java/spark-lineage/build.gradle b/metadata-integration/java/spark-lineage/build.gradle index 1b3c87288abf8..8db8a09f8cc81 100644 --- a/metadata-integration/java/spark-lineage/build.gradle +++ b/metadata-integration/java/spark-lineage/build.gradle @@ -48,7 +48,7 @@ dependencies { provided(externalDependency.sparkSql) provided(externalDependency.sparkHive) - implementation externalDependency.httpAsyncClient + implementation externalDependency.httpClient // Tests need a concrete log4j available. Providing it here testImplementation 'org.apache.logging.log4j:log4j-api:2.17.1' @@ -106,7 +106,7 @@ shadowJar { relocate 'com.fasterxml.jackson', 'datahub.shaded.jackson' relocate 'org.slf4j','datahub.shaded.org.slf4j' - relocate 'org.apache.http','datahub.spark2.shaded.http' + relocate 'org.apache.hc','datahub.spark2.shaded.http' relocate 'org.apache.commons.codec', 'datahub.spark2.shaded.o.a.c.codec' relocate 'org.apache.commons.compress', 'datahub.spark2.shaded.o.a.c.compress' relocate 'org.apache.commons.io', 'datahub.spark2.shaded.o.a.c.io' diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index d6bb1fb2401e8..d8c5c3317a2ec 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -336,7 +336,9 @@ public AutoCompleteResult autoComplete( IndexConvention indexConvention = opContext.getSearchContext().getIndexConvention(); AutocompleteRequestHandler builder = AutocompleteRequestHandler.getBuilder( - entitySpec, opContext.getRetrieverContext().get().getAspectRetriever()); + entitySpec, + customSearchConfiguration, + opContext.getRetrieverContext().get().getAspectRetriever()); SearchRequest req = builder.getSearchRequest( opContext, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java index 0acedc5d49171..62525bdd35b3f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java @@ -4,10 +4,14 @@ import static com.linkedin.metadata.search.utils.ESAccessControlUtil.restrictUrn; import static com.linkedin.metadata.search.utils.ESUtils.applyDefaultSearchFilters; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringArray; import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.config.search.custom.QueryConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation; @@ -18,9 +22,9 @@ import com.linkedin.metadata.search.utils.ESUtils; import io.datahubproject.metadata.context.OperationContext; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -35,7 +39,9 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; @@ -51,9 +57,17 @@ public class AutocompleteRequestHandler { private final AspectRetriever aspectRetriever; + private final CustomizedQueryHandler customizedQueryHandler; + + private final EntitySpec entitySpec; + public AutocompleteRequestHandler( - @Nonnull EntitySpec entitySpec, @Nonnull AspectRetriever aspectRetriever) { + @Nonnull EntitySpec entitySpec, + @Nullable CustomSearchConfiguration customSearchConfiguration, + @Nonnull AspectRetriever aspectRetriever) { + this.entitySpec = entitySpec; List fieldSpecs = entitySpec.getSearchableFieldSpecs(); + this.customizedQueryHandler = CustomizedQueryHandler.builder(customSearchConfiguration).build(); _defaultAutocompleteFields = Stream.concat( fieldSpecs.stream() @@ -80,9 +94,13 @@ public AutocompleteRequestHandler( } public static AutocompleteRequestHandler getBuilder( - @Nonnull EntitySpec entitySpec, @Nonnull AspectRetriever aspectRetriever) { + @Nonnull EntitySpec entitySpec, + @Nullable CustomSearchConfiguration customSearchConfiguration, + @Nonnull AspectRetriever aspectRetriever) { return AUTOCOMPLETE_QUERY_BUILDER_BY_ENTITY_NAME.computeIfAbsent( - entitySpec, k -> new AutocompleteRequestHandler(entitySpec, aspectRetriever)); + entitySpec, + k -> + new AutocompleteRequestHandler(entitySpec, customSearchConfiguration, aspectRetriever)); } public SearchRequest getSearchRequest( @@ -94,24 +112,90 @@ public SearchRequest getSearchRequest( SearchRequest searchRequest = new SearchRequest(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(limit); - // apply default filters - BoolQueryBuilder boolQueryBuilder = - applyDefaultSearchFilters(opContext, filter, getQuery(input, field)); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.postFilter( - ESUtils.buildFilterQuery(filter, false, searchableFieldTypes, aspectRetriever)); + AutocompleteConfiguration customAutocompleteConfig = + customizedQueryHandler.lookupAutocompleteConfig(input).orElse(null); + QueryConfiguration customQueryConfig = + customizedQueryHandler.lookupQueryConfig(input).orElse(null); + + // Initial query with input filters + BoolQueryBuilder baseQuery = + ESUtils.buildFilterQuery(filter, false, searchableFieldTypes, aspectRetriever); + + // Add autocomplete query + baseQuery.should(getQuery(opContext.getObjectMapper(), customAutocompleteConfig, input, field)); + + // Apply default filters + BoolQueryBuilder queryWithDefaultFilters = + applyDefaultSearchFilters(opContext, filter, baseQuery); + + // Apply scoring + FunctionScoreQueryBuilder functionScoreQueryBuilder = + Optional.ofNullable(customAutocompleteConfig) + .flatMap( + cac -> + CustomizedQueryHandler.functionScoreQueryBuilder( + opContext.getObjectMapper(), + cac, + queryWithDefaultFilters, + customQueryConfig)) + .orElse( + SearchQueryBuilder.buildScoreFunctions( + opContext, customQueryConfig, List.of(entitySpec), queryWithDefaultFilters)); + searchSourceBuilder.query(functionScoreQueryBuilder); + + ESUtils.buildSortOrder(searchSourceBuilder, null, List.of(entitySpec)); + + // wire inner non-scored query searchSourceBuilder.highlighter(getHighlights(field)); searchRequest.source(searchSourceBuilder); return searchRequest; } - private BoolQueryBuilder getQuery(@Nonnull String query, @Nullable String field) { - return getQuery(getAutocompleteFields(field), query); + private BoolQueryBuilder getQuery( + @Nonnull ObjectMapper objectMapper, + @Nullable AutocompleteConfiguration customAutocompleteConfig, + @Nonnull String query, + @Nullable String field) { + return getQuery(objectMapper, customAutocompleteConfig, getAutocompleteFields(field), query); + } + + public BoolQueryBuilder getQuery( + @Nonnull ObjectMapper objectMapper, + @Nullable AutocompleteConfiguration customAutocompleteConfig, + List autocompleteFields, + @Nonnull String query) { + + BoolQueryBuilder finalQuery = + Optional.ofNullable(customAutocompleteConfig) + .flatMap(cac -> CustomizedQueryHandler.boolQueryBuilder(objectMapper, cac, query)) + .orElse(QueryBuilders.boolQuery()) + .minimumShouldMatch(1); + + getAutocompleteQuery(customAutocompleteConfig, autocompleteFields, query) + .ifPresent(finalQuery::should); + + return finalQuery; + } + + private Optional getAutocompleteQuery( + @Nullable AutocompleteConfiguration customConfig, + List autocompleteFields, + @Nonnull String query) { + Optional result = Optional.empty(); + + if (customConfig == null || customConfig.isDefaultQuery()) { + result = Optional.of(defaultQuery(autocompleteFields, query)); + } + + return result; } - public static BoolQueryBuilder getQuery(List autocompleteFields, @Nonnull String query) { + private static BoolQueryBuilder defaultQuery( + List autocompleteFields, @Nonnull String query) { BoolQueryBuilder finalQuery = QueryBuilders.boolQuery(); + finalQuery.minimumShouldMatch(1); + // Search for exact matches with higher boost and ngram matches MultiMatchQueryBuilder autocompleteQueryBuilder = QueryBuilders.multiMatchQuery(query).type(MultiMatchQueryBuilder.Type.BOOL_PREFIX); @@ -154,6 +238,12 @@ private HighlightBuilder getHighlights(@Nullable String field) { .field(fieldName + ".*") .field(fieldName + ".ngram") .field(fieldName + ".delimited")); + + // set field match req false for ngram + highlightBuilder.fields().stream() + .filter(f -> f.name().contains("ngram")) + .forEach(f -> f.requireFieldMatch(false).noMatchSize(200)); + return highlightBuilder; } @@ -168,8 +258,9 @@ public AutoCompleteResult extractResult( @Nonnull OperationContext opContext, @Nonnull SearchResponse searchResponse, @Nonnull String input) { - Set results = new LinkedHashSet<>(); - Set entityResults = new HashSet<>(); + // use lists to preserve ranking + List results = new ArrayList<>(); + List entityResults = new ArrayList<>(); for (SearchHit hit : searchResponse.getHits()) { Optional matchedFieldValue = @@ -181,13 +272,15 @@ public AutoCompleteResult extractResult( if (matchedUrn.isPresent()) { Urn autoCompleteUrn = Urn.createFromString(matchedUrn.get()); if (!restrictUrn(opContext, autoCompleteUrn)) { - entityResults.add( - new AutoCompleteEntity().setUrn(Urn.createFromString(matchedUrn.get()))); - matchedFieldValue.ifPresent(results::add); + matchedFieldValue.ifPresent( + value -> { + entityResults.add(new AutoCompleteEntity().setUrn(autoCompleteUrn)); + results.add(value); + }); } } } catch (URISyntaxException e) { - throw new RuntimeException(String.format("Failed to create urn %s", matchedUrn.get()), e); + log.warn(String.format("Failed to create urn %s", matchedUrn.get())); } } return new AutoCompleteResult() diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java index 478d633fe3c55..0dbdf80860f7f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java @@ -1,26 +1,54 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration; +import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.config.search.custom.QueryConfiguration; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.opensearch.search.SearchModule; @Slf4j @Builder(builderMethodName = "hiddenBuilder") @Getter public class CustomizedQueryHandler { + private static final NamedXContentRegistry X_CONTENT_REGISTRY; + + static { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); + } + private CustomSearchConfiguration customSearchConfiguration; @Builder.Default private List> queryConfigurations = List.of(); + @Builder.Default + private List> autocompleteConfigurations = + List.of(); + public Optional lookupQueryConfig(String query) { return queryConfigurations.stream() .filter(e -> e.getKey().matcher(query).matches()) @@ -28,6 +56,129 @@ public Optional lookupQueryConfig(String query) { .findFirst(); } + public Optional lookupAutocompleteConfig(String query) { + return autocompleteConfigurations.stream() + .filter(e -> e.getKey().matcher(query).matches()) + .map(Map.Entry::getValue) + .findFirst(); + } + + public static String unquote(String query) { + return query.replaceAll("[\"']", ""); + } + + public static boolean isQuoted(String query) { + return Stream.of("\"", "'").anyMatch(query::contains); + } + + public static Optional boolQueryBuilder( + @Nonnull ObjectMapper objectMapper, + QueryConfiguration customQueryConfiguration, + String query) { + if (customQueryConfiguration.getBoolQuery() != null) { + log.debug( + "Using custom query configuration queryRegex: {}", + customQueryConfiguration.getQueryRegex()); + } + return Optional.ofNullable(customQueryConfiguration.getBoolQuery()) + .map(bq -> toBoolQueryBuilder(objectMapper, query, bq)); + } + + public static Optional boolQueryBuilder( + @Nonnull ObjectMapper objectMapper, + AutocompleteConfiguration customAutocompleteConfiguration, + String query) { + if (customAutocompleteConfiguration.getBoolQuery() != null) { + log.debug( + "Using custom query autocomplete queryRegex: {}", + customAutocompleteConfiguration.getQueryRegex()); + } + return Optional.ofNullable(customAutocompleteConfiguration.getBoolQuery()) + .map(bq -> toBoolQueryBuilder(objectMapper, query, bq)); + } + + private static BoolQueryBuilder toBoolQueryBuilder( + @Nonnull ObjectMapper objectMapper, String query, BoolQueryConfiguration boolQuery) { + try { + String jsonFragment = + objectMapper + .writeValueAsString(boolQuery) + .replace("\"{{query_string}}\"", objectMapper.writeValueAsString(query)) + .replace( + "\"{{unquoted_query_string}}\"", objectMapper.writeValueAsString(unquote(query))); + XContentParser parser = + XContentType.JSON + .xContent() + .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, jsonFragment); + return BoolQueryBuilder.fromXContent(parser); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static FunctionScoreQueryBuilder functionScoreQueryBuilder( + @Nonnull ObjectMapper objectMapper, + @Nonnull QueryConfiguration customQueryConfiguration, + QueryBuilder queryBuilder) { + return toFunctionScoreQueryBuilder( + objectMapper, queryBuilder, customQueryConfiguration.getFunctionScore()); + } + + public static Optional functionScoreQueryBuilder( + @Nonnull ObjectMapper objectMapper, + @Nonnull AutocompleteConfiguration customAutocompleteConfiguration, + QueryBuilder queryBuilder, + @Nullable QueryConfiguration customQueryConfiguration) { + + Optional result = Optional.empty(); + + if ((customAutocompleteConfiguration.getFunctionScore() == null + || customAutocompleteConfiguration.getFunctionScore().isEmpty()) + && customAutocompleteConfiguration.isInheritFunctionScore() + && customQueryConfiguration != null) { + log.debug( + "Inheriting query configuration for autocomplete function scoring: " + + customQueryConfiguration); + // inherit if not overridden + result = + Optional.of( + toFunctionScoreQueryBuilder( + objectMapper, queryBuilder, customQueryConfiguration.getFunctionScore())); + } else if (customAutocompleteConfiguration.getFunctionScore() != null + && !customAutocompleteConfiguration.getFunctionScore().isEmpty()) { + log.debug("Applying custom autocomplete function scores."); + result = + Optional.of( + toFunctionScoreQueryBuilder( + objectMapper, queryBuilder, customAutocompleteConfiguration.getFunctionScore())); + } + + return result; + } + + private static FunctionScoreQueryBuilder toFunctionScoreQueryBuilder( + @Nonnull ObjectMapper objectMapper, + @Nonnull QueryBuilder queryBuilder, + @Nonnull Map params) { + try { + HashMap body = new HashMap<>(params); + if (!body.isEmpty()) { + log.debug("Using custom scoring functions: {}", body); + } + + body.put("query", objectMapper.readValue(queryBuilder.toString(), Map.class)); + + String jsonFragment = objectMapper.writeValueAsString(Map.of("function_score", body)); + XContentParser parser = + XContentType.JSON + .xContent() + .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, jsonFragment); + return (FunctionScoreQueryBuilder) FunctionScoreQueryBuilder.parseInnerQueryBuilder(parser); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public static CustomizedQueryHandlerBuilder builder( @Nullable CustomSearchConfiguration customSearchConfiguration) { CustomizedQueryHandlerBuilder builder = @@ -38,7 +189,12 @@ public static CustomizedQueryHandlerBuilder builder( customSearchConfiguration.getQueryConfigurations().stream() .map(cfg -> Map.entry(Pattern.compile(cfg.getQueryRegex()), cfg)) .collect(Collectors.toList())); + builder.autocompleteConfigurations( + customSearchConfiguration.getAutocompleteConfigurations().stream() + .map(cfg -> Map.entry(Pattern.compile(cfg.getQueryRegex()), cfg)) + .collect(Collectors.toList())); } + return builder; } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index 048987ef5aa6a..33195d4ea807d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -3,18 +3,14 @@ import static com.linkedin.metadata.Constants.SKIP_REFERENCE_ASPECT; import static com.linkedin.metadata.models.SearchableFieldSpecExtractor.PRIMARY_URN_SEARCH_PROPERTIES; import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.*; -import static com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig.*; +import static com.linkedin.metadata.search.elasticsearch.query.request.CustomizedQueryHandler.isQuoted; +import static com.linkedin.metadata.search.elasticsearch.query.request.CustomizedQueryHandler.unquote; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.StreamReadConstraints; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.config.search.ExactMatchConfiguration; import com.linkedin.metadata.config.search.PartialConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.WordGramConfiguration; -import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.config.search.custom.QueryConfiguration; import com.linkedin.metadata.models.AspectSpec; @@ -28,10 +24,8 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.utils.ESUtils; import io.datahubproject.metadata.context.OperationContext; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -39,18 +33,12 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.common.lucene.search.function.FieldValueFactorFunction; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilder; @@ -60,32 +48,9 @@ import org.opensearch.index.query.functionscore.FieldValueFactorFunctionBuilder; import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.opensearch.index.query.functionscore.ScoreFunctionBuilders; -import org.opensearch.search.SearchModule; @Slf4j public class SearchQueryBuilder { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - static { - OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); - int maxSize = - Integer.parseInt( - System.getenv() - .getOrDefault( - Constants.INGESTION_MAX_SERIALIZED_STRING_LENGTH, - Constants.MAX_JACKSON_STRING_SIZE)); - OBJECT_MAPPER - .getFactory() - .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); - } - - private static final NamedXContentRegistry X_CONTENT_REGISTRY; - - static { - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); - } - public static final String STRUCTURED_QUERY_PREFIX = "\\\\/q "; private final ExactMatchConfiguration exactMatchConfiguration; private final PartialConfiguration partialConfiguration; @@ -112,7 +77,7 @@ public QueryBuilder buildQuery( final QueryBuilder queryBuilder = buildInternalQuery(opContext, customQueryConfig, entitySpecs, query, fulltext); - return buildScoreFunctions(customQueryConfig, entitySpecs, queryBuilder); + return buildScoreFunctions(opContext, customQueryConfig, entitySpecs, queryBuilder); } /** @@ -133,7 +98,10 @@ private QueryBuilder buildInternalQuery( final String sanitizedQuery = query.replaceFirst("^:+", ""); final BoolQueryBuilder finalQuery = Optional.ofNullable(customQueryConfig) - .flatMap(cqc -> boolQueryBuilder(cqc, sanitizedQuery)) + .flatMap( + cqc -> + CustomizedQueryHandler.boolQueryBuilder( + opContext.getObjectMapper(), cqc, sanitizedQuery)) .orElse(QueryBuilders.boolQuery()) .minimumShouldMatch(1); @@ -326,14 +294,6 @@ private Set getStandardFields( return fields; } - private static String unquote(String query) { - return query.replaceAll("[\"']", ""); - } - - private static boolean isQuoted(String query) { - return Stream.of("\"", "'").anyMatch(query::contains); - } - private Optional getSimpleQuery( @Nonnull EntityRegistry entityRegistry, @Nullable QueryConfiguration customQueryConfig, @@ -410,13 +370,20 @@ private Optional getPrefixAndExactMatchQuery( getStandardFields(entityRegistry, entitySpecs) .forEach( searchFieldConfig -> { + boolean caseSensitivityEnabled = + exactMatchConfiguration.getCaseSensitivityFactor() > 0.0f; + float caseSensitivityFactor = + caseSensitivityEnabled + ? exactMatchConfiguration.getCaseSensitivityFactor() + : 1.0f; + if (searchFieldConfig.isDelimitedSubfield() && isPrefixQuery) { finalQuery.should( QueryBuilders.matchPhrasePrefixQuery(searchFieldConfig.fieldName(), query) .boost( searchFieldConfig.boost() * exactMatchConfiguration.getPrefixFactor() - * exactMatchConfiguration.getCaseSensitivityFactor()) + * caseSensitivityFactor) .queryName(searchFieldConfig.shortName())); // less than exact } @@ -425,13 +392,16 @@ private Optional getPrefixAndExactMatchQuery( // The non-.keyword field removes case information // Exact match case-sensitive - finalQuery.should( - QueryBuilders.termQuery( - ESUtils.toKeywordField(searchFieldConfig.fieldName(), false), - unquotedQuery) - .caseInsensitive(false) - .boost(searchFieldConfig.boost() * exactMatchConfiguration.getExactFactor()) - .queryName(searchFieldConfig.shortName())); + if (caseSensitivityEnabled) { + finalQuery.should( + QueryBuilders.termQuery( + ESUtils.toKeywordField(searchFieldConfig.fieldName(), false), + unquotedQuery) + .caseInsensitive(false) + .boost( + searchFieldConfig.boost() * exactMatchConfiguration.getExactFactor()) + .queryName(searchFieldConfig.shortName())); + } // Exact match case-insensitive finalQuery.should( @@ -442,7 +412,7 @@ private Optional getPrefixAndExactMatchQuery( .boost( searchFieldConfig.boost() * exactMatchConfiguration.getExactFactor() - * exactMatchConfiguration.getCaseSensitivityFactor()) + * caseSensitivityFactor) .queryName(searchFieldConfig.fieldName())); } @@ -485,14 +455,16 @@ private Optional getStructuredQuery( return result; } - private FunctionScoreQueryBuilder buildScoreFunctions( + static FunctionScoreQueryBuilder buildScoreFunctions( + @Nonnull OperationContext opContext, @Nullable QueryConfiguration customQueryConfig, @Nonnull List entitySpecs, @Nonnull QueryBuilder queryBuilder) { if (customQueryConfig != null) { // Prefer configuration function scoring over annotation scoring - return functionScoreQueryBuilder(customQueryConfig, queryBuilder); + return CustomizedQueryHandler.functionScoreQueryBuilder( + opContext.getObjectMapper(), customQueryConfig, queryBuilder); } else { return QueryBuilders.functionScoreQuery( queryBuilder, buildAnnotationScoreFunctions(entitySpecs)) @@ -586,62 +558,6 @@ private static FieldValueFactorFunction.Modifier mapModifier( } } - public FunctionScoreQueryBuilder functionScoreQueryBuilder( - QueryConfiguration customQueryConfiguration, QueryBuilder queryBuilder) { - return toFunctionScoreQueryBuilder(queryBuilder, customQueryConfiguration.getFunctionScore()); - } - - public Optional boolQueryBuilder( - QueryConfiguration customQueryConfiguration, String query) { - if (customQueryConfiguration.getBoolQuery() != null) { - log.debug( - "Using custom query configuration queryRegex: {}", - customQueryConfiguration.getQueryRegex()); - } - return Optional.ofNullable(customQueryConfiguration.getBoolQuery()) - .map(bq -> toBoolQueryBuilder(query, bq)); - } - - private BoolQueryBuilder toBoolQueryBuilder(String query, BoolQueryConfiguration boolQuery) { - try { - String jsonFragment = - OBJECT_MAPPER - .writeValueAsString(boolQuery) - .replace("\"{{query_string}}\"", OBJECT_MAPPER.writeValueAsString(query)) - .replace( - "\"{{unquoted_query_string}}\"", - OBJECT_MAPPER.writeValueAsString(unquote(query))); - XContentParser parser = - XContentType.JSON - .xContent() - .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, jsonFragment); - return BoolQueryBuilder.fromXContent(parser); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private FunctionScoreQueryBuilder toFunctionScoreQueryBuilder( - QueryBuilder queryBuilder, Map params) { - try { - HashMap body = new HashMap<>(params); - if (!body.isEmpty()) { - log.debug("Using custom scoring functions: {}", body); - } - - body.put("query", OBJECT_MAPPER.readValue(queryBuilder.toString(), Map.class)); - - String jsonFragment = OBJECT_MAPPER.writeValueAsString(Map.of("function_score", body)); - XContentParser parser = - XContentType.JSON - .xContent() - .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, jsonFragment); - return (FunctionScoreQueryBuilder) FunctionScoreQueryBuilder.parseInnerQueryBuilder(parser); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - public float getWordGramFactor(String fieldName) { if (fieldName.endsWith("Grams2")) { return wordGramConfiguration.getTwoGramFactor(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/EditableSchemaMetadataChangeEventGenerator.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/EditableSchemaMetadataChangeEventGenerator.java index 1f094bb6ca989..4850fde426f00 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/EditableSchemaMetadataChangeEventGenerator.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/EditableSchemaMetadataChangeEventGenerator.java @@ -5,6 +5,7 @@ import com.datahub.util.RecordUtils; import com.github.fge.jsonpatch.JsonPatch; +import com.google.common.collect.ImmutableMap; import com.linkedin.common.AuditStamp; import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTerms; @@ -166,6 +167,7 @@ private static ChangeEvent getDocumentationChangeEvent( targetFieldInfo.getFieldPath(), datasetFieldUrn, targetFieldDescription)) + .parameters(ImmutableMap.of("description", targetFieldDescription)) .auditStamp(auditStamp) .build(); } @@ -183,6 +185,7 @@ private static ChangeEvent getDocumentationChangeEvent( Optional.ofNullable(targetFieldInfo).map(EditableSchemaFieldInfo::getFieldPath), datasetFieldUrn, baseFieldDescription)) + .parameters(ImmutableMap.of("description", baseFieldDescription)) .auditStamp(auditStamp) .build(); } @@ -203,6 +206,7 @@ private static ChangeEvent getDocumentationChangeEvent( datasetFieldUrn, baseFieldDescription, targetFieldDescription)) + .parameters(ImmutableMap.of("description", targetFieldDescription)) .auditStamp(auditStamp) .build(); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java index 1fd5d6e2c0f7a..483ab806c8462 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java @@ -4,6 +4,7 @@ import com.datahub.util.RecordUtils; import com.github.fge.jsonpatch.JsonPatch; +import com.google.common.collect.ImmutableMap; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.Urn; @@ -62,6 +63,7 @@ private static ChangeEvent getDescriptionChange( .description( String.format( FIELD_DESCRIPTION_ADDED_FORMAT, targetDescription, targetField.getFieldPath())) + .parameters(ImmutableMap.of("description", targetDescription)) .auditStamp(auditStamp) .build(); } @@ -75,6 +77,7 @@ private static ChangeEvent getDescriptionChange( .description( String.format( FIELD_DESCRIPTION_REMOVED_FORMAT, baseDescription, baseField.getFieldPath())) + .parameters(ImmutableMap.of("description", baseDescription)) .auditStamp(auditStamp) .build(); } @@ -91,6 +94,7 @@ private static ChangeEvent getDescriptionChange( baseField.getFieldPath(), baseDescription, targetDescription)) + .parameters(ImmutableMap.of("description", targetDescription)) .auditStamp(auditStamp) .build(); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java index 326f814c13dd2..2f68f17dae241 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java @@ -5,6 +5,11 @@ import static org.testng.Assert.assertTrue; import com.linkedin.metadata.TestEntitySpecBuilder; +import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration; +import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.config.search.custom.QueryConfiguration; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.query.request.AutocompleteRequestHandler; import io.datahubproject.metadata.context.OperationContext; @@ -12,10 +17,16 @@ import java.util.List; import java.util.Map; import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.lucene.search.function.FieldValueFactorFunction; import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.opensearch.index.query.functionscore.ScoreFunctionBuilders; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.testng.annotations.Test; @@ -23,10 +34,50 @@ public class AutocompleteRequestHandlerTest { private AutocompleteRequestHandler handler = AutocompleteRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), TestOperationContexts.emptyAspectRetriever(null)); + TestEntitySpecBuilder.getSpec(), + CustomSearchConfiguration.builder().build(), + TestOperationContexts.emptyAspectRetriever(null)); private OperationContext mockOpContext = TestOperationContexts.systemContextNoSearchAuthorization(mock(EntityRegistry.class)); + private static final QueryConfiguration TEST_QUERY_CONFIG = + QueryConfiguration.builder() + .queryRegex(".*") + .simpleQuery(true) + .exactMatchQuery(true) + .prefixMatchQuery(true) + .boolQuery( + BoolQueryConfiguration.builder() + .must(List.of(Map.of("term", Map.of("name", "{{query_string}}")))) + .build()) + .functionScore( + Map.of( + "score_mode", + "avg", + "boost_mode", + "multiply", + "functions", + List.of( + Map.of( + "weight", + 1, + "filter", + Map.of("match_all", Map.of())), + Map.of( + "weight", + 0.5, + "filter", + Map.of( + "term", Map.of("materialized", Map.of("value", true)))), + Map.of( + "weight", + 1.5, + "filter", + Map.of( + "term", + Map.of("deprecated", Map.of("value", false))))))) + .build(); + @Test public void testDefaultAutocompleteRequest() { // When field is null @@ -34,7 +85,9 @@ public void testDefaultAutocompleteRequest() { handler.getSearchRequest(mockOpContext, "input", null, null, 10); SearchSourceBuilder sourceBuilder = autocompleteRequest.source(); assertEquals(sourceBuilder.size(), 10); - BoolQueryBuilder query = (BoolQueryBuilder) sourceBuilder.query(); + BoolQueryBuilder wrapper = + (BoolQueryBuilder) ((FunctionScoreQueryBuilder) sourceBuilder.query()).query(); + BoolQueryBuilder query = (BoolQueryBuilder) extractNestedQuery(wrapper); assertEquals(query.should().size(), 3); MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(2); @@ -49,8 +102,8 @@ public void testDefaultAutocompleteRequest() { (MatchPhrasePrefixQueryBuilder) query.should().get(0); assertEquals("keyPart1.delimited", prefixQuery.fieldName()); - assertEquals(query.mustNot().size(), 1); - MatchQueryBuilder removedFilter = (MatchQueryBuilder) query.mustNot().get(0); + assertEquals(wrapper.mustNot().size(), 1); + MatchQueryBuilder removedFilter = (MatchQueryBuilder) wrapper.mustNot().get(0); assertEquals(removedFilter.fieldName(), "removed"); assertEquals(removedFilter.value(), true); HighlightBuilder highlightBuilder = sourceBuilder.highlighter(); @@ -73,7 +126,10 @@ public void testAutocompleteRequestWithField() { handler.getSearchRequest(mockOpContext, "input", "field", null, 10); SearchSourceBuilder sourceBuilder = autocompleteRequest.source(); assertEquals(sourceBuilder.size(), 10); - BoolQueryBuilder query = (BoolQueryBuilder) sourceBuilder.query(); + BoolQueryBuilder wrapper = + (BoolQueryBuilder) ((FunctionScoreQueryBuilder) sourceBuilder.query()).query(); + assertEquals(wrapper.should().size(), 1); + BoolQueryBuilder query = (BoolQueryBuilder) extractNestedQuery(wrapper); assertEquals(query.should().size(), 2); MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(1); @@ -88,7 +144,7 @@ public void testAutocompleteRequestWithField() { (MatchPhrasePrefixQueryBuilder) query.should().get(0); assertEquals("field.delimited", prefixQuery.fieldName()); - MatchQueryBuilder removedFilter = (MatchQueryBuilder) query.mustNot().get(0); + MatchQueryBuilder removedFilter = (MatchQueryBuilder) wrapper.mustNot().get(0); assertEquals(removedFilter.fieldName(), "removed"); assertEquals(removedFilter.value(), true); HighlightBuilder highlightBuilder = sourceBuilder.highlighter(); @@ -99,4 +155,272 @@ public void testAutocompleteRequestWithField() { assertEquals(highlightedFields.get(2).name(), "field.ngram"); assertEquals(highlightedFields.get(3).name(), "field.delimited"); } + + @Test + public void testCustomConfigWithDefault() { + // Exclude Default query + AutocompleteRequestHandler withoutDefaultQuery = + AutocompleteRequestHandler.getBuilder( + TestEntitySpecBuilder.getSpec(), + CustomSearchConfiguration.builder() + .autocompleteConfigurations( + List.of( + AutocompleteConfiguration.builder() + .queryRegex(".*") + .defaultQuery(false) + .boolQuery( + BoolQueryConfiguration.builder() + .should(List.of(Map.of("match_all", Map.of()))) + .build()) + .build())) + .build(), + mock(AspectRetriever.class)); + + SearchRequest autocompleteRequest = + withoutDefaultQuery.getSearchRequest(mockOpContext, "input", null, null, 10); + SearchSourceBuilder sourceBuilder = autocompleteRequest.source(); + FunctionScoreQueryBuilder wrapper = (FunctionScoreQueryBuilder) sourceBuilder.query(); + assertEquals(((BoolQueryBuilder) wrapper.query()).should().size(), 1); + QueryBuilder customQuery = extractNestedQuery((BoolQueryBuilder) wrapper.query()); + assertEquals(customQuery, QueryBuilders.matchAllQuery()); + + // Include Default query + AutocompleteRequestHandler withDefaultQuery = + AutocompleteRequestHandler.getBuilder( + TestEntitySpecBuilder.getSpec(), + CustomSearchConfiguration.builder() + .autocompleteConfigurations( + List.of( + AutocompleteConfiguration.builder() + .queryRegex(".*") + .defaultQuery(true) + .boolQuery( + BoolQueryConfiguration.builder() + .should(List.of(Map.of("match_all", Map.of()))) + .build()) + .build())) + .build(), + mock(AspectRetriever.class)); + + autocompleteRequest = withDefaultQuery.getSearchRequest(mockOpContext, "input", null, null, 10); + sourceBuilder = autocompleteRequest.source(); + wrapper = (FunctionScoreQueryBuilder) sourceBuilder.query(); + BoolQueryBuilder query = + ((BoolQueryBuilder) ((BoolQueryBuilder) wrapper.query()).should().get(0)); + assertEquals(query.should().size(), 2); + + List shouldQueries = query.should(); + + // Default + BoolQueryBuilder defaultQuery = + (BoolQueryBuilder) + shouldQueries.stream().filter(qb -> qb instanceof BoolQueryBuilder).findFirst().get(); + assertEquals(defaultQuery.should().size(), 3); + + // Custom + customQuery = + shouldQueries.stream().filter(qb -> qb instanceof MatchAllQueryBuilder).findFirst().get(); + assertEquals(customQuery, QueryBuilders.matchAllQuery()); + } + + @Test + public void testCustomConfigWithInheritedQueryFunctionScores() { + // Pickup scoring functions from non-autocomplete + AutocompleteRequestHandler withInherit = + AutocompleteRequestHandler.getBuilder( + TestEntitySpecBuilder.getSpec(), + CustomSearchConfiguration.builder() + .queryConfigurations(List.of(TEST_QUERY_CONFIG)) + .autocompleteConfigurations( + List.of( + AutocompleteConfiguration.builder() + .queryRegex(".*") + .defaultQuery(false) + .inheritFunctionScore(true) + .boolQuery( + BoolQueryConfiguration.builder() + .should(List.of(Map.of("match_all", Map.of()))) + .build()) + .build())) + .build(), + mock(AspectRetriever.class)); + + SearchRequest autocompleteRequest = + withInherit.getSearchRequest(mockOpContext, "input", null, null, 10); + SearchSourceBuilder sourceBuilder = autocompleteRequest.source(); + FunctionScoreQueryBuilder wrapper = (FunctionScoreQueryBuilder) sourceBuilder.query(); + assertEquals(((BoolQueryBuilder) wrapper.query()).should().size(), 1); + + QueryBuilder customQuery = extractNestedQuery(((BoolQueryBuilder) wrapper.query())); + assertEquals(customQuery, QueryBuilders.matchAllQuery()); + + FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedQueryConfigurationScoreFunctions = { + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + ScoreFunctionBuilders.weightFactorFunction(1f)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + QueryBuilders.termQuery("materialized", true), + ScoreFunctionBuilders.weightFactorFunction(0.5f)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + QueryBuilders.termQuery("deprecated", false), + ScoreFunctionBuilders.weightFactorFunction(1.5f)) + }; + assertEquals(wrapper.filterFunctionBuilders(), expectedQueryConfigurationScoreFunctions); + + // no search query customization + AutocompleteRequestHandler noQueryCustomization = + AutocompleteRequestHandler.getBuilder( + TestEntitySpecBuilder.getSpec(), + CustomSearchConfiguration.builder() + .autocompleteConfigurations( + List.of( + AutocompleteConfiguration.builder() + .queryRegex(".*") + .defaultQuery(false) + .boolQuery( + BoolQueryConfiguration.builder() + .should(List.of(Map.of("match_all", Map.of()))) + .build()) + .build())) + .build(), + mock(AspectRetriever.class)); + + autocompleteRequest = + noQueryCustomization.getSearchRequest(mockOpContext, "input", null, null, 10); + sourceBuilder = autocompleteRequest.source(); + wrapper = (FunctionScoreQueryBuilder) sourceBuilder.query(); + assertEquals(((BoolQueryBuilder) wrapper.query()).should().size(), 1); + + customQuery = extractNestedQuery((BoolQueryBuilder) wrapper.query()); + assertEquals(customQuery, QueryBuilders.matchAllQuery()); + + // PDL annotation based on default behavior of query builder + FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedDefaultScoreFunctions = { + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + ScoreFunctionBuilders.weightFactorFunction(1f)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + ScoreFunctionBuilders.fieldValueFactorFunction("feature2") + .modifier(FieldValueFactorFunction.Modifier.NONE) + .missing(0.0)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + ScoreFunctionBuilders.fieldValueFactorFunction("feature1") + .modifier(FieldValueFactorFunction.Modifier.LOG1P) + .missing(0.0)) + }; + assertEquals(wrapper.filterFunctionBuilders(), expectedDefaultScoreFunctions); + } + + @Test + public void testCustomConfigWithFunctionScores() { + // Scoring functions explicit autocomplete override + AutocompleteRequestHandler explicitNoInherit = + AutocompleteRequestHandler.getBuilder( + TestEntitySpecBuilder.getSpec(), + CustomSearchConfiguration.builder() + .queryConfigurations(List.of(TEST_QUERY_CONFIG)) // should be ignored + .autocompleteConfigurations( + List.of( + AutocompleteConfiguration.builder() + .queryRegex(".*") + .defaultQuery(false) + .inheritFunctionScore(false) + .boolQuery( + BoolQueryConfiguration.builder() + .should(List.of(Map.of("match_all", Map.of()))) + .build()) + .functionScore( + Map.of( + "score_mode", + "avg", + "boost_mode", + "multiply", + "functions", + List.of( + Map.of( + "weight", + 1.5, + "filter", + Map.of( + "term", + Map.of( + "deprecated", Map.of("value", false))))))) + .build())) + .build(), + mock(AspectRetriever.class)); + + SearchRequest autocompleteRequest = + explicitNoInherit.getSearchRequest(mockOpContext, "input", null, null, 10); + SearchSourceBuilder sourceBuilder = autocompleteRequest.source(); + FunctionScoreQueryBuilder wrapper = (FunctionScoreQueryBuilder) sourceBuilder.query(); + assertEquals(((BoolQueryBuilder) wrapper.query()).should().size(), 1); + + QueryBuilder customQuery = extractNestedQuery((BoolQueryBuilder) wrapper.query()); + assertEquals(customQuery, QueryBuilders.matchAllQuery()); + + FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedCustomScoreFunctions = { + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + QueryBuilders.termQuery("deprecated", false), + ScoreFunctionBuilders.weightFactorFunction(1.5f)) + }; + assertEquals(wrapper.filterFunctionBuilders(), expectedCustomScoreFunctions); + + // Pickup scoring functions explicit autocomplete override (even though default query and + // inherit enabled) + AutocompleteRequestHandler explicit = + AutocompleteRequestHandler.getBuilder( + TestEntitySpecBuilder.getSpec(), + CustomSearchConfiguration.builder() + .queryConfigurations(List.of(TEST_QUERY_CONFIG)) // should be ignored + .autocompleteConfigurations( + List.of( + AutocompleteConfiguration.builder() + .queryRegex(".*") + .defaultQuery(true) + .inheritFunctionScore(true) + .boolQuery( + BoolQueryConfiguration.builder() + .should(List.of(Map.of("match_all", Map.of()))) + .build()) + .functionScore( + Map.of( + "score_mode", + "avg", + "boost_mode", + "multiply", + "functions", + List.of( + Map.of( + "weight", + 1.5, + "filter", + Map.of( + "term", + Map.of( + "deprecated", Map.of("value", false))))))) + .build())) + .build(), + mock(AspectRetriever.class)); + + autocompleteRequest = explicit.getSearchRequest(mockOpContext, "input", null, null, 10); + sourceBuilder = autocompleteRequest.source(); + wrapper = (FunctionScoreQueryBuilder) sourceBuilder.query(); + BoolQueryBuilder query = + ((BoolQueryBuilder) ((BoolQueryBuilder) wrapper.query()).should().get(0)); + assertEquals(query.should().size(), 2); + + customQuery = query.should().get(0); + assertEquals(customQuery, QueryBuilders.matchAllQuery()); + + // standard query still present + assertEquals(((BoolQueryBuilder) query.should().get(1)).should().size(), 3); + + // custom functions included + assertEquals(wrapper.filterFunctionBuilders(), expectedCustomScoreFunctions); + } + + private static QueryBuilder extractNestedQuery(BoolQueryBuilder nested) { + assertEquals(nested.should().size(), 1); + BoolQueryBuilder firstLevel = (BoolQueryBuilder) nested.should().get(0); + assertEquals(firstLevel.should().size(), 1); + return firstLevel.should().get(0); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java index 47d18fe0d299c..4e4c8acf300e4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java @@ -178,8 +178,8 @@ public void functionScoreQueryBuilderTest() { * Test select star */ FunctionScoreQueryBuilder selectStarTest = - SEARCH_QUERY_BUILDER.functionScoreQueryBuilder( - test.lookupQueryConfig("*").get(), inputQuery); + CustomizedQueryHandler.functionScoreQueryBuilder( + new ObjectMapper(), test.lookupQueryConfig("*").get(), inputQuery); FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedSelectStarScoreFunctions = { new FunctionScoreQueryBuilder.FilterFunctionBuilder( @@ -202,8 +202,8 @@ public void functionScoreQueryBuilderTest() { * Test default (non-select start) */ FunctionScoreQueryBuilder defaultTest = - SEARCH_QUERY_BUILDER.functionScoreQueryBuilder( - test.lookupQueryConfig("foobar").get(), inputQuery); + CustomizedQueryHandler.functionScoreQueryBuilder( + new ObjectMapper(), test.lookupQueryConfig("foobar").get(), inputQuery); FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedDefaultScoreFunctions = { new FunctionScoreQueryBuilder.FilterFunctionBuilder( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java index 39dcd120f2842..dbb5bdb0b7d01 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java @@ -29,6 +29,7 @@ import com.linkedin.metadata.search.elasticsearch.query.request.SearchQueryBuilder; import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.test.metadata.context.TestOperationContexts; import io.datahubproject.test.search.config.SearchCommonTestConfiguration; import java.io.IOException; import java.util.List; @@ -91,15 +92,14 @@ public class SearchQueryBuilderTest extends AbstractTestNGSpringContextTests { public static final SearchQueryBuilder TEST_BUILDER = new SearchQueryBuilder(testQueryConfig, null); + public OperationContext opContext = TestOperationContexts.systemContextNoSearchAuthorization(); + @Test public void testQueryBuilderFulltext() { FunctionScoreQueryBuilder result = (FunctionScoreQueryBuilder) TEST_BUILDER.buildQuery( - mock(OperationContext.class), - ImmutableList.of(TestEntitySpecBuilder.getSpec()), - "testQuery", - true); + opContext, ImmutableList.of(TestEntitySpecBuilder.getSpec()), "testQuery", true); BoolQueryBuilder mainQuery = (BoolQueryBuilder) result.query(); List shouldQueries = mainQuery.should(); assertEquals(shouldQueries.size(), 2); @@ -200,10 +200,7 @@ public void testQueryBuilderStructured() { FunctionScoreQueryBuilder result = (FunctionScoreQueryBuilder) TEST_BUILDER.buildQuery( - mock(OperationContext.class), - ImmutableList.of(TestEntitySpecBuilder.getSpec()), - "testQuery", - false); + opContext, ImmutableList.of(TestEntitySpecBuilder.getSpec()), "testQuery", false); BoolQueryBuilder mainQuery = (BoolQueryBuilder) result.query(); List shouldQueries = mainQuery.should(); assertEquals(shouldQueries.size(), 2); @@ -246,10 +243,7 @@ public void testCustomSelectAll() { FunctionScoreQueryBuilder result = (FunctionScoreQueryBuilder) TEST_CUSTOM_BUILDER.buildQuery( - mock(OperationContext.class), - ImmutableList.of(TestEntitySpecBuilder.getSpec()), - triggerQuery, - true); + opContext, ImmutableList.of(TestEntitySpecBuilder.getSpec()), triggerQuery, true); BoolQueryBuilder mainQuery = (BoolQueryBuilder) result.query(); List shouldQueries = mainQuery.should(); @@ -263,10 +257,7 @@ public void testCustomExactMatch() { FunctionScoreQueryBuilder result = (FunctionScoreQueryBuilder) TEST_CUSTOM_BUILDER.buildQuery( - mock(OperationContext.class), - ImmutableList.of(TestEntitySpecBuilder.getSpec()), - triggerQuery, - true); + opContext, ImmutableList.of(TestEntitySpecBuilder.getSpec()), triggerQuery, true); BoolQueryBuilder mainQuery = (BoolQueryBuilder) result.query(); List shouldQueries = mainQuery.should(); @@ -302,10 +293,7 @@ public void testCustomDefault() { FunctionScoreQueryBuilder result = (FunctionScoreQueryBuilder) TEST_CUSTOM_BUILDER.buildQuery( - mock(OperationContext.class), - ImmutableList.of(TestEntitySpecBuilder.getSpec()), - triggerQuery, - true); + opContext, ImmutableList.of(TestEntitySpecBuilder.getSpec()), triggerQuery, true); BoolQueryBuilder mainQuery = (BoolQueryBuilder) result.query(); List shouldQueries = mainQuery.should(); diff --git a/metadata-io/src/test/resources/search_config_test.yml b/metadata-io/src/test/resources/search_config_test.yml index 787d7f22de431..2ec81eddcab0e 100644 --- a/metadata-io/src/test/resources/search_config_test.yml +++ b/metadata-io/src/test/resources/search_config_test.yml @@ -53,3 +53,28 @@ queryConfigurations: weight: 1.5 score_mode: avg boost_mode: multiply + +autocompleteConfigurations: + - queryRegex: .* + defaultQuery: true + boolQuery: + must: + - term: + removed: 'false' + functionScore: + functions: + - filter: + match_all: {} + weight: 1 + - filter: + term: + materialized: + value: true + weight: 0.5 + - filter: + term: + deprecated: + value: false + weight: 1.5 + score_mode: avg + boost_mode: multiply \ No newline at end of file diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHookTest.java index bb2a96693d214..e9f013c5a227f 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHookTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHookTest.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.kafka.hook.event; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.timeline.eventgenerator.ChangeEventGeneratorUtils.getSchemaFieldUrn; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @@ -36,6 +37,7 @@ import com.linkedin.dataprocess.DataProcessRunStatus; import com.linkedin.dataprocess.RunResultType; import com.linkedin.dataset.DatasetProperties; +import com.linkedin.dataset.EditableDatasetProperties; import com.linkedin.domain.Domains; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; @@ -50,22 +52,14 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.timeline.data.ChangeCategory; import com.linkedin.metadata.timeline.data.ChangeOperation; -import com.linkedin.metadata.timeline.eventgenerator.AssertionRunEventChangeEventGenerator; -import com.linkedin.metadata.timeline.eventgenerator.DataProcessInstanceRunEventChangeEventGenerator; -import com.linkedin.metadata.timeline.eventgenerator.DeprecationChangeEventGenerator; -import com.linkedin.metadata.timeline.eventgenerator.EntityChangeEventGeneratorRegistry; -import com.linkedin.metadata.timeline.eventgenerator.EntityKeyChangeEventGenerator; -import com.linkedin.metadata.timeline.eventgenerator.GlobalTagsChangeEventGenerator; -import com.linkedin.metadata.timeline.eventgenerator.GlossaryTermsChangeEventGenerator; -import com.linkedin.metadata.timeline.eventgenerator.OwnershipChangeEventGenerator; -import com.linkedin.metadata.timeline.eventgenerator.SingleDomainChangeEventGenerator; -import com.linkedin.metadata.timeline.eventgenerator.StatusChangeEventGenerator; +import com.linkedin.metadata.timeline.eventgenerator.*; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.PlatformEvent; import com.linkedin.mxe.PlatformEventHeader; import com.linkedin.platform.event.v1.EntityChangeEvent; import com.linkedin.platform.event.v1.Parameters; +import com.linkedin.schema.*; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; import java.net.URISyntaxException; @@ -77,7 +71,7 @@ /** * Tests the {@link EntityChangeEventGeneratorHook}. * - *

TODO: Include Schema Field Tests, description update tests. + *

TODO: Include more Schema Field Tests for tags, terms and schema-changes. */ public class EntityChangeEventGeneratorHookTest { private static final long EVENT_TIME = 123L; @@ -675,6 +669,381 @@ public void testInvokeIneligibleAspect() throws Exception { Mockito.verifyNoMoreInteractions(_mockClient); } + @Test + public void testDatasetPropertiesAdd() throws Exception { + final String newDescription = "New desc"; + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(DATASET_PROPERTIES_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + event.setAspect( + GenericRecordUtils.serializeAspect(new DatasetProperties().setDescription(newDescription))); + _entityChangeEventHook.invoke(event); + + PlatformEvent platformEvent = + createChangeEvent( + DATASET_ENTITY_NAME, + Urn.createFromString(TEST_DATASET_URN), + ChangeCategory.DOCUMENTATION, + ChangeOperation.ADD, + null, + ImmutableMap.of("description", newDescription), + actorUrn); + verifyProducePlatformEvent(_mockClient, platformEvent); + } + + @Test + public void testDatasetDescriptionAdd() throws Exception { + final String newDescription = "New desc"; + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(DATASET_PROPERTIES_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + event.setAspect( + GenericRecordUtils.serializeAspect(new DatasetProperties().setDescription(newDescription))); + event.setPreviousAspectValue(GenericRecordUtils.serializeAspect(new DatasetProperties())); + + _entityChangeEventHook.invoke(event); + + PlatformEvent platformEvent = + createChangeEvent( + DATASET_ENTITY_NAME, + Urn.createFromString(TEST_DATASET_URN), + ChangeCategory.DOCUMENTATION, + ChangeOperation.ADD, + null, + ImmutableMap.of("description", newDescription), + actorUrn); + verifyProducePlatformEvent(_mockClient, platformEvent); + } + + @Test + public void testDatasetDescriptionModify() throws Exception { + final String newDescription = "New desc"; + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(DATASET_PROPERTIES_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + event.setAspect( + GenericRecordUtils.serializeAspect(new DatasetProperties().setDescription(newDescription))); + event.setPreviousAspectValue( + GenericRecordUtils.serializeAspect(new DatasetProperties().setDescription("Old desc"))); + + _entityChangeEventHook.invoke(event); + + PlatformEvent platformEvent = + createChangeEvent( + DATASET_ENTITY_NAME, + Urn.createFromString(TEST_DATASET_URN), + ChangeCategory.DOCUMENTATION, + ChangeOperation.MODIFY, + null, + ImmutableMap.of("description", newDescription), + actorUrn); + verifyProducePlatformEvent(_mockClient, platformEvent); + } + + @Test + public void testDatasetDescriptionRemove() throws Exception { + final String oldDescription = "Old desc"; + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(DATASET_PROPERTIES_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + event.setAspect(GenericRecordUtils.serializeAspect(new DatasetProperties())); + event.setPreviousAspectValue( + GenericRecordUtils.serializeAspect(new DatasetProperties().setDescription(oldDescription))); + + _entityChangeEventHook.invoke(event); + + PlatformEvent platformEvent = + createChangeEvent( + DATASET_ENTITY_NAME, + Urn.createFromString(TEST_DATASET_URN), + ChangeCategory.DOCUMENTATION, + ChangeOperation.REMOVE, + null, + ImmutableMap.of("description", oldDescription), + actorUrn); + verifyProducePlatformEvent(_mockClient, platformEvent); + } + + @Test + public void testEditableDatasetPropertiesAdd() throws Exception { + final String newDescription = "New desc"; + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + event.setAspect( + GenericRecordUtils.serializeAspect( + new EditableDatasetProperties().setDescription(newDescription))); + _entityChangeEventHook.invoke(event); + + PlatformEvent platformEvent = + createChangeEvent( + DATASET_ENTITY_NAME, + Urn.createFromString(TEST_DATASET_URN), + ChangeCategory.DOCUMENTATION, + ChangeOperation.ADD, + null, + ImmutableMap.of("description", newDescription), + actorUrn); + verifyProducePlatformEvent(_mockClient, platformEvent); + } + + @Test + public void testEditableDatasetDescriptionAdd() throws Exception { + final String newDescription = "New desc"; + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + event.setAspect( + GenericRecordUtils.serializeAspect( + new EditableDatasetProperties().setDescription(newDescription))); + event.setPreviousAspectValue( + GenericRecordUtils.serializeAspect(new EditableDatasetProperties())); + + _entityChangeEventHook.invoke(event); + + PlatformEvent platformEvent = + createChangeEvent( + DATASET_ENTITY_NAME, + Urn.createFromString(TEST_DATASET_URN), + ChangeCategory.DOCUMENTATION, + ChangeOperation.ADD, + null, + ImmutableMap.of("description", newDescription), + actorUrn); + verifyProducePlatformEvent(_mockClient, platformEvent); + } + + @Test + public void testEditableDatasetDescriptionModify() throws Exception { + final String newDescription = "New desc"; + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + event.setAspect( + GenericRecordUtils.serializeAspect( + new EditableDatasetProperties().setDescription(newDescription))); + event.setPreviousAspectValue( + GenericRecordUtils.serializeAspect( + new EditableDatasetProperties().setDescription("Old desc"))); + + _entityChangeEventHook.invoke(event); + + PlatformEvent platformEvent = + createChangeEvent( + DATASET_ENTITY_NAME, + Urn.createFromString(TEST_DATASET_URN), + ChangeCategory.DOCUMENTATION, + ChangeOperation.MODIFY, + null, + ImmutableMap.of("description", newDescription), + actorUrn); + verifyProducePlatformEvent(_mockClient, platformEvent); + } + + @Test + public void testEditableDatasetDescriptionRemove() throws Exception { + final String oldDescription = "Old desc"; + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + event.setAspect(GenericRecordUtils.serializeAspect(new EditableDatasetProperties())); + event.setPreviousAspectValue( + GenericRecordUtils.serializeAspect( + new EditableDatasetProperties().setDescription(oldDescription))); + + _entityChangeEventHook.invoke(event); + + PlatformEvent platformEvent = + createChangeEvent( + DATASET_ENTITY_NAME, + Urn.createFromString(TEST_DATASET_URN), + ChangeCategory.DOCUMENTATION, + ChangeOperation.REMOVE, + null, + ImmutableMap.of("description", oldDescription), + actorUrn); + verifyProducePlatformEvent(_mockClient, platformEvent); + } + + @Test + public void testSchemaFieldDescriptionChanges() throws Exception { + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(SCHEMA_METADATA_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + SchemaField field1 = new SchemaField().setNativeDataType("string").setFieldPath("c1"); + SchemaField field2 = new SchemaField().setNativeDataType("string").setFieldPath("c2"); + SchemaField field3 = new SchemaField().setNativeDataType("string").setFieldPath("c3"); + SchemaField field4 = new SchemaField().setNativeDataType("string").setFieldPath("c4"); + + SchemaFieldArray oldFields = new SchemaFieldArray(); + SchemaFieldArray newFields = new SchemaFieldArray(); + + oldFields.add(field1.clone()); + newFields.add(field1.clone().setDescription("c1Desc")); + + oldFields.add(field2.clone().setDescription("oldC2Desc")); + newFields.add(field2.clone().setDescription("newC2Desc")); + + oldFields.add(field3.clone().setDescription("c3Desc")); + newFields.add(field3.clone()); + + oldFields.add(field4.clone().setDescription("c4Desc")); + newFields.add(field4.clone().setDescription("c4Desc")); + + event.setPreviousAspectValue( + GenericRecordUtils.serializeAspect(new SchemaMetadata().setFields(oldFields))); + event.setAspect(GenericRecordUtils.serializeAspect(new SchemaMetadata().setFields(newFields))); + + _entityChangeEventHook.invoke(event); + + verifyProducePlatformEvent( + _mockClient, + createChangeEvent( + SCHEMA_FIELD_ENTITY_NAME, + getSchemaFieldUrn(Urn.createFromString(TEST_DATASET_URN), "c1"), + ChangeCategory.DOCUMENTATION, + ChangeOperation.ADD, + null, + ImmutableMap.of("description", "c1Desc"), + actorUrn), + false); + + verifyProducePlatformEvent( + _mockClient, + createChangeEvent( + SCHEMA_FIELD_ENTITY_NAME, + getSchemaFieldUrn(Urn.createFromString(TEST_DATASET_URN), "c2"), + ChangeCategory.DOCUMENTATION, + ChangeOperation.MODIFY, + null, + ImmutableMap.of("description", "newC2Desc"), + actorUrn), + false); + + verifyProducePlatformEvent( + _mockClient, + createChangeEvent( + SCHEMA_FIELD_ENTITY_NAME, + getSchemaFieldUrn(Urn.createFromString(TEST_DATASET_URN), "c3"), + ChangeCategory.DOCUMENTATION, + ChangeOperation.REMOVE, + null, + ImmutableMap.of("description", "c3Desc"), + actorUrn), + true); + } + + @Test + public void testEditableSchemaFieldDescriptionChanges() throws Exception { + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(EDITABLE_SCHEMA_METADATA_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + event.setEntityUrn(Urn.createFromString(TEST_DATASET_URN)); + event.setCreated(new AuditStamp().setActor(actorUrn).setTime(EVENT_TIME)); + + EditableSchemaFieldInfo field1 = new EditableSchemaFieldInfo().setFieldPath("c1"); + EditableSchemaFieldInfo field2 = new EditableSchemaFieldInfo().setFieldPath("c2"); + EditableSchemaFieldInfo field3 = new EditableSchemaFieldInfo().setFieldPath("c3"); + EditableSchemaFieldInfo field4 = new EditableSchemaFieldInfo().setFieldPath("c4"); + + EditableSchemaFieldInfoArray oldFields = new EditableSchemaFieldInfoArray(); + EditableSchemaFieldInfoArray newFields = new EditableSchemaFieldInfoArray(); + + oldFields.add(field1.clone()); + newFields.add(field1.clone().setDescription("c1Desc")); + + oldFields.add(field2.clone().setDescription("oldC2Desc")); + newFields.add(field2.clone().setDescription("newC2Desc")); + + oldFields.add(field3.clone().setDescription("c3Desc")); + newFields.add(field3.clone()); + + oldFields.add(field4.clone().setDescription("c4Desc")); + newFields.add(field4.clone().setDescription("c4Desc")); + + event.setPreviousAspectValue( + GenericRecordUtils.serializeAspect( + new EditableSchemaMetadata().setEditableSchemaFieldInfo(oldFields))); + event.setAspect( + GenericRecordUtils.serializeAspect( + new EditableSchemaMetadata().setEditableSchemaFieldInfo(newFields))); + + _entityChangeEventHook.invoke(event); + + verifyProducePlatformEvent( + _mockClient, + createChangeEvent( + SCHEMA_FIELD_ENTITY_NAME, + getSchemaFieldUrn(Urn.createFromString(TEST_DATASET_URN), "c1"), + ChangeCategory.DOCUMENTATION, + ChangeOperation.ADD, + null, + ImmutableMap.of("description", "c1Desc"), + actorUrn), + false); + + verifyProducePlatformEvent( + _mockClient, + createChangeEvent( + SCHEMA_FIELD_ENTITY_NAME, + getSchemaFieldUrn(Urn.createFromString(TEST_DATASET_URN), "c2"), + ChangeCategory.DOCUMENTATION, + ChangeOperation.MODIFY, + null, + ImmutableMap.of("description", "newC2Desc"), + actorUrn), + false); + + verifyProducePlatformEvent( + _mockClient, + createChangeEvent( + SCHEMA_FIELD_ENTITY_NAME, + getSchemaFieldUrn(Urn.createFromString(TEST_DATASET_URN), "c3"), + ChangeCategory.DOCUMENTATION, + ChangeOperation.REMOVE, + null, + ImmutableMap.of("description", "c3Desc"), + actorUrn), + true); + } + private PlatformEvent createChangeEvent( String entityType, Urn entityUrn, @@ -714,8 +1083,13 @@ private EntityChangeEventGeneratorRegistry createEntityChangeEventGeneratorRegis registry.register(OWNERSHIP_ASPECT_NAME, new OwnershipChangeEventGenerator()); registry.register(STATUS_ASPECT_NAME, new StatusChangeEventGenerator()); registry.register(DEPRECATION_ASPECT_NAME, new DeprecationChangeEventGenerator()); - - // TODO Add Dataset Schema Field related change generators. + registry.register(DATASET_PROPERTIES_ASPECT_NAME, new DatasetPropertiesChangeEventGenerator()); + registry.register( + EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, + new EditableDatasetPropertiesChangeEventGenerator()); + registry.register(SCHEMA_METADATA_ASPECT_NAME, new SchemaMetadataChangeEventGenerator()); + registry.register( + EDITABLE_SCHEMA_METADATA_ASPECT_NAME, new EditableSchemaMetadataChangeEventGenerator()); // Entity Lifecycle change event generators registry.register(DATASET_KEY_ASPECT_NAME, new EntityKeyChangeEventGenerator<>()); @@ -756,6 +1130,23 @@ private OperationContext createMockOperationContext() { AspectSpec mockDatasetKey = createMockAspectSpec(DatasetKey.class); Mockito.when(datasetSpec.getAspectSpec(eq(DATASET_KEY_ASPECT_NAME))).thenReturn(mockDatasetKey); + AspectSpec mockDatasetProperties = createMockAspectSpec(DatasetProperties.class); + Mockito.when(datasetSpec.getAspectSpec(eq(DATASET_PROPERTIES_ASPECT_NAME))) + .thenReturn(mockDatasetProperties); + + AspectSpec mockEditableDatasetProperties = + createMockAspectSpec(EditableDatasetProperties.class); + Mockito.when(datasetSpec.getAspectSpec(eq(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME))) + .thenReturn(mockEditableDatasetProperties); + + AspectSpec mockSchemaMetadata = createMockAspectSpec(SchemaMetadata.class); + Mockito.when(datasetSpec.getAspectSpec(eq(SCHEMA_METADATA_ASPECT_NAME))) + .thenReturn(mockSchemaMetadata); + + AspectSpec mockEditableSchemaMetadata = createMockAspectSpec(EditableSchemaMetadata.class); + Mockito.when(datasetSpec.getAspectSpec(eq(EDITABLE_SCHEMA_METADATA_ASPECT_NAME))) + .thenReturn(mockEditableSchemaMetadata); + Mockito.when(registry.getEntitySpec(eq(DATASET_ENTITY_NAME))).thenReturn(datasetSpec); // Build Assertion Entity Spec diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ObjectMapperContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ObjectMapperContext.java new file mode 100644 index 0000000000000..2e96e48338a66 --- /dev/null +++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ObjectMapperContext.java @@ -0,0 +1,48 @@ +package io.datahubproject.metadata.context; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.metadata.Constants; +import java.util.Optional; +import javax.annotation.Nonnull; +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +public class ObjectMapperContext implements ContextInterface { + + public static ObjectMapper defaultMapper = new ObjectMapper(); + + static { + defaultMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + int maxSize = + Integer.parseInt( + System.getenv() + .getOrDefault( + Constants.INGESTION_MAX_SERIALIZED_STRING_LENGTH, + Constants.MAX_JACKSON_STRING_SIZE)); + defaultMapper + .getFactory() + .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); + } + + public static ObjectMapperContext DEFAULT = ObjectMapperContext.builder().build(); + + @Nonnull private final ObjectMapper objectMapper; + + @Override + public Optional getCacheKeyComponent() { + return Optional.empty(); + } + + public static class ObjectMapperContextBuilder { + public ObjectMapperContext build() { + if (this.objectMapper == null) { + objectMapper(defaultMapper); + } + return new ObjectMapperContext(this.objectMapper); + } + } +} diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java index 2e4da5abe7f82..56247d61337e8 100644 --- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java +++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java @@ -2,6 +2,7 @@ import com.datahub.authentication.Authentication; import com.datahub.plugins.auth.authorization.Authorizer; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; @@ -120,6 +121,24 @@ public static OperationContext asSystem( @Nullable ServicesRegistryContext servicesRegistryContext, @Nullable IndexConvention indexConvention, @Nullable RetrieverContext retrieverContext) { + return asSystem( + config, + systemAuthentication, + entityRegistry, + servicesRegistryContext, + indexConvention, + retrieverContext, + ObjectMapperContext.DEFAULT); + } + + public static OperationContext asSystem( + @Nonnull OperationContextConfig config, + @Nonnull Authentication systemAuthentication, + @Nullable EntityRegistry entityRegistry, + @Nullable ServicesRegistryContext servicesRegistryContext, + @Nullable IndexConvention indexConvention, + @Nullable RetrieverContext retrieverContext, + @Nonnull ObjectMapperContext objectMapperContext) { ActorContext systemActorContext = ActorContext.builder().systemAuth(true).authentication(systemAuthentication).build(); @@ -139,6 +158,7 @@ public static OperationContext asSystem( // Authorizer.EMPTY doesn't actually apply to system auth .authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build()) .retrieverContext(retrieverContext) + .objectMapperContext(objectMapperContext) .build(systemAuthentication); } @@ -152,6 +172,7 @@ public static OperationContext asSystem( @Nullable private final RequestContext requestContext; @Nullable private final ViewAuthorizationContext viewAuthorizationContext; @Nullable private final RetrieverContext retrieverContext; + @Nonnull private final ObjectMapperContext objectMapperContext; public OperationContext withSearchFlags( @Nonnull Function flagDefaults) { @@ -298,6 +319,7 @@ public String getGlobalContextId() { getRetrieverContext().isPresent() ? getRetrieverContext().get() : EmptyContext.EMPTY) + .add(getObjectMapperContext()) .build() .stream() .map(ContextInterface::getCacheKeyComponent) @@ -360,6 +382,11 @@ public String getRequestID() { return Optional.ofNullable(requestContext).map(RequestContext::getRequestID).orElse(""); } + @Nonnull + public ObjectMapper getObjectMapper() { + return objectMapperContext.getObjectMapper(); + } + public static class OperationContextBuilder { @Nonnull @@ -392,7 +419,10 @@ public OperationContext build(@Nonnull ActorContext sessionActor) { this.servicesRegistryContext, this.requestContext, this.viewAuthorizationContext, - this.retrieverContext); + this.retrieverContext, + this.objectMapperContext != null + ? this.objectMapperContext + : ObjectMapperContext.DEFAULT); } private OperationContext build() { diff --git a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java index bb6aa1ed231d7..71eaca71a3641 100644 --- a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java +++ b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java @@ -123,9 +123,7 @@ CompletableFuture> generateSessionTokenForUser( try { bodyJson = mapper.readTree(jsonStr); } catch (JsonProcessingException e) { - log.error( - String.format( - "Failed to parse json while attempting to generate session token %s", jsonStr)); + log.error("Failed to parse json while attempting to generate session token {}", jsonStr, e); return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } if (bodyJson == null) { @@ -139,7 +137,7 @@ CompletableFuture> generateSessionTokenForUser( return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } - log.debug(String.format("Attempting to generate session token for user %s", userId.asText())); + log.info("Attempting to generate session token for user {}", userId.asText()); final String actorId = AuthenticationContext.getAuthentication().getActor().getId(); return CompletableFuture.supplyAsync( () -> { @@ -147,14 +145,20 @@ CompletableFuture> generateSessionTokenForUser( if (isAuthorizedToGenerateSessionToken(actorId)) { try { // 2. Generate a new DataHub JWT + final long sessionTokenDurationMs = + _configProvider.getAuthentication().getSessionTokenDurationMs(); final String token = _statelessTokenService.generateAccessToken( TokenType.SESSION, new Actor(ActorType.USER, userId.asText()), - _configProvider.getAuthentication().getSessionTokenDurationMs()); + sessionTokenDurationMs); + log.info( + "Successfully generated session token for user: {}, duration: {} ms", + userId.asText(), + sessionTokenDurationMs); return new ResponseEntity<>(buildTokenResponse(token), HttpStatus.OK); } catch (Exception e) { - log.error("Failed to generate session token for user", e); + log.error("Failed to generate session token for user: {}", userId.asText(), e); return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); } } @@ -189,8 +193,7 @@ CompletableFuture> signUp(final HttpEntity httpEn try { bodyJson = mapper.readTree(jsonStr); } catch (JsonProcessingException e) { - log.error( - String.format("Failed to parse json while attempting to create native user %s", jsonStr)); + log.debug("Failed to parse json while attempting to create native user", e); return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } if (bodyJson == null) { @@ -229,13 +232,13 @@ CompletableFuture> signUp(final HttpEntity httpEn String passwordString = password.asText(); String inviteTokenString = inviteToken.asText(); Authentication auth = AuthenticationContext.getAuthentication(); - log.debug(String.format("Attempting to create native user %s", userUrnString)); + log.info("Attempting to create native user {}", userUrnString); return CompletableFuture.supplyAsync( () -> { try { Urn inviteTokenUrn = _inviteTokenService.getInviteTokenUrn(inviteTokenString); if (!_inviteTokenService.isInviteTokenValid(systemOperationContext, inviteTokenUrn)) { - log.error(String.format("Invalid invite token %s", inviteTokenString)); + log.error("Invalid invite token {}", inviteTokenString); return new ResponseEntity<>(HttpStatus.BAD_REQUEST); } @@ -247,10 +250,10 @@ CompletableFuture> signUp(final HttpEntity httpEn titleString, passwordString); String response = buildSignUpResponse(); + log.info("Created native user {}", userUrnString); return new ResponseEntity<>(response, HttpStatus.OK); } catch (Exception e) { - log.error( - String.format("Failed to create credentials for native user %s", userUrnString), e); + log.error("Failed to create credentials for native user {}", userUrnString, e); return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); } }); @@ -279,8 +282,7 @@ CompletableFuture> resetNativeUserCredentials( try { bodyJson = mapper.readTree(jsonStr); } catch (JsonProcessingException e) { - log.error( - String.format("Failed to parse json while attempting to create native user %s", jsonStr)); + log.debug("Failed to parse json while attempting to create native user", e); return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } if (bodyJson == null) { @@ -300,17 +302,17 @@ CompletableFuture> resetNativeUserCredentials( String passwordString = password.asText(); String resetTokenString = resetToken.asText(); Authentication auth = AuthenticationContext.getAuthentication(); - log.debug(String.format("Attempting to reset credentials for native user %s", userUrnString)); + log.info("Attempting to reset credentials for native user {}", userUrnString); return CompletableFuture.supplyAsync( () -> { try { _nativeUserService.resetCorpUserCredentials( systemOperationContext, userUrnString, passwordString, resetTokenString); String response = buildResetNativeUserCredentialsResponse(); + log.info("Reset credentials for native user {}", userUrnString); return new ResponseEntity<>(response, HttpStatus.OK); } catch (Exception e) { - log.error( - String.format("Failed to reset credentials for native user %s", userUrnString), e); + log.error("Failed to reset credentials for native user {}", userUrnString, e); return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); } }); @@ -338,9 +340,7 @@ CompletableFuture> verifyNativeUserCredentials( try { bodyJson = mapper.readTree(jsonStr); } catch (JsonProcessingException e) { - log.error( - String.format( - "Failed to parse json while attempting to verify native user password %s", jsonStr)); + log.debug("Failed to parse json while attempting to verify native user password", e); return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } if (bodyJson == null) { @@ -357,7 +357,7 @@ CompletableFuture> verifyNativeUserCredentials( String userUrnString = userUrn.asText(); String passwordString = password.asText(); - log.debug(String.format("Attempting to verify credentials for native user %s", userUrnString)); + log.info("Attempting to verify credentials for native user {}", userUrnString); return CompletableFuture.supplyAsync( () -> { try { @@ -365,10 +365,13 @@ CompletableFuture> verifyNativeUserCredentials( _nativeUserService.doesPasswordMatch( systemOperationContext, userUrnString, passwordString); String response = buildVerifyNativeUserPasswordResponse(doesPasswordMatch); + log.info( + "Verified credentials for native user: {}, result: {}", + userUrnString, + doesPasswordMatch); return new ResponseEntity<>(response, HttpStatus.OK); } catch (Exception e) { - log.error( - String.format("Failed to verify credentials for native user %s", userUrnString), e); + log.error("Failed to verify credentials for native user {}", userUrnString, e); return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); } }); @@ -383,9 +386,7 @@ CompletableFuture> track(final HttpEntity httpEnt try { bodyJson = mapper.readTree(jsonStr); } catch (JsonProcessingException e) { - log.error( - String.format( - "Failed to parse json while attempting to track analytics event %s", jsonStr)); + log.error("Failed to parse json while attempting to track analytics event {}", jsonStr); return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } if (bodyJson == null) { diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java index 2a492c746cc9c..c8925581899c4 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java @@ -30,9 +30,15 @@ public CustomSearchConfiguration resolve(ObjectMapper mapper) throws IOException log.info("Custom search configuration found in classpath: {}", file); return mapper.readValue(stream, CustomSearchConfiguration.class); } catch (FileNotFoundException e) { + log.info("Custom search configuration was NOT found in the classpath."); try (InputStream stream = new FileSystemResource(file).getInputStream()) { log.info("Custom search configuration found in filesystem: {}", file); return mapper.readValue(stream, CustomSearchConfiguration.class); + } catch (Exception e2) { + log.warn( + "Custom search enabled, however there was an error loading configuration: " + file, + e2); + return null; } } } else { diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/AutocompleteConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/AutocompleteConfiguration.java new file mode 100644 index 0000000000000..6a7565c2a55cf --- /dev/null +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/AutocompleteConfiguration.java @@ -0,0 +1,34 @@ +package com.linkedin.metadata.config.search.custom; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import java.util.Collections; +import java.util.Map; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Builder(toBuilder = true) +@Getter +@ToString +@EqualsAndHashCode +@JsonDeserialize(builder = AutocompleteConfiguration.AutocompleteConfigurationBuilder.class) +public class AutocompleteConfiguration { + // match this configuration based on query string regex match + private String queryRegex; + // include the default autocomplete query + @Builder.Default private boolean defaultQuery = true; + // override or extend default autocomplete query + private BoolQueryConfiguration boolQuery; + // inherit the query configuration's function score (disabled if functionScore exists) + @Builder.Default private boolean inheritFunctionScore = true; + + // additional function scores to apply for ranking + @Builder.Default private Map functionScore = Collections.emptyMap(); + + @JsonPOJOBuilder(withPrefix = "") + public static class AutocompleteConfigurationBuilder {} +} diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java index e6756ca8f0da8..d2a908050cf30 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import java.util.Collections; import java.util.List; import lombok.Builder; import lombok.EqualsAndHashCode; @@ -13,7 +14,10 @@ @JsonDeserialize(builder = CustomSearchConfiguration.CustomSearchConfigurationBuilder.class) public class CustomSearchConfiguration { - private List queryConfigurations; + @Builder.Default private List queryConfigurations = Collections.emptyList(); + + @Builder.Default + private List autocompleteConfigurations = Collections.emptyList(); @JsonPOJOBuilder(withPrefix = "") public static class CustomSearchConfigurationBuilder {} diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java index e3a9d076dbef2..c1801b0947016 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import java.util.Collections; import java.util.Map; import lombok.Builder; import lombok.EqualsAndHashCode; @@ -29,7 +30,7 @@ public class QueryConfiguration { @Builder.Default private boolean exactMatchQuery = true; @Builder.Default private boolean prefixMatchQuery = true; private BoolQueryConfiguration boolQuery; - private Map functionScore; + @Builder.Default private Map functionScore = Collections.emptyMap(); @JsonPOJOBuilder(withPrefix = "") public static class QueryConfigurationBuilder {} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java index b4da40871cdd4..89e69174c1502 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java @@ -9,7 +9,7 @@ public class RestoreIndicesArgs implements Cloneable { public static final int DEFAULT_BATCH_SIZE = 500; public static final int DEFAULT_NUM_THREADS = 1; - public static final int DEFAULT_BATCH_DELAY_MS = 1; + public static final int DEFAULT_BATCH_DELAY_MS = 1000; public static final long DEFAULT_GE_PIT_EPOCH_MS = 0; public int start = 0; diff --git a/smoke-test/build.gradle b/smoke-test/build.gradle index 7e9c50f121548..9800cf65fc452 100644 --- a/smoke-test/build.gradle +++ b/smoke-test/build.gradle @@ -44,6 +44,11 @@ task yarnInstall(type: YarnTask) { environment = ['NODE_OPTIONS': '--openssl-legacy-provider'] args = ['install', '--cwd', "${project.rootDir}/smoke-test/tests/cypress"] } +task cypressLint(type: YarnTask, dependsOn: yarnInstall) { + environment = ['NODE_OPTIONS': '--openssl-legacy-provider'] + // TODO: Run a full lint instead of just format. + args = ['--cwd', "${project.rootDir}/smoke-test/tests/cypress", 'run', 'format'] +} task installDev(type: Exec) { inputs.file file('pyproject.toml') @@ -58,7 +63,7 @@ task installDev(type: Exec) { "touch ${venv_name}/.build_install_dev_sentinel" } -task lint(type: Exec, dependsOn: installDev) { +task pythonLint(type: Exec, dependsOn: installDev) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "black --check --diff tests/ && " + @@ -66,7 +71,7 @@ task lint(type: Exec, dependsOn: installDev) { "ruff --statistics tests/ && " + "mypy tests/" } -task lintFix(type: Exec, dependsOn: installDev) { +task pythonLintFix(type: Exec, dependsOn: installDev) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "black tests/ && " + @@ -135,6 +140,21 @@ task cypressDev(type: Exec, dependsOn: [installDev, ':metadata-ingestion:install environment 'DATAHUB_KAFKA_SCHEMA_REGISTRY_URL', 'http://localhost:8080/schema-registry/api/' environment 'KAFKA_BROKER_CONTAINER', 'datahub-kafka-broker-1' + workingDir = project.projectDir + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && set -x && " + + "./cypress-dev.sh" +} + +/** + * The following will install Cypress data in an already running stack. + */ +task cypressData(type: Exec, dependsOn: [installDev, ':metadata-ingestion:installDev']) { + environment 'RUN_QUICKSTART', 'false' + environment 'DATAHUB_KAFKA_SCHEMA_REGISTRY_URL', 'http://localhost:8080/schema-registry/api/' + environment 'KAFKA_BROKER_CONTAINER', 'datahub-kafka-broker-1' + environment 'RUN_UI', 'false' + workingDir = project.projectDir commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + diff --git a/smoke-test/cypress-dev.sh b/smoke-test/cypress-dev.sh index 090189af3f1a7..2b31c574d0578 100755 --- a/smoke-test/cypress-dev.sh +++ b/smoke-test/cypress-dev.sh @@ -19,5 +19,7 @@ yarn install source "$DIR/set-cypress-creds.sh" -npx cypress open \ - --env "ADMIN_DISPLAYNAME=$CYPRESS_ADMIN_DISPLAYNAME,ADMIN_USERNAME=$CYPRESS_ADMIN_USERNAME,ADMIN_PASSWORD=$CYPRESS_ADMIN_PASSWORD" +if [ "${RUN_UI:-true}" == "true" ]; then + npx cypress open \ + --env "ADMIN_DISPLAYNAME=$CYPRESS_ADMIN_DISPLAYNAME,ADMIN_USERNAME=$CYPRESS_ADMIN_USERNAME,ADMIN_PASSWORD=$CYPRESS_ADMIN_PASSWORD" +fi diff --git a/smoke-test/tests/cypress/.eslintrc.js b/smoke-test/tests/cypress/.eslintrc.js new file mode 100644 index 0000000000000..2dfa99ac5a374 --- /dev/null +++ b/smoke-test/tests/cypress/.eslintrc.js @@ -0,0 +1,31 @@ +module.exports = { + env: { + es2021: true, + node: true, + }, + plugins: ["cypress"], + extends: ["airbnb-base", "plugin:cypress/recommended", "prettier"], + overrides: [ + { + env: { + node: true, + }, + files: [".eslintrc.{js,cjs}"], + parserOptions: { + sourceType: "script", + }, + }, + ], + parserOptions: { + ecmaVersion: "latest", + sourceType: "module", + }, + rules: { + camelcase: "off", + "import/prefer-default-export": "off", + // TODO: These should be upgraded to warnings and fixed. + "cypress/no-unnecessary-waiting": "off", + "cypress/unsafe-to-chain-command": "off", + "no-unused-vars": "off", + }, +}; diff --git a/smoke-test/tests/cypress/cypress.config.js b/smoke-test/tests/cypress/cypress.config.js index 3eb65825378b9..7c3863ad869e3 100644 --- a/smoke-test/tests/cypress/cypress.config.js +++ b/smoke-test/tests/cypress/cypress.config.js @@ -1,10 +1,11 @@ -const { defineConfig } = require('cypress') +// eslint-disable-next-line global-require +const { defineConfig } = require("cypress"); module.exports = defineConfig({ chromeWebSecurity: false, viewportHeight: 960, viewportWidth: 1536, - projectId: 'hkrxk5', + projectId: "hkrxk5", defaultCommandTimeout: 10000, retries: { runMode: 2, @@ -15,10 +16,11 @@ module.exports = defineConfig({ // We've imported your old cypress plugins here. // You may want to clean this up later by importing these. setupNodeEvents(on, config) { - return require('./cypress/plugins/index.js')(on, config) + // eslint-disable-next-line global-require + return require("./cypress/plugins/index")(on, config); }, - baseUrl: 'http://localhost:9002/', - specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}', + baseUrl: "http://localhost:9002/", + specPattern: "cypress/e2e/**/*.{js,jsx,ts,tsx}", experimentalStudio: true, }, -}) +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/analytics/analytics.js b/smoke-test/tests/cypress/cypress/e2e/analytics/analytics.js index 0e5105717d2be..6b5452ff07d88 100644 --- a/smoke-test/tests/cypress/cypress/e2e/analytics/analytics.js +++ b/smoke-test/tests/cypress/cypress/e2e/analytics/analytics.js @@ -1,5 +1,5 @@ -describe('analytics', () => { - it('can go to a chart and see analytics in tab views', () => { +describe("analytics", () => { + it("can go to a chart and see analytics in tab views", () => { cy.login(); cy.goToChart("urn:li:chart:(looker,cypress_baz1)"); @@ -9,8 +9,8 @@ describe('analytics', () => { cy.goToAnalytics(); cy.contains("Tab Views By Entity Type (Past Week)").scrollIntoView({ - ensureScrollable: false - }) + ensureScrollable: false, + }); cy.waitTextPresent("dashboards"); }); -}) +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/auto_complete/auto_complete.js b/smoke-test/tests/cypress/cypress/e2e/auto_complete/auto_complete.js index adff124ca857c..df7d20eb6f933 100644 --- a/smoke-test/tests/cypress/cypress/e2e/auto_complete/auto_complete.js +++ b/smoke-test/tests/cypress/cypress/e2e/auto_complete/auto_complete.js @@ -29,7 +29,7 @@ describe("auto-complete", () => { cy.get('[data-testid^="auto-complete-option"]').first().click(); cy.url().should( "include", - "dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)" + "dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", ); cy.wait(2000); }); @@ -43,7 +43,7 @@ describe("auto-complete", () => { cy.get('[data-testid="quick-filter-DASHBOARD"]').click(); cy.wait(2000); cy.get('[data-testid="auto-complete-entity-name-Baz Chart 2').should( - "not.exist" + "not.exist", ); cy.contains("Baz Dashboard"); cy.wait(1000); @@ -58,7 +58,7 @@ describe("auto-complete", () => { cy.focused().type("{enter}"); cy.url().should( "include", - "?filter_platform___false___EQUAL___0=urn%3Ali%3AdataPlatform%3Abigquery" + "?filter_platform___false___EQUAL___0=urn%3Ali%3AdataPlatform%3Abigquery", ); }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js b/smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js index d951b15d4a592..3a6759ae87afc 100644 --- a/smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js +++ b/smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js @@ -83,7 +83,7 @@ describe("search", () => { cy.url().should("include", "/browse/dataset"); cy.url().should( "not.include", - "search?filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET" + "search?filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET", ); }); @@ -91,7 +91,7 @@ describe("search", () => { setBrowseFeatureFlag(true); cy.login(); cy.visit( - "/dataset/urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)" + "/dataset/urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", ); cy.get('[data-testid="browse-path-cypress_project"]').click({ force: true, @@ -99,15 +99,15 @@ describe("search", () => { cy.url().should("not.include", "/browse/dataset"); cy.url().should( "include", - "filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET" + "filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET", ); cy.url().should( "include", - "filter_platform___false___EQUAL___1=urn%3Ali%3AdataPlatform%3Abigquery" + "filter_platform___false___EQUAL___1=urn%3Ali%3AdataPlatform%3Abigquery", ); cy.url().should( "include", - "filter_browsePathV2___false___EQUAL___2=%E2%90%9Fcypress_project" + "filter_browsePathV2___false___EQUAL___2=%E2%90%9Fcypress_project", ); }); @@ -135,7 +135,7 @@ describe("search", () => { cy.url().should("not.include", "/browse/dataset"); cy.url().should( "include", - "search?filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET" + "search?filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET", ); cy.get("[data-testid=browse-platform-BigQuery]"); }); @@ -156,15 +156,15 @@ describe("search", () => { cy.url().should( "include", - "filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET" + "filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET", ); cy.url().should( "include", - "filter_platform___false___EQUAL___1=urn%3Ali%3AdataPlatform%3Abigquery" + "filter_platform___false___EQUAL___1=urn%3Ali%3AdataPlatform%3Abigquery", ); cy.url().should( "include", - "filter_browsePathV2___false___EQUAL___2=%E2%90%9Fcypress_project%E2%90%9Fjaffle_shop" + "filter_browsePathV2___false___EQUAL___2=%E2%90%9Fcypress_project%E2%90%9Fjaffle_shop", ); // close each of the levels, ensuring its children aren't visible anymore @@ -175,7 +175,7 @@ describe("search", () => { cy.get("[data-testid=browse-platform-BigQuery]").click({ force: true }); cy.get("[data-testid=browse-node-expand-cypress_project]").should( - "not.be.visible" + "not.be.visible", ); cy.get("[data-testid=browse-entity-Datasets]").click({ force: true }); @@ -197,30 +197,30 @@ describe("search", () => { cy.url().should( "include", - "filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET" + "filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET", ); cy.url().should( "include", - "filter_platform___false___EQUAL___1=urn%3Ali%3AdataPlatform%3Abigquery" + "filter_platform___false___EQUAL___1=urn%3Ali%3AdataPlatform%3Abigquery", ); cy.url().should( "include", - "filter_browsePathV2___false___EQUAL___2=%E2%90%9Fcypress_project%E2%90%9Fjaffle_shop" + "filter_browsePathV2___false___EQUAL___2=%E2%90%9Fcypress_project%E2%90%9Fjaffle_shop", ); cy.get("[data-testid=browse-node-jaffle_shop]").click({ force: true }); cy.url().should( "not.include", - "filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET" + "filter__entityType%E2%90%9EtypeNames___false___EQUAL___0=DATASET", ); cy.url().should( "not.include", - "filter_platform___false___EQUAL___1=urn%3Ali%3AdataPlatform%3Abigquery" + "filter_platform___false___EQUAL___1=urn%3Ali%3AdataPlatform%3Abigquery", ); cy.url().should( "not.include", - "filter_browsePathV2___false___EQUAL___2=%E2%90%9Fcypress_project%E2%90%9Fjaffle_shop" + "filter_browsePathV2___false___EQUAL___2=%E2%90%9Fcypress_project%E2%90%9Fjaffle_shop", ); }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/businessAttribute/attribute_mutations.js b/smoke-test/tests/cypress/cypress/e2e/businessAttribute/attribute_mutations.js index decee024f050b..a0aabe385b742 100644 --- a/smoke-test/tests/cypress/cypress/e2e/businessAttribute/attribute_mutations.js +++ b/smoke-test/tests/cypress/cypress/e2e/businessAttribute/attribute_mutations.js @@ -1,119 +1,127 @@ import { aliasQuery, hasOperationName } from "../utils"; describe("attribute list adding tags and terms", () => { - let businessAttributeEntityEnabled; + let businessAttributeEntityEnabled; - beforeEach(() => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - aliasQuery(req, "appConfig"); - }); - }); - - const setBusinessAttributeFeatureFlag = () => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - if (hasOperationName(req, "appConfig")) { - req.reply((res) => { - businessAttributeEntityEnabled = res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; - return res; - }); - } - }).as('apiCall'); - }; - - - it("can create and add a tag to business attribute and visit new tag page", () => { - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit("/business-attribute"); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.wait(3000); - cy.waitTextVisible("Business Attribute"); - cy.wait(3000); - - cy.mouseover('[data-testid="schema-field-cypressTestAttribute-tags"]'); - cy.get('[data-testid="schema-field-cypressTestAttribute-tags"]').within(() => - cy.contains("Add Tags").click() - ); - - cy.enterTextInTestId("tag-term-modal-input", "CypressAddTagToAttribute"); - - cy.contains("Create CypressAddTagToAttribute").click({ force: true }); - - cy.get("textarea").type("CypressAddTagToAttribute Test Description"); - - cy.contains(/Create$/).click({ force: true }); - - // wait a breath for elasticsearch to index the tag being applied to the business attribute- if we navigate too quick ES - // wont know and we'll see applied to 0 entities - cy.wait(3000); - - // go to tag drawer - cy.contains("CypressAddTagToAttribute").click({ force: true }); - - cy.wait(3000); - - // Click the Tag Details to launch full profile - cy.contains("Tag Details").click({ force: true }); - - cy.wait(3000); - - // title of tag page - cy.contains("CypressAddTagToAttribute"); - - // description of tag page - cy.contains("CypressAddTagToAttribute Test Description"); - - cy.wait(3000); - // used by panel - click to search - cy.contains("1 Business Attributes").click({ force: true }); - - // verify business attribute shows up in search now - cy.contains("of 1 result").click({ force: true }); - cy.contains("cypressTestAttribute").click({ force: true }); - cy.get('[data-testid="tag-CypressAddTagToAttribute"]').within(() => - cy.get("span[aria-label=close]").click() - ); - cy.contains("Yes").click(); - - cy.contains("CypressAddTagToAttribute").should("not.exist"); - - cy.goToTag("urn:li:tag:CypressAddTagToAttribute", "CypressAddTagToAttribute"); - cy.deleteFromDropdown(); + beforeEach(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); + }); + }); + + const setBusinessAttributeFeatureFlag = () => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + businessAttributeEntityEnabled = + res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; + return res; }); + } + }).as("apiCall"); + }; - }); + it("can create and add a tag to business attribute and visit new tag page", () => { + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute"); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); + cy.mouseover('[data-testid="schema-field-cypressTestAttribute-tags"]'); + cy.get('[data-testid="schema-field-cypressTestAttribute-tags"]').within( + () => cy.contains("Add Tags").click(), + ); - it("can add and remove terms from a business attribute", () => { - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit("/business-attribute/" + "urn:li:businessAttribute:cypressTestAttribute"); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.wait(3000); - cy.waitTextVisible("cypressTestAttribute"); - cy.wait(3000); - cy.clickOptionWithText("Add Terms"); - cy.selectOptionInTagTermModal("CypressTerm"); - cy.contains("CypressTerm"); - - cy.goToBusinessAttributeList(); - cy.get('[data-testid="schema-field-cypressTestAttribute-terms"]').contains("CypressTerm"); - - cy.get('[data-testid="schema-field-cypressTestAttribute-terms"]').within(() => - cy - .get("span[aria-label=close]") - .trigger("mouseover", { force: true }) - .click({ force: true }) - ); - cy.contains("Yes").click({ force: true }); - - cy.get('[data-testid="schema-field-cypressTestAttribute-terms"]').contains("CypressTerm").should("not.exist"); - }); + cy.enterTextInTestId("tag-term-modal-input", "CypressAddTagToAttribute"); + + cy.contains("Create CypressAddTagToAttribute").click({ force: true }); + + cy.get("textarea").type("CypressAddTagToAttribute Test Description"); + + cy.contains(/Create$/).click({ force: true }); + + // wait a breath for elasticsearch to index the tag being applied to the business attribute- if we navigate too quick ES + // wont know and we'll see applied to 0 entities + cy.wait(3000); + + // go to tag drawer + cy.contains("CypressAddTagToAttribute").click({ force: true }); + + cy.wait(3000); + + // Click the Tag Details to launch full profile + cy.contains("Tag Details").click({ force: true }); + + cy.wait(3000); + + // title of tag page + cy.contains("CypressAddTagToAttribute"); + + // description of tag page + cy.contains("CypressAddTagToAttribute Test Description"); + + cy.wait(3000); + // used by panel - click to search + cy.contains("1 Business Attributes").click({ force: true }); + + // verify business attribute shows up in search now + cy.contains("of 1 result").click({ force: true }); + cy.contains("cypressTestAttribute").click({ force: true }); + cy.get('[data-testid="tag-CypressAddTagToAttribute"]').within(() => + cy.get("span[aria-label=close]").click(), + ); + cy.contains("Yes").click(); + + cy.contains("CypressAddTagToAttribute").should("not.exist"); + + cy.goToTag( + "urn:li:tag:CypressAddTagToAttribute", + "CypressAddTagToAttribute", + ); + cy.deleteFromDropdown(); + }); + }); + + it("can add and remove terms from a business attribute", () => { + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit( + "/business-attribute/" + "urn:li:businessAttribute:cypressTestAttribute", + ); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("cypressTestAttribute"); + cy.wait(3000); + cy.clickOptionWithText("Add Terms"); + cy.selectOptionInTagTermModal("CypressTerm"); + cy.contains("CypressTerm"); + + cy.goToBusinessAttributeList(); + cy.get( + '[data-testid="schema-field-cypressTestAttribute-terms"]', + ).contains("CypressTerm"); + + cy.get('[data-testid="schema-field-cypressTestAttribute-terms"]').within( + () => + cy + .get("span[aria-label=close]") + .trigger("mouseover", { force: true }) + .click({ force: true }), + ); + cy.contains("Yes").click({ force: true }); + + cy.get('[data-testid="schema-field-cypressTestAttribute-terms"]') + .contains("CypressTerm") + .should("not.exist"); }); + }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/businessAttribute/businessAttribute.js b/smoke-test/tests/cypress/cypress/e2e/businessAttribute/businessAttribute.js index 0657dc238a154..a106915463d3d 100644 --- a/smoke-test/tests/cypress/cypress/e2e/businessAttribute/businessAttribute.js +++ b/smoke-test/tests/cypress/cypress/e2e/businessAttribute/businessAttribute.js @@ -1,197 +1,215 @@ import { aliasQuery, hasOperationName } from "../utils"; describe("businessAttribute", () => { - let businessAttributeEntityEnabled; + let businessAttributeEntityEnabled; - beforeEach(() => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - aliasQuery(req, "appConfig"); - }); - }); - - const setBusinessAttributeFeatureFlag = () => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - if (hasOperationName(req, "appConfig")) { - req.reply((res) => { - businessAttributeEntityEnabled = res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; - return res; - }); - } - }).as('apiCall'); - }; - - it('go to business attribute page, create attribute ', function () { - const urn="urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; - const businessAttribute="CypressBusinessAttribute"; - const datasetName = "cypress_logging_events"; - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit("/business-attribute"); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.wait(3000); - cy.waitTextVisible("Business Attribute"); - cy.wait(3000); - cy.clickOptionWithText("Create Business Attribute"); - cy.addBusinessAttributeViaModal(businessAttribute, "Create Business Attribute", businessAttribute, "create-business-attribute-button"); - - cy.wait(3000); - cy.goToBusinessAttributeList() - - cy.wait(3000) - cy.contains(businessAttribute).should("be.visible"); - - cy.addAttributeToDataset(urn, datasetName, businessAttribute); - - cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within(() => - cy - .get("span[aria-label=close]") - .trigger("mouseover", { force: true }) - .click({ force: true }) - ); - cy.contains("Yes").click({ force: true }); - - cy.get('[data-testid="schema-field-event_name-businessAttribute"]').contains("CypressBusinessAttribute").should("not.exist"); - - cy.goToBusinessAttributeList(); - cy.clickOptionWithText(businessAttribute); - cy.deleteFromDropdown(); - - cy.goToBusinessAttributeList(); - cy.ensureTextNotPresent(businessAttribute); - }); + beforeEach(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); }); - - it('Inheriting tags and terms from business attribute to dataset ', function () { - const urn="urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; - const businessAttribute="CypressAttribute"; - const datasetName = "cypress_logging_events"; - const term="CypressTerm"; - const tag="Cypress"; - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit("/dataset/" + urn); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.wait(5000); - cy.waitTextVisible(datasetName); - cy.clickOptionWithText("event_name"); - cy.contains("Business Attribute"); - cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within(() => - cy.contains("Add Attribute").click() - ); - cy.selectOptionInAttributeModal(businessAttribute); - cy.contains(businessAttribute); - cy.contains(term); - cy.contains(tag); + }); + + const setBusinessAttributeFeatureFlag = () => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + businessAttributeEntityEnabled = + res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; + return res; }); + } + }).as("apiCall"); + }; + + it("go to business attribute page, create attribute ", () => { + const urn = + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; + const businessAttribute = "CypressBusinessAttribute"; + const datasetName = "cypress_logging_events"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute"); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); + cy.clickOptionWithText("Create Business Attribute"); + cy.addBusinessAttributeViaModal( + businessAttribute, + "Create Business Attribute", + businessAttribute, + "create-business-attribute-button", + ); + + cy.wait(3000); + cy.goToBusinessAttributeList(); + + cy.wait(3000); + cy.contains(businessAttribute).should("be.visible"); + + cy.addAttributeToDataset(urn, datasetName, businessAttribute); + + cy.get( + '[data-testid="schema-field-event_name-businessAttribute"]', + ).within(() => + cy + .get("span[aria-label=close]") + .trigger("mouseover", { force: true }) + .click({ force: true }), + ); + cy.contains("Yes").click({ force: true }); + + cy.get('[data-testid="schema-field-event_name-businessAttribute"]') + .contains("CypressBusinessAttribute") + .should("not.exist"); + + cy.goToBusinessAttributeList(); + cy.clickOptionWithText(businessAttribute); + cy.deleteFromDropdown(); + + cy.goToBusinessAttributeList(); + cy.ensureTextNotPresent(businessAttribute); }); - - it("can visit related entities", () => { - const businessAttribute="CypressAttribute"; - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit("/business-attribute"); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.wait(3000); - cy.waitTextVisible("Business Attribute"); - cy.wait(3000); - cy.clickOptionWithText(businessAttribute); - cy.clickOptionWithText("Related Entities"); - //cy.visit("/business-attribute/urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449/Related%20Entities"); - //cy.wait(5000); - cy.contains("of 0").should("not.exist"); - cy.contains(/of [0-9]+/); - }); + }); + + it("Inheriting tags and terms from business attribute to dataset ", () => { + const urn = + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; + const businessAttribute = "CypressAttribute"; + const datasetName = "cypress_logging_events"; + const term = "CypressTerm"; + const tag = "Cypress"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit(`/dataset/${urn}`); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(5000); + cy.waitTextVisible(datasetName); + cy.clickOptionWithText("event_name"); + cy.contains("Business Attribute"); + cy.get( + '[data-testid="schema-field-event_name-businessAttribute"]', + ).within(() => cy.contains("Add Attribute").click()); + cy.selectOptionInAttributeModal(businessAttribute); + cy.contains(businessAttribute); + cy.contains(term); + cy.contains(tag); }); - - - it("can search related entities by query", () => { - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit("/business-attribute/urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449/Related%20Entities"); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.get('[placeholder="Filter entities..."]').click().type( - "event_n{enter}" - ); - cy.wait(5000); - cy.contains("of 0").should("not.exist"); - cy.contains(/of 1/); - cy.contains("event_name"); - }); + }); + + it("can visit related entities", () => { + const businessAttribute = "CypressAttribute"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute"); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); + cy.clickOptionWithText(businessAttribute); + cy.clickOptionWithText("Related Entities"); + // cy.visit("/business-attribute/urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449/Related%20Entities"); + // cy.wait(5000); + cy.contains("of 0").should("not.exist"); + cy.contains(/of [0-9]+/); }); - - it("remove business attribute from dataset", () => { - const urn="urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; - const datasetName = "cypress_logging_events"; - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit("/dataset/" + urn); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.wait(5000); - cy.waitTextVisible(datasetName); - - cy.wait(3000); - cy.get('body').then(($body) => { - if ($body.find('button[aria-label="Close"]').length > 0) { - cy.get('button[aria-label="Close"]').click(); - } - }); - cy.clickOptionWithText("event_name"); - cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within(() => - cy - .get("span[aria-label=close]") - .trigger("mouseover", { force: true }) - .click({ force: true }) - ); - cy.contains("Yes").click({ force: true }); - - cy.get('[data-testid="schema-field-event_name-businessAttribute"]').contains("CypressAttribute").should("not.exist"); - }); + }); + + it("can search related entities by query", () => { + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit( + "/business-attribute/urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449/Related%20Entities", + ); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.get('[placeholder="Filter entities..."]') + .click() + .type("event_n{enter}"); + cy.wait(5000); + cy.contains("of 0").should("not.exist"); + cy.contains(/of 1/); + cy.contains("event_name"); }); - - it("update the data type of a business attribute", () => { - const businessAttribute="cypressTestAttribute"; - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit("/business-attribute"); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.wait(3000); - cy.waitTextVisible("Business Attribute"); - cy.wait(3000); - - cy.clickOptionWithText(businessAttribute); - - cy.get('[data-testid="edit-data-type-button"]').within(() => - cy - .get("span[aria-label=edit]") - .trigger("mouseover", { force: true }) - .click({ force: true }) - ); - - cy.get('[data-testid="add-data-type-option"]').get('.ant-select-selection-search-input').click({multiple: true}); - - cy.get('.ant-select-item-option-content') - .contains('STRING') - .click(); - - cy.contains("STRING"); - }); + }); + + it("remove business attribute from dataset", () => { + const urn = + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; + const datasetName = "cypress_logging_events"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit(`/dataset/${urn}`); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(5000); + cy.waitTextVisible(datasetName); + + cy.wait(3000); + cy.get("body").then(($body) => { + if ($body.find('button[aria-label="Close"]').length > 0) { + cy.get('button[aria-label="Close"]').click(); + } + }); + cy.clickOptionWithText("event_name"); + cy.get( + '[data-testid="schema-field-event_name-businessAttribute"]', + ).within(() => + cy + .get("span[aria-label=close]") + .trigger("mouseover", { force: true }) + .click({ force: true }), + ); + cy.contains("Yes").click({ force: true }); + + cy.get('[data-testid="schema-field-event_name-businessAttribute"]') + .contains("CypressAttribute") + .should("not.exist"); + }); + }); + + it("update the data type of a business attribute", () => { + const businessAttribute = "cypressTestAttribute"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute"); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); + + cy.clickOptionWithText(businessAttribute); + + cy.get('[data-testid="edit-data-type-button"]').within(() => + cy + .get("span[aria-label=edit]") + .trigger("mouseover", { force: true }) + .click({ force: true }), + ); + + cy.get('[data-testid="add-data-type-option"]') + .get(".ant-select-selection-search-input") + .click({ multiple: true }); + + cy.get(".ant-select-item-option-content").contains("STRING").click(); + + cy.contains("STRING"); }); + }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js index 3152174e17072..9c320eaf77ae4 100644 --- a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js +++ b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js @@ -1,225 +1,293 @@ const domainName = "CypressNestedDomain"; -//Delete Unecessary Existing Domains +// Delete Unecessary Existing Domains const deleteExisitingDomain = () => { - cy.get('a[href*="urn:li"] span[class^="ant-typography"]') - .should('be.visible') - .its('length') - .then((length) => { - for (let i = 0; i < length - 1; i++) { - cy.get('a[href*="urn:li"] span[class^="ant-typography"]') - .should('be.visible') - .first() - .click({ force: true }); - deleteFromDomainDropdown(); - } - }); - cy.waitTextVisible('Marketing'); - } + cy.get('a[href*="urn:li"] span[class^="ant-typography"]') + .should("be.visible") + .its("length") + .then((length) => { + for (let i = 0; i < length - 1; i++) { + cy.get('a[href*="urn:li"] span[class^="ant-typography"]') + .should("be.visible") + .first() + .click({ force: true }); + deleteFromDomainDropdown(); + } + }); + cy.waitTextVisible("Marketing"); +}; const createDomain = () => { - cy.get('.anticon-plus').first().click() - cy.waitTextVisible('Create New Domain') - cy.get('[data-testid="create-domain-name"]').click().type(domainName); - cy.clickOptionWithTestId("create-domain-button"); - cy.waitTextVisible("Created domain!"); - } + cy.get(".anticon-plus").first().click(); + cy.waitTextVisible("Create New Domain"); + cy.get('[data-testid="create-domain-name"]').click().type(domainName); + cy.clickOptionWithTestId("create-domain-button"); + cy.waitTextVisible("Created domain!"); +}; const moveDomaintoRootLevel = () => { - cy.clickOptionWithText(domainName); - cy.openThreeDotDropdown(); - cy.clickOptionWithTestId("entity-menu-move-button"); - cy.get('[data-testid="move-domain-modal"]').contains("Marketing").click({force: true}); - cy.waitTextVisible('Marketing') - cy.clickOptionWithTestId("move-domain-modal-move-button") - } + cy.clickOptionWithText(domainName); + cy.openThreeDotDropdown(); + cy.clickOptionWithTestId("entity-menu-move-button"); + cy.get('[data-testid="move-domain-modal"]') + .contains("Marketing") + .click({ force: true }); + cy.waitTextVisible("Marketing"); + cy.clickOptionWithTestId("move-domain-modal-move-button"); +}; const moveDomaintoParent = () => { - cy.get('[data-testid="domain-list-item"]').contains("Marketing").prev().click(); - cy.clickOptionWithText(domainName); - cy.waitTextVisible(domainName) - cy.openThreeDotDropdown(); - cy.clickOptionWithTestId("entity-menu-move-button"); - cy.clickOptionWithTestId("move-domain-modal-move-button") - } - -const getDomainList = (domainName) =>{ - cy.contains('span.ant-typography-ellipsis', domainName) - .parent('[data-testid="domain-list-item"]') - .find('[aria-label="right"]') - .click(); - } + cy.get('[data-testid="domain-list-item"]') + .contains("Marketing") + .prev() + .click(); + cy.clickOptionWithText(domainName); + cy.waitTextVisible(domainName); + cy.openThreeDotDropdown(); + cy.clickOptionWithTestId("entity-menu-move-button"); + cy.clickOptionWithTestId("move-domain-modal-move-button"); +}; + +const getDomainList = (domainName) => { + cy.contains("span.ant-typography-ellipsis", domainName) + .parent('[data-testid="domain-list-item"]') + .find('[aria-label="right"]') + .click(); +}; const deleteFromDomainDropdown = () => { - cy.clickOptionWithText('Filters') - cy.openThreeDotDropdown(); - cy.clickOptionWithTestId("entity-menu-delete-button"); - cy.waitTextVisible("Are you sure you want to remove this Domain?"); - cy.clickOptionWithText("Yes"); - } + cy.clickOptionWithText("Filters"); + cy.openThreeDotDropdown(); + cy.clickOptionWithTestId("entity-menu-delete-button"); + cy.waitTextVisible("Are you sure you want to remove this Domain?"); + cy.clickOptionWithText("Yes"); +}; const deleteDomain = () => { - cy.clickOptionWithText(domainName).waitTextVisible('Domains'); - deleteFromDomainDropdown() - } - -const verifyEditAndPerformAddAndRemoveActionForDomain = (entity, action, text, body) =>{ - cy.clickOptionWithText(entity) - cy.clickOptionWithText(action) - cy.get('[data-testid="tag-term-modal-input"]').type(text) - cy.get('[data-testid="tag-term-option"]').contains(text).click() - cy.clickOptionWithText(body) - cy.get('[data-testid="add-tag-term-from-modal-btn"]').click() - cy.waitTextVisible(text) - } - -const clearAndType = (text) =>{ - cy.get('[role="textbox"]').click().clear().type(text) - } - -const clearAndDelete = () =>{ - cy.clickOptionWithText("Edit") - cy.get('[role="textbox"]').click().clear() - cy.clickOptionWithTestId("description-editor-save-button") - cy.waitTextVisible('No documentation') - cy.mouseover('.ant-list-item-meta-content') - cy.get('[aria-label="delete"]').click() - cy.waitTextVisible('Link Removed') - } + cy.clickOptionWithText(domainName).waitTextVisible("Domains"); + deleteFromDomainDropdown(); +}; + +const verifyEditAndPerformAddAndRemoveActionForDomain = ( + entity, + action, + text, + body, +) => { + cy.clickOptionWithText(entity); + cy.clickOptionWithText(action); + cy.get('[data-testid="tag-term-modal-input"]').type(text); + cy.get('[data-testid="tag-term-option"]').contains(text).click(); + cy.clickOptionWithText(body); + cy.get('[data-testid="add-tag-term-from-modal-btn"]').click(); + cy.waitTextVisible(text); +}; + +const clearAndType = (text) => { + cy.get('[role="textbox"]').click().clear().type(text); +}; + +const clearAndDelete = () => { + cy.clickOptionWithText("Edit"); + cy.get('[role="textbox"]').click().clear(); + cy.clickOptionWithTestId("description-editor-save-button"); + cy.waitTextVisible("No documentation"); + cy.mouseover(".ant-list-item-meta-content"); + cy.get('[aria-label="delete"]').click(); + cy.waitTextVisible("Link Removed"); +}; describe("Verify nested domains test functionalities", () => { - beforeEach (() => { + beforeEach(() => { cy.loginWithCredentials(); cy.goToDomainList(); }); - - it("Verify Create a new domain", () => { - deleteExisitingDomain() - cy.get('a[href*="urn:li"] span[class^="ant-typography"]') - .should('be.visible') - createDomain(); - cy.waitTextVisible("Domains"); - }); - - it ("verify Move domain root level to parent level", () => { - cy.waitTextVisible(domainName) - moveDomaintoRootLevel(); - cy.waitTextVisible("Moved Domain!") - cy.goToDomainList(); - cy.waitTextVisible("1 sub-domain"); - }); - it("Verify Move domain parent level to root level", () => { - moveDomaintoParent(); - cy.waitTextVisible("Moved Domain!") - cy.goToDomainList(); - cy.waitTextVisible(domainName); - }); + it("Verify Create a new domain", () => { + deleteExisitingDomain(); + cy.get('a[href*="urn:li"] span[class^="ant-typography"]').should( + "be.visible", + ); + createDomain(); + cy.waitTextVisible("Domains"); + }); - it("Verify Documentation tab by adding editing Description and adding link", () => { - cy.clickOptionWithText(domainName) - cy.clickOptionWithId('#rc-tabs-0-tab-Documentation') - cy.clickFirstOptionWithText("Add Documentation") - clearAndType("Test added") - cy.clickOptionWithTestId("description-editor-save-button") - cy.waitTextVisible('Description Updated') - cy.waitTextVisible('Test added') - cy.clickFirstOptionWithTestId("add-link-button") - cy.waitTextVisible("Add Link") - cy.enterTextInTestId("add-link-modal-url", 'www.test.com') - cy.enterTextInTestId("add-link-modal-label", 'Test Label') - cy.clickOptionWithTestId("add-link-modal-add-button") - cy.waitTextVisible("Test Label") - cy.goToDomainList(); - cy.waitTextVisible("Test added") - cy.clickOptionWithText(domainName) - cy.clickOptionWithText("Documentation") - clearAndDelete() - }) - - it("Verify Right side panel functionalities", () => { - cy.clickOptionWithText(domainName) - cy.waitTextVisible("Filters") - cy.clickOptionWithText("Add Documentation") - clearAndType("Test documentation") - cy.clickOptionWithTestId("description-editor-save-button") - cy.ensureTextNotPresent("Add Documentation") - cy.waitTextVisible('Test documentation') - cy.clickFirstOptionWithSpecificTestId("add-link-button", 1) - cy.waitTextVisible("URL") - cy.enterTextInTestId("add-link-modal-url", 'www.test.com') - cy.enterTextInTestId("add-link-modal-label", 'Test Label') - cy.clickOptionWithTestId("add-link-modal-add-button") - cy.waitTextVisible("Test Label") - cy.clickOptionWithTestId("add-owners-button") - cy.waitTextVisible("Find a user or group") - cy.clickTextOptionWithClass(".rc-virtual-list-holder-inner", Cypress.env('ADMIN_DISPLAYNAME')) - cy.clickOptionWithText("Find a user or group") - cy.clickOptionWithId('#addOwnerButton') - cy.waitTextVisible(Cypress.env('ADMIN_DISPLAYNAME')) - cy.goToDomainList(); - cy.waitTextVisible("Test documentation") - cy.waitTextVisible(Cypress.env('ADMIN_DISPLAYNAME')) - cy.clickOptionWithText(domainName) - cy.clickOptionWithText("Documentation") - clearAndDelete() - }) - - it("Verify Edit Domain Name", () => { - cy.clickFirstOptionWithText(domainName) - cy.clickOptionWithText('Filters') - - //edit name - cy.get('.anticon-edit').eq(0).click().then(() => { - cy.get('.ant-typography-edit-content').type(" Edited").type('{enter}'); - }); - cy.waitTextVisible(domainName + " Edited") - }) - - it("Verify Remove the domain", () => { - deleteDomain(); - cy.goToDomainList(); - cy.ensureTextNotPresent(domainName); - }); + it("Verify Documentation tab by adding editing Description and adding link", () => { + cy.clickOptionWithText(domainName); + cy.clickOptionWithId("#rc-tabs-0-tab-Documentation"); + cy.clickFirstOptionWithText("Add Documentation"); + clearAndType("Test added"); + cy.clickOptionWithTestId("description-editor-save-button"); + cy.waitTextVisible("Description Updated"); + cy.waitTextVisible("Test added"); + cy.clickFirstOptionWithTestId("add-link-button"); + cy.waitTextVisible("Add Link"); + cy.enterTextInTestId("add-link-modal-url", "www.test.com"); + cy.enterTextInTestId("add-link-modal-label", "Test Label"); + cy.clickOptionWithTestId("add-link-modal-add-button"); + cy.waitTextVisible("Test Label"); + cy.goToDomainList(); + cy.waitTextVisible("Test added"); + cy.clickOptionWithText(domainName); + cy.clickOptionWithText("Documentation"); + clearAndDelete(); + }); + + it("Verify Right side panel functionalities", () => { + cy.clickOptionWithText(domainName); + cy.waitTextVisible("Filters"); + cy.clickOptionWithText("Add Documentation"); + clearAndType("Test documentation"); + cy.clickOptionWithTestId("description-editor-save-button"); + cy.ensureTextNotPresent("Add Documentation"); + cy.waitTextVisible("Test documentation"); + cy.clickFirstOptionWithSpecificTestId("add-link-button", 1); + cy.waitTextVisible("URL"); + cy.enterTextInTestId("add-link-modal-url", "www.test.com"); + cy.enterTextInTestId("add-link-modal-label", "Test Label"); + cy.clickOptionWithTestId("add-link-modal-add-button"); + cy.waitTextVisible("Test Label"); + cy.clickOptionWithTestId("add-owners-button"); + cy.waitTextVisible("Find a user or group"); + cy.clickTextOptionWithClass( + ".rc-virtual-list-holder-inner", + Cypress.env("ADMIN_DISPLAYNAME"), + ); + cy.clickOptionWithText("Find a user or group"); + cy.clickOptionWithId("#addOwnerButton"); + cy.waitTextVisible(Cypress.env("ADMIN_DISPLAYNAME")); + cy.goToDomainList(); + cy.waitTextVisible("Test documentation"); + cy.waitTextVisible(Cypress.env("ADMIN_DISPLAYNAME")); + cy.clickOptionWithText(domainName); + cy.clickOptionWithText("Documentation"); + clearAndDelete(); + }); - it('Verify Add and delete sub domain', () => { - cy.clickFirstOptionWithText('Marketing') - cy.clickOptionWithText('Filters') - createDomain(); - cy.ensureTextNotPresent('Created domain!') - getDomainList('Marketing') - cy.clickOptionWithText(domainName) - deleteFromDomainDropdown() - cy.ensureTextNotPresent(domainName) - }) - - it('Verify entities tab with adding and deleting assets and performing some actions', () => { - cy.clickFirstOptionWithText('Marketing'); - cy.clickOptionWithText('Add assets'); - cy.waitTextVisible("Add assets to Domain"); - cy.enterTextInSpecificTestId("search-bar", 3, 'Baz Chart 1') - cy.clickOptionWithSpecificClass('.ant-checkbox', 1) - cy.clickOptionWithId('#continueButton') - cy.waitTextVisible("Added assets to Domain!") - cy.openThreeDotMenu() - cy.clickOptionWithText("Edit") - cy.clickOptionWithSpecificClass('.ant-checkbox', 1) - verifyEditAndPerformAddAndRemoveActionForDomain('Tags', 'Add tags', 'Cypress', 'Add Tags') - cy.clickOptionWithText('Baz Chart 1') - cy.waitTextVisible("Cypress") - cy.waitTextVisible("Marketing") - cy.go('back') - cy.openThreeDotMenu() - cy.clickOptionWithText("Edit") - cy.clickOptionWithSpecificClass('.ant-checkbox', 1) - verifyEditAndPerformAddAndRemoveActionForDomain('Tags', 'Remove tags', 'Cypress', 'Remove Tags') - cy.clickTextOptionWithClass('.ant-dropdown-trigger', 'Domain') - cy.clickOptionWithText('Unset Domain') - cy.clickOptionWithText("Yes"); - cy.clickOptionWithText('Baz Chart 1') - cy.waitTextVisible('Dashboards') - cy.reload() - cy.ensureTextNotPresent("Cypress") - cy.ensureTextNotPresent("Marketing") - }) + it("Verify Move domain parent level to root level", () => { + moveDomaintoParent(); + cy.waitTextVisible("Moved Domain!"); + cy.goToDomainList(); + cy.waitTextVisible(domainName); + }); + + it("Verify Documentation tab by adding editing Description and adding link", () => { + cy.clickOptionWithText(domainName); + cy.clickOptionWithId("#rc-tabs-0-tab-Documentation"); + cy.clickFirstOptionWithText("Add Documentation"); + clearAndType("Test added"); + cy.clickOptionWithTestId("description-editor-save-button"); + cy.waitTextVisible("Description Updated"); + cy.waitTextVisible("Test added"); + cy.clickFirstOptionWithTestId("add-link-button"); + cy.waitTextVisible("Add Link"); + cy.enterTextInTestId("add-link-modal-url", "www.test.com"); + cy.enterTextInTestId("add-link-modal-label", "Test Label"); + cy.clickOptionWithTestId("add-link-modal-add-button"); + cy.waitTextVisible("Test Label"); + cy.goToDomainList(); + cy.waitTextVisible("Test added"); + cy.clickOptionWithText(domainName); + cy.clickOptionWithText("Documentation"); + clearAndDelete(); + }); + + it("Verify Right side panel functionalities", () => { + cy.clickOptionWithText(domainName); + cy.waitTextVisible("Filters"); + cy.clickOptionWithText("Add Documentation"); + clearAndType("Test documentation"); + cy.clickOptionWithTestId("description-editor-save-button"); + cy.ensureTextNotPresent("Add Documentation"); + cy.waitTextVisible("Test documentation"); + cy.clickFirstOptionWithSpecificTestId("add-link-button", 1); + cy.waitTextVisible("URL"); + cy.enterTextInTestId("add-link-modal-url", "www.test.com"); + cy.enterTextInTestId("add-link-modal-label", "Test Label"); + cy.clickOptionWithTestId("add-link-modal-add-button"); + cy.waitTextVisible("Test Label"); + cy.clickOptionWithTestId("add-owners-button"); + cy.waitTextVisible("Find a user or group"); + cy.clickTextOptionWithClass(".rc-virtual-list-holder-inner", "DataHub"); + cy.clickOptionWithText("Find a user or group"); + cy.clickOptionWithId("#addOwnerButton"); + cy.waitTextVisible("DataHub"); + cy.goToDomainList(); + cy.waitTextVisible("Test documentation"); + cy.waitTextVisible("DataHub"); + cy.clickOptionWithText(domainName); + cy.clickOptionWithText("Documentation"); + clearAndDelete(); + }); + + it("Verify Edit Domain Name", () => { + cy.clickFirstOptionWithText(domainName); + cy.clickOptionWithText("Filters"); + + // edit name + cy.get(".anticon-edit") + .eq(0) + .click() + .then(() => { + cy.get(".ant-typography-edit-content").type(" Edited").type("{enter}"); + }); + cy.waitTextVisible(`${domainName} Edited`); + }); + + it("Verify Remove the domain", () => { + deleteDomain(); + cy.goToDomainList(); + cy.ensureTextNotPresent(domainName); + }); + + it("Verify Add and delete sub domain", () => { + cy.clickFirstOptionWithText("Marketing"); + cy.clickOptionWithText("Filters"); + createDomain(); + cy.ensureTextNotPresent("Created domain!"); + getDomainList("Marketing"); + cy.clickOptionWithText(domainName); + deleteFromDomainDropdown(); + cy.ensureTextNotPresent(domainName); + }); + + it("Verify entities tab with adding and deleting assets and performing some actions", () => { + cy.clickFirstOptionWithText("Marketing"); + cy.clickOptionWithText("Add assets"); + cy.waitTextVisible("Add assets to Domain"); + cy.enterTextInSpecificTestId("search-bar", 3, "Baz Chart 1"); + cy.clickOptionWithSpecificClass(".ant-checkbox", 1); + cy.clickOptionWithId("#continueButton"); + cy.waitTextVisible("Added assets to Domain!"); + cy.openThreeDotMenu(); + cy.clickOptionWithText("Edit"); + cy.clickOptionWithSpecificClass(".ant-checkbox", 1); + verifyEditAndPerformAddAndRemoveActionForDomain( + "Tags", + "Add tags", + "Cypress", + "Add Tags", + ); + cy.clickOptionWithText("Baz Chart 1"); + cy.waitTextVisible("Cypress"); + cy.waitTextVisible("Marketing"); + cy.go("back"); + cy.openThreeDotMenu(); + cy.clickOptionWithText("Edit"); + cy.clickOptionWithSpecificClass(".ant-checkbox", 1); + verifyEditAndPerformAddAndRemoveActionForDomain( + "Tags", + "Remove tags", + "Cypress", + "Remove Tags", + ); + cy.clickTextOptionWithClass(".ant-dropdown-trigger", "Domain"); + cy.clickOptionWithText("Unset Domain"); + cy.clickOptionWithText("Yes"); + cy.clickOptionWithText("Baz Chart 1"); + cy.waitTextVisible("Dashboards"); + cy.reload(); + cy.ensureTextNotPresent("Cypress"); + cy.ensureTextNotPresent("Marketing"); + }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary.js b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary.js index b0e24d5346fea..d4746032ac607 100644 --- a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary.js +++ b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary.js @@ -1,27 +1,38 @@ -const urn = "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; +const urn = + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; const datasetName = "cypress_logging_events"; const glossaryTerm = "CypressGlossaryTerm"; const glossaryTermGroup = "CypressGlossaryGroup"; describe("glossary", () => { - it("go to glossary page, create terms, term group", () => { - cy.loginWithCredentials(); - cy.goToGlossaryList(); - cy.clickOptionWithText("Add Term"); - cy.addViaModal(glossaryTerm, "Create Glossary Term", glossaryTerm, "glossary-entity-modal-create-button"); - cy.clickOptionWithText("Add Term Group"); - cy.addViaModal(glossaryTermGroup, "Create Term Group", glossaryTermGroup, "glossary-entity-modal-create-button"); - cy.addTermToDataset(urn, datasetName, glossaryTerm); - cy.waitTextVisible(glossaryTerm) - cy.goToGlossaryList(); - cy.clickOptionWithText(glossaryTerm); - cy.deleteFromDropdown(); - cy.goToDataset(urn, datasetName); - cy.ensureTextNotPresent(glossaryTerm); - cy.goToGlossaryList(); - cy.clickOptionWithText(glossaryTermGroup); - cy.deleteFromDropdown(); - cy.goToGlossaryList(); - cy.ensureTextNotPresent(glossaryTermGroup); - }); + it("go to glossary page, create terms, term group", () => { + cy.loginWithCredentials(); + cy.goToGlossaryList(); + cy.clickOptionWithText("Add Term"); + cy.addViaModal( + glossaryTerm, + "Create Glossary Term", + glossaryTerm, + "glossary-entity-modal-create-button", + ); + cy.clickOptionWithText("Add Term Group"); + cy.addViaModal( + glossaryTermGroup, + "Create Term Group", + glossaryTermGroup, + "glossary-entity-modal-create-button", + ); + cy.addTermToDataset(urn, datasetName, glossaryTerm); + cy.waitTextVisible(glossaryTerm); + cy.goToGlossaryList(); + cy.clickOptionWithText(glossaryTerm); + cy.deleteFromDropdown(); + cy.goToDataset(urn, datasetName); + cy.ensureTextNotPresent(glossaryTerm); + cy.goToGlossaryList(); + cy.clickOptionWithText(glossaryTermGroup); + cy.deleteFromDropdown(); + cy.goToGlossaryList(); + cy.ensureTextNotPresent(glossaryTermGroup); + }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/glossary/glossaryTerm.js b/smoke-test/tests/cypress/cypress/e2e/glossary/glossaryTerm.js index 211a93393cec9..943347a403784 100644 --- a/smoke-test/tests/cypress/cypress/e2e/glossary/glossaryTerm.js +++ b/smoke-test/tests/cypress/cypress/e2e/glossary/glossaryTerm.js @@ -1,11 +1,12 @@ const glossaryTerms = { - glossaryTermUrl:"/glossaryTerm/urn:li:glossaryTerm:CypressNode.CypressColumnInfoType/Related%20Entities", - hdfsDataset:"SampleCypressHdfsDataset", - hiveDataset:"cypress_logging_events" + glossaryTermUrl: + "/glossaryTerm/urn:li:glossaryTerm:CypressNode.CypressColumnInfoType/Related%20Entities", + hdfsDataset: "SampleCypressHdfsDataset", + hiveDataset: "cypress_logging_events", }; const applyTagFilter = (tag) => { - cy.get('[aria-label="filter"]').should('be.visible').click() + cy.get('[aria-label="filter"]').should("be.visible").click(); cy.waitTextVisible("Filter"); cy.get(`[data-testid="facet-tags-${tag}"]`).click({ force: true }); }; @@ -13,7 +14,7 @@ const applyTagFilter = (tag) => { const applyAdvancedSearchFilter = (filterType, value) => { cy.get('[aria-label="filter"]').click(); cy.get('[id="search-results-advanced-search"]').click(); - cy.clickOptionWithText('Add Filter'); + cy.clickOptionWithText("Add Filter"); if (filterType === "Tag") { applyTagFilterInSearch(value); @@ -24,17 +25,17 @@ const applyAdvancedSearchFilter = (filterType, value) => { const applyBasicSearchFilter = () => { cy.waitTextVisible("Basic"); - cy.clickOptionWithText('Add Filter'); + cy.clickOptionWithText("Add Filter"); }; const searchByConceptsWithLogicalOperator = (concept1, concept2, operator) => { cy.waitTextVisible("Filters"); applyBasicSearchFilter(); applyTagFilterInSearch(concept1); - cy.clickOptionWithText('Add Filter'); + cy.clickOptionWithText("Add Filter"); applyDescriptionFilterInAdvancedSearch(concept2); cy.get('[title="all filters"]').click(); - cy.clickOptionWithText(operator) + cy.clickOptionWithText(operator); }; // Helper function to apply tag filter in basic search @@ -45,7 +46,9 @@ const applyTagFilterInSearch = (tag) => { // Helper function to apply description filter in advanced search const applyDescriptionFilterInAdvancedSearch = (value) => { - cy.get('[data-testid="adv-search-add-filter-description"]').click({ force: true }); + cy.get('[data-testid="adv-search-add-filter-description"]').click({ + force: true, + }); cy.get('[data-testid="edit-text-input"]').type(value); cy.get('[data-testid="edit-text-done-btn"]').click({ force: true }); }; @@ -57,7 +60,10 @@ describe("glossaryTerm", () => { }); it("can search related entities by query", () => { - cy.get('[placeholder="Filter entities..."]').should("be.visible").click().type("logging{enter}"); + cy.get('[placeholder="Filter entities..."]') + .should("be.visible") + .click() + .type("logging{enter}"); cy.waitTextVisible(glossaryTerms.hiveDataset); cy.contains(glossaryTerms.hdfsDataset).should("not.exist"); }); @@ -73,21 +79,21 @@ describe("glossaryTerm", () => { cy.waitTextVisible(glossaryTerms.hdfsDataset); applyAdvancedSearchFilter("Tag", "Cypress2"); cy.waitTextVisible(glossaryTerms.hdfsDataset); - cy.clickOptionWithText(glossaryTerms.hdfsDataset) + cy.clickOptionWithText(glossaryTerms.hdfsDataset); cy.waitTextVisible("Cypress 2"); }); it("can search related entities by AND-ing two concepts using search", () => { cy.waitTextVisible(glossaryTerms.hdfsDataset); applyAdvancedSearchFilter(); - cy.clickOptionWithText('Add Filter'); + cy.clickOptionWithText("Add Filter"); cy.get('[data-testid="adv-search-add-filter-description"]').click({ force: true, }); cy.get('[data-testid="edit-text-input"]').type("my hdfs dataset"); cy.get('[data-testid="edit-text-done-btn"]').click({ force: true }); cy.waitTextVisible(glossaryTerms.hdfsDataset); - cy.clickOptionWithText(glossaryTerms.hdfsDataset) + cy.clickOptionWithText(glossaryTerms.hdfsDataset); cy.waitTextVisible("my hdfs dataset"); }); @@ -99,4 +105,4 @@ describe("glossaryTerm", () => { cy.waitTextVisible(glossaryTerms.hdfsDataset); cy.waitTextVisible(glossaryTerms.hiveDataset); }); -}); \ No newline at end of file +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js index f6f1ff5949d25..553c0fb1626bc 100644 --- a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js +++ b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js @@ -10,17 +10,29 @@ const createTerm = (glossaryTerm) => { }; const navigateToParentAndCheckTermGroup = (parentGroup, termGroup) => { - cy.get('[data-testid="glossary-browser-sidebar"]').contains(parentGroup).click(); - cy.get('*[class^="GlossaryEntitiesList"]').contains(termGroup).should("be.visible"); + cy.get('[data-testid="glossary-browser-sidebar"]') + .contains(parentGroup) + .click(); + cy.get('*[class^="GlossaryEntitiesList"]') + .contains(termGroup) + .should("be.visible"); }; -const moveGlossaryEntityToGroup = (sourceEntity, targetEntity, confirmationMsg) => { +const moveGlossaryEntityToGroup = ( + sourceEntity, + targetEntity, + confirmationMsg, +) => { cy.clickOptionWithText(sourceEntity); - cy.get('[data-testid="entity-header-dropdown"]').should('be.visible'); + cy.get('[data-testid="entity-header-dropdown"]').should("be.visible"); cy.openThreeDotDropdown(); cy.clickOptionWithText("Move"); - cy.get('[data-testid="move-glossary-entity-modal"]').contains(targetEntity).click({ force: true }); - cy.get('[data-testid="move-glossary-entity-modal"]').contains(targetEntity).should("be.visible"); + cy.get('[data-testid="move-glossary-entity-modal"]') + .contains(targetEntity) + .click({ force: true }); + cy.get('[data-testid="move-glossary-entity-modal"]') + .contains(targetEntity) + .should("be.visible"); cy.clickOptionWithTestId("glossary-entity-modal-move-button"); cy.waitTextVisible(confirmationMsg); }; @@ -42,7 +54,11 @@ describe("glossary sidebar navigation test", () => { cy.createGlossaryTermGroup(glossaryTermGroup); cy.clickOptionWithTestId("add-term-button"); createTerm(glossaryTerm); - moveGlossaryEntityToGroup(glossaryTerm, glossaryTermGroup, `Moved Glossary Term!`); + moveGlossaryEntityToGroup( + glossaryTerm, + glossaryTermGroup, + `Moved Glossary Term!`, + ); navigateToParentAndCheckTermGroup(glossaryTermGroup, glossaryTerm); // Create another term and move it to the same term group @@ -50,24 +66,42 @@ describe("glossary sidebar navigation test", () => { cy.openThreeDotDropdown(); cy.clickOptionWithTestId("entity-menu-add-term-button"); createTerm(glossarySecondTerm); - moveGlossaryEntityToGroup(glossarySecondTerm, glossaryTermGroup, `Moved Glossary Term!`); + moveGlossaryEntityToGroup( + glossarySecondTerm, + glossaryTermGroup, + `Moved Glossary Term!`, + ); navigateToParentAndCheckTermGroup(glossaryTermGroup, glossarySecondTerm); // Switch between terms and ensure the "Properties" tab is active cy.clickOptionWithText(glossaryTerm); - cy.get('[data-testid="entity-tab-headers-test-id"]').contains("Properties").click({ force: true }); - cy.get('[data-node-key="Properties"]').contains("Properties").should("have.attr", "aria-selected", "true"); + cy.get('[data-testid="entity-tab-headers-test-id"]') + .contains("Properties") + .click({ force: true }); + cy.get('[data-node-key="Properties"]') + .contains("Properties") + .should("have.attr", "aria-selected", "true"); cy.clickOptionWithText(glossarySecondTerm); - cy.get('[data-node-key="Properties"]').contains("Properties").should("have.attr", "aria-selected", "true"); + cy.get('[data-node-key="Properties"]') + .contains("Properties") + .should("have.attr", "aria-selected", "true"); // Move a term group from the root level to be under a parent term group cy.goToGlossaryList(); - moveGlossaryEntityToGroup(glossaryTermGroup, glossaryParentGroup, 'Moved Term Group!'); + moveGlossaryEntityToGroup( + glossaryTermGroup, + glossaryParentGroup, + "Moved Term Group!", + ); navigateToParentAndCheckTermGroup(glossaryParentGroup, glossaryTermGroup); // Delete glossary terms and term group deleteGlossaryTerm(glossaryParentGroup, glossaryTermGroup, glossaryTerm); - deleteGlossaryTerm(glossaryParentGroup, glossaryTermGroup, glossarySecondTerm); + deleteGlossaryTerm( + glossaryParentGroup, + glossaryTermGroup, + glossarySecondTerm, + ); cy.goToGlossaryList(); cy.clickOptionWithText(glossaryParentGroup); diff --git a/smoke-test/tests/cypress/cypress/e2e/home/home.js b/smoke-test/tests/cypress/cypress/e2e/home/home.js index 05140486e189b..8b40cfaae41af 100644 --- a/smoke-test/tests/cypress/cypress/e2e/home/home.js +++ b/smoke-test/tests/cypress/cypress/e2e/home/home.js @@ -1,39 +1,44 @@ import { aliasQuery, hasOperationName } from "../utils"; -describe('home', () => { - let businessAttributeEntityEnabled; +describe("home", () => { + let businessAttributeEntityEnabled; - beforeEach(() => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - aliasQuery(req, "appConfig"); - }); + beforeEach(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); }); - - const setBusinessAttributeFeatureFlag = () => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - if (hasOperationName(req, "appConfig")) { - req.reply((res) => { - businessAttributeEntityEnabled = res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; - return res; - }); - } - }).as('apiCall'); - }; - it('home page shows ', () => { - setBusinessAttributeFeatureFlag(); - cy.login(); - cy.visit('/'); - // cy.get('img[src="/assets/platforms/datahublogo.png"]').should('exist'); - cy.get('[data-testid="entity-type-browse-card-DATASET"]').should('exist'); - cy.get('[data-testid="entity-type-browse-card-DASHBOARD"]').should('exist'); - cy.get('[data-testid="entity-type-browse-card-CHART"]').should('exist'); - cy.get('[data-testid="entity-type-browse-card-DATA_FLOW"]').should('exist'); - cy.get('[data-testid="entity-type-browse-card-GLOSSARY_TERM"]').should('exist'); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.get('[data-testid="entity-type-browse-card-BUSINESS_ATTRIBUTE"]').should('exist'); - }); + }); + + const setBusinessAttributeFeatureFlag = () => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + businessAttributeEntityEnabled = + res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; + return res; + }); + } + }).as("apiCall"); + }; + it("home page shows ", () => { + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/"); + // cy.get('img[src="/assets/platforms/datahublogo.png"]').should('exist'); + cy.get('[data-testid="entity-type-browse-card-DATASET"]').should("exist"); + cy.get('[data-testid="entity-type-browse-card-DASHBOARD"]').should("exist"); + cy.get('[data-testid="entity-type-browse-card-CHART"]').should("exist"); + cy.get('[data-testid="entity-type-browse-card-DATA_FLOW"]').should("exist"); + cy.get('[data-testid="entity-type-browse-card-GLOSSARY_TERM"]').should( + "exist", + ); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.get( + '[data-testid="entity-type-browse-card-BUSINESS_ATTRIBUTE"]', + ).should("exist"); }); - }) + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js b/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js index ed4167b87c506..325902cbc91ab 100644 --- a/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js +++ b/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js @@ -1,83 +1,90 @@ -const test_dataset = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)"; +const test_dataset = + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)"; const first_degree = [ - "urn:li:chart:(looker,cypress_baz1)", - "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", - "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + "urn:li:chart:(looker,cypress_baz1)", + "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)", ]; const second_degree = [ - "urn:li:chart:(looker,cypress_baz2)", - "urn:li:dashboard:(looker,cypress_baz)", - "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", - "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)" + "urn:li:chart:(looker,cypress_baz2)", + "urn:li:dashboard:(looker,cypress_baz)", + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)", ]; const third_degree_plus = [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", - "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", - "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)" + "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)", + "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)", + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)", ]; const downloadCsvFile = (filename) => { - cy.get('[data-testid="three-dot-menu"]').click(); - cy.get('[data-testid="download-as-csv-menu-item"]').click(); - cy.get('[data-testid="download-as-csv-input"]').clear().type(filename); - cy.get('[data-testid="csv-modal-download-button"]').click().wait(5000); - cy.ensureTextNotPresent("Creating CSV to download"); + cy.get('[data-testid="three-dot-menu"]').click(); + cy.get('[data-testid="download-as-csv-menu-item"]').click(); + cy.get('[data-testid="download-as-csv-input"]').clear().type(filename); + cy.get('[data-testid="csv-modal-download-button"]').click().wait(5000); + cy.ensureTextNotPresent("Creating CSV to download"); }; describe("download lineage results to .csv file", () => { - beforeEach(() => { - cy.on('uncaught:exception', (err, runnable) => { return false; }); - }); + beforeEach(() => { + cy.on("uncaught:exception", (err, runnable) => false); + }); - it("download and verify lineage results for 1st, 2nd and 3+ degree of dependencies", () => { - cy.loginWithCredentials(); - cy.goToDataset(test_dataset,"SampleCypressKafkaDataset"); - cy.openEntityTab("Lineage"); + it("download and verify lineage results for 1st, 2nd and 3+ degree of dependencies", () => { + cy.loginWithCredentials(); + cy.goToDataset(test_dataset, "SampleCypressKafkaDataset"); + cy.openEntityTab("Lineage"); - // Verify 1st degree of dependencies - cy.contains(/1 - [3-4] of [3-4]/); - downloadCsvFile("first_degree_results.csv"); - let first_degree_csv = cy.readFile('cypress/downloads/first_degree_results.csv'); - first_degree.forEach(function (urn) { - first_degree_csv.should('contain', urn) - }); - second_degree.forEach(function (urn) { - first_degree_csv.should('not.contain', urn) - }); - third_degree_plus.forEach(function (urn) { - first_degree_csv.should('not.contain', urn); - }); + // Verify 1st degree of dependencies + cy.contains(/1 - [3-4] of [3-4]/); + downloadCsvFile("first_degree_results.csv"); + const first_degree_csv = cy.readFile( + "cypress/downloads/first_degree_results.csv", + ); + first_degree.forEach((urn) => { + first_degree_csv.should("contain", urn); + }); + second_degree.forEach((urn) => { + first_degree_csv.should("not.contain", urn); + }); + third_degree_plus.forEach((urn) => { + first_degree_csv.should("not.contain", urn); + }); - // Verify 1st and 2nd degree of dependencies - cy.get('[data-testid="facet-degree-2"]').click().wait(5000); - cy.contains(/1 - [7-8] of [7-8]/); - downloadCsvFile("second_degree_results.csv"); - let second_degree_csv = cy.readFile('cypress/downloads/second_degree_results.csv'); - first_degree.forEach(function (urn) { - second_degree_csv.should('contain', urn) - }); - second_degree.forEach(function (urn) { - second_degree_csv.should('contain', urn) - }); - third_degree_plus.forEach(function (urn) { - second_degree_csv.should('not.contain', urn); - }); + // Verify 1st and 2nd degree of dependencies + cy.get('[data-testid="facet-degree-2"]').click().wait(5000); + cy.contains(/1 - [7-8] of [7-8]/); + downloadCsvFile("second_degree_results.csv"); + const second_degree_csv = cy.readFile( + "cypress/downloads/second_degree_results.csv", + ); + first_degree.forEach((urn) => { + second_degree_csv.should("contain", urn); + }); + second_degree.forEach((urn) => { + second_degree_csv.should("contain", urn); + }); + third_degree_plus.forEach((urn) => { + second_degree_csv.should("not.contain", urn); + }); - // Verify 1st 2nd and 3+ degree of dependencies(Verify multi page download) - cy.get('[data-testid="facet-degree-3+"]').click().wait(5000); - cy.contains(/1 - 10 of 1[3-4]/); - downloadCsvFile("third_plus_degree_results.csv"); - let third_degree_csv = cy.readFile('cypress/downloads/third_plus_degree_results.csv'); - first_degree.forEach(function (urn) { - third_degree_csv.should('contain', urn) - }); - second_degree.forEach(function (urn) { - third_degree_csv.should('contain', urn) - }); - third_degree_plus.forEach(function (urn) { - third_degree_csv.should('contain', urn); - }); + // Verify 1st 2nd and 3+ degree of dependencies(Verify multi page download) + cy.get('[data-testid="facet-degree-3+"]').click().wait(5000); + cy.contains(/1 - 10 of 1[3-4]/); + downloadCsvFile("third_plus_degree_results.csv"); + const third_degree_csv = cy.readFile( + "cypress/downloads/third_plus_degree_results.csv", + ); + first_degree.forEach((urn) => { + third_degree_csv.should("contain", urn); + }); + second_degree.forEach((urn) => { + third_degree_csv.should("contain", urn); + }); + third_degree_plus.forEach((urn) => { + third_degree_csv.should("contain", urn); }); -}); \ No newline at end of file + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/impact_analysis.js b/smoke-test/tests/cypress/cypress/e2e/lineage/impact_analysis.js index 784ccf8f0f87d..1b4367fcb8bbc 100644 --- a/smoke-test/tests/cypress/cypress/e2e/lineage/impact_analysis.js +++ b/smoke-test/tests/cypress/cypress/e2e/lineage/impact_analysis.js @@ -2,38 +2,38 @@ import { getTimestampMillisNumDaysAgo } from "../../support/commands"; const JAN_1_2021_TIMESTAMP = 1609553357755; const JAN_1_2022_TIMESTAMP = 1641089357755; -const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)'; +const DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)"; const TIMESTAMP_MILLIS_14_DAYS_AGO = getTimestampMillisNumDaysAgo(14); const TIMESTAMP_MILLIS_7_DAYS_AGO = getTimestampMillisNumDaysAgo(7); const TIMESTAMP_MILLIS_NOW = getTimestampMillisNumDaysAgo(0); -const GNP_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:snowflake,economic_data.gnp,PROD)"; -const TRANSACTION_ETL_URN = "urn:li:dataJob:(urn:li:dataFlow:(airflow,bq_etl,prod),transaction_etl)"; -const MONTHLY_TEMPERATURE_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.monthly_temperature,PROD)"; - +const GNP_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:snowflake,economic_data.gnp,PROD)"; +const TRANSACTION_ETL_URN = + "urn:li:dataJob:(urn:li:dataFlow:(airflow,bq_etl,prod),transaction_etl)"; +const MONTHLY_TEMPERATURE_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.monthly_temperature,PROD)"; const startAtDataSetLineage = () => { - cy.login(); - cy.goToDataset( - DATASET_URN, - "SampleCypressKafkaDataset" - ); - cy.openEntityTab("Lineage") -} + cy.login(); + cy.goToDataset(DATASET_URN, "SampleCypressKafkaDataset"); + cy.openEntityTab("Lineage"); +}; describe("impact analysis", () => { beforeEach(() => { - cy.on('uncaught:exception', (err, runnable) => { return false; }); + cy.on("uncaught:exception", (err, runnable) => false); }); it("can see 1 hop of lineage by default", () => { - startAtDataSetLineage() + startAtDataSetLineage(); cy.ensureTextNotPresent("User Creations"); cy.ensureTextNotPresent("User Deletions"); }); it("can see lineage multiple hops away", () => { - startAtDataSetLineage() + startAtDataSetLineage(); // click to show more relationships now that we default to 1 degree of dependency cy.clickOptionWithText("3+"); @@ -42,7 +42,7 @@ describe("impact analysis", () => { }); it("can filter the lineage results as well", () => { - startAtDataSetLineage() + startAtDataSetLineage(); // click to show more relationships now that we default to 1 degree of dependency cy.clickOptionWithText("3+"); @@ -50,11 +50,11 @@ describe("impact analysis", () => { cy.clickOptionWithText("Add Filter"); - cy.clickOptionWithTestId('adv-search-add-filter-description'); + cy.clickOptionWithTestId("adv-search-add-filter-description"); cy.get('[data-testid="edit-text-input"]').type("fct_users_deleted"); - cy.clickOptionWithTestId('edit-text-done-btn'); + cy.clickOptionWithTestId("edit-text-done-btn"); cy.ensureTextNotPresent("User Creations"); cy.waitTextVisible("User Deletions"); @@ -63,7 +63,7 @@ describe("impact analysis", () => { it("can view column level impact analysis and turn it off", () => { cy.login(); cy.visit( - `/dataset/${DATASET_URN}/Lineage?column=%5Bversion%3D2.0%5D.%5Btype%3Dboolean%5D.field_bar&is_lineage_mode=false` + `/dataset/${DATASET_URN}/Lineage?column=%5Bversion%3D2.0%5D.%5Btype%3Dboolean%5D.field_bar&is_lineage_mode=false`, ); // impact analysis can take a beat- don't want to time out here @@ -85,11 +85,10 @@ describe("impact analysis", () => { cy.contains("Baz Chart 1"); }); - it("can filter lineage edges by time", () => { cy.login(); cy.visit( - `/dataset/${DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${JAN_1_2021_TIMESTAMP}&end_time_millis=${JAN_1_2022_TIMESTAMP}` + `/dataset/${DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${JAN_1_2021_TIMESTAMP}&end_time_millis=${JAN_1_2022_TIMESTAMP}`, ); // impact analysis can take a beat- don't want to time out here @@ -105,7 +104,7 @@ describe("impact analysis", () => { cy.login(); // Between 14 days ago and 7 days ago, only transactions was an input cy.visit( - `/tasks/${TRANSACTION_ETL_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_14_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}` + `/tasks/${TRANSACTION_ETL_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_14_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}`, ); // Downstream cy.contains("aggregated"); @@ -115,7 +114,7 @@ describe("impact analysis", () => { cy.contains("user_profile").should("not.exist"); // 1 day ago, factor_income was removed from the join cy.visit( - `/tasks/${TRANSACTION_ETL_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_NOW}` + `/tasks/${TRANSACTION_ETL_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_NOW}`, ); // Downstream cy.contains("aggregated"); @@ -129,14 +128,14 @@ describe("impact analysis", () => { cy.login(); // Between 14 days ago and 7 days ago, only temperature_etl_1 was an iput cy.visit( - `/dataset/${MONTHLY_TEMPERATURE_DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_14_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}` + `/dataset/${MONTHLY_TEMPERATURE_DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_14_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}`, ); cy.lineageTabClickOnUpstream(); cy.contains("temperature_etl_1"); cy.contains("temperature_etl_2").should("not.exist"); // Since 7 days ago, temperature_etl_1 has been replaced by temperature_etl_2 cy.visit( - `/dataset/${MONTHLY_TEMPERATURE_DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_NOW}` + `/dataset/${MONTHLY_TEMPERATURE_DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_NOW}`, ); cy.lineageTabClickOnUpstream(); cy.contains("temperature_etl_1").should("not.exist"); @@ -147,14 +146,14 @@ describe("impact analysis", () => { cy.login(); // 8 days ago, both gdp and factor_income were joined to create gnp cy.visit( - `/dataset/${GNP_DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_14_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_NOW}` + `/dataset/${GNP_DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_14_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_NOW}`, ); cy.lineageTabClickOnUpstream(); cy.contains("gdp"); cy.contains("factor_income"); // 1 day ago, factor_income was removed from the join cy.visit( - `/dataset/${GNP_DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_NOW}` + `/dataset/${GNP_DATASET_URN}/Lineage?filter_degree___false___EQUAL___0=1&is_lineage_mode=false&page=1&unionType=0&start_time_millis=${TIMESTAMP_MILLIS_7_DAYS_AGO}&end_time_millis=${TIMESTAMP_MILLIS_NOW}`, ); cy.lineageTabClickOnUpstream(); cy.contains("gdp"); diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_level.js b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_level.js index 2a8fe045f154e..cedf3c6c051be 100644 --- a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_level.js +++ b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_level.js @@ -1,51 +1,50 @@ -const DATASET_ENTITY_TYPE = 'dataset'; -const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)'; +const DATASET_ENTITY_TYPE = "dataset"; +const DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)"; describe("column-level lineage graph test", () => { - - it("navigate to lineage graph view and verify that column-level lineage is showing correctly", () => { - cy.login(); - cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN); - //verify columns not shown by default - cy.waitTextVisible("SampleCypressHdfs"); - cy.waitTextVisible("SampleCypressHive"); - cy.waitTextVisible("cypress_logging"); - cy.ensureTextNotPresent("shipment_info"); - cy.ensureTextNotPresent("field_foo"); - cy.ensureTextNotPresent("field_baz"); - cy.ensureTextNotPresent("event_name"); - cy.ensureTextNotPresent("event_data"); - cy.ensureTextNotPresent("timestamp"); - cy.ensureTextNotPresent("browser"); - cy.clickOptionWithTestId("column-toggle") - //verify columns appear and belong co correct dataset - cy.waitTextVisible("shipment_info"); - cy.waitTextVisible("shipment_info.date"); - cy.waitTextVisible("shipment_info.target"); - cy.waitTextVisible("shipment_info.destination"); - cy.waitTextVisible("shipment_info.geo_info"); - cy.waitTextVisible("field_foo"); - cy.waitTextVisible("field_baz"); - cy.waitTextVisible("event_name"); - cy.waitTextVisible("event_data"); - cy.waitTextVisible("timestamp"); - cy.waitTextVisible("browser"); - //verify columns can be hidden and shown again - cy.contains("Hide").click({ force:true }); - cy.ensureTextNotPresent("field_foo"); - cy.ensureTextNotPresent("field_baz"); - cy.get("[aria-label='down']").eq(1).click({ force:true }); - cy.waitTextVisible("field_foo"); - cy.waitTextVisible("field_baz"); - //verify columns can be disabled successfully - cy.clickOptionWithTestId("column-toggle") - cy.ensureTextNotPresent("shipment_info"); - cy.ensureTextNotPresent("field_foo"); - cy.ensureTextNotPresent("field_baz"); - cy.ensureTextNotPresent("event_name"); - cy.ensureTextNotPresent("event_data"); - cy.ensureTextNotPresent("timestamp"); - cy.ensureTextNotPresent("browser"); - }); - -}); \ No newline at end of file + it("navigate to lineage graph view and verify that column-level lineage is showing correctly", () => { + cy.login(); + cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN); + // verify columns not shown by default + cy.waitTextVisible("SampleCypressHdfs"); + cy.waitTextVisible("SampleCypressHive"); + cy.waitTextVisible("cypress_logging"); + cy.ensureTextNotPresent("shipment_info"); + cy.ensureTextNotPresent("field_foo"); + cy.ensureTextNotPresent("field_baz"); + cy.ensureTextNotPresent("event_name"); + cy.ensureTextNotPresent("event_data"); + cy.ensureTextNotPresent("timestamp"); + cy.ensureTextNotPresent("browser"); + cy.clickOptionWithTestId("column-toggle"); + // verify columns appear and belong co correct dataset + cy.waitTextVisible("shipment_info"); + cy.waitTextVisible("shipment_info.date"); + cy.waitTextVisible("shipment_info.target"); + cy.waitTextVisible("shipment_info.destination"); + cy.waitTextVisible("shipment_info.geo_info"); + cy.waitTextVisible("field_foo"); + cy.waitTextVisible("field_baz"); + cy.waitTextVisible("event_name"); + cy.waitTextVisible("event_data"); + cy.waitTextVisible("timestamp"); + cy.waitTextVisible("browser"); + // verify columns can be hidden and shown again + cy.contains("Hide").click({ force: true }); + cy.ensureTextNotPresent("field_foo"); + cy.ensureTextNotPresent("field_baz"); + cy.get("[aria-label='down']").eq(1).click({ force: true }); + cy.waitTextVisible("field_foo"); + cy.waitTextVisible("field_baz"); + // verify columns can be disabled successfully + cy.clickOptionWithTestId("column-toggle"); + cy.ensureTextNotPresent("shipment_info"); + cy.ensureTextNotPresent("field_foo"); + cy.ensureTextNotPresent("field_baz"); + cy.ensureTextNotPresent("event_name"); + cy.ensureTextNotPresent("event_data"); + cy.ensureTextNotPresent("timestamp"); + cy.ensureTextNotPresent("browser"); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js index 37ca62c8d1229..7691b04d91f32 100644 --- a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js +++ b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js @@ -1,68 +1,89 @@ import { aliasQuery } from "../utils"; -const DATASET_ENTITY_TYPE = 'dataset'; -const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)'; -const DOWNSTREAM_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)"; -const upstreamColumn = '[data-testid="node-urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)-Upstream"] text'; -const downstreamColumn = '[data-testid="node-urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)-Downstream"] text'; + +const DATASET_ENTITY_TYPE = "dataset"; +const DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)"; +const DOWNSTREAM_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)"; +const upstreamColumn = + '[data-testid="node-urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)-Upstream"] text'; +const downstreamColumn = + '[data-testid="node-urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)-Downstream"] text'; const verifyColumnPathModal = (from, to) => { - cy.get('[data-testid="entity-paths-modal"]').contains(from).should("be.visible"); - cy.get('[data-testid="entity-paths-modal"]').contains(to).should("be.visible"); + cy.get('[data-testid="entity-paths-modal"]') + .contains(from) + .should("be.visible"); + cy.get('[data-testid="entity-paths-modal"]') + .contains(to) + .should("be.visible"); }; describe("column-Level lineage and impact analysis path test", () => { - beforeEach(() => { - cy.on('uncaught:exception', (err, runnable) => { return false; }); - cy.intercept("POST", "/api/v2/graphql", (req) => { - aliasQuery(req, "appConfig"); - }); - }); + beforeEach(() => { + cy.on("uncaught:exception", (err, runnable) => false); + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); + }); + }); - it("verify column-level lineage path at lineage praph and impact analysis ", () => { - // Open dataset with column-level lineage configured an navigate to lineage tab -> visualize lineage - cy.loginWithCredentials(); - cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN); + it("verify column-level lineage path at lineage praph and impact analysis ", () => { + // Open dataset with column-level lineage configured an navigate to lineage tab -> visualize lineage + cy.loginWithCredentials(); + cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN); - // Enable “show columns” toggle - cy.waitTextVisible("SampleCypressHdfs"); - cy.clickOptionWithTestId("column-toggle"); - cy.waitTextVisible("shipment_info"); + // Enable “show columns” toggle + cy.waitTextVisible("SampleCypressHdfs"); + cy.clickOptionWithTestId("column-toggle"); + cy.waitTextVisible("shipment_info"); - // Verify functionality of column lineage - cy.get(upstreamColumn).eq(3).click(); - cy.get(upstreamColumn).eq(3).prev().should('not.have.attr', 'fill', 'white'); - cy.get(downstreamColumn).eq(2).prev().should('not.have.attr', 'stroke', 'transparent'); - cy.get(downstreamColumn).eq(2).click(); - cy.get(downstreamColumn).eq(2).prev().should('not.have.attr', 'fill', 'white'); - cy.get(upstreamColumn).eq(3).prev().should('not.have.attr', 'stroke', 'transparent'); + // Verify functionality of column lineage + cy.get(upstreamColumn).eq(3).click(); + cy.get(upstreamColumn) + .eq(3) + .prev() + .should("not.have.attr", "fill", "white"); + cy.get(downstreamColumn) + .eq(2) + .prev() + .should("not.have.attr", "stroke", "transparent"); + cy.get(downstreamColumn).eq(2).click(); + cy.get(downstreamColumn) + .eq(2) + .prev() + .should("not.have.attr", "fill", "white"); + cy.get(upstreamColumn) + .eq(3) + .prev() + .should("not.have.attr", "stroke", "transparent"); - // Open dataset impact analysis view, enable column lineage - cy.goToDataset(DATASET_URN, "SampleCypressHdfsDataset"); - cy.openEntityTab("Lineage"); - cy.clickOptionWithText("Column Lineage"); - cy.clickOptionWithText("Downstream"); + // Open dataset impact analysis view, enable column lineage + cy.goToDataset(DATASET_URN, "SampleCypressHdfsDataset"); + cy.openEntityTab("Lineage"); + cy.clickOptionWithText("Column Lineage"); + cy.clickOptionWithText("Downstream"); - // Verify upstream column lineage, test column path modal - cy.clickOptionWithText("Upstream"); - cy.waitTextVisible("SampleCypressKafkaDataset"); - cy.ensureTextNotPresent("field_bar"); - cy.contains("Select column").click({ force: true}).wait(1000); - cy.get(".rc-virtual-list").contains("shipment_info").click(); - cy.waitTextVisible("field_bar"); - cy.clickOptionWithText("field_bar"); - verifyColumnPathModal("shipment_info", "field_bar"); - cy.get('[data-testid="entity-paths-modal"] [data-icon="close"]').click(); - - // Verify downstream column lineage, test column path modal - cy.goToDataset(DOWNSTREAM_DATASET_URN, "SampleCypressKafkaDataset"); - cy.openEntityTab("Lineage"); - cy.clickOptionWithText("Column Lineage"); - cy.ensureTextNotPresent("shipment_info"); - cy.contains("Select column").click({ force: true}).wait(1000); - cy.get(".rc-virtual-list").contains("field_bar").click(); - cy.waitTextVisible("shipment_info"); - cy.clickOptionWithText("shipment_info"); - verifyColumnPathModal("shipment_info", "field_bar"); - cy.get('[data-testid="entity-paths-modal"] [data-icon="close"]').click(); - }); -}); \ No newline at end of file + // Verify upstream column lineage, test column path modal + cy.clickOptionWithText("Upstream"); + cy.waitTextVisible("SampleCypressKafkaDataset"); + cy.ensureTextNotPresent("field_bar"); + cy.contains("Select column").click({ force: true }).wait(1000); + cy.get(".rc-virtual-list").contains("shipment_info").click(); + cy.waitTextVisible("field_bar"); + cy.clickOptionWithText("field_bar"); + verifyColumnPathModal("shipment_info", "field_bar"); + cy.get('[data-testid="entity-paths-modal"] [data-icon="close"]').click(); + + // Verify downstream column lineage, test column path modal + cy.goToDataset(DOWNSTREAM_DATASET_URN, "SampleCypressKafkaDataset"); + cy.openEntityTab("Lineage"); + cy.clickOptionWithText("Column Lineage"); + cy.ensureTextNotPresent("shipment_info"); + cy.contains("Select column").click({ force: true }).wait(1000); + cy.get(".rc-virtual-list").contains("field_bar").click(); + cy.waitTextVisible("shipment_info"); + cy.clickOptionWithText("shipment_info"); + verifyColumnPathModal("shipment_info", "field_bar"); + cy.get('[data-testid="entity-paths-modal"] [data-icon="close"]').click(); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_graph.js b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_graph.js index 85db210649c27..e66afdda11ca4 100644 --- a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_graph.js +++ b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_graph.js @@ -1,80 +1,118 @@ import { getTimestampMillisNumDaysAgo } from "../../support/commands"; -const DATASET_ENTITY_TYPE = 'dataset'; -const TASKS_ENTITY_TYPE = 'tasks'; -const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)'; +const DATASET_ENTITY_TYPE = "dataset"; +const TASKS_ENTITY_TYPE = "tasks"; +const DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)"; const JAN_1_2021_TIMESTAMP = 1609553357755; const JAN_1_2022_TIMESTAMP = 1641089357755; const TIMESTAMP_MILLIS_14_DAYS_AGO = getTimestampMillisNumDaysAgo(14); const TIMESTAMP_MILLIS_7_DAYS_AGO = getTimestampMillisNumDaysAgo(7); const TIMESTAMP_MILLIS_NOW = getTimestampMillisNumDaysAgo(0); -const GNP_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:snowflake,economic_data.gnp,PROD)"; -const TRANSACTION_ETL_URN = "urn:li:dataJob:(urn:li:dataFlow:(airflow,bq_etl,prod),transaction_etl)"; -const MONTHLY_TEMPERATURE_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.monthly_temperature,PROD)"; +const GNP_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:snowflake,economic_data.gnp,PROD)"; +const TRANSACTION_ETL_URN = + "urn:li:dataJob:(urn:li:dataFlow:(airflow,bq_etl,prod),transaction_etl)"; +const MONTHLY_TEMPERATURE_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.monthly_temperature,PROD)"; describe("lineage_graph", () => { - it("can see full history", () => { - cy.login(); - cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN); + it("can see full history", () => { + cy.login(); + cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN); - cy.contains("SampleCypressKafka"); - cy.contains("SampleCypressHdfs"); - cy.contains("Baz Chart 1"); - cy.contains("some-cypress"); - }); + cy.contains("SampleCypressKafka"); + cy.contains("SampleCypressHdfs"); + cy.contains("Baz Chart 1"); + cy.contains("some-cypress"); + }); - it("cannot see any lineage edges for 2021", () => { - cy.login(); - cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN, JAN_1_2021_TIMESTAMP, JAN_1_2022_TIMESTAMP); + it("cannot see any lineage edges for 2021", () => { + cy.login(); + cy.goToEntityLineageGraph( + DATASET_ENTITY_TYPE, + DATASET_URN, + JAN_1_2021_TIMESTAMP, + JAN_1_2022_TIMESTAMP, + ); - cy.contains("SampleCypressKafka"); - cy.contains("SampleCypressHdfs").should("not.exist"); - cy.contains("Baz Chart 1").should("not.exist"); - cy.contains("some-cypress").should("not.exist"); - }); + cy.contains("SampleCypressKafka"); + cy.contains("SampleCypressHdfs").should("not.exist"); + cy.contains("Baz Chart 1").should("not.exist"); + cy.contains("some-cypress").should("not.exist"); + }); - it("can see when the inputs to a data job change", () => { - cy.login(); - // Between 14 days ago and 7 days ago, only transactions was an input - cy.goToEntityLineageGraph(TASKS_ENTITY_TYPE, TRANSACTION_ETL_URN, TIMESTAMP_MILLIS_14_DAYS_AGO, TIMESTAMP_MILLIS_7_DAYS_AGO); - cy.contains("transaction_etl"); - cy.contains("aggregated"); - cy.contains("transactions"); - cy.contains("user_profile").should("not.exist"); - // 1 day ago, user_profile was also added as an input - cy.goToEntityLineageGraph(TASKS_ENTITY_TYPE, TRANSACTION_ETL_URN, TIMESTAMP_MILLIS_7_DAYS_AGO, TIMESTAMP_MILLIS_NOW); - cy.contains("transaction_etl"); - cy.contains("aggregated"); - cy.contains("transactions"); - cy.contains("user_profile"); - }); - - it("can see when a data job is replaced", () => { - cy.login(); - // Between 14 days ago and 7 days ago, only temperature_etl_1 was an iput - cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, MONTHLY_TEMPERATURE_DATASET_URN, TIMESTAMP_MILLIS_14_DAYS_AGO, TIMESTAMP_MILLIS_7_DAYS_AGO); - cy.contains("monthly_temperature"); - cy.contains("temperature_etl_1"); - cy.contains("temperature_etl_2").should("not.exist"); - // Since 7 days ago, temperature_etl_1 has been replaced by temperature_etl_2 - cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, MONTHLY_TEMPERATURE_DATASET_URN, TIMESTAMP_MILLIS_7_DAYS_AGO, TIMESTAMP_MILLIS_NOW); - cy.contains("monthly_temperature"); - cy.contains("temperature_etl_1").should("not.exist"); - cy.contains("temperature_etl_2"); - }); + it("can see when the inputs to a data job change", () => { + cy.login(); + // Between 14 days ago and 7 days ago, only transactions was an input + cy.goToEntityLineageGraph( + TASKS_ENTITY_TYPE, + TRANSACTION_ETL_URN, + TIMESTAMP_MILLIS_14_DAYS_AGO, + TIMESTAMP_MILLIS_7_DAYS_AGO, + ); + cy.contains("transaction_etl"); + cy.contains("aggregated"); + cy.contains("transactions"); + cy.contains("user_profile").should("not.exist"); + // 1 day ago, user_profile was also added as an input + cy.goToEntityLineageGraph( + TASKS_ENTITY_TYPE, + TRANSACTION_ETL_URN, + TIMESTAMP_MILLIS_7_DAYS_AGO, + TIMESTAMP_MILLIS_NOW, + ); + cy.contains("transaction_etl"); + cy.contains("aggregated"); + cy.contains("transactions"); + cy.contains("user_profile"); + }); + + it("can see when a data job is replaced", () => { + cy.login(); + // Between 14 days ago and 7 days ago, only temperature_etl_1 was an iput + cy.goToEntityLineageGraph( + DATASET_ENTITY_TYPE, + MONTHLY_TEMPERATURE_DATASET_URN, + TIMESTAMP_MILLIS_14_DAYS_AGO, + TIMESTAMP_MILLIS_7_DAYS_AGO, + ); + cy.contains("monthly_temperature"); + cy.contains("temperature_etl_1"); + cy.contains("temperature_etl_2").should("not.exist"); + // Since 7 days ago, temperature_etl_1 has been replaced by temperature_etl_2 + cy.goToEntityLineageGraph( + DATASET_ENTITY_TYPE, + MONTHLY_TEMPERATURE_DATASET_URN, + TIMESTAMP_MILLIS_7_DAYS_AGO, + TIMESTAMP_MILLIS_NOW, + ); + cy.contains("monthly_temperature"); + cy.contains("temperature_etl_1").should("not.exist"); + cy.contains("temperature_etl_2"); + }); - it("can see when a dataset join changes", () => { - cy.login(); - // 8 days ago, both gdp and factor_income were joined to create gnp - cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, GNP_DATASET_URN, TIMESTAMP_MILLIS_14_DAYS_AGO, TIMESTAMP_MILLIS_NOW); - cy.contains("gnp"); - cy.contains("gdp"); - cy.contains("factor_income"); - // 1 day ago, factor_income was removed from the join - cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, GNP_DATASET_URN, TIMESTAMP_MILLIS_7_DAYS_AGO, TIMESTAMP_MILLIS_NOW); - cy.contains("gnp"); - cy.contains("gdp"); - cy.contains("factor_income").should("not.exist"); - }); + it("can see when a dataset join changes", () => { + cy.login(); + // 8 days ago, both gdp and factor_income were joined to create gnp + cy.goToEntityLineageGraph( + DATASET_ENTITY_TYPE, + GNP_DATASET_URN, + TIMESTAMP_MILLIS_14_DAYS_AGO, + TIMESTAMP_MILLIS_NOW, + ); + cy.contains("gnp"); + cy.contains("gdp"); + cy.contains("factor_income"); + // 1 day ago, factor_income was removed from the join + cy.goToEntityLineageGraph( + DATASET_ENTITY_TYPE, + GNP_DATASET_URN, + TIMESTAMP_MILLIS_7_DAYS_AGO, + TIMESTAMP_MILLIS_NOW, + ); + cy.contains("gnp"); + cy.contains("gdp"); + cy.contains("factor_income").should("not.exist"); }); - \ No newline at end of file +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/login/login.js b/smoke-test/tests/cypress/cypress/e2e/login/login.js index cfeb2619593ff..ad11e80a0caed 100644 --- a/smoke-test/tests/cypress/cypress/e2e/login/login.js +++ b/smoke-test/tests/cypress/cypress/e2e/login/login.js @@ -1,9 +1,9 @@ -describe('login', () => { - it('logs in', () => { - cy.visit('/'); - cy.get('input[data-testid=username]').type('datahub'); - cy.get('input[data-testid=password]').type('datahub'); - cy.contains('Sign In').click(); - cy.contains('Welcome back, ' + Cypress.env('ADMIN_DISPLAYNAME')); +describe("login", () => { + it("logs in", () => { + cy.visit("/"); + cy.get("input[data-testid=username]").type("datahub"); + cy.get("input[data-testid=password]").type("datahub"); + cy.contains("Sign In").click(); + cy.contains(`Welcome back, ${Cypress.env("ADMIN_DISPLAYNAME")}`); }); -}) +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/ml/feature_table.js b/smoke-test/tests/cypress/cypress/e2e/ml/feature_table.js index 229bc86cf3a33..9cf3ec1987da9 100644 --- a/smoke-test/tests/cypress/cypress/e2e/ml/feature_table.js +++ b/smoke-test/tests/cypress/cypress/e2e/ml/feature_table.js @@ -1,54 +1,59 @@ -describe('features', () => { - it('can visit feature tables and see features', () => { - cy.visit('/') - cy.login(); - cy.visit('/featureTables/urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)/Features?is_lineage_mode=false'); - - // the feature table descriptions should be there - cy.contains('Yet another test feature group'); - cy.contains('this is a description from source system'); - - // additional properties are visible - cy.contains('CypressPrimaryKeyTag'); - cy.contains('CypressFeatureTag'); - - // navigate to sources - cy.contains('Sources').click(); - - // feature & primary key sources are visible - cy.contains('SampleCypressHdfsDataset'); - cy.contains('SampleCypressKafkaDataset'); - - // navigate to properties - cy.contains('Properties').click(); - - // custom properties are visible - cy.contains('status'); - cy.contains('Created'); - - }); - - it('can visit feature page', () => { - cy.visit('/') - cy.login(); - cy.visit('/features/urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)/Feature%20Tables?is_lineage_mode=false'); - - // Shows the parent table - cy.contains('cypress-feature-table'); - - // Has upstream & downstream lineage - cy.contains('1 upstream, 1 downstream'); - }); - - it('can visit primary key page', () => { - cy.visit('/') - cy.login(); - cy.visit('/mlPrimaryKeys/urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)/Feature%20Tables?is_lineage_mode=false'); - - // Shows the parent table - cy.contains('cypress-feature-table'); - - // Has upstream from its sources - cy.contains('1 upstream, 0 downstream'); - }); -}) +describe("features", () => { + it("can visit feature tables and see features", () => { + cy.visit("/"); + cy.login(); + cy.visit( + "/featureTables/urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,cypress-feature-table)/Features?is_lineage_mode=false", + ); + + // the feature table descriptions should be there + cy.contains("Yet another test feature group"); + cy.contains("this is a description from source system"); + + // additional properties are visible + cy.contains("CypressPrimaryKeyTag"); + cy.contains("CypressFeatureTag"); + + // navigate to sources + cy.contains("Sources").click(); + + // feature & primary key sources are visible + cy.contains("SampleCypressHdfsDataset"); + cy.contains("SampleCypressKafkaDataset"); + + // navigate to properties + cy.contains("Properties").click(); + + // custom properties are visible + cy.contains("status"); + cy.contains("Created"); + }); + + it("can visit feature page", () => { + cy.visit("/"); + cy.login(); + cy.visit( + "/features/urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)/Feature%20Tables?is_lineage_mode=false", + ); + + // Shows the parent table + cy.contains("cypress-feature-table"); + + // Has upstream & downstream lineage + cy.contains("1 upstream, 1 downstream"); + }); + + it("can visit primary key page", () => { + cy.visit("/"); + cy.login(); + cy.visit( + "/mlPrimaryKeys/urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)/Feature%20Tables?is_lineage_mode=false", + ); + + // Shows the parent table + cy.contains("cypress-feature-table"); + + // Has upstream from its sources + cy.contains("1 upstream, 0 downstream"); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/ml/model.js b/smoke-test/tests/cypress/cypress/e2e/ml/model.js index 25f98b15d9f6b..ecebaa7649f4b 100644 --- a/smoke-test/tests/cypress/cypress/e2e/ml/model.js +++ b/smoke-test/tests/cypress/cypress/e2e/ml/model.js @@ -1,30 +1,36 @@ -describe('models', () => { - it('can visit models and groups', () => { - cy.visit('/') - cy.login(); - cy.visit('/mlModels/urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)/Summary?is_lineage_mode=false'); +describe("models", () => { + it("can visit models and groups", () => { + cy.visit("/"); + cy.login(); + cy.visit( + "/mlModels/urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)/Summary?is_lineage_mode=false", + ); - cy.contains('ml model description'); + cy.contains("ml model description"); - // the model has metrics & hyper params - cy.contains('another-metric'); - cy.contains('parameter-1'); + // the model has metrics & hyper params + cy.contains("another-metric"); + cy.contains("parameter-1"); - // the model has features - cy.contains('Features').click(); - cy.contains('some-cypress-feature-1'); + // the model has features + cy.contains("Features").click(); + cy.contains("some-cypress-feature-1"); - // the model has a group - cy.visit('/mlModels/urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)/Group?is_lineage_mode=false'); - cy.contains('cypress-model-package-group'); - }); + // the model has a group + cy.visit( + "/mlModels/urn:li:mlModel:(urn:li:dataPlatform:sagemaker,cypress-model,PROD)/Group?is_lineage_mode=false", + ); + cy.contains("cypress-model-package-group"); + }); - it('can visit models and groups', () => { - cy.visit('/') - cy.login(); - cy.visit('/mlModelGroup/urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)'); - // the model group has its model - cy.contains('cypress-model'); - cy.contains('Just a model package group.'); - }); -}) + it("can visit models and groups", () => { + cy.visit("/"); + cy.login(); + cy.visit( + "/mlModelGroup/urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,cypress-model-package-group,PROD)", + ); + // the model group has its model + cy.contains("cypress-model"); + cy.contains("Just a model package group."); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/add_users.js b/smoke-test/tests/cypress/cypress/e2e/mutations/add_users.js index ba225ba37884b..411f7d574bf63 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/add_users.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/add_users.js @@ -1,7 +1,7 @@ const tryToSignUp = () => { - let number = Math.floor(Math.random() * 100000); - let name = `Example Name ${number}`; - let email = `example${number}@example.com`; + const number = Math.floor(Math.random() * 100000); + const name = `Example Name ${number}`; + const email = `example${number}@example.com`; cy.enterTextInTestId("email", email); cy.enterTextInTestId("name", name); cy.enterTextInTestId("password", "Example password"); @@ -15,7 +15,7 @@ const tryToSignUp = () => { }; describe("add_user", () => { - let registeredEmail = ""; + let registeredEmail = ""; it("go to user link and invite a user", () => { cy.login(); @@ -53,7 +53,7 @@ describe("add_user", () => { cy.get('[data-testid="reset-menu-item"]').should( "have.attr", "aria-disabled", - "true" + "true", ); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_health.js b/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_health.js index 2ecf8b1833b24..072574e0f57aa 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_health.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_health.js @@ -1,15 +1,18 @@ -const urn = "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)"; +const urn = + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_health_test,PROD)"; const datasetName = "cypress_health_test"; describe("dataset health test", () => { - it("go to dataset with failing assertions and active incidents and verify health of dataset", () => { - cy.login(); - cy.goToDataset(urn, datasetName); - // Ensure that the “Health” badge is present and there is an active incident warning - cy.get(`[href="/dataset/${urn}/Validation"]`).should("be.visible"); - cy.get(`[href="/dataset/${urn}/Validation"] span`).trigger("mouseover", { force: true }); - cy.waitTextVisible("This asset may be unhealthy"); - cy.waitTextVisible("Assertions 1 of 1 assertions are failing"); - cy.waitTextVisible("1 active incident"); + it("go to dataset with failing assertions and active incidents and verify health of dataset", () => { + cy.login(); + cy.goToDataset(urn, datasetName); + // Ensure that the “Health” badge is present and there is an active incident warning + cy.get(`[href="/dataset/${urn}/Validation"]`).should("be.visible"); + cy.get(`[href="/dataset/${urn}/Validation"] span`).trigger("mouseover", { + force: true, }); -}); \ No newline at end of file + cy.waitTextVisible("This asset may be unhealthy"); + cy.waitTextVisible("Assertions 1 of 1 assertions are failing"); + cy.waitTextVisible("1 active incident"); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js b/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js index 552c3d460ade9..452e2eb3408d3 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js @@ -5,64 +5,105 @@ const password = "Example password"; const group_name = `Test group ${test_id}`; const addOwner = (owner, type, elementId) => { - cy.clickOptionWithTestId("add-owners-button"); - cy.contains("Search for users or groups...").click({ force: true }); - cy.focused().type(owner); - cy.get('.ant-select-item').contains(owner).click(); - cy.focused().blur(); - cy.waitTextVisible(owner); - cy.get('[role="dialog"]').contains("Technical Owner").click(); - cy.get('[role="listbox"]').parent().contains(type).click(); - cy.get('[role="dialog"]').contains(type).should("be.visible"); - cy.clickOptionWithText("Done"); - cy.waitTextVisible("Owners Added"); - cy.waitTextVisible(type); - cy.waitTextVisible(owner).wait(3000); - cy.clickOptionWithText(owner); - cy.waitTextVisible("SampleCypressHiveDataset"); - cy.goToDataset("urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", "SampleCypressHiveDataset"); - cy.get(elementId).next().click(); - cy.clickOptionWithText("Yes"); - cy.waitTextVisible("Owner Removed"); - cy.ensureTextNotPresent(owner); - cy.ensureTextNotPresent(type); -} + cy.clickOptionWithTestId("add-owners-button"); + cy.contains("Search for users or groups...").click({ force: true }); + cy.focused().type(owner); + cy.get(".ant-select-item").contains(owner).click(); + cy.focused().blur(); + cy.waitTextVisible(owner); + cy.get('[role="dialog"]').contains("Technical Owner").click(); + cy.get('[role="listbox"]').parent().contains(type).click(); + cy.get('[role="dialog"]').contains(type).should("be.visible"); + cy.clickOptionWithText("Done"); + cy.waitTextVisible("Owners Added"); + cy.waitTextVisible(type); + cy.waitTextVisible(owner).wait(3000); + cy.clickOptionWithText(owner); + cy.waitTextVisible("SampleCypressHiveDataset"); + cy.goToDataset( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "SampleCypressHiveDataset", + ); + cy.get(elementId).next().click(); + cy.clickOptionWithText("Yes"); + cy.waitTextVisible("Owner Removed"); + cy.ensureTextNotPresent(owner); + cy.ensureTextNotPresent(type); +}; describe("add, remove ownership for dataset", () => { - beforeEach(() => { - cy.on('uncaught:exception', (err, runnable) => { return false; }); - }); + beforeEach(() => { + cy.on("uncaught:exception", (err, runnable) => false); + }); - it("create test user and test group, add user to a group", () => { - cy.loginWithCredentials(); - cy.createUser(username, password, email); - cy.createGroup(group_name, "Test group description", test_id); - cy.addGroupMember(group_name, `/group/urn:li:corpGroup:${test_id}/assets`, username); - }); + it("create test user and test group, add user to a group", () => { + cy.loginWithCredentials(); + cy.createUser(username, password, email); + cy.createGroup(group_name, "Test group description", test_id); + cy.addGroupMember( + group_name, + `/group/urn:li:corpGroup:${test_id}/assets`, + username, + ); + }); - it("open test dataset page, add and remove user ownership(test every type)", () => { - cy.loginWithCredentials(); - cy.goToDataset("urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", "SampleCypressHiveDataset"); - //business owner - addOwner(username, "Business Owner", `[href="/user/urn:li:corpuser:example${test_id}@example.com"]`); - //data steward - addOwner(username, "Data Steward", `[href="/user/urn:li:corpuser:example${test_id}@example.com"]`); - //none - addOwner(username, "None", `[href="/user/urn:li:corpuser:example${test_id}@example.com"]`); - //technical owner - addOwner(username, "Technical Owner", `[href="/user/urn:li:corpuser:example${test_id}@example.com"]`); - }); + it("open test dataset page, add and remove user ownership(test every type)", () => { + cy.loginWithCredentials(); + cy.goToDataset( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "SampleCypressHiveDataset", + ); + // business owner + addOwner( + username, + "Business Owner", + `[href="/user/urn:li:corpuser:example${test_id}@example.com"]`, + ); + // data steward + addOwner( + username, + "Data Steward", + `[href="/user/urn:li:corpuser:example${test_id}@example.com"]`, + ); + // none + addOwner( + username, + "None", + `[href="/user/urn:li:corpuser:example${test_id}@example.com"]`, + ); + // technical owner + addOwner( + username, + "Technical Owner", + `[href="/user/urn:li:corpuser:example${test_id}@example.com"]`, + ); + }); - it("open test dataset page, add and remove group ownership(test every type)", () => { - cy.loginWithCredentials(); - cy.goToDataset("urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", "SampleCypressHiveDataset"); - //business owner - addOwner(group_name, "Business Owner", `[href="/group/urn:li:corpGroup:${test_id}"]`); - //data steward - addOwner(group_name, "Data Steward", `[href="/group/urn:li:corpGroup:${test_id}"]`); - //none - addOwner(group_name, "None", `[href="/group/urn:li:corpGroup:${test_id}"]`); - //technical owner - addOwner(group_name, "Technical Owner", `[href="/group/urn:li:corpGroup:${test_id}"]`); - }); -}); \ No newline at end of file + it("open test dataset page, add and remove group ownership(test every type)", () => { + cy.loginWithCredentials(); + cy.goToDataset( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "SampleCypressHiveDataset", + ); + // business owner + addOwner( + group_name, + "Business Owner", + `[href="/group/urn:li:corpGroup:${test_id}"]`, + ); + // data steward + addOwner( + group_name, + "Data Steward", + `[href="/group/urn:li:corpGroup:${test_id}"]`, + ); + // none + addOwner(group_name, "None", `[href="/group/urn:li:corpGroup:${test_id}"]`); + // technical owner + addOwner( + group_name, + "Technical Owner", + `[href="/group/urn:li:corpGroup:${test_id}"]`, + ); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/deprecations.js b/smoke-test/tests/cypress/cypress/e2e/mutations/deprecations.js index 2fa11654a3c3e..5c1d329b9ab0b 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/deprecations.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/deprecations.js @@ -1,29 +1,30 @@ describe("dataset deprecation", () => { - it("go to dataset and check deprecation works", () => { - const urn = "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; - const datasetName = "cypress_logging_events"; - cy.login(); - cy.goToDataset(urn, datasetName); - cy.openThreeDotDropdown(); - cy.clickOptionWithText("Mark as deprecated"); - cy.addViaFormModal("test deprecation", "Add Deprecation Details"); - cy.waitTextVisible("Deprecation Updated"); - cy.waitTextVisible("DEPRECATED") - cy.openThreeDotDropdown(); - cy.clickOptionWithText("Mark as un-deprecated"); - cy.waitTextVisible("Deprecation Updated"); - cy.ensureTextNotPresent("DEPRECATED"); - cy.openThreeDotDropdown(); - cy.clickOptionWithText("Mark as deprecated"); - cy.addViaFormModal("test deprecation", "Add Deprecation Details"); - cy.waitTextVisible("Deprecation Updated"); - cy.waitTextVisible("DEPRECATED"); - cy.contains("DEPRECATED").trigger("mouseover", { force: true }); - cy.waitTextVisible("Deprecation note"); - cy.get("[role='tooltip']").contains("Mark as un-deprecated").click(); - cy.waitTextVisible("Confirm Mark as un-deprecated"); - cy.get("button").contains("Yes").click(); - cy.waitTextVisible("Marked assets as un-deprecated!"); - cy.ensureTextNotPresent("DEPRECATED"); - }); + it("go to dataset and check deprecation works", () => { + const urn = + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; + const datasetName = "cypress_logging_events"; + cy.login(); + cy.goToDataset(urn, datasetName); + cy.openThreeDotDropdown(); + cy.clickOptionWithText("Mark as deprecated"); + cy.addViaFormModal("test deprecation", "Add Deprecation Details"); + cy.waitTextVisible("Deprecation Updated"); + cy.waitTextVisible("DEPRECATED"); + cy.openThreeDotDropdown(); + cy.clickOptionWithText("Mark as un-deprecated"); + cy.waitTextVisible("Deprecation Updated"); + cy.ensureTextNotPresent("DEPRECATED"); + cy.openThreeDotDropdown(); + cy.clickOptionWithText("Mark as deprecated"); + cy.addViaFormModal("test deprecation", "Add Deprecation Details"); + cy.waitTextVisible("Deprecation Updated"); + cy.waitTextVisible("DEPRECATED"); + cy.contains("DEPRECATED").trigger("mouseover", { force: true }); + cy.waitTextVisible("Deprecation note"); + cy.get("[role='tooltip']").contains("Mark as un-deprecated").click(); + cy.waitTextVisible("Confirm Mark as un-deprecated"); + cy.get("button").contains("Yes").click(); + cy.waitTextVisible("Marked assets as un-deprecated!"); + cy.ensureTextNotPresent("DEPRECATED"); + }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/domains.js b/smoke-test/tests/cypress/cypress/e2e/mutations/domains.js index 3de0e9b4b893e..694c4009a77f4 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/domains.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/domains.js @@ -1,76 +1,78 @@ import { aliasQuery, hasOperationName } from "../utils"; const test_domain_id = Math.floor(Math.random() * 100000); -const test_domain = `CypressDomainTest ${test_domain_id}` -const test_domain_urn = `urn:li:domain:${test_domain_id}` - +const test_domain = `CypressDomainTest ${test_domain_id}`; +const test_domain_urn = `urn:li:domain:${test_domain_id}`; describe("add remove domain", () => { - beforeEach(() => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - aliasQuery(req, "appConfig"); - }); - }); - - const setDomainsFeatureFlag = (isOn) => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - if (hasOperationName(req, "appConfig")) { - req.reply((res) => { - res.body.data.appConfig.featureFlags.nestedDomainsEnabled = isOn; - }); - } + beforeEach(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); + }); + }); + + const setDomainsFeatureFlag = (isOn) => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + res.body.data.appConfig.featureFlags.nestedDomainsEnabled = isOn; }); - }; + } + }); + }; - it("create domain", () => { - cy.loginWithCredentials(); - cy.goToDomainList(); - cy.clickOptionWithText("New Domain"); - cy.waitTextVisible("Create New Domain"); - cy.get('[data-testid="create-domain-name"]').click().type(test_domain) - cy.clickOptionWithText('Advanced') - cy.get('[data-testid="create-domain-id"]').click().type(test_domain_id) - cy.get('[data-testid="create-domain-button"]').click() - cy.waitTextVisible(test_domain) - }) + it("create domain", () => { + cy.loginWithCredentials(); + cy.goToDomainList(); + cy.clickOptionWithText("New Domain"); + cy.waitTextVisible("Create New Domain"); + cy.get('[data-testid="create-domain-name"]').click().type(test_domain); + cy.clickOptionWithText("Advanced"); + cy.get('[data-testid="create-domain-id"]').click().type(test_domain_id); + cy.get('[data-testid="create-domain-button"]').click(); + cy.waitTextVisible(test_domain); + }); - it("add entities to domain", () => { - setDomainsFeatureFlag(false); - cy.loginWithCredentials(); - cy.goToDomainList(); - cy.clickOptionWithText(test_domain); - cy.waitTextVisible("Add assets") - cy.clickOptionWithText("Add assets") - cy.get(".ant-modal-content").within(() => { - cy.get('[data-testid="search-input"]').click().invoke("val", "cypress_project.jaffle_shop.").type("customer") - cy.contains("BigQuery", {timeout: 30000 }) - cy.get(".ant-checkbox-input").first().click() - cy.get("#continueButton").click() - }) - cy.waitTextVisible("Added assets to Domain!") - }) + it("add entities to domain", () => { + setDomainsFeatureFlag(false); + cy.loginWithCredentials(); + cy.goToDomainList(); + cy.clickOptionWithText(test_domain); + cy.waitTextVisible("Add assets"); + cy.clickOptionWithText("Add assets"); + cy.get(".ant-modal-content").within(() => { + cy.get('[data-testid="search-input"]') + .click() + .invoke("val", "cypress_project.jaffle_shop.") + .type("customer"); + cy.contains("BigQuery", { timeout: 30000 }); + cy.get(".ant-checkbox-input").first().click(); + cy.get("#continueButton").click(); + }); + cy.waitTextVisible("Added assets to Domain!"); + }); - it("remove entity from domain", () => { - setDomainsFeatureFlag(false); - cy.loginWithCredentials(); - cy.goToDomainList(); - cy.removeDomainFromDataset( - "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", - "customers", - test_domain_urn - ) - }) + it("remove entity from domain", () => { + setDomainsFeatureFlag(false); + cy.loginWithCredentials(); + cy.goToDomainList(); + cy.removeDomainFromDataset( + "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + "customers", + test_domain_urn, + ); + }); - it("delete a domain and ensure dangling reference is deleted on entities", () => { - setDomainsFeatureFlag(false); - cy.loginWithCredentials(); - cy.goToDomainList(); - cy.get('[data-testid="dropdown-menu-' + test_domain_urn + '"]').click(); - cy.clickOptionWithText("Delete"); - cy.clickOptionWithText("Yes"); - cy.ensureTextNotPresent(test_domain) - cy.goToContainer("urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb") - cy.waitTextVisible("customers") - cy.ensureTextNotPresent(test_domain) - }) -}); \ No newline at end of file + it("delete a domain and ensure dangling reference is deleted on entities", () => { + setDomainsFeatureFlag(false); + cy.loginWithCredentials(); + cy.goToDomainList(); + cy.get(`[data-testid="dropdown-menu-${test_domain_urn}"]`).click(); + cy.clickOptionWithText("Delete"); + cy.clickOptionWithText("Yes"); + cy.ensureTextNotPresent(test_domain); + cy.goToContainer("urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb"); + cy.waitTextVisible("customers"); + cy.ensureTextNotPresent(test_domain); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js b/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js index c6d2b205250e0..4d14683ac4b13 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js @@ -5,10 +5,10 @@ const correct_url = "https://www.linkedin.com"; describe("edit documentation and link to dataset", () => { it("open test dataset page, edit documentation", () => { - //edit documentation and verify changes saved + // edit documentation and verify changes saved cy.loginWithCredentials(); cy.visit( - "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema" + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema", ); cy.openEntityTab("Documentation"); cy.waitTextVisible("my hive dataset"); @@ -19,7 +19,7 @@ describe("edit documentation and link to dataset", () => { cy.clickOptionWithTestId("description-editor-save-button"); cy.waitTextVisible("Description Updated"); cy.waitTextVisible(documentation_edited); - //return documentation to original state + // return documentation to original state cy.clickOptionWithTestId("edit-documentation-button"); cy.focused().clear().wait(1000); cy.focused().type("my hive dataset"); @@ -31,7 +31,7 @@ describe("edit documentation and link to dataset", () => { it("open test dataset page, remove and add dataset link", () => { cy.loginWithCredentials(); cy.visit( - "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema" + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema", ); cy.openEntityTab("Documentation"); cy.contains("Sample doc").trigger("mouseover", { force: true }); @@ -76,7 +76,7 @@ describe("edit documentation and link to dataset", () => { it("edit field documentation", () => { cy.loginWithCredentials(); cy.visit( - "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema" + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema", ); cy.clickOptionWithText("field_foo"); cy.clickOptionWithTestId("edit-field-description"); @@ -96,4 +96,4 @@ describe("edit documentation and link to dataset", () => { cy.waitTextVisible("Foo field description has changed"); cy.waitTextVisible("(edited)"); }); -}); \ No newline at end of file +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js b/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js index 6c5dd77810644..8f50262b41d2c 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js @@ -1,4 +1,3 @@ - const number = Math.floor(Math.random() * 100000); const accound_id = `account${number}`; const warehouse_id = `warehouse${number}`; @@ -8,61 +7,67 @@ const role = `role${number}`; const ingestion_source_name = `ingestion source ${number}`; describe("ingestion source creation flow", () => { - it("create a ingestion source using ui, verify ingestion source details saved correctly, remove ingestion source", () => { - // Go to ingestion page, create a snowflake source - cy.loginWithCredentials(); - cy.goToIngestionPage(); - cy.clickOptionWithTestId("create-ingestion-source-button"); - cy.clickOptionWithText("Snowflake"); - cy.waitTextVisible("Snowflake Recipe"); - cy.get("#account_id").type(accound_id); - cy.get("#warehouse").type(warehouse_id); - cy.get("#username").type(username); - cy.get("#password").type(password); - cy.focused().blur(); - cy.get("#role").type(role); + it("create a ingestion source using ui, verify ingestion source details saved correctly, remove ingestion source", () => { + // Go to ingestion page, create a snowflake source + cy.loginWithCredentials(); + cy.goToIngestionPage(); + cy.clickOptionWithTestId("create-ingestion-source-button"); + cy.clickOptionWithText("Snowflake"); + cy.waitTextVisible("Snowflake Recipe"); + cy.get("#account_id").type(accound_id); + cy.get("#warehouse").type(warehouse_id); + cy.get("#username").type(username); + cy.get("#password").type(password); + cy.focused().blur(); + cy.get("#role").type(role); - // Verify yaml recipe is generated correctly - cy.clickOptionWithTestId("recipe-builder-yaml-button"); - cy.waitTextVisible("account_id"); - cy.waitTextVisible(accound_id); - cy.waitTextVisible(warehouse_id); - cy.waitTextVisible(username); - cy.waitTextVisible(password); - cy.waitTextVisible(role); + // Verify yaml recipe is generated correctly + cy.clickOptionWithTestId("recipe-builder-yaml-button"); + cy.waitTextVisible("account_id"); + cy.waitTextVisible(accound_id); + cy.waitTextVisible(warehouse_id); + cy.waitTextVisible(username); + cy.waitTextVisible(password); + cy.waitTextVisible(role); - // Finish creating source - cy.clickOptionWithTestId("recipe-builder-next-button"); - cy.waitTextVisible("Configure an Ingestion Schedule"); - cy.clickOptionWithTestId("ingestion-schedule-next-button"); - cy.waitTextVisible("Give this ingestion source a name."); - cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); - cy.clickOptionWithTestId("ingestion-source-save-button"); - cy.waitTextVisible("Successfully created ingestion source!").wait(5000) - cy.waitTextVisible(ingestion_source_name); - cy.get('[data-testid="ingestion-source-table-status"]').contains("Pending...").should("be.visible"); + // Finish creating source + cy.clickOptionWithTestId("recipe-builder-next-button"); + cy.waitTextVisible("Configure an Ingestion Schedule"); + cy.clickOptionWithTestId("ingestion-schedule-next-button"); + cy.waitTextVisible("Give this ingestion source a name."); + cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); + cy.clickOptionWithTestId("ingestion-source-save-button"); + cy.waitTextVisible("Successfully created ingestion source!").wait(5000); + cy.waitTextVisible(ingestion_source_name); + cy.get('[data-testid="ingestion-source-table-status"]') + .contains("Pending...") + .should("be.visible"); - // Verify ingestion source details are saved correctly - cy.get('[data-testid="ingestion-source-table-edit-button"]').first().click(); - cy.waitTextVisible("Edit Ingestion Source"); - cy.get("#account_id").should("have.value", accound_id); - cy.get("#warehouse").should("have.value", warehouse_id); - cy.get("#username").should("have.value", username); - cy.get("#password").should("have.value", password); - cy.get("#role").should("have.value", role); - cy.get("button").contains("Next").click(); - cy.waitTextVisible("Configure an Ingestion Schedule"); - cy.clickOptionWithTestId("ingestion-schedule-next-button"); - cy.get('[data-testid="source-name-input"]').clear().type(ingestion_source_name + " EDITED"); - cy.clickOptionWithTestId("ingestion-source-save-button"); - cy.waitTextVisible("Successfully updated ingestion source!"); - cy.waitTextVisible(ingestion_source_name + " EDITED"); + // Verify ingestion source details are saved correctly + cy.get('[data-testid="ingestion-source-table-edit-button"]') + .first() + .click(); + cy.waitTextVisible("Edit Ingestion Source"); + cy.get("#account_id").should("have.value", accound_id); + cy.get("#warehouse").should("have.value", warehouse_id); + cy.get("#username").should("have.value", username); + cy.get("#password").should("have.value", password); + cy.get("#role").should("have.value", role); + cy.get("button").contains("Next").click(); + cy.waitTextVisible("Configure an Ingestion Schedule"); + cy.clickOptionWithTestId("ingestion-schedule-next-button"); + cy.get('[data-testid="source-name-input"]') + .clear() + .type(`${ingestion_source_name} EDITED`); + cy.clickOptionWithTestId("ingestion-source-save-button"); + cy.waitTextVisible("Successfully updated ingestion source!"); + cy.waitTextVisible(`${ingestion_source_name} EDITED`); - // Remove ingestion source - cy.get('[data-testid="delete-button"]').first().click(); - cy.waitTextVisible("Confirm Ingestion Source Removal"); - cy.get("button").contains("Yes").click(); - cy.waitTextVisible("Removed ingestion source."); - cy.ensureTextNotPresent(ingestion_source_name + " EDITED") - }) -}); \ No newline at end of file + // Remove ingestion source + cy.get('[data-testid="delete-button"]').first().click(); + cy.waitTextVisible("Confirm Ingestion Source Removal"); + cy.get("button").contains("Yes").click(); + cy.waitTextVisible("Removed ingestion source."); + cy.ensureTextNotPresent(`${ingestion_source_name} EDITED`); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js index c355aaabc336a..d23b0ca7523b8 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js @@ -1,39 +1,40 @@ function readyToTypeEditor() { - return cy.get('.monaco-editor textarea:first') - .click().focused(); + return cy.get(".monaco-editor textarea:first").click().focused(); } describe("run managed ingestion", () => { - it("create run managed ingestion source", () => { - let number = Math.floor(Math.random() * 100000); - let testName = `cypress test source ${number}` - let cli_version = "0.12.0"; - cy.login(); - cy.goToIngestionPage(); - cy.clickOptionWithText("Create new source"); - cy.clickOptionWithTextToScrollintoView("Other"); + it("create run managed ingestion source", () => { + const number = Math.floor(Math.random() * 100000); + const testName = `cypress test source ${number}`; + const cli_version = "0.12.0"; + cy.login(); + cy.goToIngestionPage(); + cy.clickOptionWithText("Create new source"); + cy.clickOptionWithTextToScrollintoView("Other"); - cy.waitTextVisible("source-type"); - readyToTypeEditor().type('{ctrl}a').clear() - readyToTypeEditor().type("source:{enter}"); - readyToTypeEditor().type(" type: demo-data"); - readyToTypeEditor().type("{enter}"); - // no space because the editor starts new line at same indentation - readyToTypeEditor().type("config: {}"); - cy.clickOptionWithText("Next") - cy.clickOptionWithText("Next") + cy.waitTextVisible("source-type"); + readyToTypeEditor().type("{ctrl}a").clear(); + readyToTypeEditor().type("source:{enter}"); + readyToTypeEditor().type(" type: demo-data"); + readyToTypeEditor().type("{enter}"); + // no space because the editor starts new line at same indentation + readyToTypeEditor().type("config: {}"); + cy.clickOptionWithText("Next"); + cy.clickOptionWithText("Next"); - cy.enterTextInTestId('source-name-input', testName) - cy.clickOptionWithText("Advanced") - cy.enterTextInTestId('cli-version-input', cli_version) - cy.clickOptionWithTextToScrollintoView("Save & Run") - cy.waitTextVisible(testName) + cy.enterTextInTestId("source-name-input", testName); + cy.clickOptionWithText("Advanced"); + cy.enterTextInTestId("cli-version-input", cli_version); + cy.clickOptionWithTextToScrollintoView("Save & Run"); + cy.waitTextVisible(testName); - cy.contains(testName).parent().within(() => { - cy.contains("Succeeded", {timeout: 180000}) - cy.clickOptionWithTestId("delete-button"); - }) - cy.clickOptionWithText("Yes") - cy.ensureTextNotPresent(testName) - }) + cy.contains(testName) + .parent() + .within(() => { + cy.contains("Succeeded", { timeout: 180000 }); + cy.clickOptionWithTestId("delete-button"); + }); + cy.clickOptionWithText("Yes"); + cy.ensureTextNotPresent(testName); + }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js index 77fd63b9cae02..57eccc3211096 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js @@ -7,104 +7,110 @@ const role = `role${number}`; const ingestion_source_name = `ingestion source ${number}`; describe("managing secrets for ingestion creation", () => { - it("create a secret, create ingestion source using a secret, remove a secret", () => { - // Navigate to the manage ingestion page → secrets - cy.loginWithCredentials(); - cy.goToIngestionPage(); - cy.openEntityTab("Secrets"); + it("create a secret, create ingestion source using a secret, remove a secret", () => { + // Navigate to the manage ingestion page → secrets + cy.loginWithCredentials(); + cy.goToIngestionPage(); + cy.openEntityTab("Secrets"); - // Create a new secret - cy.clickOptionWithTestId("create-secret-button"); - cy.enterTextInTestId('secret-modal-name-input', `secretname${number}`); - cy.enterTextInTestId('secret-modal-value-input', `secretvalue${number}`); - cy.enterTextInTestId('secret-modal-description-input', `secretdescription${number}`); - cy.clickOptionWithTestId("secret-modal-create-button"); - cy.waitTextVisible("Successfully created Secret!"); - cy.waitTextVisible(`secretname${number}`); - cy.waitTextVisible(`secretdescription${number}`).wait(5000) + // Create a new secret + cy.clickOptionWithTestId("create-secret-button"); + cy.enterTextInTestId("secret-modal-name-input", `secretname${number}`); + cy.enterTextInTestId("secret-modal-value-input", `secretvalue${number}`); + cy.enterTextInTestId( + "secret-modal-description-input", + `secretdescription${number}`, + ); + cy.clickOptionWithTestId("secret-modal-create-button"); + cy.waitTextVisible("Successfully created Secret!"); + cy.waitTextVisible(`secretname${number}`); + cy.waitTextVisible(`secretdescription${number}`).wait(5000); - // Create an ingestion source using a secret - cy.goToIngestionPage(); - cy.get("#ingestion-create-source").click(); - cy.clickOptionWithText("Snowflake"); - cy.waitTextVisible("Snowflake Recipe"); - cy.get("#account_id").type(accound_id); - cy.get("#warehouse").type(warehouse_id); - cy.get("#username").type(username); - cy.get("#password").click().wait(1000); - cy.contains(`secretname${number}`).click({force: true}); - cy.focused().blur(); - cy.get("#role").type(role); - cy.get("button").contains("Next").click(); - cy.waitTextVisible("Configure an Ingestion Schedule"); - cy.get("button").contains("Next").click(); - cy.waitTextVisible("Give this ingestion source a name."); - cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); - cy.get("button").contains("Save").click(); - cy.waitTextVisible("Successfully created ingestion source!").wait(5000) - cy.waitTextVisible(ingestion_source_name); - cy.get("button").contains("Pending...").should("be.visible"); + // Create an ingestion source using a secret + cy.goToIngestionPage(); + cy.get("#ingestion-create-source").click(); + cy.clickOptionWithText("Snowflake"); + cy.waitTextVisible("Snowflake Recipe"); + cy.get("#account_id").type(accound_id); + cy.get("#warehouse").type(warehouse_id); + cy.get("#username").type(username); + cy.get("#password").click().wait(1000); + cy.contains(`secretname${number}`).click({ force: true }); + cy.focused().blur(); + cy.get("#role").type(role); + cy.get("button").contains("Next").click(); + cy.waitTextVisible("Configure an Ingestion Schedule"); + cy.get("button").contains("Next").click(); + cy.waitTextVisible("Give this ingestion source a name."); + cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); + cy.get("button").contains("Save").click(); + cy.waitTextVisible("Successfully created ingestion source!").wait(5000); + cy.waitTextVisible(ingestion_source_name); + cy.get("button").contains("Pending...").should("be.visible"); - // Remove a secret - cy.openEntityTab("Secrets"); - cy.waitTextVisible(`secretname${number}`); - cy.get('[data-icon="delete"]').first().click(); - cy.waitTextVisible("Confirm Secret Removal"); - cy.get("button").contains("Yes").click(); - cy.waitTextVisible("Removed secret."); - cy.ensureTextNotPresent(`secretname${number}`); - cy.ensureTextNotPresent(`secretdescription${number}`); + // Remove a secret + cy.openEntityTab("Secrets"); + cy.waitTextVisible(`secretname${number}`); + cy.get('[data-icon="delete"]').first().click(); + cy.waitTextVisible("Confirm Secret Removal"); + cy.get("button").contains("Yes").click(); + cy.waitTextVisible("Removed secret."); + cy.ensureTextNotPresent(`secretname${number}`); + cy.ensureTextNotPresent(`secretdescription${number}`); - // Remove ingestion source - cy.goToIngestionPage(); - cy.get('[data-testid="delete-button"]').first().click(); - cy.waitTextVisible("Confirm Ingestion Source Removal"); - cy.get("button").contains("Yes").click(); - cy.waitTextVisible("Removed ingestion source."); - cy.ensureTextNotPresent(ingestion_source_name) + // Remove ingestion source + cy.goToIngestionPage(); + cy.get('[data-testid="delete-button"]').first().click(); + cy.waitTextVisible("Confirm Ingestion Source Removal"); + cy.get("button").contains("Yes").click(); + cy.waitTextVisible("Removed ingestion source."); + cy.ensureTextNotPresent(ingestion_source_name); - // Verify secret is not present during ingestion source creation for password dropdown - cy.clickOptionWithText("Create new source"); - cy.clickOptionWithText("Snowflake"); - cy.waitTextVisible("Snowflake Recipe"); - cy.get("#account_id").type(accound_id); - cy.get("#warehouse").type(warehouse_id); - cy.get("#username").type(username); - cy.get("#password").click().wait(1000); - cy.ensureTextNotPresent(`secretname${number}`); + // Verify secret is not present during ingestion source creation for password dropdown + cy.clickOptionWithText("Create new source"); + cy.clickOptionWithText("Snowflake"); + cy.waitTextVisible("Snowflake Recipe"); + cy.get("#account_id").type(accound_id); + cy.get("#warehouse").type(warehouse_id); + cy.get("#username").type(username); + cy.get("#password").click().wait(1000); + cy.ensureTextNotPresent(`secretname${number}`); - // Verify secret can be added during ingestion source creation and used successfully - cy.clickOptionWithText("Create Secret"); - cy.enterTextInTestId('secret-modal-name-input', `secretname${number}`) - cy.enterTextInTestId('secret-modal-value-input', `secretvalue${number}`) - cy.enterTextInTestId('secret-modal-description-input', `secretdescription${number}`) - cy.clickOptionWithTestId("secret-modal-create-button"); - cy.waitTextVisible("Created secret!"); - cy.get("#role").type(role); - cy.get("button").contains("Next").click(); - cy.waitTextVisible("Configure an Ingestion Schedule"); - cy.get("button").contains("Next").click(); - cy.waitTextVisible("Give this ingestion source a name."); - cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); - cy.get("button").contains("Save").click(); - cy.waitTextVisible("Successfully created ingestion source!").wait(5000)//prevent issue with missing form data - cy.waitTextVisible(ingestion_source_name); - cy.get("button").contains("Pending...").should("be.visible"); + // Verify secret can be added during ingestion source creation and used successfully + cy.clickOptionWithText("Create Secret"); + cy.enterTextInTestId("secret-modal-name-input", `secretname${number}`); + cy.enterTextInTestId("secret-modal-value-input", `secretvalue${number}`); + cy.enterTextInTestId( + "secret-modal-description-input", + `secretdescription${number}`, + ); + cy.clickOptionWithTestId("secret-modal-create-button"); + cy.waitTextVisible("Created secret!"); + cy.get("#role").type(role); + cy.get("button").contains("Next").click(); + cy.waitTextVisible("Configure an Ingestion Schedule"); + cy.get("button").contains("Next").click(); + cy.waitTextVisible("Give this ingestion source a name."); + cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); + cy.get("button").contains("Save").click(); + cy.waitTextVisible("Successfully created ingestion source!").wait(5000); // prevent issue with missing form data + cy.waitTextVisible(ingestion_source_name); + cy.get("button").contains("Pending...").should("be.visible"); - //Remove ingestion source and secret - cy.goToIngestionPage(); - cy.get('[data-testid="delete-button"]').first().click(); - cy.waitTextVisible("Confirm Ingestion Source Removal"); - cy.get("button").contains("Yes").click(); - cy.waitTextVisible("Removed ingestion source."); - cy.ensureTextNotPresent(ingestion_source_name) - cy.clickOptionWithText("Secrets"); - cy.waitTextVisible(`secretname${number}`); - cy.get('[data-icon="delete"]').first().click(); - cy.waitTextVisible("Confirm Secret Removal"); - cy.get("button").contains("Yes").click(); - cy.waitTextVisible("Removed secret."); - cy.ensureTextNotPresent(`secretname${number}`); - cy.ensureTextNotPresent(`secretdescription${number}`); - }) -}); \ No newline at end of file + // Remove ingestion source and secret + cy.goToIngestionPage(); + cy.get('[data-testid="delete-button"]').first().click(); + cy.waitTextVisible("Confirm Ingestion Source Removal"); + cy.get("button").contains("Yes").click(); + cy.waitTextVisible("Removed ingestion source."); + cy.ensureTextNotPresent(ingestion_source_name); + cy.clickOptionWithText("Secrets"); + cy.waitTextVisible(`secretname${number}`); + cy.get('[data-icon="delete"]').first().click(); + cy.waitTextVisible("Confirm Secret Removal"); + cy.get("button").contains("Yes").click(); + cy.waitTextVisible("Removed secret."); + cy.ensureTextNotPresent(`secretname${number}`); + cy.ensureTextNotPresent(`secretdescription${number}`); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/mutations.js b/smoke-test/tests/cypress/cypress/e2e/mutations/mutations.js index e2a74a15d3dfc..cf19a34b71761 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/mutations.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/mutations.js @@ -4,35 +4,36 @@ describe("mutations", () => { let businessAttributeEntityEnabled; beforeEach(() => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - aliasQuery(req, "appConfig"); - }); - }); - - const setBusinessAttributeFeatureFlag = () => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - if (hasOperationName(req, "appConfig")) { - req.reply((res) => { - businessAttributeEntityEnabled = res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; - return res; - }); - } - }).as('apiCall'); - }; + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); + }); + }); + + const setBusinessAttributeFeatureFlag = () => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + businessAttributeEntityEnabled = + res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; + return res; + }); + } + }).as("apiCall"); + }; before(() => { // warm up elastic by issuing a `*` search cy.login(); - //Commented below function, and used individual commands below with wait + // Commented below function, and used individual commands below with wait // cy.goToStarSearchList(); cy.visit("/search?query=%2A"); - cy.wait(3000) - cy.waitTextVisible("Showing") - cy.waitTextVisible("results") + cy.wait(3000); + cy.waitTextVisible("Showing"); + cy.waitTextVisible("results"); cy.wait(2000); - cy.get('body').then(($body) => { + cy.get("body").then(($body) => { if ($body.find('button[aria-label="Close"]').length > 0) { - cy.get('button[aria-label="Close"]').click(); + cy.get('button[aria-label="Close"]').click(); } }); cy.wait(2000); @@ -41,10 +42,13 @@ describe("mutations", () => { it("can create and add a tag to dataset and visit new tag page", () => { // cy.deleteUrn("urn:li:tag:CypressTestAddTag"); cy.login(); - cy.goToDataset("urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", "cypress_logging_events"); - cy.get('body').then(($body) => { + cy.goToDataset( + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "cypress_logging_events", + ); + cy.get("body").then(($body) => { if ($body.find('button[aria-label="Close"]').length > 0) { - cy.get('button[aria-label="Close"]').click(); + cy.get('button[aria-label="Close"]').click(); } }); cy.contains("Add Tag").click({ force: true }); @@ -80,13 +84,13 @@ describe("mutations", () => { // used by panel - click to search cy.wait(3000); - cy.contains("1 Datasets").click({ force: true }); + cy.contains("1 Datasets").click({ force: true }); // verify dataset shows up in search now cy.contains("of 1 result").click({ force: true }); cy.contains("cypress_logging_events").click({ force: true }); cy.get('[data-testid="tag-CypressTestAddTag"]').within(() => - cy.get("span[aria-label=close]").click() + cy.get("span[aria-label=close]").click(), ); cy.contains("Yes").click(); @@ -99,11 +103,11 @@ describe("mutations", () => { cy.addTermToDataset( "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", "cypress_logging_events", - "CypressTerm" - ) + "CypressTerm", + ); cy.get( - 'a[href="/glossaryTerm/urn:li:glossaryTerm:CypressNode.CypressTerm"]' + 'a[href="/glossaryTerm/urn:li:glossaryTerm:CypressNode.CypressTerm"]', ).within(() => cy.get("span[aria-label=close]").click()); cy.contains("Yes").click(); @@ -113,10 +117,13 @@ describe("mutations", () => { it("can add and remove tags from a dataset field", () => { cy.login(); cy.viewport(2000, 800); - cy.goToDataset("urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", "cypress_logging_events"); + cy.goToDataset( + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "cypress_logging_events", + ); cy.clickOptionWithText("event_name"); cy.get('[data-testid="schema-field-event_name-tags"]').within(() => - cy.contains("Add Tag").click() + cy.contains("Add Tag").click(), ); cy.enterTextInTestId("tag-term-modal-input", "CypressTestAddTag2"); @@ -159,7 +166,7 @@ describe("mutations", () => { cy .get("span[aria-label=close]") .trigger("mouseover", { force: true }) - .click({ force: true }) + .click({ force: true }), ); cy.contains("Yes").click({ force: true }); @@ -172,10 +179,13 @@ describe("mutations", () => { cy.login(); // make space for the glossary term column cy.viewport(2000, 800); - cy.goToDataset("urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", "cypress_logging_events"); + cy.goToDataset( + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + "cypress_logging_events", + ); cy.clickOptionWithText("event_name"); cy.get('[data-testid="schema-field-event_name-terms"]').within(() => - cy.contains("Add Term").click({ force: true }) + cy.contains("Add Term").click({ force: true }), ); cy.selectOptionInTagTermModal("CypressTerm"); @@ -186,7 +196,7 @@ describe("mutations", () => { cy .get("span[aria-label=close]") .trigger("mouseover", { force: true }) - .click({ force: true }) + .click({ force: true }), ); cy.contains("Yes").click({ force: true }); @@ -195,39 +205,42 @@ describe("mutations", () => { it("can add and remove business attribute from a dataset field", () => { setBusinessAttributeFeatureFlag(); - cy.login(); - // make space for the glossary term column - cy.viewport(2000, 800); - cy.visit("/dataset/" + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"); - cy.wait('@apiCall').then(() => { - if (!businessAttributeEntityEnabled) { - return; - } - cy.wait(5000); - cy.waitTextVisible("cypress_logging_events"); - cy.clickOptionWithText("event_data"); - cy.get('[data-testid="schema-field-event_data-businessAttribute"]').trigger( - "mouseover", - { force: true } - ); - cy.get('[data-testid="schema-field-event_data-businessAttribute"]').within(() => - cy.contains("Add Attribute").click({ force: true }) - ); - - cy.selectOptionInAttributeModal("cypressTestAttribute"); - cy.wait(2000); - cy.contains("cypressTestAttribute"); - - cy.get('[data-testid="schema-field-event_data-businessAttribute"]'). - within(() => - cy - .get("span[aria-label=close]") - .trigger("mouseover", { force: true }) - .click({ force: true }) - ); - cy.contains("Yes").click({ force: true }); - - cy.contains("cypressTestAttribute").should("not.exist"); - }); - }); + cy.login(); + // make space for the glossary term column + cy.viewport(2000, 800); + cy.visit( + "/dataset/" + + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", + ); + cy.wait("@apiCall").then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(5000); + cy.waitTextVisible("cypress_logging_events"); + cy.clickOptionWithText("event_data"); + cy.get( + '[data-testid="schema-field-event_data-businessAttribute"]', + ).trigger("mouseover", { force: true }); + cy.get( + '[data-testid="schema-field-event_data-businessAttribute"]', + ).within(() => cy.contains("Add Attribute").click({ force: true })); + + cy.selectOptionInAttributeModal("cypressTestAttribute"); + cy.wait(2000); + cy.contains("cypressTestAttribute"); + + cy.get( + '[data-testid="schema-field-event_data-businessAttribute"]', + ).within(() => + cy + .get("span[aria-label=close]") + .trigger("mouseover", { force: true }) + .click({ force: true }), + ); + cy.contains("Yes").click({ force: true }); + + cy.contains("cypressTestAttribute").should("not.exist"); + }); + }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/operations/operations.js b/smoke-test/tests/cypress/cypress/e2e/operations/operations.js index 8b95656bfe8ba..4d416380c164d 100644 --- a/smoke-test/tests/cypress/cypress/e2e/operations/operations.js +++ b/smoke-test/tests/cypress/cypress/e2e/operations/operations.js @@ -1,11 +1,12 @@ -describe('operations', () => { - it('can visit dataset with operation aspect and verify last updated is present', () => { - cy.login(); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)/Stats?is_lineage_mode=false'); - cy.contains('test-project.bigquery_usage_logs.cypress_logging_events'); - - // Last updated text is present - cy.contains('Last Updated') - }); - }) - \ No newline at end of file +describe("operations", () => { + it("can visit dataset with operation aspect and verify last updated is present", () => { + cy.login(); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:bigquery,test-project.bigquery_usage_logs.cypress_logging_events,PROD)/Stats?is_lineage_mode=false", + ); + cy.contains("test-project.bigquery_usage_logs.cypress_logging_events"); + + // Last updated text is present + cy.contains("Last Updated"); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/ownership/manage_ownership.js b/smoke-test/tests/cypress/cypress/e2e/ownership/manage_ownership.js index 6d34b7604b74a..1c10c4a8edcea 100644 --- a/smoke-test/tests/cypress/cypress/e2e/ownership/manage_ownership.js +++ b/smoke-test/tests/cypress/cypress/e2e/ownership/manage_ownership.js @@ -8,32 +8,32 @@ describe("manage ownership", () => { cy.clickOptionWithText("Create new Ownership Type"); cy.get('[data-testid="ownership-type-name-input"]').clear("T"); cy.get('[data-testid="ownership-type-name-input"]').type( - "Test Ownership Type" + "Test Ownership Type", ); cy.get('[data-testid="ownership-type-description-input"]').clear("T"); cy.get('[data-testid="ownership-type-description-input"]').type( - "This is a test ownership type description." + "This is a test ownership type description.", ); cy.get('[data-testid="ownership-builder-save"]').click(); cy.wait(3000); cy.waitTextVisible("Test Ownership Type"); cy.get( - '[data-row-key="Test Ownership Type"] > :nth-child(3) > .anticon > svg' + '[data-row-key="Test Ownership Type"] > :nth-child(3) > .anticon > svg', ).click(); cy.clickOptionWithText("Edit"); cy.get('[data-testid="ownership-type-description-input"]').clear( - "This is an test ownership type description." + "This is an test ownership type description.", ); cy.get('[data-testid="ownership-type-description-input"]').type( - "This is an edited test ownership type description." + "This is an edited test ownership type description.", ); cy.get('[data-testid="ownership-builder-save"] > span').click(); cy.wait(3000); cy.waitTextVisible("This is an edited test ownership type description."); cy.get( - '[data-row-key="Test Ownership Type"] > :nth-child(3) > .anticon > svg' + '[data-row-key="Test Ownership Type"] > :nth-child(3) > .anticon > svg', ).click(); cy.clickOptionWithText("Delete"); cy.get(".ant-popover-buttons > .ant-btn-primary").click(); diff --git a/smoke-test/tests/cypress/cypress/e2e/query/query_tab.js b/smoke-test/tests/cypress/cypress/e2e/query/query_tab.js index 015ce8c058eb8..f03aa6afda4a2 100644 --- a/smoke-test/tests/cypress/cypress/e2e/query/query_tab.js +++ b/smoke-test/tests/cypress/cypress/e2e/query/query_tab.js @@ -1,71 +1,82 @@ -const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)'; -const runId = Date.now() +const DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)"; +const runId = Date.now(); const addNewQuery = () => { cy.get('[data-testid="add-query-button"]').click(); - cy.get('[data-mode-id="sql"]').click() - .type(` + Test Query-${runId}`); - cy.get('[data-testid="query-builder-title-input"]').click() - .type(`Test Table-${runId}`); - cy.get('.ProseMirror').click() - .type(`Test Description-${runId}`); + cy.get('[data-mode-id="sql"]').click().type(` + Test Query-${runId}`); + cy.get('[data-testid="query-builder-title-input"]') + .click() + .type(`Test Table-${runId}`); + cy.get(".ProseMirror").click().type(`Test Description-${runId}`); cy.get('[data-testid="query-builder-save-button"]').click(); cy.waitTextVisible("Created Query!"); -} +}; const editQuery = () => { - cy.get('[data-testid="query-edit-button-0"]').click() - cy.get('[data-mode-id="sql"]').click() - .type(` + Edited Query-${runId}`); - cy.get('[data-testid="query-builder-title-input"]').click().clear() - .type(`Edited Table-${runId}`); - cy.get('.ProseMirror').click().clear() - .type(`Edited Description-${runId}`); + cy.get('[data-testid="query-edit-button-0"]').click(); + cy.get('[data-mode-id="sql"]').click().type(` + Edited Query-${runId}`); + cy.get('[data-testid="query-builder-title-input"]') + .click() + .clear() + .type(`Edited Table-${runId}`); + cy.get(".ProseMirror").click().clear().type(`Edited Description-${runId}`); cy.get('[data-testid="query-builder-save-button"]').click(); cy.waitTextVisible("Edited Query!"); - } +}; - const deleteQuery = () => { - cy.get('[data-testid="query-more-button-0"]').click(); - cy.clickOptionWithText("Delete"); - cy.clickOptionWithText('Yes') - cy.waitTextVisible("Deleted Query!"); - } +const deleteQuery = () => { + cy.get('[data-testid="query-more-button-0"]').click(); + cy.clickOptionWithText("Delete"); + cy.clickOptionWithText("Yes"); + cy.waitTextVisible("Deleted Query!"); +}; - const verifyViewCardDetails = (query,title,description) => { - cy.get('[data-testid="query-content-0"]').scrollIntoView().should('be.visible').click() - cy.get('.ant-modal-content').waitTextVisible(query); - cy.get('.ant-modal-content').waitTextVisible(title); - cy.get('.ant-modal-content').waitTextVisible(description); +const verifyViewCardDetails = (query, title, description) => { + cy.get('[data-testid="query-content-0"]') + .scrollIntoView() + .should("be.visible") + .click(); + cy.get(".ant-modal-content").waitTextVisible(query); + cy.get(".ant-modal-content").waitTextVisible(title); + cy.get(".ant-modal-content").waitTextVisible(description); }; describe("manage queries", () => { beforeEach(() => { cy.loginWithCredentials(); - cy.goToDataset(DATASET_URN,"SampleCypressHdfsDataset"); - cy.openEntityTab("Queries") - }) - + cy.goToDataset(DATASET_URN, "SampleCypressHdfsDataset"); + cy.openEntityTab("Queries"); + }); + it("go to queries tab on dataset page then create query and verify & view the card", () => { - cy.waitTextVisible("Highlighted Queries"); - cy.ensureTextNotPresent("Recent Queries"); - addNewQuery(); - cy.waitTextVisible(`+ Test Query-${runId}`); - cy.waitTextVisible(`Test Table-${runId}`); - cy.waitTextVisible(`Test Description-${runId}`); - cy.waitTextVisible("Created on"); - verifyViewCardDetails(`+ Test Query-${runId}`,`Test Table-${runId}`,`Test Description-${runId}`) + cy.waitTextVisible("Highlighted Queries"); + cy.ensureTextNotPresent("Recent Queries"); + addNewQuery(); + cy.waitTextVisible(`+ Test Query-${runId}`); + cy.waitTextVisible(`Test Table-${runId}`); + cy.waitTextVisible(`Test Description-${runId}`); + cy.waitTextVisible("Created on"); + verifyViewCardDetails( + `+ Test Query-${runId}`, + `Test Table-${runId}`, + `Test Description-${runId}`, + ); }); - it("go to queries tab on dataset page then edit the query and verify edited Query card", () => { - editQuery(); - verifyViewCardDetails(`+ Test Query-${runId} + Edited Query-${runId}`,`Edited Table-${runId}`,`Edited Description-${runId}`) - }); + it("go to queries tab on dataset page then edit the query and verify edited Query card", () => { + editQuery(); + verifyViewCardDetails( + `+ Test Query-${runId} + Edited Query-${runId}`, + `Edited Table-${runId}`, + `Edited Description-${runId}`, + ); + }); - it("go to queries tab on dataset page then delete the query and verify that query should be gone", () => { - deleteQuery(); - cy.ensureTextNotPresent(`+ Test Query-${runId} + Edited Query-${runId}`); - cy.ensureTextNotPresent(`Edited Table-${runId}`); - cy.ensureTextNotPresent(`Edited Description-${runId}`); - }); -}); \ No newline at end of file + it("go to queries tab on dataset page then delete the query and verify that query should be gone", () => { + deleteQuery(); + cy.ensureTextNotPresent(`+ Test Query-${runId} + Edited Query-${runId}`); + cy.ensureTextNotPresent(`Edited Table-${runId}`); + cy.ensureTextNotPresent(`Edited Description-${runId}`); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/schema_blame/schema_blame.js b/smoke-test/tests/cypress/cypress/e2e/schema_blame/schema_blame.js index 2218cbd95cf9d..4d5d4f324e0aa 100644 --- a/smoke-test/tests/cypress/cypress/e2e/schema_blame/schema_blame.js +++ b/smoke-test/tests/cypress/cypress/e2e/schema_blame/schema_blame.js @@ -1,55 +1,105 @@ -describe('schema blame', () => { - Cypress.on('uncaught:exception', (err, runnable) => { - return false; - }); - - it('can activate the blame view and verify for the latest version of a dataset', () => { - cy.login(); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema'); - cy.wait(10000); - - // Verify which fields are present, along with checking descriptions and tags - cy.contains('field_foo'); - cy.contains('field_baz'); - cy.contains('field_bar').should('not.exist'); - cy.clickOptionWithText("field_foo"); - cy.contains('Foo field description has changed'); - cy.contains('Baz field description'); - cy.get('[data-testid="schema-field-field_foo-tags"]').contains('Legacy'); - - // Make sure the schema blame is accurate - cy.get('[data-testid="schema-blame-button"]').click({ force: true }); - cy.wait(3000); - - cy.get('[data-testid="field_foo-schema-blame-description"]').contains("Modified in v1.0.0"); - cy.get('[data-testid="field_baz-schema-blame-description"]').contains("Added in v1.0.0"); - - // Verify the "view blame prior to" button changes state by modifying the URL - cy.get('[data-testid="field_foo-view-prior-blame-button"]').click({force: true}); - cy.wait(3000); - - cy.url().should('include', 'semantic_version=1.0.0'); +describe("schema blame", () => { + Cypress.on("uncaught:exception", (err, runnable) => false); + + it("can activate the blame view and verify for the latest version of a dataset", () => { + cy.login(); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema", + ); + cy.wait(10000); + + // Verify which fields are present, along with checking descriptions and tags + cy.contains("field_foo"); + cy.contains("field_baz"); + cy.contains("field_bar").should("not.exist"); + cy.clickOptionWithText("field_foo"); + cy.contains("Foo field description has changed"); + cy.contains("Baz field description"); + cy.get('[data-testid="schema-field-field_foo-tags"]').contains("Legacy"); + + // Make sure the schema blame is accurate + cy.get('[data-testid="schema-blame-button"]').click({ force: true }); + cy.wait(3000); + + cy.get('[data-testid="field_foo-schema-blame-description"]').contains( + "Modified in v1.0.0", + ); + cy.get('[data-testid="field_baz-schema-blame-description"]').contains( + "Added in v1.0.0", + ); + + // Verify the "view blame prior to" button changes state by modifying the URL + cy.get('[data-testid="field_foo-view-prior-blame-button"]').click({ + force: true, + }); + cy.wait(3000); + + cy.url().should("include", "semantic_version=1.0.0"); + }); + + it("can activate the blame view and verify for an older version of a dataset", () => { + cy.login(); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema", + ); + cy.wait(10000); + + // Verify which fields are present, along with checking descriptions and tags + cy.contains("field_foo"); + cy.contains("field_baz"); + cy.contains("field_bar").should("not.exist"); + cy.contains("Foo field description has changed"); + cy.contains("Baz field description"); + cy.clickOptionWithText("field_foo"); + cy.get('[data-testid="schema-field-field_foo-tags"]').contains("Legacy"); + + // Make sure the schema blame is accurate + cy.get('[data-testid="schema-blame-button"]').click({ force: true }); + cy.wait(3000); + + cy.get('[data-testid="field_foo-schema-blame-description"]').contains( + "Modified in v1.0.0", + ); + cy.get('[data-testid="field_baz-schema-blame-description"]').contains( + "Added in v1.0.0", + ); + + // Verify the "view blame prior to" button changes state by modifying the URL + cy.get('[data-testid="field_foo-view-prior-blame-button"]').click({ + force: true, + }); + cy.wait(3000); + + cy.url().should("include", "semantic_version=1.0.0"); }); - it('can activate the blame view and verify for an older version of a dataset', () => { + it("can activate the blame view and verify for an older version of a dataset", () => { cy.login(); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema?semantic_version=0.0.0'); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema?semantic_version=0.0.0", + ); cy.wait(10000); - // Verify which fields are present, along with checking descriptions and tags - cy.contains('field_foo'); - cy.contains('field_bar'); - cy.contains('field_baz').should('not.exist'); - cy.contains('Foo field description'); - cy.contains('Bar field description'); + // Verify which fields are present, along with checking descriptions and tags + cy.contains("field_foo"); + cy.contains("field_bar"); + cy.contains("field_baz").should("not.exist"); + cy.contains("Foo field description"); + cy.contains("Bar field description"); cy.clickOptionWithText("field_foo"); - cy.get('[data-testid="schema-field-field_foo-tags"]').contains('Legacy').should('not.exist'); + cy.get('[data-testid="schema-field-field_foo-tags"]') + .contains("Legacy") + .should("not.exist"); // Make sure the schema blame is accurate cy.get('[data-testid="schema-blame-button"]').click({ force: true }); cy.wait(3000); - cy.get('[data-testid="field_foo-schema-blame-description"]').contains("Added in v0.0.0"); - cy.get('[data-testid="field_bar-schema-blame-description"]').contains("Added in v0.0.0"); + cy.get('[data-testid="field_foo-schema-blame-description"]').contains( + "Added in v0.0.0", + ); + cy.get('[data-testid="field_bar-schema-blame-description"]').contains( + "Added in v0.0.0", + ); + }); }); -}) \ No newline at end of file diff --git a/smoke-test/tests/cypress/cypress/e2e/search/query_and_filter_search.js b/smoke-test/tests/cypress/cypress/e2e/search/query_and_filter_search.js index 4306ecfbcec28..32e61d37f7562 100644 --- a/smoke-test/tests/cypress/cypress/e2e/search/query_and_filter_search.js +++ b/smoke-test/tests/cypress/cypress/e2e/search/query_and_filter_search.js @@ -9,81 +9,79 @@ const selectFilteredEntity = (textToClick, entity, url) => { cy.get("[data-testid=update-filters]").click({ force: true }); cy.url().should("include", `${url}`); cy.get("[data-testid=update-filters]").should("not.be.visible"); - cy.get('.ant-pagination-next').scrollIntoView().should('be.visible'); + cy.get(".ant-pagination-next").scrollIntoView().should("be.visible"); }; const verifyFilteredEntity = (text) => { - cy.get('.ant-typography').contains(text).should('be.visible'); + cy.get(".ant-typography").contains(text).should("be.visible"); }; const clickAndVerifyEntity = (entity) => { - cy.get('[class*="entityUrn-urn"]').first() - .find('a[href*="urn:li"] span[class^="ant-typography"]').last().invoke('text') + cy.get('[class*="entityUrn-urn"]') + .first() + .find('a[href*="urn:li"] span[class^="ant-typography"]') + .last() + .invoke("text") .then((text) => { cy.contains(text).click(); verifyFilteredEntity(text); verifyFilteredEntity(entity); }); - } +}; describe("auto-complete dropdown, filter plus query search test", () => { - beforeEach(() => { - cy.loginWithCredentials(); - cy.visit('/'); + cy.loginWithCredentials(); + cy.visit("/"); }); - - it("Verify the 'filter by type' section + query", () => { - //Dashboard + it("Verify the 'filter by type' section + query", () => { + // Dashboard searchToExecute("*"); selectFilteredEntity("Type", "Dashboards", "filter__entityType"); - clickAndVerifyEntity('Dashboard') + clickAndVerifyEntity("Dashboard"); - //Ml Models + // Ml Models searchToExecute("*"); selectFilteredEntity("Type", "ML Models", "filter__entityType"); - clickAndVerifyEntity('ML Model'); + clickAndVerifyEntity("ML Model"); - //Piplines + // Piplines searchToExecute("*"); selectFilteredEntity("Type", "Pipelines", "filter__entityType"); - clickAndVerifyEntity('Pipeline'); + clickAndVerifyEntity("Pipeline"); }); it("Verify the 'filter by Glossary term' section + query", () => { - - //Glossary Term - searchToExecute("*"); - selectFilteredEntity("Type", "Glossary Terms", "filter__entityType"); - clickAndVerifyEntity('Glossary Term') -}); + // Glossary Term + searchToExecute("*"); + selectFilteredEntity("Type", "Glossary Terms", "filter__entityType"); + clickAndVerifyEntity("Glossary Term"); + }); it("Verify the 'filter by platform' section + query", () => { - - //Hive + // Hive searchToExecute("*"); selectFilteredEntity("Platform", "Hive", "filter_platform"); - clickAndVerifyEntity('Hive') - - //HDFS + clickAndVerifyEntity("Hive"); + + // HDFS searchToExecute("*"); selectFilteredEntity("Platform", "HDFS", "filter_platform"); - clickAndVerifyEntity('HDFS') + clickAndVerifyEntity("HDFS"); - //Airflow + // Airflow searchToExecute("*"); selectFilteredEntity("Platform", "Airflow", "filter_platform"); - clickAndVerifyEntity('Airflow') + clickAndVerifyEntity("Airflow"); }); it("Verify the 'filter by tag' section + query", () => { - - //CypressFeatureTag + // CypressFeatureTag searchToExecute("*"); selectFilteredEntity("Tag", "CypressFeatureTag", "filter_tags"); - clickAndVerifyEntity('Tags') + clickAndVerifyEntity("Tags"); cy.mouseover('[data-testid="tag-CypressFeatureTag"]'); - verifyFilteredEntity('CypressFeatureTag'); + verifyFilteredEntity("CypressFeatureTag"); }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/search/search.js b/smoke-test/tests/cypress/cypress/e2e/search/search.js index bc11dbee5142c..ea46b2d8d012b 100644 --- a/smoke-test/tests/cypress/cypress/e2e/search/search.js +++ b/smoke-test/tests/cypress/cypress/e2e/search/search.js @@ -13,7 +13,7 @@ describe("search", () => { cy.visit("/"); // random string that is unlikely to accidentally have a match cy.get("input[data-testid=search-input]").type( - "zzzzzzzzzzzzzqqqqqqqqqqqqqzzzzzzqzqzqzqzq{enter}" + "zzzzzzzzzzzzzqqqqqqqqqqqqqzzzzzzqzqzqzqzq{enter}", ); cy.wait(5000); cy.contains("of 0 results"); @@ -22,7 +22,7 @@ describe("search", () => { it("can search, find a result, and visit the dataset page", () => { cy.login(); cy.visit( - "/search?filter_entity=DATASET&filter_tags=urn%3Ali%3Atag%3ACypress&page=1&query=users_created" + "/search?filter_entity=DATASET&filter_tags=urn%3Ali%3Atag%3ACypress&page=1&query=users_created", ); cy.contains("of 2 result"); @@ -51,7 +51,7 @@ describe("search", () => { it("can search and get glossary term facets with proper labels", () => { cy.login(); cy.visit( - "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)" + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", ); cy.contains("cypress_logging_events"); diff --git a/smoke-test/tests/cypress/cypress/e2e/search/searchFilters.js b/smoke-test/tests/cypress/cypress/e2e/search/searchFilters.js index f86418879ed04..761901c0c1175 100644 --- a/smoke-test/tests/cypress/cypress/e2e/search/searchFilters.js +++ b/smoke-test/tests/cypress/cypress/e2e/search/searchFilters.js @@ -54,7 +54,7 @@ describe("search", () => { cy.get("[data-testid=update-filters").click({ force: true }); cy.url().should( "include", - "filter_tags___false___EQUAL___0=urn%3Ali%3Atag%3ACypress" + "filter_tags___false___EQUAL___0=urn%3Ali%3Atag%3ACypress", ); cy.get("[data-testid=update-filters").should("not.exist"); @@ -64,7 +64,7 @@ describe("search", () => { cy.get("[data-testid=update-filters").click({ force: true }); cy.url().should( "include", - "filter__entityType%E2%90%9EtypeNames___false___EQUAL___1=DATASET" + "filter__entityType%E2%90%9EtypeNames___false___EQUAL___1=DATASET", ); // ensure expected entity is in search results @@ -78,7 +78,7 @@ describe("search", () => { cy.get("[data-testid=remove-filter-Datasets").click({ force: true }); cy.url().should( "not.include", - "filter__entityType%E2%90%9EtypeNames___false___EQUAL___1=DATASET" + "filter__entityType%E2%90%9EtypeNames___false___EQUAL___1=DATASET", ); cy.get("[data-testid=active-filter-Datasets").should("not.exist"); @@ -86,7 +86,7 @@ describe("search", () => { cy.get("[data-testid=clear-all-filters").click({ force: true }); cy.url().should( "not.include", - "filter_tags___false___EQUAL___0=urn%3Ali%3Atag%3ACypress" + "filter_tags___false___EQUAL___0=urn%3Ali%3Atag%3ACypress", ); cy.get("[data-testid=active-filter-Cypress").should("not.exist"); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/homePagePost.js b/smoke-test/tests/cypress/cypress/e2e/settings/homePagePost.js index 843a15d7430af..f9f76331ee7d7 100644 --- a/smoke-test/tests/cypress/cypress/e2e/settings/homePagePost.js +++ b/smoke-test/tests/cypress/cypress/e2e/settings/homePagePost.js @@ -1,85 +1,106 @@ const homePageRedirection = () => { - cy.visit('/'); - cy.waitTextPresent("Welcome back"); + cy.visit("/"); + cy.waitTextPresent("Welcome back"); }; const addOrEditAnnouncement = (text, title, description, testId) => { - cy.waitTextPresent(text); - cy.get('[data-testid="create-post-title"]').clear().type(title); - cy.get('[id="description"]').clear().type(description); - cy.get(`[data-testid="${testId}-post-button"]`).click({ force: true }); - cy.reload(); - homePageRedirection(); + cy.waitTextPresent(text); + cy.get('[data-testid="create-post-title"]').clear().type(title); + cy.get('[id="description"]').clear().type(description); + cy.get(`[data-testid="${testId}-post-button"]`).click({ force: true }); + cy.reload(); + homePageRedirection(); }; const addOrEditLink = (text, title, url, imagesURL, testId) => { - cy.waitTextPresent(text); - cy.get('[data-testid="create-post-title"]').clear().type(title); - cy.get('[data-testid="create-post-link"]').clear().type(url); - cy.get('[data-testid="create-post-media-location"]').clear().type(imagesURL); - cy.get(`[data-testid="${testId}-post-button"]`).click({ force: true }); - cy.reload(); - homePageRedirection(); + cy.waitTextPresent(text); + cy.get('[data-testid="create-post-title"]').clear().type(title); + cy.get('[data-testid="create-post-link"]').clear().type(url); + cy.get('[data-testid="create-post-media-location"]').clear().type(imagesURL); + cy.get(`[data-testid="${testId}-post-button"]`).click({ force: true }); + cy.reload(); + homePageRedirection(); }; -const clickOnNewPost = () =>{ - cy.get('[id="posts-create-post"]').click({ force: true }); -} +const clickOnNewPost = () => { + cy.get('[id="posts-create-post"]').click({ force: true }); +}; const clickOnMoreOption = () => { - cy.get('[aria-label="more"]').first().click(); -} + cy.get('[aria-label="more"]').first().click(); +}; describe("create announcement and link post", () => { - beforeEach(() => { - cy.loginWithCredentials(); - cy.goToHomePagePostSettings(); - }); + beforeEach(() => { + cy.loginWithCredentials(); + cy.goToHomePagePostSettings(); + }); - it("create announcement post and verify", () => { - clickOnNewPost() - addOrEditAnnouncement("Create new Post", "Test Announcement Title", "Add Description to post announcement", "create"); - cy.waitTextPresent("Test Announcement Title"); - }); + it("create announcement post and verify", () => { + clickOnNewPost(); + addOrEditAnnouncement( + "Create new Post", + "Test Announcement Title", + "Add Description to post announcement", + "create", + ); + cy.waitTextPresent("Test Announcement Title"); + }); - it("edit announced post and verify", () => { - clickOnMoreOption() - cy.clickOptionWithText("Edit"); - addOrEditAnnouncement("Edit Post", "Test Announcement Title Edited", "Decription Edited", "update"); - cy.waitTextPresent("Test Announcement Title Edited"); - }); + it("edit announced post and verify", () => { + clickOnMoreOption(); + cy.clickOptionWithText("Edit"); + addOrEditAnnouncement( + "Edit Post", + "Test Announcement Title Edited", + "Decription Edited", + "update", + ); + cy.waitTextPresent("Test Announcement Title Edited"); + }); - it("delete announced post and verify", () => { - clickOnMoreOption() - cy.clickOptionWithText("Delete"); - cy.clickOptionWithText("Yes"); - cy.reload(); - homePageRedirection(); - cy.ensureTextNotPresent("Test Announcement Title Edited"); - }); + it("delete announced post and verify", () => { + clickOnMoreOption(); + cy.clickOptionWithText("Delete"); + cy.clickOptionWithText("Yes"); + cy.reload(); + homePageRedirection(); + cy.ensureTextNotPresent("Test Announcement Title Edited"); + }); - it("create link post and verify", () => { - clickOnNewPost() - cy.waitTextPresent('Create new Post'); - cy.contains('label', 'Link').click(); - addOrEditLink("Create new Post", "Test Link Title", 'https://www.example.com', 'https://www.example.com/images/example-image.jpg', "create"); - cy.waitTextPresent("Test Link Title"); - }); + it("create link post and verify", () => { + clickOnNewPost(); + cy.waitTextPresent("Create new Post"); + cy.contains("label", "Link").click(); + addOrEditLink( + "Create new Post", + "Test Link Title", + "https://www.example.com", + "https://www.example.com/images/example-image.jpg", + "create", + ); + cy.waitTextPresent("Test Link Title"); + }); - it("edit linked post and verify", () => { - clickOnMoreOption() - cy.clickOptionWithText("Edit"); - addOrEditLink("Edit Post", "Test Link Edited Title", 'https://www.updatedexample.com', 'https://www.updatedexample.com/images/example-image.jpg', "update"); - cy.waitTextPresent("Test Link Edited Title"); - }); + it("edit linked post and verify", () => { + clickOnMoreOption(); + cy.clickOptionWithText("Edit"); + addOrEditLink( + "Edit Post", + "Test Link Edited Title", + "https://www.updatedexample.com", + "https://www.updatedexample.com/images/example-image.jpg", + "update", + ); + cy.waitTextPresent("Test Link Edited Title"); + }); - it("delete linked post and verify", () => { - clickOnMoreOption() - cy.clickOptionWithText("Delete"); - cy.clickOptionWithText("Yes"); - cy.reload(); - homePageRedirection(); - cy.ensureTextNotPresent("Test Link Edited Title"); - }); + it("delete linked post and verify", () => { + clickOnMoreOption(); + cy.clickOptionWithText("Delete"); + cy.clickOptionWithText("Yes"); + cy.reload(); + homePageRedirection(); + cy.ensureTextNotPresent("Test Link Edited Title"); + }); }); - diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/manage_access_tokens.js b/smoke-test/tests/cypress/cypress/e2e/settings/manage_access_tokens.js index 7a77c2b77df5b..61ed8417cd450 100644 --- a/smoke-test/tests/cypress/cypress/e2e/settings/manage_access_tokens.js +++ b/smoke-test/tests/cypress/cypress/e2e/settings/manage_access_tokens.js @@ -1,43 +1,49 @@ import { aliasQuery, hasOperationName } from "../utils"; + const test_id = Math.floor(Math.random() * 100000); describe("manage access tokens", () => { - before(() => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - aliasQuery(req, "appConfig"); - }); + before(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); }); - - const setTokenAuthEnabledFlag = (isOn) => { - cy.intercept("POST", "/api/v2/graphql", (req) => { - if (hasOperationName(req, "appConfig")) { - req.reply((res) => { - res.body.data.appConfig.authConfig.tokenAuthEnabled = isOn; - }); - } - }); - }; + }); - it("create and revoke access token", () => { - //create access token, verify token on ui - setTokenAuthEnabledFlag(true); - cy.loginWithCredentials(); - cy.goToAccessTokenSettings(); - cy.clickOptionWithTestId("add-token-button"); - cy.enterTextInTestId("create-access-token-name", "Token Name" + test_id); - cy.enterTextInTestId("create-access-token-description", "Token Description" + test_id); - cy.clickOptionWithTestId("create-access-token-button"); - cy.waitTextVisible("New Personal Access Token"); - cy.get('[data-testid="access-token-value"]').should("be.visible"); - cy.get('[data-testid="access-token-value"]').invoke('text').should('match', /^[a-zA-Z0-9-_]+\.[a-zA-Z0-9-_]+\.[a-zA-Z0-9-_]+$/); - cy.clickOptionWithTestId("access-token-modal-close-button"); - //revoke access token, verify token removed from ui - cy.waitTextVisible("Token Name" + test_id); - cy.waitTextVisible("Token Description" + test_id); - cy.clickOptionWithTestId("revoke-token-button"); - cy.waitTextVisible("Are you sure you want to revoke this token?"); - cy.clickOptionWithText("Yes"); - cy.ensureTextNotPresent("Token Name" + test_id); - cy.ensureTextNotPresent("Token Description" + test_id); + const setTokenAuthEnabledFlag = (isOn) => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + res.body.data.appConfig.authConfig.tokenAuthEnabled = isOn; + }); + } }); -}); \ No newline at end of file + }; + + it("create and revoke access token", () => { + // create access token, verify token on ui + setTokenAuthEnabledFlag(true); + cy.loginWithCredentials(); + cy.goToAccessTokenSettings(); + cy.clickOptionWithTestId("add-token-button"); + cy.enterTextInTestId("create-access-token-name", `Token Name${test_id}`); + cy.enterTextInTestId( + "create-access-token-description", + `Token Description${test_id}`, + ); + cy.clickOptionWithTestId("create-access-token-button"); + cy.waitTextVisible("New Personal Access Token"); + cy.get('[data-testid="access-token-value"]').should("be.visible"); + cy.get('[data-testid="access-token-value"]') + .invoke("text") + .should("match", /^[a-zA-Z0-9-_]+\.[a-zA-Z0-9-_]+\.[a-zA-Z0-9-_]+$/); + cy.clickOptionWithTestId("access-token-modal-close-button"); + // revoke access token, verify token removed from ui + cy.waitTextVisible(`Token Name${test_id}`); + cy.waitTextVisible(`Token Description${test_id}`); + cy.clickOptionWithTestId("revoke-token-button"); + cy.waitTextVisible("Are you sure you want to revoke this token?"); + cy.clickOptionWithText("Yes"); + cy.ensureTextNotPresent(`Token Name${test_id}`); + cy.ensureTextNotPresent(`Token Description${test_id}`); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/manage_policies.js b/smoke-test/tests/cypress/cypress/e2e/settings/manage_policies.js index 0e69a4e7f287a..98d66f4fc5102 100644 --- a/smoke-test/tests/cypress/cypress/e2e/settings/manage_policies.js +++ b/smoke-test/tests/cypress/cypress/e2e/settings/manage_policies.js @@ -4,19 +4,15 @@ const platform_policy_edited = `Platform test policy ${test_id} EDITED`; const metadata_policy_name = `Metadata test policy ${test_id}`; const metadata_policy_edited = `Metadata test policy ${test_id} EDITED`; - - function searchAndToggleMetadataPolicyStatus(metadataPolicyName, targetStatus) { - cy.get('[data-testid="search-input"]').should('be.visible'); + cy.get('[data-testid="search-input"]').should("be.visible"); cy.get('[data-testid="search-input"]').eq(1).type(metadataPolicyName); - cy.contains('tr', metadataPolicyName).as('metadataPolicyRow'); + cy.contains("tr", metadataPolicyName).as("metadataPolicyRow"); cy.contains(targetStatus).click(); } function clickFocusAndType(Id, text) { - cy.clickOptionWithTestId(Id) - .focused().clear() - .type(text); + cy.clickOptionWithTestId(Id).focused().clear().type(text); } function updateAndSave(Id, groupName, text) { @@ -30,20 +26,26 @@ function clickOnButton(saveButton) { } function createPolicy(decription, policyName) { - clickFocusAndType("policy-description", decription) + clickFocusAndType("policy-description", decription); clickOnButton("nextButton"); - updateAndSave("privileges", "All", "All Privileges", "nextButton") + updateAndSave("privileges", "All", "All Privileges", "nextButton"); clickOnButton("nextButton"); - updateAndSave("users", "All", "All Users") - updateAndSave("groups", "All", "All Groups") + updateAndSave("users", "All", "All Users"); + updateAndSave("groups", "All", "All Groups"); clickOnButton("saveButton"); cy.waitTextVisible("Successfully saved policy."); cy.waitTextVisible(policyName); } -function editPolicy(policyName, editPolicy, description, policyEdited, visibleDiscription) { - searchAndToggleMetadataPolicyStatus(policyName, 'EDIT') - cy.clickOptionWithTestId("policy-name") +function editPolicy( + policyName, + editPolicy, + description, + policyEdited, + visibleDiscription, +) { + searchAndToggleMetadataPolicyStatus(policyName, "EDIT"); + cy.clickOptionWithTestId("policy-name"); cy.focused().clear().type(editPolicy); cy.clickOptionWithTestId("policy-description"); cy.focused().clear().type(description); @@ -52,15 +54,15 @@ function editPolicy(policyName, editPolicy, description, policyEdited, visibleDi clickOnButton("saveButton"); cy.waitTextVisible("Successfully saved policy."); cy.waitTextVisible(policyEdited); - cy.waitTextVisible(visibleDiscription);; + cy.waitTextVisible(visibleDiscription); } function deletePolicy(policyEdited, deletePolicy) { - searchAndToggleMetadataPolicyStatus(policyEdited, 'DEACTIVATE') - cy.waitTextVisible("Successfully deactivated policy.") - cy.contains('DEACTIVATE').should('not.exist') - cy.contains('ACTIVATE').click(); - cy.waitTextVisible("Successfully activated policy.") + searchAndToggleMetadataPolicyStatus(policyEdited, "DEACTIVATE"); + cy.waitTextVisible("Successfully deactivated policy."); + cy.contains("DEACTIVATE").should("not.exist"); + cy.contains("ACTIVATE").click(); + cy.waitTextVisible("Successfully activated policy."); cy.get("[data-icon='delete']").click(); cy.waitTextVisible(deletePolicy); cy.clickOptionWithText("Yes"); @@ -77,37 +79,58 @@ describe("create and manage platform and metadata policies", () => { it("create platform policy", () => { cy.waitTextVisible("Manage Permissions"); cy.clickOptionWithText("Create new policy"); - clickFocusAndType("policy-name", platform_policy_name) + clickFocusAndType("policy-name", platform_policy_name); cy.get('[data-testid="policy-type"] [title="Metadata"]').click(); cy.clickOptionWithTestId("platform"); - createPolicy(`Platform policy description ${test_id}`, platform_policy_name) + createPolicy( + `Platform policy description ${test_id}`, + platform_policy_name, + ); }); it("edit platform policy", () => { - editPolicy(`${platform_policy_name}`, platform_policy_edited, + editPolicy( + `${platform_policy_name}`, + platform_policy_edited, + `Platform policy description ${test_id} EDITED`, + platform_policy_edited, `Platform policy description ${test_id} EDITED`, - platform_policy_edited, `Platform policy description ${test_id} EDITED`) + ); }); it("deactivate and activate platform policy", () => { - deletePolicy(`${platform_policy_edited}`, `Delete ${platform_policy_edited}`, `${platform_policy_edited}`) + deletePolicy( + `${platform_policy_edited}`, + `Delete ${platform_policy_edited}`, + `${platform_policy_edited}`, + ); }); it("create metadata policy", () => { cy.clickOptionWithText("Create new policy"); - clickFocusAndType("policy-name", metadata_policy_name) - cy.get('[data-testid="policy-type"]').should('have.text', 'Metadata'); - createPolicy(`Metadata policy description ${test_id}`, metadata_policy_name) + clickFocusAndType("policy-name", metadata_policy_name); + cy.get('[data-testid="policy-type"]').should("have.text", "Metadata"); + createPolicy( + `Metadata policy description ${test_id}`, + metadata_policy_name, + ); }); it("edit metadata policy", () => { - editPolicy(`${metadata_policy_name}`, metadata_policy_edited, + editPolicy( + `${metadata_policy_name}`, + metadata_policy_edited, `Metadata policy description ${test_id} EDITED`, - metadata_policy_edited, `Metadata policy description ${test_id} EDITED`) + metadata_policy_edited, + `Metadata policy description ${test_id} EDITED`, + ); }); it("deactivate and activate metadata policy", () => { - deletePolicy(`${metadata_policy_name}`, `Delete ${metadata_policy_name}`, `${metadata_policy_edited}`) + deletePolicy( + `${metadata_policy_name}`, + `Delete ${metadata_policy_name}`, + `${metadata_policy_edited}`, + ); }); - -}); \ No newline at end of file +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js b/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js index d9f69cd9a5ec4..247a9c8b9b273 100644 --- a/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js +++ b/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js @@ -1,127 +1,132 @@ const test_id = Math.floor(Math.random() * 100000); const username = `Example Name ${test_id}`; -const email = `example${test_id}@example.com` -const password = "Example password" +const email = `example${test_id}@example.com`; +const password = "Example password"; const group_name = `Test group ${test_id}`; describe("create and manage group", () => { - it("add test user", () => { - cy.loginWithCredentials(); - cy.visit("/settings/identities/users"); - cy.waitTextVisible("Invite Users"); - cy.clickOptionWithText("Invite Users"); - cy.waitTextVisible(/signup\?invite_token=\w{32}/).then(($elem) => { - const inviteLink = $elem.text(); - cy.visit("/settings/identities/users"); - cy.logout(); - cy.visit(inviteLink); - cy.enterTextInTestId("email", email); - cy.enterTextInTestId("name", username); - cy.enterTextInTestId("password", password); - cy.enterTextInTestId("confirmPassword", password); - cy.mouseover("#title").click(); - cy.waitTextVisible("Other").click(); - cy.get("[type=submit]").click(); - cy.waitTextVisible("Welcome to DataHub"); - cy.hideOnboardingTour(); - cy.waitTextVisible(username); - }) + it("add test user", () => { + cy.loginWithCredentials(); + cy.visit("/settings/identities/users"); + cy.waitTextVisible("Invite Users"); + cy.clickOptionWithText("Invite Users"); + cy.waitTextVisible(/signup\?invite_token=\w{32}/).then(($elem) => { + const inviteLink = $elem.text(); + cy.visit("/settings/identities/users"); + cy.logout(); + cy.visit(inviteLink); + cy.enterTextInTestId("email", email); + cy.enterTextInTestId("name", username); + cy.enterTextInTestId("password", password); + cy.enterTextInTestId("confirmPassword", password); + cy.mouseover("#title").click(); + cy.waitTextVisible("Other").click(); + cy.get("[type=submit]").click(); + cy.waitTextVisible("Welcome to DataHub"); + cy.hideOnboardingTour(); + cy.waitTextVisible(username); }); + }); - it("create a group", () => { - cy.loginWithCredentials(); - cy.visit("/settings/identities/groups") - cy.waitTextVisible("Create group"); - cy.clickOptionWithText("Create group"); - cy.waitTextVisible("Create new group"); - cy.get("#name").type(group_name); - cy.get("#description").type("Test group description"); - cy.contains("Advanced").click(); - cy.waitTextVisible("Group Id"); - cy.get("#groupId").type(test_id); - cy.get("#createGroupButton").click(); - cy.waitTextVisible("Created group!"); - cy.waitTextVisible(group_name); - }); + it("create a group", () => { + cy.loginWithCredentials(); + cy.visit("/settings/identities/groups"); + cy.waitTextVisible("Create group"); + cy.clickOptionWithText("Create group"); + cy.waitTextVisible("Create new group"); + cy.get("#name").type(group_name); + cy.get("#description").type("Test group description"); + cy.contains("Advanced").click(); + cy.waitTextVisible("Group Id"); + cy.get("#groupId").type(test_id); + cy.get("#createGroupButton").click(); + cy.waitTextVisible("Created group!"); + cy.waitTextVisible(group_name); + }); - it("add test user to a group", () => { - cy.loginWithCredentials(); - cy.visit("/settings/identities/users"); - cy.get(".ant-tabs-tab-btn").contains("Groups").click(); - cy.clickOptionWithText(group_name); - cy.get(".ant-typography").contains(group_name).should("be.visible"); - cy.get(".ant-tabs-tab").contains("Members").click(); - cy.waitTextVisible("No members in this group yet."); - cy.clickOptionWithText("Add Member"); - cy.contains("Search for users...").click({ force: true }); - cy.focused().type(username); - cy.get(".ant-select-item-option").contains(username).click(); - cy.focused().blur(); - cy.contains(username).should("have.length", 1); - cy.get('[role="dialog"] button').contains("Add").click({ force: true }); - cy.waitTextVisible("Group members added!"); - cy.contains(username, {timeout: 10000}).should("be.visible"); - }); + it("add test user to a group", () => { + cy.loginWithCredentials(); + cy.visit("/settings/identities/users"); + cy.get(".ant-tabs-tab-btn").contains("Groups").click(); + cy.clickOptionWithText(group_name); + cy.get(".ant-typography").contains(group_name).should("be.visible"); + cy.get(".ant-tabs-tab").contains("Members").click(); + cy.waitTextVisible("No members in this group yet."); + cy.clickOptionWithText("Add Member"); + cy.contains("Search for users...").click({ force: true }); + cy.focused().type(username); + cy.get(".ant-select-item-option").contains(username).click(); + cy.focused().blur(); + cy.contains(username).should("have.length", 1); + cy.get('[role="dialog"] button').contains("Add").click({ force: true }); + cy.waitTextVisible("Group members added!"); + cy.contains(username, { timeout: 10000 }).should("be.visible"); + }); - it("update group info", () => { - cy.loginWithCredentials(); - cy.visit("/settings/identities/groups"); - cy.clickOptionWithText(group_name); - cy.contains(group_name).find('[aria-label="Edit"]').click(); - cy.focused().clear().type(`Test group EDITED ${test_id}{enter}`); - cy.waitTextVisible("Name Updated"); - cy.contains(`Test group EDITED ${test_id}`).should("be.visible"); - cy.get('[data-testid="edit-icon"]').click(); - cy.waitTextVisible("Edit Description"); - cy.get("#description").should("be.visible").type(" EDITED"); - cy.get("#updateGroupButton").click(); - cy.waitTextVisible("Changes saved."); - cy.contains("Test group description EDITED").should("be.visible"); - cy.clickOptionWithText("Add Owners"); - cy.get('[id="owner"]').click({ force: true }); - cy.focused().type(username); - cy.get(".ant-select-item-option").contains(username, { matchCase: false }).click(); - cy.focused().blur(); - cy.contains(username, { matchCase: false }).should("have.length", 1); - cy.get('[role="dialog"] button').contains("Done").click(); - cy.waitTextVisible("Owners Added"); - cy.contains(username, { matchCase: false }).should("be.visible"); - cy.clickOptionWithText("Edit Group"); - cy.waitTextVisible("Edit Profile"); - cy.get("#email").type(`${test_id}@testemail.com`); - cy.get("#slack").type(`#${test_id}`); - cy.clickOptionWithText("Save Changes"); - cy.waitTextVisible("Changes saved."); - cy.waitTextVisible(`${test_id}@testemail.com`); - cy.waitTextVisible(`#${test_id}`); - }); + it("update group info", () => { + cy.loginWithCredentials(); + cy.visit("/settings/identities/groups"); + cy.clickOptionWithText(group_name); + cy.contains(group_name).find('[aria-label="Edit"]').click(); + cy.focused().clear().type(`Test group EDITED ${test_id}{enter}`); + cy.waitTextVisible("Name Updated"); + cy.contains(`Test group EDITED ${test_id}`).should("be.visible"); + cy.get('[data-testid="edit-icon"]').click(); + cy.waitTextVisible("Edit Description"); + cy.get("#description").should("be.visible").type(" EDITED"); + cy.get("#updateGroupButton").click(); + cy.waitTextVisible("Changes saved."); + cy.contains("Test group description EDITED").should("be.visible"); + cy.clickOptionWithText("Add Owners"); + cy.get('[id="owner"]').click({ force: true }); + cy.focused().type(username); + cy.get(".ant-select-item-option") + .contains(username, { matchCase: false }) + .click(); + cy.focused().blur(); + cy.contains(username, { matchCase: false }).should("have.length", 1); + cy.get('[role="dialog"] button').contains("Done").click(); + cy.waitTextVisible("Owners Added"); + cy.contains(username, { matchCase: false }).should("be.visible"); + cy.clickOptionWithText("Edit Group"); + cy.waitTextVisible("Edit Profile"); + cy.get("#email").type(`${test_id}@testemail.com`); + cy.get("#slack").type(`#${test_id}`); + cy.clickOptionWithText("Save Changes"); + cy.waitTextVisible("Changes saved."); + cy.waitTextVisible(`${test_id}@testemail.com`); + cy.waitTextVisible(`#${test_id}`); + }); - it("test User verify group participation", () => { - cy.loginWithCredentials(); - cy.visit("/settings/identities/groups"); - cy.hideOnboardingTour(); - cy.clickOptionWithText(`Test group EDITED ${test_id}`); - cy.get(".ant-tabs-tab").contains("Members").click(); - cy.waitTextVisible(username); - }); + it("test User verify group participation", () => { + cy.loginWithCredentials(); + cy.visit("/settings/identities/groups"); + cy.hideOnboardingTour(); + cy.clickOptionWithText(`Test group EDITED ${test_id}`); + cy.get(".ant-tabs-tab").contains("Members").click(); + cy.waitTextVisible(username); + }); - it("assign role to group ", () => { - cy.loginWithCredentials(); - cy.visit("/settings/identities/groups"); - cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).next().click() - cy.get('.ant-select-item-option').contains('Admin').click() - cy.get('button.ant-btn-primary').contains('OK').click(); - cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).waitTextVisible('Admin'); - }); - - it("remove group", () => { - cy.loginWithCredentials(); - cy.visit("/settings/identities/groups"); - cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).openThreeDotDropdown() - cy.clickOptionWithText("Delete"); - cy.clickOptionWithText("Yes"); - cy.waitTextVisible("Deleted Group!"); - cy.ensureTextNotPresent(`Test group EDITED ${test_id}`); - }); + it("assign role to group ", () => { + cy.loginWithCredentials(); + cy.visit("/settings/identities/groups"); + cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).next().click(); + cy.get(".ant-select-item-option").contains("Admin").click(); + cy.get("button.ant-btn-primary").contains("OK").click(); + cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).waitTextVisible( + "Admin", + ); + }); -}); \ No newline at end of file + it("remove group", () => { + cy.loginWithCredentials(); + cy.visit("/settings/identities/groups"); + cy.get( + `[href="/group/urn:li:corpGroup:${test_id}"]`, + ).openThreeDotDropdown(); + cy.clickOptionWithText("Delete"); + cy.clickOptionWithText("Yes"); + cy.waitTextVisible("Deleted Group!"); + cy.ensureTextNotPresent(`Test group EDITED ${test_id}`); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js index f89b70b7a7d23..b6b6fea1a7d70 100644 --- a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js +++ b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js @@ -1,132 +1,157 @@ -describe('siblings', () => { - it('will merge metadata to non-primary sibling', () => { +describe("siblings", () => { + it("will merge metadata to non-primary sibling", () => { cy.login(); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false", + ); // check merged platforms - cy.contains('dbt & BigQuery'); + cy.contains("dbt & BigQuery"); // check merged schema (from dbt) - cy.contains('This is a unique identifier for a customer'); + cy.contains("This is a unique identifier for a customer"); // check merged profile (from bigquery) - cy.contains('Stats').click({ force: true }); + cy.contains("Stats").click({ force: true }); cy.get('[data-testid="table-stats-rowcount"]').contains("100"); - }); + }); - it('will merge metadata to primary sibling', () => { + it("will merge metadata to primary sibling", () => { cy.login(); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false", + ); // check merged platforms - cy.contains('dbt & BigQuery'); + cy.contains("dbt & BigQuery"); // check merged schema (from dbt) - cy.contains('This is a unique identifier for a customer'); + cy.contains("This is a unique identifier for a customer"); // check merged profile (from bigquery) - cy.contains('Stats').click({ force: true }); + cy.contains("Stats").click({ force: true }); cy.get('[data-testid="table-stats-rowcount"]').contains("100"); }); - it('can view individual nodes', () => { + it("can view individual nodes", () => { cy.login(); - const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/ - cy.on('uncaught:exception', (err) => { - /* returning false here prevents Cypress from failing the test */ - if (resizeObserverLoopErrRe.test(err.message)) { - return false - } - }) + const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/; + cy.on("uncaught:exception", (err) => { + /* returning false here prevents Cypress from failing the test */ + if (resizeObserverLoopErrRe.test(err.message)) { + return false; + } + }); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false", + ); // navigate to the bq entity - cy.clickOptionWithTestId('compact-entity-link-urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)'); + cy.clickOptionWithTestId( + "compact-entity-link-urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + ); // check merged platforms is not shown - cy.get('[data-testid="entity-header-test-id"]').contains('dbt & BigQuery').should('not.exist'); - cy.get('[data-testid="entity-header-test-id"]').contains('BigQuery'); + cy.get('[data-testid="entity-header-test-id"]') + .contains("dbt & BigQuery") + .should("not.exist"); + cy.get('[data-testid="entity-header-test-id"]').contains("BigQuery"); // check dbt schema descriptions not shown - cy.contains('This is a unique identifier for a customer').should('not.exist'); + cy.contains("This is a unique identifier for a customer").should( + "not.exist", + ); // check merged profile still there (from bigquery) - cy.contains('Stats').click({ force: true }); + cy.contains("Stats").click({ force: true }); cy.get('[data-testid="table-stats-rowcount"]').contains("100"); }); - it('can mutate at individual node or combined node level', () => { + it("can mutate at individual node or combined node level", () => { cy.login(); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false", + ); // navigate to the bq entity - cy.clickOptionWithTestId('compact-entity-link-urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)'); + cy.clickOptionWithTestId( + "compact-entity-link-urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + ); - cy.clickOptionWithText('Add Term'); + cy.clickOptionWithText("Add Term"); - cy.selectOptionInTagTermModal('CypressTerm'); + cy.selectOptionInTagTermModal("CypressTerm"); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false", + ); - cy.get('a[href="/glossaryTerm/urn:li:glossaryTerm:CypressNode.CypressTerm"]').within(() => cy.get('span[aria-label=close]').click()); - cy.clickOptionWithText('Yes'); + cy.get( + 'a[href="/glossaryTerm/urn:li:glossaryTerm:CypressNode.CypressTerm"]', + ).within(() => cy.get("span[aria-label=close]").click()); + cy.clickOptionWithText("Yes"); - cy.contains('CypressTerm').should('not.exist'); + cy.contains("CypressTerm").should("not.exist"); }); - it('will combine results in search', () => { + it("will combine results in search", () => { cy.login(); - cy.visit('/search?page=1&query=raw_orders'); + cy.visit("/search?page=1&query=raw_orders"); - cy.contains('Showing 1 - 10 of '); + cy.contains("Showing 1 - 10 of "); - cy.get('.test-search-result').should('have.length', 5); - cy.get('.test-search-result-sibling-section').should('have.length', 5); + cy.get(".test-search-result").should("have.length", 5); + cy.get(".test-search-result-sibling-section").should("have.length", 5); - cy.get('.test-search-result-sibling-section').get('.test-mini-preview-class:contains(raw_orders)').should('have.length', 2); + cy.get(".test-search-result-sibling-section") + .get(".test-mini-preview-class:contains(raw_orders)") + .should("have.length", 2); }); - it('will combine results in lineage', () => { + it("will combine results in lineage", () => { cy.login(); - cy.visit('dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)/?is_lineage_mode=true'); + cy.visit( + "dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)/?is_lineage_mode=true", + ); // check the subtypes - cy.get('text:contains(Table)').should('have.length', 2); - cy.get('text:contains(Seed)').should('have.length', 1); + cy.get("text:contains(Table)").should("have.length", 2); + cy.get("text:contains(Seed)").should("have.length", 1); // check the names - cy.get('text:contains(raw_orders)').should('have.length', 1); - cy.get('text:contains(customers)').should('have.length', 1); + cy.get("text:contains(raw_orders)").should("have.length", 1); + cy.get("text:contains(customers)").should("have.length", 1); // center counts twice since we secretely render two center nodes - cy.get('text:contains(stg_orders)').should('have.length', 2); + cy.get("text:contains(stg_orders)").should("have.length", 2); // check the platform - cy.get('svg').get('text:contains(dbt & BigQuery)').should('have.length', 5); + cy.get("svg").get("text:contains(dbt & BigQuery)").should("have.length", 5); }); - it('can separate results in lineage if flag is set', () => { + it("can separate results in lineage if flag is set", () => { cy.login(); - cy.visit('dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)/?is_lineage_mode=true'); + cy.visit( + "dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)/?is_lineage_mode=true", + ); - cy.clickOptionWithTestId('compress-lineage-toggle'); + cy.clickOptionWithTestId("compress-lineage-toggle"); // check the subtypes - cy.get('[data-testid="Seed"]').should('have.length', 1); + cy.get('[data-testid="Seed"]').should("have.length", 1); // center counts twice since we secretely render two center nodes, plus the downstream bigquery - cy.get('[data-testid="View"]').should('have.length', 3); - cy.get('[data-testid="Table"]').should('have.length', 0); - + cy.get('[data-testid="View"]').should("have.length", 3); + cy.get('[data-testid="Table"]').should("have.length", 0); // check the names - cy.get('text:contains(raw_orders)').should('have.length', 1); + cy.get("text:contains(raw_orders)").should("have.length", 1); // center counts twice since we secretely render two center nodes, plus the downstream bigquery - cy.get('text:contains(stg_orders)').should('have.length', 3); + cy.get("text:contains(stg_orders)").should("have.length", 3); // check the platform - cy.get('svg').get('text:contains(dbt & BigQuery)').should('have.length', 0); - cy.get('svg').get('text:contains(dbt)').should('have.length', 3); - cy.get('svg').get('text:contains(BigQuery)').should('have.length', 1); + cy.get("svg").get("text:contains(dbt & BigQuery)").should("have.length", 0); + cy.get("svg").get("text:contains(dbt)").should("have.length", 3); + cy.get("svg").get("text:contains(BigQuery)").should("have.length", 1); }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/task_runs/task_runs.js b/smoke-test/tests/cypress/cypress/e2e/task_runs/task_runs.js index ffe1884201c42..536310e70a564 100644 --- a/smoke-test/tests/cypress/cypress/e2e/task_runs/task_runs.js +++ b/smoke-test/tests/cypress/cypress/e2e/task_runs/task_runs.js @@ -1,39 +1,42 @@ -describe('task runs', () => { - it('can visit dataset with runs aspect and verify the task run is present', () => { - cy.visit('/') - cy.login(); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)/Runs'); - - // the run data should not be there since the run wrote - cy.contains('manual__2022-03-30T11:35:08.970522+00:00') - cy.contains('Failed'); - - // inputs - cy.contains('fct_cypress_users_created_no_tag'); - - // outputs - cy.contains('SampleCypressHiveDataset'); - cy.contains('cypress_logging_events'); - - // task name - cy.contains('User Creations'); - }); - - it('can visit task with runs aspect and verify the task run is present', () => { - cy.visit('/') - cy.login(); - cy.visit('/tasks/urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)/Runs?is_lineage_mode=false'); - - // Verify the run data is there - cy.contains('manual__2022-03-30T11:35:08.970522+00:00'); - cy.contains('Failed'); - - // inputs - cy.contains('fct_cypress_users_created_no_tag'); - - // outputs - cy.contains('SampleCypressHiveDataset'); - cy.contains('cypress_logging_events'); - }); - -}) +describe("task runs", () => { + it("can visit dataset with runs aspect and verify the task run is present", () => { + cy.visit("/"); + cy.login(); + cy.visit( + "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)/Runs", + ); + + // the run data should not be there since the run wrote + cy.contains("manual__2022-03-30T11:35:08.970522+00:00"); + cy.contains("Failed"); + + // inputs + cy.contains("fct_cypress_users_created_no_tag"); + + // outputs + cy.contains("SampleCypressHiveDataset"); + cy.contains("cypress_logging_events"); + + // task name + cy.contains("User Creations"); + }); + + it("can visit task with runs aspect and verify the task run is present", () => { + cy.visit("/"); + cy.login(); + cy.visit( + "/tasks/urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)/Runs?is_lineage_mode=false", + ); + + // Verify the run data is there + cy.contains("manual__2022-03-30T11:35:08.970522+00:00"); + cy.contains("Failed"); + + // inputs + cy.contains("fct_cypress_users_created_no_tag"); + + // outputs + cy.contains("SampleCypressHiveDataset"); + cy.contains("cypress_logging_events"); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/views/manage_views.js b/smoke-test/tests/cypress/cypress/e2e/views/manage_views.js index 9f3039ab4de77..92b63348c1bde 100644 --- a/smoke-test/tests/cypress/cypress/e2e/views/manage_views.js +++ b/smoke-test/tests/cypress/cypress/e2e/views/manage_views.js @@ -1,36 +1,44 @@ describe("manage views", () => { - it("go to views settings page, create, edit, make default, delete a view", () => { - const viewName = "Test View" - - cy.login(); - cy.goToViewsSettings(); - - cy.clickOptionWithText("Create new View"); - cy.get(".ant-input-affix-wrapper > input[type='text']").first().type(viewName); - cy.clickOptionWithTestId("view-builder-save"); - - // Confirm that the test has been created. - cy.waitTextVisible("Test View"); - - // Now edit the View - cy.clickFirstOptionWithTestId("views-table-dropdown"); - cy.get('[data-testid="view-dropdown-edit"]').click({ force: true }); - cy.get(".ant-input-affix-wrapper > input[type='text']").first().clear().type("New View Name"); - cy.clickOptionWithTestId("view-builder-save"); - cy.waitTextVisible("New View Name"); - - // Now make the view the default - cy.clickFirstOptionWithTestId("views-table-dropdown"); - cy.get('[data-testid="view-dropdown-set-user-default"]').click({ force: true }); - - // Now unset as the default - cy.clickFirstOptionWithTestId("views-table-dropdown"); - cy.get('[data-testid="view-dropdown-remove-user-default"]').click({ force: true }); - - // Now delete the View - cy.clickFirstOptionWithTestId("views-table-dropdown"); - cy.get('[data-testid="view-dropdown-delete"]').click({ force: true }); - cy.clickOptionWithText("Yes"); + it("go to views settings page, create, edit, make default, delete a view", () => { + const viewName = "Test View"; + + cy.login(); + cy.goToViewsSettings(); + + cy.clickOptionWithText("Create new View"); + cy.get(".ant-input-affix-wrapper > input[type='text']") + .first() + .type(viewName); + cy.clickOptionWithTestId("view-builder-save"); + + // Confirm that the test has been created. + cy.waitTextVisible("Test View"); + + // Now edit the View + cy.clickFirstOptionWithTestId("views-table-dropdown"); + cy.get('[data-testid="view-dropdown-edit"]').click({ force: true }); + cy.get(".ant-input-affix-wrapper > input[type='text']") + .first() + .clear() + .type("New View Name"); + cy.clickOptionWithTestId("view-builder-save"); + cy.waitTextVisible("New View Name"); + + // Now make the view the default + cy.clickFirstOptionWithTestId("views-table-dropdown"); + cy.get('[data-testid="view-dropdown-set-user-default"]').click({ + force: true, + }); + // Now unset as the default + cy.clickFirstOptionWithTestId("views-table-dropdown"); + cy.get('[data-testid="view-dropdown-remove-user-default"]').click({ + force: true, }); + + // Now delete the View + cy.clickFirstOptionWithTestId("views-table-dropdown"); + cy.get('[data-testid="view-dropdown-delete"]').click({ force: true }); + cy.clickOptionWithText("Yes"); + }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/views/view_select.js b/smoke-test/tests/cypress/cypress/e2e/views/view_select.js index 752f4c768ba40..09520818a2190 100644 --- a/smoke-test/tests/cypress/cypress/e2e/views/view_select.js +++ b/smoke-test/tests/cypress/cypress/e2e/views/view_select.js @@ -8,17 +8,16 @@ function openViewEditDropDownAndClickId(data_id) { describe("view select", () => { it("click view select, create view, clear view, make defaults, clear view", () => { cy.login(); - let randomNumber = Math.floor(Math.random() * 100000); + const randomNumber = Math.floor(Math.random() * 100000); const viewName = `Test View ${randomNumber}`; const newViewName = `New View Name ${randomNumber}`; // Resize Observer Loop warning can be safely ignored - ref. https://github.com/cypress-io/cypress/issues/22113 const resizeObserverLoopErrRe = "ResizeObserver loop limit exceeded"; - cy.on("uncaught:exception", (err) => { - if (err.message.includes(resizeObserverLoopErrRe)) { - return false; - } - }); + cy.on( + "uncaught:exception", + (err) => !err.message.includes(resizeObserverLoopErrRe), + ); cy.goToStarSearchList(); diff --git a/smoke-test/tests/cypress/cypress/plugins/index.js b/smoke-test/tests/cypress/cypress/plugins/index.js index 4dff56c3fcb20..161e6895aa260 100644 --- a/smoke-test/tests/cypress/cypress/plugins/index.js +++ b/smoke-test/tests/cypress/cypress/plugins/index.js @@ -19,5 +19,7 @@ module.exports = (on, config) => { // `on` is used to hook into various events Cypress emits // `config` is the resolved Cypress config - require('cypress-timestamps/plugin')(on); -} + + // eslint-disable-next-line global-require + require("cypress-timestamps/plugin")(on); +}; diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js index c670e1b573245..b6aeccfeb81a5 100644 --- a/smoke-test/tests/cypress/cypress/support/commands.js +++ b/smoke-test/tests/cypress/cypress/support/commands.js @@ -13,48 +13,52 @@ import dayjs from "dayjs"; -function selectorWithtestId (id) { - return '[data-testid="' + id +'"]'; +function selectorWithtestId(id) { + return `[data-testid="${id}"]`; } -export function getTimestampMillisNumDaysAgo (numDays) { - return dayjs().subtract(numDays, 'day').valueOf(); +export function getTimestampMillisNumDaysAgo(numDays) { + return dayjs().subtract(numDays, "day").valueOf(); } - -Cypress.Commands.add('login', () => { +Cypress.Commands.add("login", () => { cy.request({ - method: 'POST', - url: '/logIn', + method: "POST", + url: "/logIn", body: { - username: Cypress.env('ADMIN_USERNAME'), - password: Cypress.env('ADMIN_PASSWORD'), + username: Cypress.env("ADMIN_USERNAME"), + password: Cypress.env("ADMIN_PASSWORD"), }, retryOnStatusCodeFailure: true, }); -}) +}); Cypress.Commands.add("loginWithCredentials", (username, password) => { - cy.visit('/'); - if (username,password) { - cy.get('input[data-testid=username]').type(username); - cy.get('input[data-testid=password]').type(password); + cy.visit("/"); + if ((username, password)) { + cy.get("input[data-testid=username]").type(username); + cy.get("input[data-testid=password]").type(password); } else { - cy.get('input[data-testid=username]').type(Cypress.env('ADMIN_USERNAME')); - cy.get('input[data-testid=password]').type(Cypress.env('ADMIN_PASSWORD')); + cy.get("input[data-testid=username]").type(Cypress.env("ADMIN_USERNAME")); + cy.get("input[data-testid=password]").type(Cypress.env("ADMIN_PASSWORD")); } - cy.contains('Sign In').click(); - cy.contains('Welcome back'); + cy.contains("Sign In").click(); + cy.contains("Welcome back"); }); -Cypress.Commands.add('deleteUrn', (urn) => { - cy.request({ method: 'POST', url: 'http://localhost:8080/entities?action=delete', body: { - urn - }, headers: { - "X-RestLi-Protocol-Version": "2.0.0", - "Content-Type": "application/json", - }}) -}) +Cypress.Commands.add("deleteUrn", (urn) => { + cy.request({ + method: "POST", + url: "http://localhost:8080/entities?action=delete", + body: { + urn, + }, + headers: { + "X-RestLi-Protocol-Version": "2.0.0", + "Content-Type": "application/json", + }, + }); +}); Cypress.Commands.add("logout", () => { cy.get(selectorWithtestId("manage-account-menu")).click(); @@ -107,91 +111,83 @@ Cypress.Commands.add("goToIngestionPage", () => { }); Cypress.Commands.add("goToDataset", (urn, dataset_name) => { - cy.visit( - "/dataset/" + urn - ); + cy.visit(`/dataset/${urn}`); cy.wait(5000); cy.waitTextVisible(dataset_name); }); Cypress.Commands.add("goToBusinessAttribute", (urn, attribute_name) => { - cy.visit( - "/business-attribute/" + urn - ); + cy.visit(`/business-attribute/${urn}`); cy.wait(5000); cy.waitTextVisible(attribute_name); }); Cypress.Commands.add("goToTag", (urn, tag_name) => { - cy.visit( - "/tag/" + urn - ); + cy.visit(`/tag/${urn}`); cy.wait(5000); cy.waitTextVisible(tag_name); }); Cypress.Commands.add("goToEntityLineageGraph", (entity_type, urn) => { - cy.visit( - `/${entity_type}/${urn}?is_lineage_mode=true` - ); -}) + cy.visit(`/${entity_type}/${urn}?is_lineage_mode=true`); +}); -Cypress.Commands.add("goToEntityLineageGraph", (entity_type, urn, start_time_millis, end_time_millis) => { - cy.visit( - `/${entity_type}/${urn}?is_lineage_mode=true&start_time_millis=${start_time_millis}&end_time_millis=${end_time_millis}` - ); -}) +Cypress.Commands.add( + "goToEntityLineageGraph", + (entity_type, urn, start_time_millis, end_time_millis) => { + cy.visit( + `/${entity_type}/${urn}?is_lineage_mode=true&start_time_millis=${start_time_millis}&end_time_millis=${end_time_millis}`, + ); + }, +); Cypress.Commands.add("lineageTabClickOnUpstream", () => { - cy.get('[data-testid="lineage-tab-direction-select-option-downstream"] > b').click(); - cy.get('[data-testid="lineage-tab-direction-select-option-upstream"] > b').click(); -}) - + cy.get( + '[data-testid="lineage-tab-direction-select-option-downstream"] > b', + ).click(); + cy.get( + '[data-testid="lineage-tab-direction-select-option-upstream"] > b', + ).click(); +}); Cypress.Commands.add("goToChart", (urn) => { - cy.visit( - "/chart/" + urn - ); -}) + cy.visit(`/chart/${urn}`); +}); Cypress.Commands.add("goToContainer", (urn) => { - cy.visit( - "/container/" + urn - ); -}) + cy.visit(`/container/${urn}`); +}); Cypress.Commands.add("goToDomain", (urn) => { - cy.visit( - "/domain/" + urn - ); -}) + cy.visit(`/domain/${urn}`); +}); Cypress.Commands.add("goToAnalytics", () => { cy.visit("/analytics"); - cy.contains("Data Landscape Summary", {timeout: 10000}); + cy.contains("Data Landscape Summary", { timeout: 10000 }); }); Cypress.Commands.add("goToUserList", () => { cy.visit("/settings/identities/users"); cy.waitTextVisible("Manage Users & Groups"); -}) +}); Cypress.Commands.add("goToStarSearchList", () => { - cy.visit("/search?query=%2A") - cy.waitTextVisible("Showing") - cy.waitTextVisible("results") -}) + cy.visit("/search?query=%2A"); + cy.waitTextVisible("Showing"); + cy.waitTextVisible("results"); +}); Cypress.Commands.add("openThreeDotDropdown", () => { - cy.clickOptionWithTestId("entity-header-dropdown") + cy.clickOptionWithTestId("entity-header-dropdown"); }); Cypress.Commands.add("openThreeDotMenu", () => { - cy.clickOptionWithTestId("three-dot-menu") + cy.clickOptionWithTestId("three-dot-menu"); }); Cypress.Commands.add("clickOptionWithText", (text) => { - cy.contains(text).should('be.visible').click(); + cy.contains(text).should("be.visible").click(); }); Cypress.Commands.add("clickFirstOptionWithText", (text) => { @@ -210,97 +206,100 @@ Cypress.Commands.add("deleteFromDropdown", () => { Cypress.Commands.add("addViaFormModal", (text, modelHeader) => { cy.waitTextVisible(modelHeader); - cy.get('.ProseMirror-focused').type(text); + cy.get(".ProseMirror-focused").type(text); cy.get(".ant-modal-footer > button:nth-child(2)").click(); }); Cypress.Commands.add("addViaModal", (text, modelHeader, value, dataTestId) => { cy.waitTextVisible(modelHeader); cy.get(".ant-input-affix-wrapper > input[type='text']").first().type(text); - cy.get('[data-testid="' + dataTestId + '"]').click(); - cy.contains(value).should('be.visible'); + cy.get(`[data-testid="${dataTestId}"]`).click(); + cy.contains(value).should("be.visible"); }); -Cypress.Commands.add("addBusinessAttributeViaModal", (text, modelHeader, value, dataTestId) => { - cy.waitTextVisible(modelHeader); - cy.get(".ant-input-affix-wrapper > input[type='text']").first().type(text); - cy.get('[data-testid="' + dataTestId + '"]').click(); - cy.wait(3000); - cy.contains(value).should('be.visible'); -}); +Cypress.Commands.add( + "addBusinessAttributeViaModal", + (text, modelHeader, value, dataTestId) => { + cy.waitTextVisible(modelHeader); + cy.get(".ant-input-affix-wrapper > input[type='text']").first().type(text); + cy.get(`[data-testid="${dataTestId}"]`).click(); + cy.wait(3000); + cy.contains(value).should("be.visible"); + }, +); Cypress.Commands.add("ensureTextNotPresent", (text) => { cy.contains(text).should("not.exist"); }); Cypress.Commands.add("waitTextPresent", (text) => { - cy.contains(text).should('exist'); - cy.contains(text).should('have.length.above', 0); + cy.contains(text).should("exist"); + cy.contains(text).should("have.length.above", 0); return cy.contains(text); -}) +}); Cypress.Commands.add("waitTextVisible", (text) => { - cy.contains(text).should('exist'); - cy.contains(text).should('be.visible'); - cy.contains(text).should('have.length.above', 0); + cy.contains(text).should("exist"); + cy.contains(text).should("be.visible"); + cy.contains(text).should("have.length.above", 0); return cy.contains(text); -}) +}); Cypress.Commands.add("openMultiSelect", (data_id) => { - let selector = `${selectorWithtestId(data_id)}` - cy.get(`.ant-select${selector} > .ant-select-selector > .ant-select-selection-search`).click(); -}) + const selector = `${selectorWithtestId(data_id)}`; + cy.get( + `.ant-select${selector} > .ant-select-selector > .ant-select-selection-search`, + ).click(); +}); -Cypress.Commands.add( 'multiSelect', (within_data_id , text) => { +Cypress.Commands.add("multiSelect", (within_data_id, text) => { cy.openMultiSelect(within_data_id); cy.waitTextVisible(text); cy.clickOptionWithText(text); }); -Cypress.Commands.add("getWithTestId", (id) => { - return cy.get(selectorWithtestId(id)); -}); +Cypress.Commands.add("getWithTestId", (id) => cy.get(selectorWithtestId(id))); Cypress.Commands.add("clickOptionWithId", (id) => { - cy.get(id).click() -}) + cy.get(id).click(); +}); Cypress.Commands.add("enterTextInSpecificTestId", (id, value, text) => { cy.get(selectorWithtestId(id)).eq(value).type(text); -}) +}); Cypress.Commands.add("enterTextInTestId", (id, text) => { cy.get(selectorWithtestId(id)).type(text); -}) +}); Cypress.Commands.add("clickOptionWithTestId", (id) => { cy.get(selectorWithtestId(id)).first().click({ force: true, }); -}) +}); Cypress.Commands.add("clickFirstOptionWithTestId", (id) => { cy.get(selectorWithtestId(id)).first().click({ force: true, }); -}) +}); -Cypress.Commands.add("clickFirstOptionWithSpecificTestId", (id,value) => { +Cypress.Commands.add("clickFirstOptionWithSpecificTestId", (id, value) => { cy.get(selectorWithtestId(id)).eq(value).click({ force: true, }); -}) +}); Cypress.Commands.add("clickOptionWithSpecificClass", (locator, value) => { - cy.get(locator).should('be.visible') + cy.get(locator).should("be.visible"); cy.get(locator).eq(value).click(); -}) +}); Cypress.Commands.add("clickTextOptionWithClass", (locator, text) => { - cy.get(locator).should('be.visible').contains(text).click({force:true}) -}) + cy.get(locator).should("be.visible").contains(text).click({ force: true }); +}); Cypress.Commands.add("hideOnboardingTour", () => { - cy.get('body').type("{ctrl} {meta} h"); + cy.get("body").type("{ctrl} {meta} h"); }); Cypress.Commands.add("clearView", (viewName) => { @@ -308,75 +307,83 @@ Cypress.Commands.add("clearView", (viewName) => { cy.clickOptionWithTestId("view-select-clear"); cy.get("input[data-testid='search-input']").click(); cy.contains(viewName).should("not.be.visible"); -}) +}); -Cypress.Commands.add('addTermToDataset', (urn, dataset_name, term) => { +Cypress.Commands.add("addTermToDataset", (urn, dataset_name, term) => { cy.goToDataset(urn, dataset_name); cy.clickOptionWithText("Add Term"); cy.selectOptionInTagTermModal(term); cy.contains(term); }); -Cypress.Commands.add('addTermToBusinessAttribute', (urn, attribute_name, term) => { - cy.goToBusinessAttribute(urn, attribute_name); - cy.clickOptionWithText("Add Terms"); - cy.selectOptionInTagTermModal(term); - cy.contains(term); -}); - -Cypress.Commands.add('addAttributeToDataset', (urn, dataset_name, businessAttribute) => { - cy.goToDataset(urn, dataset_name); - cy.clickOptionWithText("event_name"); - cy.contains("Business Attribute"); - cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within(() => - cy.contains("Add Attribute").click() - ); - cy.selectOptionInAttributeModal(businessAttribute); - cy.contains(businessAttribute); -}); - -Cypress.Commands.add('selectOptionInTagTermModal', (text) => { +Cypress.Commands.add( + "addTermToBusinessAttribute", + (urn, attribute_name, term) => { + cy.goToBusinessAttribute(urn, attribute_name); + cy.clickOptionWithText("Add Terms"); + cy.selectOptionInTagTermModal(term); + cy.contains(term); + }, +); + +Cypress.Commands.add( + "addAttributeToDataset", + (urn, dataset_name, businessAttribute) => { + cy.goToDataset(urn, dataset_name); + cy.clickOptionWithText("event_name"); + cy.contains("Business Attribute"); + cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within( + () => cy.contains("Add Attribute").click(), + ); + cy.selectOptionInAttributeModal(businessAttribute); + cy.contains(businessAttribute); + }, +); + +Cypress.Commands.add("selectOptionInTagTermModal", (text) => { cy.enterTextInTestId("tag-term-modal-input", text); cy.clickOptionWithTestId("tag-term-option"); - let btn_id = "add-tag-term-from-modal-btn"; + const btn_id = "add-tag-term-from-modal-btn"; cy.clickOptionWithTestId(btn_id); cy.get(selectorWithtestId(btn_id)).should("not.exist"); }); -Cypress.Commands.add('selectOptionInAttributeModal', (text) => { +Cypress.Commands.add("selectOptionInAttributeModal", (text) => { cy.enterTextInTestId("business-attribute-modal-input", text); cy.clickOptionWithTestId("business-attribute-option"); - let btn_id = "add-attribute-from-modal-btn"; + const btn_id = "add-attribute-from-modal-btn"; cy.clickOptionWithTestId(btn_id); cy.get(selectorWithtestId(btn_id)).should("not.exist"); }); -Cypress.Commands.add("removeDomainFromDataset", (urn, dataset_name, domain_urn) => { - cy.goToDataset(urn, dataset_name); - cy.get('.sidebar-domain-section [href="/domain/' + domain_urn + '"] .anticon-close').click(); - cy.clickOptionWithText("Yes"); -}) +Cypress.Commands.add( + "removeDomainFromDataset", + (urn, dataset_name, domain_urn) => { + cy.goToDataset(urn, dataset_name); + cy.get( + `.sidebar-domain-section [href="/domain/${domain_urn}"] .anticon-close`, + ).click(); + cy.clickOptionWithText("Yes"); + }, +); Cypress.Commands.add("openEntityTab", (tab) => { - const selector = 'div[id$="' + tab + '"]:nth-child(1)' + const selector = `div[id$="${tab}"]:nth-child(1)`; cy.highlighElement(selector); - cy.get(selector).click() + cy.get(selector).click(); }); Cypress.Commands.add("highlighElement", (selector) => { cy.wait(3000); - cy.get(selector).then($button => { - $button.css('border', '1px solid magenta') - }) + cy.get(selector).then(($button) => { + $button.css("border", "1px solid magenta"); + }); cy.wait(3000); -}) +}); -Cypress.Commands.add("mouseover", (selector) => { - return cy.get(selector).trigger( - "mouseover", - { force: true } - ); -}) +Cypress.Commands.add("mouseover", (selector) => + cy.get(selector).trigger("mouseover", { force: true }), +); Cypress.Commands.add("createUser", (name, password, email) => { cy.visit("/settings/identities/users"); @@ -396,13 +403,13 @@ Cypress.Commands.add("createUser", (name, password, email) => { cy.waitTextVisible("Welcome to DataHub"); cy.hideOnboardingTour(); cy.waitTextVisible(name); - cy.logout() + cy.logout(); cy.loginWithCredentials(); - }) -}) + }); +}); Cypress.Commands.add("createGroup", (name, description, group_id) => { - cy.visit("/settings/identities/groups") + cy.visit("/settings/identities/groups"); cy.clickOptionWithText("Create group"); cy.waitTextVisible("Create new group"); cy.get("#name").type(name); @@ -413,10 +420,10 @@ Cypress.Commands.add("createGroup", (name, description, group_id) => { cy.get("#createGroupButton").click(); cy.waitTextVisible("Created group!"); cy.waitTextVisible(name); -}) +}); Cypress.Commands.add("addGroupMember", (group_name, group_urn, member_name) => { - cy.visit(group_urn) + cy.visit(group_urn); cy.clickOptionWithText(group_name); cy.contains(group_name).should("be.visible"); cy.get('[role="tab"]').contains("Members").click(); @@ -428,20 +435,21 @@ Cypress.Commands.add("addGroupMember", (group_name, group_urn, member_name) => { cy.contains(member_name).should("have.length", 1); cy.get('[role="dialog"] button').contains("Add").click({ force: true }); cy.waitTextVisible("Group members added!"); - cy.contains(member_name, {timeout: 10000}).should("be.visible"); -}) + cy.contains(member_name, { timeout: 10000 }).should("be.visible"); +}); Cypress.Commands.add("createGlossaryTermGroup", (term_group_name) => { cy.goToGlossaryList(); - cy.clickOptionWithText('Add Term Group'); + cy.clickOptionWithText("Add Term Group"); cy.waitTextVisible("Create Term Group"); cy.enterTextInTestId("create-glossary-entity-modal-name", term_group_name); cy.clickOptionWithTestId("glossary-entity-modal-create-button"); - cy.get('[data-testid="glossary-browser-sidebar"]').contains(term_group_name).should("be.visible"); + cy.get('[data-testid="glossary-browser-sidebar"]') + .contains(term_group_name) + .should("be.visible"); cy.waitTextVisible(`Created Term Group!`); }); - // // // -- This is a child command -- diff --git a/smoke-test/tests/cypress/cypress/support/e2e.js b/smoke-test/tests/cypress/cypress/support/e2e.js index 751649a8d6cd6..dee6d14ace4dd 100644 --- a/smoke-test/tests/cypress/cypress/support/e2e.js +++ b/smoke-test/tests/cypress/cypress/support/e2e.js @@ -14,14 +14,14 @@ // *********************************************************** // Import commands.js using ES2015 syntax: -import './commands' +import "./commands"; // Alternatively you can use CommonJS syntax: // require('./commands') // https://github.com/bahmutov/cypress-timestamps -require('cypress-timestamps/support')({ - terminal: true, // by default the terminal output is disabled - error: true, - commandLog: true, -}); \ No newline at end of file +require("cypress-timestamps/support")({ + terminal: true, // by default the terminal output is disabled + error: true, + commandLog: true, +}); diff --git a/smoke-test/tests/cypress/cypress_dbt_data.json b/smoke-test/tests/cypress/cypress_dbt_data.json index 087af0f3704c6..3e0c3a9a4d54a 100644 --- a/smoke-test/tests/cypress/cypress_dbt_data.json +++ b/smoke-test/tests/cypress/cypress_dbt_data.json @@ -1,5 +1,5 @@ [ -{ + { "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:b5e95fce839e7d78151ed7e0a7420d84", @@ -7,18 +7,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"bigquery\", \"instance\": \"PROD\", \"project_id\": \"cypress_project\"}, \"name\": \"cypress_project\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"bigquery\", \"instance\": \"PROD\", \"project_id\": \"cypress_project\"}, \"name\": \"cypress_project\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162350940, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162350940, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:b5e95fce839e7d78151ed7e0a7420d84", @@ -26,18 +26,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:bigquery\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:bigquery\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162350941, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162350941, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:b5e95fce839e7d78151ed7e0a7420d84", @@ -45,18 +45,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Project\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Project\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162350942, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162350942, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb", @@ -64,18 +64,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"bigquery\", \"instance\": \"PROD\", \"project_id\": \"cypress_project\", \"dataset_id\": \"jaffle_shop\"}, \"name\": \"jaffle_shop\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"bigquery\", \"instance\": \"PROD\", \"project_id\": \"cypress_project\", \"dataset_id\": \"jaffle_shop\"}, \"name\": \"jaffle_shop\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162353361, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162353361, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb", @@ -83,18 +83,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:bigquery\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:bigquery\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162353362, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162353362, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb", @@ -102,18 +102,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Dataset\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Dataset\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162353363, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162353363, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb", @@ -121,18 +121,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b5e95fce839e7d78151ed7e0a7420d84\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b5e95fce839e7d78151ed7e0a7420d84\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162353364, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162353364, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", @@ -140,223 +140,223 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162353970, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162353970, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "customers", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.customers", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_order", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "most_recent_order", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "customers", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "number_of_orders", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.customers", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_order", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "most_recent_order", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "number_of_orders", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_lifetime_value", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Float()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "customer_lifetime_value", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162353971, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162353971, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", @@ -364,18 +364,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162353980, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162353980, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", @@ -383,123 +383,123 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162354204, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354204, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "customers_source", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.customers_source", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "customers_source", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "source_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.customers_source", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "source_name", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "siour", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "siour", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162354206, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354206, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", @@ -507,18 +507,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162354211, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354211, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", @@ -526,263 +526,263 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162354420, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354420, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "orders", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.orders", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "orders", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "order_date", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.orders", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "order_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_date", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "status", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "credit_card_amount", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Float()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "coupon_amount", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Float()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "bank_transfer_amount", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Float()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "gift_card_amount", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Float()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Float()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "status", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "credit_card_amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "coupon_amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "bank_transfer_amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gift_card_amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162354421, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354421, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", @@ -790,18 +790,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162354427, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354427, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", @@ -809,143 +809,143 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162354667, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354667, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "raw_customers", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.raw_customers", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "raw_customers", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "first_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.raw_customers", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "last_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162354668, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354668, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", @@ -953,18 +953,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162354671, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354671, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", @@ -972,163 +972,163 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162354871, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354871, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "raw_orders", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.raw_orders", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "raw_orders", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "user_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.raw_orders", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_date", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "status", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "order_date", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162354873, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354873, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", @@ -1136,18 +1136,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162354879, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162354879, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", @@ -1155,163 +1155,163 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162355105, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162355105, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "raw_payments", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.raw_payments", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "raw_payments", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "order_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.raw_payments", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_method", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162355107, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162355107, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", @@ -1319,18 +1319,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162355113, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162355113, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", @@ -1338,146 +1338,146 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162355777, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162355777, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_customers`\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "is_view": "True" + }, + "externalUrl": null, + "name": "stg_customers", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.stg_customers", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_customers`\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", - "is_view": "True" - }, - "externalUrl": null, - "name": "stg_customers", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "first_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.stg_customers", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "last_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162355778, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162355778, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", @@ -1485,18 +1485,18 @@ "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162355783, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162355783, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", @@ -1504,18 +1504,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162355784, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162355784, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", @@ -1523,18 +1523,18 @@ "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_customers`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as customer_id,\\n first_name,\\n last_name\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_customers`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as customer_id,\\n first_name,\\n last_name\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162355785, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162355785, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", @@ -1542,166 +1542,166 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162356113, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356113, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_orders`\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "is_view": "True" + }, + "externalUrl": null, + "name": "stg_orders", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.stg_orders", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_orders`\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", - "is_view": "True" - }, - "externalUrl": null, - "name": "stg_orders", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "order_date", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.stg_orders", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "order_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_date", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "status", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "status", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162356115, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356115, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", @@ -1709,18 +1709,18 @@ "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162356123, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356123, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", @@ -1728,18 +1728,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162356124, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356124, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", @@ -1747,18 +1747,18 @@ "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_orders`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as order_id,\\n user_id as customer_id,\\n order_date,\\n status\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_orders`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as order_id,\\n user_id as customer_id,\\n order_date,\\n status\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162356125, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356125, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", @@ -1766,166 +1766,166 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162356440, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356440, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_payments`\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n --`amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "is_view": "True" + }, + "externalUrl": null, + "name": "stg_payments", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.stg_payments", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "order_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_payments`\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n --`amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", - "is_view": "True" - }, - "externalUrl": null, - "name": "stg_payments", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "cypress_project.jaffle_shop.stg_payments", - "platform": "urn:li:dataPlatform:bigquery", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_id", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Integer()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_method", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "String()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": true, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "Float()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1655162356441, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356441, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", @@ -1933,18 +1933,18 @@ "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162356445, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356445, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", @@ -1952,18 +1952,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162356446, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356446, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", @@ -1971,18 +1971,18 @@ "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_payments`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as payment_id,\\n order_id,\\n payment_method,\\n\\n --`amount` is currently stored in cents, so we convert it to dollars\\n amount / 100 as amount\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_payments`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as payment_id,\\n order_id,\\n payment_method,\\n\\n --`amount` is currently stored in cents, so we convert it to dollars\\n amount / 100 as amount\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162356446, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162356446, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", @@ -1990,18 +1990,18 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1655162357476, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 100, \"columnCount\": 7, \"fieldProfiles\": [{\"fieldPath\": \"customer_id\", \"uniqueCount\": 100, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"61\", \"74\", \"48\", \"75\", \"87\", \"14\", \"37\", \"55\", \"49\", \"78\", \"77\", \"10\", \"15\", \"60\", \"24\", \"45\", \"62\", \"98\", \"5\", \"97\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 79, \"uniqueProportion\": 0.79, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Timothy\", \"Harry\", \"Lillian\", \"Andrea\", \"Phillip\", \"Steve\", \"Shirley\", \"Nicholas\", \"Judy\", \"Harry\", \"Anne\", \"Henry\", \"Teresa\", \"Norma\", \"David\", \"Scott\", \"Elizabeth\", \"Nicole\", \"Katherine\", \"Shirley\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 19, \"uniqueProportion\": 0.19, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"R.\", \"A.\", \"C.\", \"H.\", \"B.\", \"F.\", \"J.\", \"R.\", \"N.\", \"H.\", \"W.\", \"W.\", \"H.\", \"W.\", \"G.\", \"B.\", \"P.\", \"M.\", \"R.\", \"D.\"]}, {\"fieldPath\": \"first_order\", \"uniqueCount\": 46, \"uniqueProportion\": 0.7419354838709677, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"2018-01-01\", \"max\": \"2018-04-07\", \"sampleValues\": [\"2018-03-01\", \"2018-03-23\", \"2018-02-26\", \"2018-01-17\", \"2018-02-04\", \"2018-03-23\", \"2018-03-16\", \"2018-03-03\", \"2018-01-24\", \"2018-02-19\", \"2018-01-18\", \"2018-04-07\", \"2018-02-02\", \"2018-04-07\", \"2018-02-13\", \"2018-01-23\", \"2018-02-06\", \"2018-01-09\", \"2018-02-16\", \"2018-02-17\"]}, {\"fieldPath\": \"most_recent_order\", \"uniqueCount\": 52, \"uniqueProportion\": 0.8387096774193549, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"2018-01-09\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-01\", \"2018-03-23\", \"2018-02-26\", \"2018-01-17\", \"2018-02-04\", \"2018-03-23\", \"2018-03-16\", \"2018-03-03\", \"2018-01-24\", \"2018-02-19\", \"2018-01-18\", \"2018-04-07\", \"2018-02-02\", \"2018-04-07\", \"2018-02-13\", \"2018-01-23\", \"2018-02-06\", \"2018-01-09\", \"2018-02-16\", \"2018-02-17\"]}, {\"fieldPath\": \"number_of_orders\", \"uniqueCount\": 4, \"uniqueProportion\": 0.06451612903225806, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"1\", \"max\": \"5\", \"mean\": \"1.5967741935483863\", \"median\": \"1.0\", \"stdev\": \"0.7779687173818426\", \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"customer_lifetime_value\", \"uniqueCount\": 35, \"uniqueProportion\": 0.5645161290322581, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"1.0\", \"max\": \"99.0\", \"mean\": \"26.967741935483883\", \"median\": \"26.5\", \"sampleValues\": [\"2.0\", \"2.0\", \"3.0\", \"3.0\", \"3.0\", \"3.0\", \"3.0\", \"4.0\", \"8.0\", \"8.0\", \"10.0\", \"10.0\", \"12.0\", \"14.0\", \"14.0\", \"15.0\", \"15.0\", \"16.0\", \"17.0\", \"18.0\"]}]}", - "contentType": "application/json" + "value": "{\"timestampMillis\": 1655162357476, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 100, \"columnCount\": 7, \"fieldProfiles\": [{\"fieldPath\": \"customer_id\", \"uniqueCount\": 100, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"61\", \"74\", \"48\", \"75\", \"87\", \"14\", \"37\", \"55\", \"49\", \"78\", \"77\", \"10\", \"15\", \"60\", \"24\", \"45\", \"62\", \"98\", \"5\", \"97\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 79, \"uniqueProportion\": 0.79, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Timothy\", \"Harry\", \"Lillian\", \"Andrea\", \"Phillip\", \"Steve\", \"Shirley\", \"Nicholas\", \"Judy\", \"Harry\", \"Anne\", \"Henry\", \"Teresa\", \"Norma\", \"David\", \"Scott\", \"Elizabeth\", \"Nicole\", \"Katherine\", \"Shirley\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 19, \"uniqueProportion\": 0.19, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"R.\", \"A.\", \"C.\", \"H.\", \"B.\", \"F.\", \"J.\", \"R.\", \"N.\", \"H.\", \"W.\", \"W.\", \"H.\", \"W.\", \"G.\", \"B.\", \"P.\", \"M.\", \"R.\", \"D.\"]}, {\"fieldPath\": \"first_order\", \"uniqueCount\": 46, \"uniqueProportion\": 0.7419354838709677, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"2018-01-01\", \"max\": \"2018-04-07\", \"sampleValues\": [\"2018-03-01\", \"2018-03-23\", \"2018-02-26\", \"2018-01-17\", \"2018-02-04\", \"2018-03-23\", \"2018-03-16\", \"2018-03-03\", \"2018-01-24\", \"2018-02-19\", \"2018-01-18\", \"2018-04-07\", \"2018-02-02\", \"2018-04-07\", \"2018-02-13\", \"2018-01-23\", \"2018-02-06\", \"2018-01-09\", \"2018-02-16\", \"2018-02-17\"]}, {\"fieldPath\": \"most_recent_order\", \"uniqueCount\": 52, \"uniqueProportion\": 0.8387096774193549, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"2018-01-09\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-01\", \"2018-03-23\", \"2018-02-26\", \"2018-01-17\", \"2018-02-04\", \"2018-03-23\", \"2018-03-16\", \"2018-03-03\", \"2018-01-24\", \"2018-02-19\", \"2018-01-18\", \"2018-04-07\", \"2018-02-02\", \"2018-04-07\", \"2018-02-13\", \"2018-01-23\", \"2018-02-06\", \"2018-01-09\", \"2018-02-16\", \"2018-02-17\"]}, {\"fieldPath\": \"number_of_orders\", \"uniqueCount\": 4, \"uniqueProportion\": 0.06451612903225806, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"1\", \"max\": \"5\", \"mean\": \"1.5967741935483863\", \"median\": \"1.0\", \"stdev\": \"0.7779687173818426\", \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"customer_lifetime_value\", \"uniqueCount\": 35, \"uniqueProportion\": 0.5645161290322581, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"1.0\", \"max\": \"99.0\", \"mean\": \"26.967741935483883\", \"median\": \"26.5\", \"sampleValues\": [\"2.0\", \"2.0\", \"3.0\", \"3.0\", \"3.0\", \"3.0\", \"3.0\", \"4.0\", \"8.0\", \"8.0\", \"10.0\", \"10.0\", \"12.0\", \"14.0\", \"14.0\", \"15.0\", \"15.0\", \"16.0\", \"17.0\", \"18.0\"]}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162378272, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162378272, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", @@ -2009,18 +2009,18 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1655162357619, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 0, \"columnCount\": 2, \"fieldProfiles\": [{\"fieldPath\": \"source_name\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}, {\"fieldPath\": \"siour\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}]}", - "contentType": "application/json" + "value": "{\"timestampMillis\": 1655162357619, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 0, \"columnCount\": 2, \"fieldProfiles\": [{\"fieldPath\": \"source_name\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}, {\"fieldPath\": \"siour\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162378286, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162378286, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", @@ -2028,18 +2028,18 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1655162357642, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 99, \"columnCount\": 9, \"fieldProfiles\": [{\"fieldPath\": \"order_id\", \"uniqueCount\": 99, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"86\", \"4\", \"9\", \"44\", \"24\", \"3\", \"62\", \"95\", \"81\", \"65\", \"94\", \"42\", \"19\", \"23\", \"58\", \"59\", \"76\", \"43\", \"93\", \"15\"]}, {\"fieldPath\": \"customer_id\", \"uniqueCount\": 62, \"uniqueProportion\": 0.6262626262626263, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"48.25252525252523\", \"median\": \"50\", \"stdev\": \"27.781341350472964\", \"sampleValues\": [\"68\", \"50\", \"53\", \"66\", \"3\", \"94\", \"57\", \"27\", \"76\", \"26\", \"63\", \"92\", \"54\", \"22\", \"22\", \"30\", \"25\", \"31\", \"66\", \"25\"]}, {\"fieldPath\": \"order_date\", \"uniqueCount\": 69, \"uniqueProportion\": 0.696969696969697, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"2018-01-01\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-26\", \"2018-01-05\", \"2018-01-12\", \"2018-02-17\", \"2018-01-27\", \"2018-01-04\", \"2018-03-05\", \"2018-04-04\", \"2018-03-23\", \"2018-03-08\", \"2018-04-03\", \"2018-02-16\", \"2018-01-22\", \"2018-01-26\", \"2018-03-01\", \"2018-03-02\", \"2018-03-20\", \"2018-02-17\", \"2018-04-03\", \"2018-01-17\"]}, {\"fieldPath\": \"status\", \"uniqueCount\": 5, \"uniqueProportion\": 0.050505050505050504, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"placed\", \"completed\", \"completed\", \"completed\", \"completed\", \"completed\", \"completed\", \"placed\", \"shipped\", \"completed\", \"placed\", \"completed\", \"completed\", \"return_pending\", \"completed\", \"completed\", \"completed\", \"completed\", \"placed\", \"completed\"]}, {\"fieldPath\": \"credit_card_amount\", \"uniqueCount\": 25, \"uniqueProportion\": 0.25252525252525254, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"30.0\", \"mean\": \"8.797979797979806\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\"]}, {\"fieldPath\": \"coupon_amount\", \"uniqueCount\": 12, \"uniqueProportion\": 0.12121212121212122, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"26.0\", \"mean\": \"1.8686868686868698\", \"median\": \"0.0\", \"sampleValues\": [\"23.0\", \"25.0\", \"0.0\", \"0.0\", \"26.0\", \"1.0\", \"0.0\", \"24.0\", \"2.0\", \"0.0\", \"7.0\", \"17.0\", \"0.0\", \"0.0\", \"18.0\", \"0.0\", \"2.0\", \"0.0\", \"0.0\", \"22.0\"]}, {\"fieldPath\": \"bank_transfer_amount\", \"uniqueCount\": 19, \"uniqueProportion\": 0.1919191919191919, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"26.0\", \"mean\": \"4.151515151515151\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\"]}, {\"fieldPath\": \"gift_card_amount\", \"uniqueCount\": 11, \"uniqueProportion\": 0.1111111111111111, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"30.0\", \"mean\": \"2.07070707070707\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"23.0\", \"11.0\", \"0.0\", \"0.0\", \"14.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"6.0\", \"23.0\", \"6.0\", \"28.0\", \"0.0\", \"18.0\", \"26.0\", \"0.0\"]}, {\"fieldPath\": \"amount\", \"uniqueCount\": 32, \"uniqueProportion\": 0.32323232323232326, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"58.0\", \"mean\": \"16.888888888888882\", \"median\": \"17.0\", \"sampleValues\": [\"23.0\", \"25.0\", \"23.0\", \"11.0\", \"26.0\", \"1.0\", \"14.0\", \"24.0\", \"2.0\", \"0.0\", \"7.0\", \"17.0\", \"6.0\", \"23.0\", \"24.0\", \"28.0\", \"2.0\", \"18.0\", \"26.0\", \"22.0\"]}]}", - "contentType": "application/json" + "value": "{\"timestampMillis\": 1655162357642, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 99, \"columnCount\": 9, \"fieldProfiles\": [{\"fieldPath\": \"order_id\", \"uniqueCount\": 99, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"86\", \"4\", \"9\", \"44\", \"24\", \"3\", \"62\", \"95\", \"81\", \"65\", \"94\", \"42\", \"19\", \"23\", \"58\", \"59\", \"76\", \"43\", \"93\", \"15\"]}, {\"fieldPath\": \"customer_id\", \"uniqueCount\": 62, \"uniqueProportion\": 0.6262626262626263, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"48.25252525252523\", \"median\": \"50\", \"stdev\": \"27.781341350472964\", \"sampleValues\": [\"68\", \"50\", \"53\", \"66\", \"3\", \"94\", \"57\", \"27\", \"76\", \"26\", \"63\", \"92\", \"54\", \"22\", \"22\", \"30\", \"25\", \"31\", \"66\", \"25\"]}, {\"fieldPath\": \"order_date\", \"uniqueCount\": 69, \"uniqueProportion\": 0.696969696969697, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"2018-01-01\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-26\", \"2018-01-05\", \"2018-01-12\", \"2018-02-17\", \"2018-01-27\", \"2018-01-04\", \"2018-03-05\", \"2018-04-04\", \"2018-03-23\", \"2018-03-08\", \"2018-04-03\", \"2018-02-16\", \"2018-01-22\", \"2018-01-26\", \"2018-03-01\", \"2018-03-02\", \"2018-03-20\", \"2018-02-17\", \"2018-04-03\", \"2018-01-17\"]}, {\"fieldPath\": \"status\", \"uniqueCount\": 5, \"uniqueProportion\": 0.050505050505050504, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"placed\", \"completed\", \"completed\", \"completed\", \"completed\", \"completed\", \"completed\", \"placed\", \"shipped\", \"completed\", \"placed\", \"completed\", \"completed\", \"return_pending\", \"completed\", \"completed\", \"completed\", \"completed\", \"placed\", \"completed\"]}, {\"fieldPath\": \"credit_card_amount\", \"uniqueCount\": 25, \"uniqueProportion\": 0.25252525252525254, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"30.0\", \"mean\": \"8.797979797979806\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\"]}, {\"fieldPath\": \"coupon_amount\", \"uniqueCount\": 12, \"uniqueProportion\": 0.12121212121212122, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"26.0\", \"mean\": \"1.8686868686868698\", \"median\": \"0.0\", \"sampleValues\": [\"23.0\", \"25.0\", \"0.0\", \"0.0\", \"26.0\", \"1.0\", \"0.0\", \"24.0\", \"2.0\", \"0.0\", \"7.0\", \"17.0\", \"0.0\", \"0.0\", \"18.0\", \"0.0\", \"2.0\", \"0.0\", \"0.0\", \"22.0\"]}, {\"fieldPath\": \"bank_transfer_amount\", \"uniqueCount\": 19, \"uniqueProportion\": 0.1919191919191919, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"26.0\", \"mean\": \"4.151515151515151\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\"]}, {\"fieldPath\": \"gift_card_amount\", \"uniqueCount\": 11, \"uniqueProportion\": 0.1111111111111111, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"30.0\", \"mean\": \"2.07070707070707\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"23.0\", \"11.0\", \"0.0\", \"0.0\", \"14.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"6.0\", \"23.0\", \"6.0\", \"28.0\", \"0.0\", \"18.0\", \"26.0\", \"0.0\"]}, {\"fieldPath\": \"amount\", \"uniqueCount\": 32, \"uniqueProportion\": 0.32323232323232326, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"58.0\", \"mean\": \"16.888888888888882\", \"median\": \"17.0\", \"sampleValues\": [\"23.0\", \"25.0\", \"23.0\", \"11.0\", \"26.0\", \"1.0\", \"14.0\", \"24.0\", \"2.0\", \"0.0\", \"7.0\", \"17.0\", \"6.0\", \"23.0\", \"24.0\", \"28.0\", \"2.0\", \"18.0\", \"26.0\", \"22.0\"]}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162388123, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162388123, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", @@ -2047,18 +2047,18 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1655162357386, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 100, \"columnCount\": 3, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 100, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"20\", \"23\", \"40\", \"59\", \"74\", \"96\", \"27\", \"45\", \"53\", \"73\", \"87\", \"4\", \"41\", \"46\", \"48\", \"64\", \"71\", \"86\", \"12\", \"82\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 79, \"uniqueProportion\": 0.79, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Anna\", \"Mildred\", \"Maria\", \"Adam\", \"Harry\", \"Jacqueline\", \"Benjamin\", \"Scott\", \"Anne\", \"Alan\", \"Phillip\", \"Jimmy\", \"Gloria\", \"Norma\", \"Lillian\", \"David\", \"Gerald\", \"Jason\", \"Amy\", \"Arthur\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 19, \"uniqueProportion\": 0.19, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"A.\", \"A.\", \"A.\", \"A.\", \"A.\", \"A.\", \"B.\", \"B.\", \"B.\", \"B.\", \"B.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"D.\", \"D.\"]}]}", - "contentType": "application/json" + "value": "{\"timestampMillis\": 1655162357386, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 100, \"columnCount\": 3, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 100, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"20\", \"23\", \"40\", \"59\", \"74\", \"96\", \"27\", \"45\", \"53\", \"73\", \"87\", \"4\", \"41\", \"46\", \"48\", \"64\", \"71\", \"86\", \"12\", \"82\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 79, \"uniqueProportion\": 0.79, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Anna\", \"Mildred\", \"Maria\", \"Adam\", \"Harry\", \"Jacqueline\", \"Benjamin\", \"Scott\", \"Anne\", \"Alan\", \"Phillip\", \"Jimmy\", \"Gloria\", \"Norma\", \"Lillian\", \"David\", \"Gerald\", \"Jason\", \"Amy\", \"Arthur\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 19, \"uniqueProportion\": 0.19, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"A.\", \"A.\", \"A.\", \"A.\", \"A.\", \"A.\", \"B.\", \"B.\", \"B.\", \"B.\", \"B.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"D.\", \"D.\"]}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162388138, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162388138, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", @@ -2066,18 +2066,18 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1655162357622, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 99, \"columnCount\": 4, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 99, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"84\", \"86\", \"87\", \"89\", \"91\", \"92\", \"93\", \"94\", \"95\", \"96\", \"97\", \"98\", \"99\", \"71\", \"72\", \"74\", \"77\", \"78\", \"79\", \"80\"]}, {\"fieldPath\": \"user_id\", \"uniqueCount\": 62, \"uniqueProportion\": 0.6262626262626263, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"48.252525252525245\", \"median\": \"50\", \"stdev\": \"27.781341350472957\", \"sampleValues\": [\"70\", \"68\", \"46\", \"21\", \"47\", \"84\", \"66\", \"63\", \"27\", \"90\", \"89\", \"41\", \"85\", \"42\", \"30\", \"9\", \"35\", \"90\", \"52\", \"11\"]}, {\"fieldPath\": \"order_date\", \"uniqueCount\": 69, \"uniqueProportion\": 0.696969696969697, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"2018-01-01\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-26\", \"2018-03-26\", \"2018-03-27\", \"2018-03-28\", \"2018-03-31\", \"2018-04-02\", \"2018-04-03\", \"2018-04-03\", \"2018-04-04\", \"2018-04-06\", \"2018-04-07\", \"2018-04-07\", \"2018-04-09\", \"2018-03-12\", \"2018-03-14\", \"2018-03-17\", \"2018-03-21\", \"2018-03-23\", \"2018-03-23\", \"2018-03-23\"]}, {\"fieldPath\": \"status\", \"uniqueCount\": 5, \"uniqueProportion\": 0.050505050505050504, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\"]}]}", - "contentType": "application/json" + "value": "{\"timestampMillis\": 1655162357622, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 99, \"columnCount\": 4, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 99, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"84\", \"86\", \"87\", \"89\", \"91\", \"92\", \"93\", \"94\", \"95\", \"96\", \"97\", \"98\", \"99\", \"71\", \"72\", \"74\", \"77\", \"78\", \"79\", \"80\"]}, {\"fieldPath\": \"user_id\", \"uniqueCount\": 62, \"uniqueProportion\": 0.6262626262626263, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"48.252525252525245\", \"median\": \"50\", \"stdev\": \"27.781341350472957\", \"sampleValues\": [\"70\", \"68\", \"46\", \"21\", \"47\", \"84\", \"66\", \"63\", \"27\", \"90\", \"89\", \"41\", \"85\", \"42\", \"30\", \"9\", \"35\", \"90\", \"52\", \"11\"]}, {\"fieldPath\": \"order_date\", \"uniqueCount\": 69, \"uniqueProportion\": 0.696969696969697, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"2018-01-01\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-26\", \"2018-03-26\", \"2018-03-27\", \"2018-03-28\", \"2018-03-31\", \"2018-04-02\", \"2018-04-03\", \"2018-04-03\", \"2018-04-04\", \"2018-04-06\", \"2018-04-07\", \"2018-04-07\", \"2018-04-09\", \"2018-03-12\", \"2018-03-14\", \"2018-03-17\", \"2018-03-21\", \"2018-03-23\", \"2018-03-23\", \"2018-03-23\"]}, {\"fieldPath\": \"status\", \"uniqueCount\": 5, \"uniqueProportion\": 0.050505050505050504, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\"]}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162388145, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162388145, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", @@ -2085,2171 +2085,2171 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1655162357609, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 113, \"columnCount\": 4, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 113, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"66\", \"27\", \"30\", \"109\", \"3\", \"17\", \"47\", \"108\", \"4\", \"86\", \"93\", \"106\", \"98\", \"48\", \"107\", \"92\", \"49\", \"22\", \"67\", \"71\"]}, {\"fieldPath\": \"order_id\", \"uniqueCount\": 99, \"uniqueProportion\": 0.8761061946902655, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"50.03539823008851\", \"median\": \"51\", \"stdev\": \"28.54317819535489\", \"sampleValues\": [\"58\", \"24\", \"25\", \"95\", \"3\", \"15\", \"42\", \"94\", \"4\", \"76\", \"81\", \"92\", \"86\", \"43\", \"93\", \"80\", \"44\", \"19\", \"58\", \"62\"]}, {\"fieldPath\": \"payment_method\", \"uniqueCount\": 4, \"uniqueProportion\": 0.035398230088495575, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\"]}, {\"fieldPath\": \"amount\", \"uniqueCount\": 30, \"uniqueProportion\": 0.26548672566371684, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0\", \"max\": \"3000\", \"mean\": \"1479.6460176991145\", \"median\": \"1500\", \"stdev\": \"919.836873351873\", \"sampleValues\": [\"1800\", \"2600\", \"1600\", \"2400\", \"100\", \"2200\", \"1700\", \"700\", \"2500\", \"200\", \"200\", \"200\", \"2300\", \"1800\", \"2600\", \"300\", \"1100\", \"600\", \"600\", \"1400\"]}]}", - "contentType": "application/json" + "value": "{\"timestampMillis\": 1655162357609, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 113, \"columnCount\": 4, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 113, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"66\", \"27\", \"30\", \"109\", \"3\", \"17\", \"47\", \"108\", \"4\", \"86\", \"93\", \"106\", \"98\", \"48\", \"107\", \"92\", \"49\", \"22\", \"67\", \"71\"]}, {\"fieldPath\": \"order_id\", \"uniqueCount\": 99, \"uniqueProportion\": 0.8761061946902655, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"50.03539823008851\", \"median\": \"51\", \"stdev\": \"28.54317819535489\", \"sampleValues\": [\"58\", \"24\", \"25\", \"95\", \"3\", \"15\", \"42\", \"94\", \"4\", \"76\", \"81\", \"92\", \"86\", \"43\", \"93\", \"80\", \"44\", \"19\", \"58\", \"62\"]}, {\"fieldPath\": \"payment_method\", \"uniqueCount\": 4, \"uniqueProportion\": 0.035398230088495575, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\"]}, {\"fieldPath\": \"amount\", \"uniqueCount\": 30, \"uniqueProportion\": 0.26548672566371684, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0\", \"max\": \"3000\", \"mean\": \"1479.6460176991145\", \"median\": \"1500\", \"stdev\": \"919.836873351873\", \"sampleValues\": [\"1800\", \"2600\", \"1600\", \"2400\", \"100\", \"2200\", \"1700\", \"700\", \"2500\", \"200\", \"200\", \"200\", \"2300\", \"1800\", \"2600\", \"300\", \"1100\", \"600\", \"600\", \"1400\"]}]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1655162388150, - "runId": "bigquery-2022_06_13-16_18_59", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1655162388150, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null } -}, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322398, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/orders.sql", - "catalog_type": "table", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "orders", - "qualifiedName": null, - "description": "This table has basic information about orders, as well as some derived facts based on payments", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.jaffle_shop.orders", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "order_id", - "jsonPath": null, - "nullable": false, - "description": "This is a unique identifier for an order", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": "Foreign key to the customers table", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_date", - "jsonPath": null, - "nullable": false, - "description": "Date (UTC) that the order was placed", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "status", - "jsonPath": null, - "nullable": false, - "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "credit_card_amount", - "jsonPath": null, - "nullable": false, - "description": "Amount of the order (AUD) paid for by credit card", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "FLOAT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "coupon_amount", - "jsonPath": null, - "nullable": false, - "description": "Amount of the order (AUD) paid for by coupon", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "FLOAT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "bank_transfer_amount", - "jsonPath": null, - "nullable": false, - "description": "Amount of the order (AUD) paid for by bank transfer", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "FLOAT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "gift_card_amount", - "jsonPath": null, - "nullable": false, - "description": "Amount of the order (AUD) paid for by gift card", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "FLOAT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": "Total amount (AUD) of the order", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "FLOAT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": true, - "viewLogic": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by 1\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n left join order_payments using (order_id)\n\n)\n\nselect * from final", - "viewLanguage": "SQL" - } + "systemMetadata": { + "lastObserved": 1655162322398, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/orders.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "orders", + "qualifiedName": null, + "description": "This table has basic information about orders, as well as some derived facts based on payments", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for an order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "Foreign key to the customers table", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) that the order was placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "credit_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by credit card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "coupon_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by coupon", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "bank_transfer_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by bank transfer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - ] + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gift_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by gift card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": "Total amount (AUD) of the order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322399, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by 1\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n left join order_payments using (order_id)\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322404, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322399, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/staging/stg_customers.sql", - "catalog_type": "view", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "stg_customers", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.jaffle_shop.stg_customers", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", - "viewLanguage": "SQL" - } + "systemMetadata": { + "lastObserved": 1655162322404, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_customers.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "stg_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - ] + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322405, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322409, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322405, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/staging/stg_payments.sql", - "catalog_type": "view", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "stg_payments", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.jaffle_shop.stg_payments", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_method", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "FLOAT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "with source as (\n \n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n --`amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", - "viewLanguage": "SQL" - } + "systemMetadata": { + "lastObserved": 1655162322409, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_payments.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "stg_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - ] + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n \n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n --`amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322410, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322414, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322410, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/staging/stg_orders.sql", - "catalog_type": "view", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "stg_orders", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.jaffle_shop.stg_orders", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "order_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "status", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", - "viewLanguage": "SQL" - } + "systemMetadata": { + "lastObserved": 1655162322414, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_orders.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "stg_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - ] + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322415, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322419, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322415, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "ephemeral", - "dbt_file_path": "models/transformers/transformers_customers.sql", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "transformers_customers", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.jaffle_shop.transformers_customers", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", - "viewLanguage": "SQL" - } - } - ] + "systemMetadata": { + "lastObserved": 1655162322419, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "ephemeral", + "dbt_file_path": "models/transformers/transformers_customers.sql", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "transformers_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.transformers_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322420, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"seed\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322423, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322420, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "seed", - "materialization": "seed", - "dbt_file_path": "data/raw_customers.csv", - "catalog_type": "table", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "raw_customers", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "seed.jaffle_shop.raw_customers", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + "systemMetadata": { + "lastObserved": 1655162322423, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "data/raw_customers.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "raw_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - ] + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322423, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"seed\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322427, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322423, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "seed", - "materialization": "seed", - "dbt_file_path": "data/raw_orders.csv", - "catalog_type": "table", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "raw_orders", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "seed.jaffle_shop.raw_orders", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "status", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + "systemMetadata": { + "lastObserved": 1655162322427, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "data/raw_orders.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "raw_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - ] + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322427, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"seed\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322431, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322427, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "seed", - "materialization": "seed", - "dbt_file_path": "data/raw_payments.csv", - "catalog_type": "table", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "raw_payments", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "seed.jaffle_shop.raw_payments", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "order_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_method", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + "systemMetadata": { + "lastObserved": 1655162322431, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "data/raw_payments.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "raw_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - ] + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322432, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322509, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322432, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/customers.sql", - "catalog_type": "table", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "customers", - "qualifiedName": null, - "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.jaffle_shop.customers", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": "This is a unique identifier for a customer", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": "Customer's first name. PII.", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": "Customer's last name. PII.", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_order", - "jsonPath": null, - "nullable": false, - "description": "Date (UTC) of a customer's first order", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "most_recent_order", - "jsonPath": null, - "nullable": false, - "description": "Date (UTC) of a customer's most recent order", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "number_of_orders", - "jsonPath": null, - "nullable": false, - "description": "Count of the number of orders a customer has placed", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_lifetime_value", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "FLOAT64", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": true, - "viewLogic": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\nephemeral_customers as (\n\n select * from {{ ref('transformers_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\nsource_customers as (\n\n select * from {{ source('jaffle_shop', 'customers_source') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by 1\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders using (order_id)\n\n group by 1\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders using (customer_id)\n\n left join customer_payments using (customer_id)\n\n)\n\nselect * from final", - "viewLanguage": "SQL" - } + "systemMetadata": { + "lastObserved": 1655162322509, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/customers.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "customers", + "qualifiedName": null, + "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for a customer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's first name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's last name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - ] + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's first order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "most_recent_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's most recent order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number_of_orders", + "jsonPath": null, + "nullable": false, + "description": "Count of the number of orders a customer has placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_lifetime_value", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\nephemeral_customers as (\n\n select * from {{ ref('transformers_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\nsource_customers as (\n\n select * from {{ source('jaffle_shop', 'customers_source') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by 1\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders using (order_id)\n\n group by 1\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders using (customer_id)\n\n left join customer_payments using (customer_id)\n\n)\n\nselect * from final", + "viewLanguage": "SQL" } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322510, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers_source,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1655162322523, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322510, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers_source,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers_source,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/schema.yml", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", - "manifest_version": "1.0.4", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "1.0.4" - }, - "externalUrl": null, - "name": "customers_source", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.jaffle_shop.jaffle_shop.customers_source", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "systemMetadata": { + "lastObserved": 1655162322523, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers_source,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/schema.yml", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "customers_source", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.jaffle_shop.jaffle_shop.customers_source", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322524, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322524, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322527, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322527, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322528, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322528, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322529, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322529, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322530, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322530, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322531, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322531, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322532, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322532, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322533, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322533, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1655162322536, - "runId": "dbt-2022_06_13-16_18_42", - "registryName": null, - "registryVersion": null, - "properties": null - } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322536, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null } + } ] diff --git a/smoke-test/tests/cypress/data.json b/smoke-test/tests/cypress/data.json index 391eba1fe9342..5253b7a33b085 100644 --- a/smoke-test/tests/cypress/data.json +++ b/smoke-test/tests/cypress/data.json @@ -202,15 +202,15 @@ ], "fineGrainedLineages": [ { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" - ], - "confidenceScore": 1.0 + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD),field_bar)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),shipment_info)" + ], + "confidenceScore": 1.0 } ] } @@ -400,7 +400,10 @@ }, { "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Cypress" }, { "tag": "urn:li:tag:Cypress2" }] + "tags": [ + { "tag": "urn:li:tag:Cypress" }, + { "tag": "urn:li:tag:Cypress2" } + ] } } ] @@ -2135,4 +2138,4 @@ }, "systemMetadata": null } -] \ No newline at end of file +] diff --git a/smoke-test/tests/cypress/package.json b/smoke-test/tests/cypress/package.json index ebc1c6b3d7a8b..490284ab4d9d0 100644 --- a/smoke-test/tests/cypress/package.json +++ b/smoke-test/tests/cypress/package.json @@ -1,11 +1,25 @@ { + "scripts": { + "format:fix": "prettier --write .", + "format": "prettier --check .", + "lint:fix": "yarn run format:fix && CI=false yarn run lint --fix", + "lint": "yarn run format && CI=false eslint ." + }, "name": "smoke-test", "version": "1.0.0", "main": "index.js", "license": "MIT", - "devDependencies": { + "dependencies": { "cypress": "12.5.1", "cypress-timestamps": "^1.2.0", "dayjs": "^1.11.7" + }, + "devDependencies": { + "eslint": "^7.32.0 || ^8.2.0", + "eslint-config-airbnb-base": "^15.0.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-cypress": "^2.15.1", + "eslint-plugin-import": "^2.25.2", + "prettier": "^3.2.5" } } diff --git a/smoke-test/tests/cypress/yarn.lock b/smoke-test/tests/cypress/yarn.lock index c5aff25ea1106..2433e9f8fae08 100644 --- a/smoke-test/tests/cypress/yarn.lock +++ b/smoke-test/tests/cypress/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@aashutoshrathi/word-wrap@^1.2.3": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf" + integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA== + "@cypress/request@^2.88.10": version "2.88.10" resolved "https://registry.npmjs.org/@cypress/request/-/request-2.88.10.tgz" @@ -34,6 +39,83 @@ debug "^3.1.0" lodash.once "^4.1.1" +"@eslint-community/eslint-utils@^4.2.0": + version "4.4.0" + resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" + integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== + dependencies: + eslint-visitor-keys "^3.3.0" + +"@eslint-community/regexpp@^4.6.1": + version "4.10.0" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.0.tgz#548f6de556857c8bb73bbee70c35dc82a2e74d63" + integrity sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA== + +"@eslint/eslintrc@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.1.4.tgz#388a269f0f25c1b6adc317b5a2c55714894c70ad" + integrity sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ== + dependencies: + ajv "^6.12.4" + debug "^4.3.2" + espree "^9.6.0" + globals "^13.19.0" + ignore "^5.2.0" + import-fresh "^3.2.1" + js-yaml "^4.1.0" + minimatch "^3.1.2" + strip-json-comments "^3.1.1" + +"@eslint/js@8.57.0": + version "8.57.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f" + integrity sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g== + +"@humanwhocodes/config-array@^0.11.14": + version "0.11.14" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" + integrity sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg== + dependencies: + "@humanwhocodes/object-schema" "^2.0.2" + debug "^4.3.1" + minimatch "^3.0.5" + +"@humanwhocodes/module-importer@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" + integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== + +"@humanwhocodes/object-schema@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz#d9fae00a2d5cb40f92cfe64b47ad749fbc38f917" + integrity sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw== + +"@nodelib/fs.scandir@2.1.5": + version "2.1.5" + resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" + integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== + dependencies: + "@nodelib/fs.stat" "2.0.5" + run-parallel "^1.1.9" + +"@nodelib/fs.stat@2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b" + integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== + +"@nodelib/fs.walk@^1.2.8": + version "1.2.8" + resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a" + integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== + dependencies: + "@nodelib/fs.scandir" "2.1.5" + fastq "^1.6.0" + +"@types/json5@^0.0.29": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" + integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== + "@types/node@*": version "16.11.11" resolved "https://registry.npmjs.org/@types/node/-/node-16.11.11.tgz" @@ -61,6 +143,21 @@ dependencies: "@types/node" "*" +"@ungap/structured-clone@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" + integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== + +acorn-jsx@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" + integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== + +acorn@^8.9.0: + version "8.11.3" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a" + integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg== + aggregate-error@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz" @@ -69,6 +166,16 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" +ajv@^6.12.4: + version "6.12.6" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" + integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== + dependencies: + fast-deep-equal "^3.1.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.2" + ansi-colors@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz" @@ -98,6 +205,86 @@ arch@^2.2.0: resolved "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz" integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== + +array-buffer-byte-length@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz#1e5583ec16763540a27ae52eed99ff899223568f" + integrity sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg== + dependencies: + call-bind "^1.0.5" + is-array-buffer "^3.0.4" + +array-includes@^3.1.7: + version "3.1.7" + resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.7.tgz#8cd2e01b26f7a3086cbc87271593fe921c62abda" + integrity sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + get-intrinsic "^1.2.1" + is-string "^1.0.7" + +array.prototype.filter@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/array.prototype.filter/-/array.prototype.filter-1.0.3.tgz#423771edeb417ff5914111fff4277ea0624c0d0e" + integrity sha512-VizNcj/RGJiUyQBgzwxzE5oHdeuXY5hSbbmKMlphj1cy1Vl7Pn2asCGbSrru6hSQjmCzqTBPVWAF/whmEOVHbw== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + es-array-method-boxes-properly "^1.0.0" + is-string "^1.0.7" + +array.prototype.findlastindex@^1.2.3: + version "1.2.4" + resolved "https://registry.yarnpkg.com/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.4.tgz#d1c50f0b3a9da191981ff8942a0aedd82794404f" + integrity sha512-hzvSHUshSpCflDR1QMUBLHGHP1VIEBegT4pix9H/Z92Xw3ySoy6c2qh7lJWTJnRJ8JCZ9bJNCgTyYaJGcJu6xQ== + dependencies: + call-bind "^1.0.5" + define-properties "^1.2.1" + es-abstract "^1.22.3" + es-errors "^1.3.0" + es-shim-unscopables "^1.0.2" + +array.prototype.flat@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz#1476217df8cff17d72ee8f3ba06738db5b387d18" + integrity sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + es-shim-unscopables "^1.0.0" + +array.prototype.flatmap@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz#c9a7c6831db8e719d6ce639190146c24bbd3e527" + integrity sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + es-shim-unscopables "^1.0.0" + +arraybuffer.prototype.slice@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz#097972f4255e41bc3425e37dc3f6421cf9aefde6" + integrity sha512-bMxMKAjg13EBSVscxTaYA4mRc5t1UAXa2kXiGTNfZ079HIWXEkKmkgFrh/nJqamaLSrXO5H4WFFkPEaLJWbs3A== + dependencies: + array-buffer-byte-length "^1.0.1" + call-bind "^1.0.5" + define-properties "^1.2.1" + es-abstract "^1.22.3" + es-errors "^1.2.1" + get-intrinsic "^1.2.3" + is-array-buffer "^3.0.4" + is-shared-array-buffer "^1.0.2" + asn1@~0.2.3: version "0.2.6" resolved "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz" @@ -130,6 +317,13 @@ at-least-node@^1.0.0: resolved "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz" integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== +available-typed-arrays@^1.0.6, available-typed-arrays@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz#a5cc375d6a03c2efc87a553f3e0b1522def14846" + integrity sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ== + dependencies: + possible-typed-array-names "^1.0.0" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz" @@ -193,12 +387,28 @@ cachedir@^2.3.0: resolved "https://registry.npmjs.org/cachedir/-/cachedir-2.3.0.tgz" integrity sha512-A+Fezp4zxnit6FanDmv9EqXNAi3vt9DWp51/71UEhXukb7QUuvtv9344h91dyAxuTLoSYJFU299qzR3tzwPAhw== +call-bind@^1.0.2, call-bind@^1.0.5, call-bind@^1.0.6, call-bind@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.7.tgz#06016599c40c56498c18769d2730be242b6fa3b9" + integrity sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w== + dependencies: + es-define-property "^1.0.0" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.4" + set-function-length "^1.2.1" + +callsites@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" + integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== + caseless@~0.12.0: version "0.12.0" resolved "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz" integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= -chalk@^4.1.0: +chalk@^4.0.0, chalk@^4.1.0: version "4.1.2" resolved "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -289,12 +499,17 @@ concat-map@0.0.1: resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= +confusing-browser-globals@^1.0.10: + version "1.0.11" + resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz#ae40e9b57cdd3915408a2805ebd3a5585608dc81" + integrity sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA== + core-util-is@1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= -cross-spawn@^7.0.0: +cross-spawn@^7.0.0, cross-spawn@^7.0.2: version "7.0.3" resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== @@ -375,7 +590,7 @@ dayjs@^1.11.7: resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2" integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ== -debug@^3.1.0: +debug@^3.1.0, debug@^3.2.7: version "3.2.7" resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== @@ -389,11 +604,55 @@ debug@^4.1.1, debug@^4.3.2: dependencies: ms "2.1.2" +debug@^4.3.1: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + +deep-is@^0.1.3: + version "0.1.4" + resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" + integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== + +define-data-property@^1.0.1, define-data-property@^1.1.2, define-data-property@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" + integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A== + dependencies: + es-define-property "^1.0.0" + es-errors "^1.3.0" + gopd "^1.0.1" + +define-properties@^1.1.3, define-properties@^1.2.0, define-properties@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.1.tgz#10781cc616eb951a80a034bafcaa7377f6af2b6c" + integrity sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg== + dependencies: + define-data-property "^1.0.1" + has-property-descriptors "^1.0.0" + object-keys "^1.1.1" + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= +doctrine@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d" + integrity sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw== + dependencies: + esutils "^2.0.2" + +doctrine@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" + integrity sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w== + dependencies: + esutils "^2.0.2" + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz" @@ -421,11 +680,256 @@ enquirer@^2.3.6: dependencies: ansi-colors "^4.1.1" +es-abstract@^1.22.1, es-abstract@^1.22.3: + version "1.22.4" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.22.4.tgz#26eb2e7538c3271141f5754d31aabfdb215f27bf" + integrity sha512-vZYJlk2u6qHYxBOTjAeg7qUxHdNfih64Uu2J8QqWgXZ2cri0ZpJAkzDUK/q593+mvKwlxyaxr6F1Q+3LKoQRgg== + dependencies: + array-buffer-byte-length "^1.0.1" + arraybuffer.prototype.slice "^1.0.3" + available-typed-arrays "^1.0.6" + call-bind "^1.0.7" + es-define-property "^1.0.0" + es-errors "^1.3.0" + es-set-tostringtag "^2.0.2" + es-to-primitive "^1.2.1" + function.prototype.name "^1.1.6" + get-intrinsic "^1.2.4" + get-symbol-description "^1.0.2" + globalthis "^1.0.3" + gopd "^1.0.1" + has-property-descriptors "^1.0.2" + has-proto "^1.0.1" + has-symbols "^1.0.3" + hasown "^2.0.1" + internal-slot "^1.0.7" + is-array-buffer "^3.0.4" + is-callable "^1.2.7" + is-negative-zero "^2.0.2" + is-regex "^1.1.4" + is-shared-array-buffer "^1.0.2" + is-string "^1.0.7" + is-typed-array "^1.1.13" + is-weakref "^1.0.2" + object-inspect "^1.13.1" + object-keys "^1.1.1" + object.assign "^4.1.5" + regexp.prototype.flags "^1.5.2" + safe-array-concat "^1.1.0" + safe-regex-test "^1.0.3" + string.prototype.trim "^1.2.8" + string.prototype.trimend "^1.0.7" + string.prototype.trimstart "^1.0.7" + typed-array-buffer "^1.0.1" + typed-array-byte-length "^1.0.0" + typed-array-byte-offset "^1.0.0" + typed-array-length "^1.0.4" + unbox-primitive "^1.0.2" + which-typed-array "^1.1.14" + +es-array-method-boxes-properly@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-array-method-boxes-properly/-/es-array-method-boxes-properly-1.0.0.tgz#873f3e84418de4ee19c5be752990b2e44718d09e" + integrity sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA== + +es-define-property@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.0.tgz#c7faefbdff8b2696cf5f46921edfb77cc4ba3845" + integrity sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ== + dependencies: + get-intrinsic "^1.2.4" + +es-errors@^1.0.0, es-errors@^1.2.1, es-errors@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" + integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== + +es-set-tostringtag@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz#8bb60f0a440c2e4281962428438d58545af39777" + integrity sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ== + dependencies: + get-intrinsic "^1.2.4" + has-tostringtag "^1.0.2" + hasown "^2.0.1" + +es-shim-unscopables@^1.0.0, es-shim-unscopables@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz#1f6942e71ecc7835ed1c8a83006d8771a63a3763" + integrity sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw== + dependencies: + hasown "^2.0.0" + +es-to-primitive@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" + integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== + dependencies: + is-callable "^1.1.4" + is-date-object "^1.0.1" + is-symbol "^1.0.2" + escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= +escape-string-regexp@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" + integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== + +eslint-config-airbnb-base@^15.0.0: + version "15.0.0" + resolved "https://registry.yarnpkg.com/eslint-config-airbnb-base/-/eslint-config-airbnb-base-15.0.0.tgz#6b09add90ac79c2f8d723a2580e07f3925afd236" + integrity sha512-xaX3z4ZZIcFLvh2oUNvcX5oEofXda7giYmuplVxoOg5A7EXJMrUyqRgR+mhDhPK8LZ4PttFOBvCYDbX3sUoUig== + dependencies: + confusing-browser-globals "^1.0.10" + object.assign "^4.1.2" + object.entries "^1.1.5" + semver "^6.3.0" + +eslint-config-prettier@^9.1.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz#31af3d94578645966c082fcb71a5846d3c94867f" + integrity sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw== + +eslint-import-resolver-node@^0.3.9: + version "0.3.9" + resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz#d4eaac52b8a2e7c3cd1903eb00f7e053356118ac" + integrity sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g== + dependencies: + debug "^3.2.7" + is-core-module "^2.13.0" + resolve "^1.22.4" + +eslint-module-utils@^2.8.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz#e439fee65fc33f6bba630ff621efc38ec0375c49" + integrity sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw== + dependencies: + debug "^3.2.7" + +eslint-plugin-cypress@^2.15.1: + version "2.15.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-cypress/-/eslint-plugin-cypress-2.15.1.tgz#336afa7e8e27451afaf65aa359c9509e0a4f3a7b" + integrity sha512-eLHLWP5Q+I4j2AWepYq0PgFEei9/s5LvjuSqWrxurkg1YZ8ltxdvMNmdSf0drnsNo57CTgYY/NIHHLRSWejR7w== + dependencies: + globals "^13.20.0" + +eslint-plugin-import@^2.25.2: + version "2.29.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz#d45b37b5ef5901d639c15270d74d46d161150643" + integrity sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw== + dependencies: + array-includes "^3.1.7" + array.prototype.findlastindex "^1.2.3" + array.prototype.flat "^1.3.2" + array.prototype.flatmap "^1.3.2" + debug "^3.2.7" + doctrine "^2.1.0" + eslint-import-resolver-node "^0.3.9" + eslint-module-utils "^2.8.0" + hasown "^2.0.0" + is-core-module "^2.13.1" + is-glob "^4.0.3" + minimatch "^3.1.2" + object.fromentries "^2.0.7" + object.groupby "^1.0.1" + object.values "^1.1.7" + semver "^6.3.1" + tsconfig-paths "^3.15.0" + +eslint-scope@^7.2.2: + version "7.2.2" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.2.2.tgz#deb4f92563390f32006894af62a22dba1c46423f" + integrity sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg== + dependencies: + esrecurse "^4.3.0" + estraverse "^5.2.0" + +eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4.3: + version "3.4.3" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" + integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== + +"eslint@^7.32.0 || ^8.2.0": + version "8.57.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.0.tgz#c786a6fd0e0b68941aaf624596fb987089195668" + integrity sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ== + dependencies: + "@eslint-community/eslint-utils" "^4.2.0" + "@eslint-community/regexpp" "^4.6.1" + "@eslint/eslintrc" "^2.1.4" + "@eslint/js" "8.57.0" + "@humanwhocodes/config-array" "^0.11.14" + "@humanwhocodes/module-importer" "^1.0.1" + "@nodelib/fs.walk" "^1.2.8" + "@ungap/structured-clone" "^1.2.0" + ajv "^6.12.4" + chalk "^4.0.0" + cross-spawn "^7.0.2" + debug "^4.3.2" + doctrine "^3.0.0" + escape-string-regexp "^4.0.0" + eslint-scope "^7.2.2" + eslint-visitor-keys "^3.4.3" + espree "^9.6.1" + esquery "^1.4.2" + esutils "^2.0.2" + fast-deep-equal "^3.1.3" + file-entry-cache "^6.0.1" + find-up "^5.0.0" + glob-parent "^6.0.2" + globals "^13.19.0" + graphemer "^1.4.0" + ignore "^5.2.0" + imurmurhash "^0.1.4" + is-glob "^4.0.0" + is-path-inside "^3.0.3" + js-yaml "^4.1.0" + json-stable-stringify-without-jsonify "^1.0.1" + levn "^0.4.1" + lodash.merge "^4.6.2" + minimatch "^3.1.2" + natural-compare "^1.4.0" + optionator "^0.9.3" + strip-ansi "^6.0.1" + text-table "^0.2.0" + +espree@^9.6.0, espree@^9.6.1: + version "9.6.1" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.6.1.tgz#a2a17b8e434690a5432f2f8018ce71d331a48c6f" + integrity sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ== + dependencies: + acorn "^8.9.0" + acorn-jsx "^5.3.2" + eslint-visitor-keys "^3.4.1" + +esquery@^1.4.2: + version "1.5.0" + resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.5.0.tgz#6ce17738de8577694edd7361c57182ac8cb0db0b" + integrity sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg== + dependencies: + estraverse "^5.1.0" + +esrecurse@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" + integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== + dependencies: + estraverse "^5.2.0" + +estraverse@^5.1.0, estraverse@^5.2.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" + integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== + +esutils@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" + integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== + eventemitter2@6.4.7: version "6.4.7" resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.7.tgz#a7f6c4d7abf28a14c1ef3442f21cb306a054271d" @@ -479,6 +983,28 @@ extsprintf@^1.2.0: resolved "https://registry.npmjs.org/extsprintf/-/extsprintf-1.4.1.tgz" integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== +fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" + integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== + +fast-json-stable-stringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + +fast-levenshtein@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" + integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== + +fastq@^1.6.0: + version "1.17.1" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.17.1.tgz#2a523f07a4e7b1e81a42b91b8bf2254107753b47" + integrity sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w== + dependencies: + reusify "^1.0.4" + fd-slicer@~1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz" @@ -493,6 +1019,42 @@ figures@^3.2.0: dependencies: escape-string-regexp "^1.0.5" +file-entry-cache@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" + integrity sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg== + dependencies: + flat-cache "^3.0.4" + +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + +flat-cache@^3.0.4: + version "3.2.0" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.2.0.tgz#2c0c2d5040c99b1632771a9d105725c0115363ee" + integrity sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw== + dependencies: + flatted "^3.2.9" + keyv "^4.5.3" + rimraf "^3.0.2" + +flatted@^3.2.9: + version "3.3.1" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" + integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== + +for-each@^0.3.3: + version "0.3.3" + resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e" + integrity sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw== + dependencies: + is-callable "^1.1.3" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" @@ -527,6 +1089,37 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + +function.prototype.name@^1.1.6: + version "1.1.6" + resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.6.tgz#cdf315b7d90ee77a4c6ee216c3c3362da07533fd" + integrity sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + functions-have-names "^1.2.3" + +functions-have-names@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" + integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== + +get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd" + integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ== + dependencies: + es-errors "^1.3.0" + function-bind "^1.1.2" + has-proto "^1.0.1" + has-symbols "^1.0.3" + hasown "^2.0.0" + get-stream@^5.0.0, get-stream@^5.1.0: version "5.2.0" resolved "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz" @@ -534,6 +1127,15 @@ get-stream@^5.0.0, get-stream@^5.1.0: dependencies: pump "^3.0.0" +get-symbol-description@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.2.tgz#533744d5aa20aca4e079c8e5daf7fd44202821f5" + integrity sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg== + dependencies: + call-bind "^1.0.5" + es-errors "^1.3.0" + get-intrinsic "^1.2.4" + getos@^3.2.1: version "3.2.1" resolved "https://registry.npmjs.org/getos/-/getos-3.2.1.tgz" @@ -548,6 +1150,13 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" +glob-parent@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3" + integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== + dependencies: + is-glob "^4.0.3" + glob@^7.1.3: version "7.2.0" resolved "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz" @@ -567,16 +1176,78 @@ global-dirs@^3.0.0: dependencies: ini "2.0.0" +globals@^13.19.0, globals@^13.20.0: + version "13.24.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-13.24.0.tgz#8432a19d78ce0c1e833949c36adb345400bb1171" + integrity sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ== + dependencies: + type-fest "^0.20.2" + +globalthis@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/globalthis/-/globalthis-1.0.3.tgz#5852882a52b80dc301b0660273e1ed082f0b6ccf" + integrity sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA== + dependencies: + define-properties "^1.1.3" + +gopd@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c" + integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA== + dependencies: + get-intrinsic "^1.1.3" + graceful-fs@^4.1.6, graceful-fs@^4.2.0: version "4.2.8" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz" integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== +graphemer@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" + integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== + +has-bigints@^1.0.1, has-bigints@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" + integrity sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ== + has-flag@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== +has-property-descriptors@^1.0.0, has-property-descriptors@^1.0.1, has-property-descriptors@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" + integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg== + dependencies: + es-define-property "^1.0.0" + +has-proto@^1.0.1, has-proto@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.3.tgz#b31ddfe9b0e6e9914536a6ab286426d0214f77fd" + integrity sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q== + +has-symbols@^1.0.2, has-symbols@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== + +has-tostringtag@^1.0.0, has-tostringtag@^1.0.1, has-tostringtag@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc" + integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw== + dependencies: + has-symbols "^1.0.3" + +hasown@^2.0.0, hasown@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.1.tgz#26f48f039de2c0f8d3356c223fb8d50253519faa" + integrity sha512-1/th4MHjnwncwXsIW6QMzlvYL9kG5e/CpVvLRZe4XPa8TOUNbCELqmvhDmnkNsAjwaG4+I8gJJL0JBvTTLO9qA== + dependencies: + function-bind "^1.1.2" + http-signature@~1.3.6: version "1.3.6" resolved "https://registry.npmjs.org/http-signature/-/http-signature-1.3.6.tgz" @@ -596,6 +1267,24 @@ ieee754@^1.1.13: resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== +ignore@^5.2.0: + version "5.3.1" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef" + integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw== + +import-fresh@^3.2.1: + version "3.3.0" + resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" + integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== + dependencies: + parent-module "^1.0.0" + resolve-from "^4.0.0" + +imurmurhash@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" + integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== + indent-string@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz" @@ -619,6 +1308,43 @@ ini@2.0.0: resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz" integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== +internal-slot@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.7.tgz#c06dcca3ed874249881007b0a5523b172a190802" + integrity sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g== + dependencies: + es-errors "^1.3.0" + hasown "^2.0.0" + side-channel "^1.0.4" + +is-array-buffer@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.4.tgz#7a1f92b3d61edd2bc65d24f130530ea93d7fae98" + integrity sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw== + dependencies: + call-bind "^1.0.2" + get-intrinsic "^1.2.1" + +is-bigint@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.4.tgz#08147a1875bc2b32005d41ccd8291dffc6691df3" + integrity sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg== + dependencies: + has-bigints "^1.0.1" + +is-boolean-object@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.2.tgz#5c6dc200246dd9321ae4b885a114bb1f75f63719" + integrity sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA== + dependencies: + call-bind "^1.0.2" + has-tostringtag "^1.0.0" + +is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: + version "1.2.7" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" + integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== + is-ci@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/is-ci/-/is-ci-3.0.1.tgz" @@ -626,11 +1352,37 @@ is-ci@^3.0.0: dependencies: ci-info "^3.2.0" +is-core-module@^2.13.0, is-core-module@^2.13.1: + version "2.13.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384" + integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw== + dependencies: + hasown "^2.0.0" + +is-date-object@^1.0.1: + version "1.0.5" + resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f" + integrity sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ== + dependencies: + has-tostringtag "^1.0.0" + +is-extglob@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== + is-fullwidth-code-point@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz" integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== +is-glob@^4.0.0, is-glob@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" + integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== + dependencies: + is-extglob "^2.1.1" + is-installed-globally@~0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.4.0.tgz" @@ -639,16 +1391,64 @@ is-installed-globally@~0.4.0: global-dirs "^3.0.0" is-path-inside "^3.0.2" -is-path-inside@^3.0.2: +is-negative-zero@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.3.tgz#ced903a027aca6381b777a5743069d7376a49747" + integrity sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw== + +is-number-object@^1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.7.tgz#59d50ada4c45251784e9904f5246c742f07a42fc" + integrity sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ== + dependencies: + has-tostringtag "^1.0.0" + +is-path-inside@^3.0.2, is-path-inside@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz" integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== +is-regex@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.4.tgz#eef5663cd59fa4c0ae339505323df6854bb15958" + integrity sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg== + dependencies: + call-bind "^1.0.2" + has-tostringtag "^1.0.0" + +is-shared-array-buffer@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.3.tgz#1237f1cba059cdb62431d378dcc37d9680181688" + integrity sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg== + dependencies: + call-bind "^1.0.7" + is-stream@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== +is-string@^1.0.5, is-string@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd" + integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg== + dependencies: + has-tostringtag "^1.0.0" + +is-symbol@^1.0.2, is-symbol@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.4.tgz#a6dac93b635b063ca6872236de88910a57af139c" + integrity sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg== + dependencies: + has-symbols "^1.0.2" + +is-typed-array@^1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.13.tgz#d6c5ca56df62334959322d7d7dd1cca50debe229" + integrity sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw== + dependencies: + which-typed-array "^1.1.14" + is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" @@ -659,6 +1459,18 @@ is-unicode-supported@^0.1.0: resolved "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz" integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== +is-weakref@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.2.tgz#9529f383a9338205e89765e0392efc2f100f06f2" + integrity sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ== + dependencies: + call-bind "^1.0.2" + +isarray@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" + integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" @@ -669,21 +1481,50 @@ isstream@~0.1.2: resolved "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + jsbn@~0.1.0: version "0.1.1" resolved "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz" integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= +json-buffer@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" + integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ== + +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== + json-schema@0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz" integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== +json-stable-stringify-without-jsonify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" + integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== + json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= +json5@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" + integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA== + dependencies: + minimist "^1.2.0" + jsonfile@^6.0.1: version "6.1.0" resolved "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz" @@ -703,11 +1544,26 @@ jsprim@^2.0.2: json-schema "0.4.0" verror "1.10.0" +keyv@^4.5.3: + version "4.5.4" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" + integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== + dependencies: + json-buffer "3.0.1" + lazy-ass@^1.6.0: version "1.6.0" resolved "https://registry.npmjs.org/lazy-ass/-/lazy-ass-1.6.0.tgz" integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM= +levn@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" + integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== + dependencies: + prelude-ls "^1.2.1" + type-check "~0.4.0" + listr2@^3.8.3: version "3.13.5" resolved "https://registry.npmjs.org/listr2/-/listr2-3.13.5.tgz" @@ -722,6 +1578,18 @@ listr2@^3.8.3: through "^2.3.8" wrap-ansi "^7.0.0" +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== + dependencies: + p-locate "^5.0.0" + +lodash.merge@^4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" + integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== + lodash.once@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz" @@ -779,13 +1647,18 @@ mimic-fn@^2.1.0: resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -minimatch@^3.0.4: +minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" +minimist@^1.2.0: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + minimist@^1.2.6: version "1.2.7" resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz" @@ -801,6 +1674,11 @@ ms@^2.1.1: resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== +natural-compare@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" + integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== + npm-run-path@^4.0.0: version "4.0.1" resolved "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz" @@ -808,6 +1686,64 @@ npm-run-path@^4.0.0: dependencies: path-key "^3.0.0" +object-inspect@^1.13.1: + version "1.13.1" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.1.tgz#b96c6109324ccfef6b12216a956ca4dc2ff94bc2" + integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ== + +object-keys@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== + +object.assign@^4.1.2, object.assign@^4.1.5: + version "4.1.5" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.5.tgz#3a833f9ab7fdb80fc9e8d2300c803d216d8fdbb0" + integrity sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ== + dependencies: + call-bind "^1.0.5" + define-properties "^1.2.1" + has-symbols "^1.0.3" + object-keys "^1.1.1" + +object.entries@^1.1.5: + version "1.1.7" + resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.7.tgz#2b47760e2a2e3a752f39dd874655c61a7f03c131" + integrity sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + +object.fromentries@^2.0.7: + version "2.0.7" + resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.7.tgz#71e95f441e9a0ea6baf682ecaaf37fa2a8d7e616" + integrity sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + +object.groupby@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.2.tgz#494800ff5bab78fd0eff2835ec859066e00192ec" + integrity sha512-bzBq58S+x+uo0VjurFT0UktpKHOZmv4/xePiOA1nbB9pMqpGK7rUPNgf+1YC+7mE+0HzhTMqNUuCqvKhj6FnBw== + dependencies: + array.prototype.filter "^1.0.3" + call-bind "^1.0.5" + define-properties "^1.2.1" + es-abstract "^1.22.3" + es-errors "^1.0.0" + +object.values@^1.1.7: + version "1.1.7" + resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.7.tgz#617ed13272e7e1071b43973aa1655d9291b8442a" + integrity sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.npmjs.org/once/-/once-1.4.0.tgz" @@ -822,11 +1758,37 @@ onetime@^5.1.0: dependencies: mimic-fn "^2.1.0" +optionator@^0.9.3: + version "0.9.3" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64" + integrity sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg== + dependencies: + "@aashutoshrathi/word-wrap" "^1.2.3" + deep-is "^0.1.3" + fast-levenshtein "^2.0.6" + levn "^0.4.1" + prelude-ls "^1.2.1" + type-check "^0.4.0" + ospath@^1.2.2: version "1.2.2" resolved "https://registry.npmjs.org/ospath/-/ospath-1.2.2.tgz" integrity sha1-EnZjl3Sj+O8lcvf+QoDg6kVQwHs= +p-limit@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== + dependencies: + yocto-queue "^0.1.0" + +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== + dependencies: + p-limit "^3.0.2" + p-map@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz" @@ -834,6 +1796,18 @@ p-map@^4.0.0: dependencies: aggregate-error "^3.0.0" +parent-module@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" + integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== + dependencies: + callsites "^3.0.0" + +path-exists@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" + integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== + path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" @@ -844,6 +1818,11 @@ path-key@^3.0.0, path-key@^3.1.0: resolved "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== +path-parse@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== + pend@~1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz" @@ -859,6 +1838,21 @@ pify@^2.2.0: resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw= +possible-typed-array-names@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f" + integrity sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q== + +prelude-ls@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" + integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== + +prettier@^3.2.5: + version "3.2.5" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.5.tgz#e52bc3090586e824964a8813b09aba6233b28368" + integrity sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A== + pretty-bytes@^5.6.0: version "5.6.0" resolved "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz" @@ -882,6 +1876,11 @@ pump@^3.0.0: end-of-stream "^1.1.0" once "^1.3.1" +punycode@^2.1.0: + version "2.3.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" + integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== + punycode@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" @@ -892,6 +1891,21 @@ qs@~6.5.2: resolved "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz" integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== +queue-microtask@^1.2.2: + version "1.2.3" + resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" + integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== + +regexp.prototype.flags@^1.5.2: + version "1.5.2" + resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz#138f644a3350f981a858c44f6bb1a61ff59be334" + integrity sha512-NcDiDkTLuPR+++OCKB0nWafEmhg/Da8aUPLPMQbK+bxKKCm1/S5he+AqYa4PlMCVBalb4/yxIRub6qkEx5yJbw== + dependencies: + call-bind "^1.0.6" + define-properties "^1.2.1" + es-errors "^1.3.0" + set-function-name "^2.0.1" + request-progress@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/request-progress/-/request-progress-3.0.0.tgz" @@ -899,6 +1913,20 @@ request-progress@^3.0.0: dependencies: throttleit "^1.0.0" +resolve-from@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" + integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== + +resolve@^1.22.4: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + restore-cursor@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz" @@ -907,18 +1935,30 @@ restore-cursor@^3.1.0: onetime "^5.1.0" signal-exit "^3.0.2" +reusify@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" + integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== + rfdc@^1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz" integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== -rimraf@^3.0.0: +rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== dependencies: glob "^7.1.3" +run-parallel@^1.1.9: + version "1.2.0" + resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" + integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== + dependencies: + queue-microtask "^1.2.2" + rxjs@^7.4.0: version "7.4.0" resolved "https://registry.npmjs.org/rxjs/-/rxjs-7.4.0.tgz" @@ -926,16 +1966,40 @@ rxjs@^7.4.0: dependencies: tslib "~2.1.0" +safe-array-concat@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.0.tgz#8d0cae9cb806d6d1c06e08ab13d847293ebe0692" + integrity sha512-ZdQ0Jeb9Ofti4hbt5lX3T2JcAamT9hfzYU1MNB+z/jaEbB6wfFfPIR/zEORmZqobkCCJhSjodobH6WHNmJ97dg== + dependencies: + call-bind "^1.0.5" + get-intrinsic "^1.2.2" + has-symbols "^1.0.3" + isarray "^2.0.5" + safe-buffer@^5.0.1, safe-buffer@^5.1.2: version "5.2.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== +safe-regex-test@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.3.tgz#a5b4c0f06e0ab50ea2c395c14d8371232924c377" + integrity sha512-CdASjNJPvRa7roO6Ra/gLYBTzYzzPyyBXxIMdGW3USQLyjWEls2RgW5UBTXaQVp+OrpeCK3bLem8smtmheoRuw== + dependencies: + call-bind "^1.0.6" + es-errors "^1.3.0" + is-regex "^1.1.4" + safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + semver@^7.3.2: version "7.3.5" resolved "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz" @@ -943,6 +2007,28 @@ semver@^7.3.2: dependencies: lru-cache "^6.0.0" +set-function-length@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.1.tgz#47cc5945f2c771e2cf261c6737cf9684a2a5e425" + integrity sha512-j4t6ccc+VsKwYHso+kElc5neZpjtq9EnRICFZtWyBsLojhmeF/ZBd/elqm22WJh/BziDe/SBiOeAt0m2mfLD0g== + dependencies: + define-data-property "^1.1.2" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.3" + gopd "^1.0.1" + has-property-descriptors "^1.0.1" + +set-function-name@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.2.tgz#16a705c5a0dc2f5e638ca96d8a8cd4e1c2b90985" + integrity sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ== + dependencies: + define-data-property "^1.1.4" + es-errors "^1.3.0" + functions-have-names "^1.2.3" + has-property-descriptors "^1.0.2" + shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz" @@ -955,6 +2041,16 @@ shebang-regex@^3.0.0: resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== +side-channel@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.5.tgz#9a84546599b48909fb6af1211708d23b1946221b" + integrity sha512-QcgiIWV4WV7qWExbN5llt6frQB/lBven9pqliLXfGPB+K9ZYXxDozp0wLkHS24kWCm+6YXH/f0HhnObZnZOBnQ== + dependencies: + call-bind "^1.0.6" + es-errors "^1.3.0" + get-intrinsic "^1.2.4" + object-inspect "^1.13.1" + signal-exit@^3.0.2: version "3.0.6" resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.6.tgz" @@ -1002,6 +2098,33 @@ string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" +string.prototype.trim@^1.2.8: + version "1.2.8" + resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz#f9ac6f8af4bd55ddfa8895e6aea92a96395393bd" + integrity sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + +string.prototype.trimend@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.7.tgz#1bb3afc5008661d73e2dc015cd4853732d6c471e" + integrity sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + +string.prototype.trimstart@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.7.tgz#d4cdb44b83a4737ffbac2d406e405d43d0184298" + integrity sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" @@ -1009,11 +2132,21 @@ strip-ansi@^6.0.0, strip-ansi@^6.0.1: dependencies: ansi-regex "^5.0.1" +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== + strip-final-newline@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz" integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== +strip-json-comments@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" + integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== + supports-color@^7.1.0: version "7.2.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz" @@ -1028,6 +2161,16 @@ supports-color@^8.1.1: dependencies: has-flag "^4.0.0" +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + +text-table@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" + integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== + throttleit@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz" @@ -1053,6 +2196,16 @@ tough-cookie@~2.5.0: psl "^1.1.28" punycode "^2.1.1" +tsconfig-paths@^3.15.0: + version "3.15.0" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz#5299ec605e55b1abb23ec939ef15edaf483070d4" + integrity sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg== + dependencies: + "@types/json5" "^0.0.29" + json5 "^1.0.2" + minimist "^1.2.6" + strip-bom "^3.0.0" + tslib@~2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz" @@ -1070,11 +2223,77 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0: resolved "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz" integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= +type-check@^0.4.0, type-check@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" + integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== + dependencies: + prelude-ls "^1.2.1" + +type-fest@^0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" + integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== + type-fest@^0.21.3: version "0.21.3" resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz" integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== +typed-array-buffer@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/typed-array-buffer/-/typed-array-buffer-1.0.2.tgz#1867c5d83b20fcb5ccf32649e5e2fc7424474ff3" + integrity sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ== + dependencies: + call-bind "^1.0.7" + es-errors "^1.3.0" + is-typed-array "^1.1.13" + +typed-array-byte-length@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/typed-array-byte-length/-/typed-array-byte-length-1.0.1.tgz#d92972d3cff99a3fa2e765a28fcdc0f1d89dec67" + integrity sha512-3iMJ9q0ao7WE9tWcaYKIptkNBuOIcZCCT0d4MRvuuH88fEoEH62IuQe0OtraD3ebQEoTRk8XCBoknUNc1Y67pw== + dependencies: + call-bind "^1.0.7" + for-each "^0.3.3" + gopd "^1.0.1" + has-proto "^1.0.3" + is-typed-array "^1.1.13" + +typed-array-byte-offset@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/typed-array-byte-offset/-/typed-array-byte-offset-1.0.2.tgz#f9ec1acb9259f395093e4567eb3c28a580d02063" + integrity sha512-Ous0vodHa56FviZucS2E63zkgtgrACj7omjwd/8lTEMEPFFyjfixMZ1ZXenpgCFBBt4EC1J2XsyVS2gkG0eTFA== + dependencies: + available-typed-arrays "^1.0.7" + call-bind "^1.0.7" + for-each "^0.3.3" + gopd "^1.0.1" + has-proto "^1.0.3" + is-typed-array "^1.1.13" + +typed-array-length@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.5.tgz#57d44da160296d8663fd63180a1802ebf25905d5" + integrity sha512-yMi0PlwuznKHxKmcpoOdeLwxBoVPkqZxd7q2FgMkmD3bNwvF5VW0+UlUQ1k1vmktTu4Yu13Q0RIxEP8+B+wloA== + dependencies: + call-bind "^1.0.7" + for-each "^0.3.3" + gopd "^1.0.1" + has-proto "^1.0.3" + is-typed-array "^1.1.13" + possible-typed-array-names "^1.0.0" + +unbox-primitive@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" + integrity sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw== + dependencies: + call-bind "^1.0.2" + has-bigints "^1.0.2" + has-symbols "^1.0.3" + which-boxed-primitive "^1.0.2" + universalify@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz" @@ -1085,6 +2304,13 @@ untildify@^4.0.0: resolved "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== +uri-js@^4.2.2: + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== + dependencies: + punycode "^2.1.0" + uuid@^8.3.2: version "8.3.2" resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" @@ -1099,6 +2325,28 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" +which-boxed-primitive@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" + integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg== + dependencies: + is-bigint "^1.0.1" + is-boolean-object "^1.1.0" + is-number-object "^1.0.4" + is-string "^1.0.5" + is-symbol "^1.0.3" + +which-typed-array@^1.1.14: + version "1.1.14" + resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.14.tgz#1f78a111aee1e131ca66164d8bdc3ab062c95a06" + integrity sha512-VnXFiIW8yNn9kIHN88xvZ4yOWchftKDsRJ8fEPacX/wl1lOvBrhsJ/OeJCXq7B0AaijRuqgzSKalJoPk+D8MPg== + dependencies: + available-typed-arrays "^1.0.6" + call-bind "^1.0.5" + for-each "^0.3.3" + gopd "^1.0.1" + has-tostringtag "^1.0.1" + which@^2.0.1: version "2.0.2" resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz" @@ -1141,3 +2389,8 @@ yauzl@^2.10.0: dependencies: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" + +yocto-queue@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" + integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==