Merge branch 'datahub-project:master' into master

acryldata · Jul 17, 2024 · efe7f38 · efe7f38
2 parents fcc4115 + 298c299
commit efe7f38
Show file tree

Hide file tree

Showing 95 changed files with 3,517 additions and 1,729 deletions.
diff --git a/...ql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java b/...ql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java
@@ -48,6 +48,7 @@ public class EntityTypeMapper {
           .put(EntityType.BUSINESS_ATTRIBUTE, Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME)
           .put(EntityType.QUERY, Constants.QUERY_ENTITY_NAME)
           .put(EntityType.POST, Constants.POST_ENTITY_NAME)
+          .put(EntityType.FORM, Constants.FORM_ENTITY_NAME)
           .build();
 
   private static final Map<String, EntityType> ENTITY_NAME_TO_TYPE =

diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx
@@ -1254,6 +1254,7 @@ export const glossaryNode5 = {
 
 export const sampleTag = {
     urn: 'urn:li:tag:abc-sample-tag',
+    type: EntityType.Tag,
     name: 'abc-sample-tag',
     description: 'sample tag description',
     ownership: {

diff --git a/...eact/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx b/...eact/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx
@@ -13,16 +13,29 @@ type Props = {
 };
 
 export const FreshnessScheduleSummary = ({ definition, evaluationSchedule }: Props) => {
-    const scheduleText =
-        definition.type === FreshnessAssertionScheduleType.Cron
-            ? `${capitalizeFirstLetter(cronstrue.toString(definition.cron?.cron as string))}.`
-            : `In the past ${
-                  definition.fixedInterval?.multiple
-              } ${definition.fixedInterval?.unit.toLocaleLowerCase()}s${
-                  (evaluationSchedule &&
-                      `, as of ${cronstrue.toString(evaluationSchedule.cron as string).toLowerCase()}`) ||
-                  ''
-              }`;
+    let scheduleText = '';
+    const cronStr = definition.cron?.cron ?? evaluationSchedule?.cron;
+    switch (definition.type) {
+        case FreshnessAssertionScheduleType.Cron:
+            scheduleText = cronStr
+                ? `${capitalizeFirstLetter(cronstrue.toString(cronStr))}.`
+                : `Unknown freshness schedule.`;
+            break;
+        case FreshnessAssertionScheduleType.SinceTheLastCheck:
+            scheduleText = cronStr
+                ? `Since the previous check, as of ${cronstrue.toString(cronStr).toLowerCase()}`
+                : 'Since the previous check';
+            break;
+        case FreshnessAssertionScheduleType.FixedInterval:
+            scheduleText = `In the past ${
+                definition.fixedInterval?.multiple
+            } ${definition.fixedInterval?.unit.toLocaleLowerCase()}s${
+                cronStr ? `, as of ${cronstrue.toString(cronStr).toLowerCase()}` : ''
+            }`;
+            break;
+        default:
+            break;
+    }
 
     return <>{scheduleText}</>;
 };
diff --git a/datahub-web-react/src/app/ingest/source/utils.ts b/datahub-web-react/src/app/ingest/source/utils.ts
@@ -47,6 +47,7 @@ export const WARNING = 'WARNING';
 export const FAILURE = 'FAILURE';
 export const CONNECTION_FAILURE = 'CONNECTION_FAILURE';
 export const CANCELLED = 'CANCELLED';
+export const ABORTED = 'ABORTED';
 export const UP_FOR_RETRY = 'UP_FOR_RETRY';
 export const ROLLING_BACK = 'ROLLING_BACK';
 export const ROLLED_BACK = 'ROLLED_BACK';
@@ -68,6 +69,7 @@ export const getExecutionRequestStatusIcon = (status: string) => {
         (status === ROLLED_BACK && WarningOutlined) ||
         (status === ROLLING_BACK && LoadingOutlined) ||
         (status === ROLLBACK_FAILED && CloseCircleOutlined) ||
+        (status === ABORTED && CloseCircleOutlined) ||
         ClockCircleOutlined
     );
 };
@@ -83,6 +85,7 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
         (status === ROLLED_BACK && 'Rolled Back') ||
         (status === ROLLING_BACK && 'Rolling Back') ||
         (status === ROLLBACK_FAILED && 'Rollback Failed') ||
+        (status === ABORTED && 'Aborted') ||
         status
     );
 };
@@ -105,6 +108,8 @@ export const getExecutionRequestSummaryText = (status: string) => {
             return 'Ingestion is in the process of rolling back.';
         case ROLLBACK_FAILED:
             return 'Ingestion rollback failed.';
+        case ABORTED:
+            return 'Ingestion job got aborted due to worker restart.';
         default:
             return 'Ingestion status not recognized.';
     }
@@ -121,6 +126,7 @@ export const getExecutionRequestStatusDisplayColor = (status: string) => {
         (status === ROLLED_BACK && 'orange') ||
         (status === ROLLING_BACK && 'orange') ||
         (status === ROLLBACK_FAILED && 'red') ||
+        (status === ABORTED && 'red') ||
         ANTD_GRAY[7]
     );
 };

diff --git a/datahub-web-react/src/graphql/container.graphql b/datahub-web-react/src/graphql/container.graphql
@@ -1,6 +1,7 @@
 query getContainer($urn: String!) {
     container(urn: $urn) {
         urn
+        type
         exists
         lastIngested
         platform {

diff --git a/datahub-web-react/src/graphql/dashboard.graphql b/datahub-web-react/src/graphql/dashboard.graphql
@@ -1,5 +1,7 @@
 query getDashboard($urn: String!) {
     dashboard(urn: $urn) {
+        urn
+        type
         ...dashboardFields
         privileges {
             ...entityPrivileges

diff --git a/datahub-web-react/src/graphql/dataJob.graphql b/datahub-web-react/src/graphql/dataJob.graphql
@@ -1,5 +1,7 @@
 query getDataJob($urn: String!) {
     dataJob(urn: $urn) {
+        urn
+        type
         ...dataJobFields
         privileges {
             ...entityPrivileges

diff --git a/datahub-web-react/src/graphql/dataPlatform.graphql b/datahub-web-react/src/graphql/dataPlatform.graphql
@@ -1,5 +1,7 @@
 query getDataPlatform($urn: String!) {
     dataPlatform(urn: $urn) {
+        urn
+        type
         ...platformFields
     }
 }

diff --git a/datahub-web-react/src/graphql/dataProduct.graphql b/datahub-web-react/src/graphql/dataProduct.graphql
@@ -1,5 +1,7 @@
 query getDataProduct($urn: String!) {
     dataProduct(urn: $urn) {
+        urn
+        type
         ...dataProductFields
         privileges {
             ...entityPrivileges

diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql
@@ -1,6 +1,7 @@
 query getDataProfiles($urn: String!, $limit: Int, $startTime: Long, $endTime: Long) {
     dataset(urn: $urn) {
         urn
+        type
         datasetProfiles(limit: $limit, startTimeMillis: $startTime, endTimeMillis: $endTime) {
             rowCount
             columnCount

diff --git a/datahub-web-react/src/graphql/mlFeature.graphql b/datahub-web-react/src/graphql/mlFeature.graphql
@@ -1,5 +1,7 @@
 query getMLFeature($urn: String!) {
     mlFeature(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLFeature
         privileges {
             ...entityPrivileges

diff --git a/datahub-web-react/src/graphql/mlFeatureTable.graphql b/datahub-web-react/src/graphql/mlFeatureTable.graphql
@@ -1,5 +1,7 @@
 query getMLFeatureTable($urn: String!) {
     mlFeatureTable(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLFeatureTable
         privileges {
             ...entityPrivileges

diff --git a/datahub-web-react/src/graphql/mlModel.graphql b/datahub-web-react/src/graphql/mlModel.graphql
@@ -1,5 +1,7 @@
 query getMLModel($urn: String!) {
     mlModel(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLModel
         features: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) {
             start

diff --git a/datahub-web-react/src/graphql/mlModelGroup.graphql b/datahub-web-react/src/graphql/mlModelGroup.graphql
@@ -1,5 +1,7 @@
 query getMLModelGroup($urn: String!) {
     mlModelGroup(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLModelGroupFields
         incoming: relationships(
             input: {

diff --git a/datahub-web-react/src/graphql/mlPrimaryKey.graphql b/datahub-web-react/src/graphql/mlPrimaryKey.graphql
@@ -1,5 +1,7 @@
 query getMLPrimaryKey($urn: String!) {
     mlPrimaryKey(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLPrimaryKey
         privileges {
             ...entityPrivileges

diff --git a/datahub-web-react/src/graphql/tag.graphql b/datahub-web-react/src/graphql/tag.graphql
@@ -1,6 +1,7 @@
 query getTag($urn: String!) {
     tag(urn: $urn) {
         urn
+        type
         name
         description
         properties {

diff --git a/datahub-web-react/src/graphql/user.graphql b/datahub-web-react/src/graphql/user.graphql
@@ -1,6 +1,7 @@
 query getUser($urn: String!, $groupsCount: Int!) {
     corpUser(urn: $urn) {
         urn
+        type
         username
         isNativeUser
         exists

diff --git a/docs/managed-datahub/managed-datahub-overview.md b/docs/managed-datahub/managed-datahub-overview.md
@@ -51,10 +51,17 @@ know.
 | Feature | DataHub | Acryl DataHub |
 | ---------------------------------------------- | ------- | ------------- |
 | Surface data quality results | ✅ | ✅ |
-| Build and enforce continuous data SLAs | ❌ | ✅ |
-| Continuous monitoring of dataset health | ❌ | ✅ |
-| Data observability alerts and notifications | ❌ | ✅ |
-| Data Incident management | ❌ | ✅ |
+| Create data contracts | ✅ | ✅ |
+| Raise and Resolve Data Incidents | ✅ | ✅ |
+| Monitor Freshness SLAs | ❌ | ✅ |
+| Monitor Table Schemas | ❌ | ✅ |
+| Monitor Table Volume | ❌ | ✅ |
+| Validate Table Columns | ❌ | ✅ |
+| Receive Notifications via Email & Slack | ❌ | ✅ |
+| Manage Data Incidents via Slack | ❌ | ✅ |
+| View Data Health Dashboard | ❌ | ✅ |
+| Evaluate data quality checks on-demand (API + UI) | ❌ | ✅ |
+| Evaluate data quality checks in your VPC | ❌ | ✅ |
 
 
 ## Enterprise Grade

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/ReadItem.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/ReadItem.java
@@ -5,6 +5,7 @@
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.mxe.GenericAspect;
 import com.linkedin.mxe.SystemMetadata;
 import java.lang.reflect.InvocationTargetException;
 import javax.annotation.Nonnull;
@@ -26,6 +27,9 @@ public interface ReadItem {
    */
   @Nonnull
   default String getAspectName() {
+    if (getAspectSpec() == null) {
+      return GenericAspect.dataSchema().getName();
+    }
     return getAspectSpec().getName();
   }
 
@@ -72,6 +76,6 @@ static <T> T getAspect(Class<T> clazz, @Nullable RecordTemplate recordTemplate)
    *
    * @return aspect's specification
    */
-  @Nonnull
+  @Nullable
   AspectSpec getAspectSpec();
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java
@@ -84,6 +84,13 @@ static void applyWriteMutationHooks(
     }
   }
 
+  default Stream<MCPItem> applyProposalMutationHooks(
+      Collection<MCPItem> proposedItems, @Nonnull RetrieverContext retrieverContext) {
+    return retrieverContext.getAspectRetriever().getEntityRegistry().getAllMutationHooks().stream()
+        .flatMap(
+            mutationHook -> mutationHook.applyProposalMutation(proposedItems, retrieverContext));
+  }
+
   default <T extends BatchItem> ValidationExceptionCollection validateProposed(
       Collection<T> mcpItems) {
     return validateProposed(mcpItems, getRetrieverContext());

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java
@@ -3,7 +3,6 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
-import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.EntitySpec;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -25,20 +24,13 @@ public boolean enabled() {
   }
 
   public boolean shouldApply(
-      @Nullable ChangeType changeType, @Nonnull Urn entityUrn, @Nonnull AspectSpec aspectSpec) {
-    return shouldApply(changeType, entityUrn.getEntityType(), aspectSpec);
+      @Nullable ChangeType changeType, @Nonnull Urn entityUrn, @Nonnull String aspectName) {
+    return shouldApply(changeType, entityUrn.getEntityType(), aspectName);
   }
 
   public boolean shouldApply(
-      @Nullable ChangeType changeType,
-      @Nonnull EntitySpec entitySpec,
-      @Nonnull AspectSpec aspectSpec) {
-    return shouldApply(changeType, entitySpec.getName(), aspectSpec.getName());
-  }
-
-  public boolean shouldApply(
-      @Nullable ChangeType changeType, @Nonnull String entityName, @Nonnull AspectSpec aspectSpec) {
-    return shouldApply(changeType, entityName, aspectSpec.getName());
+      @Nullable ChangeType changeType, @Nonnull EntitySpec entitySpec, @Nonnull String aspectName) {
+    return shouldApply(changeType, entitySpec.getName(), aspectName);
   }
 
   public boolean shouldApply(
@@ -49,8 +41,8 @@ && isChangeTypeSupported(changeType)
   }
 
   protected boolean isEntityAspectSupported(
-      @Nonnull EntitySpec entitySpec, @Nonnull AspectSpec aspectSpec) {
-    return isEntityAspectSupported(entitySpec.getName(), aspectSpec.getName());
+      @Nonnull EntitySpec entitySpec, @Nonnull String aspectName) {
+    return isEntityAspectSupported(entitySpec.getName(), aspectName);
   }
 
   protected boolean isEntityAspectSupported(

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffect.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffect.java
@@ -24,7 +24,7 @@ public final Stream<MCLItem> apply(
       @Nonnull Collection<MCLItem> batchItems, @Nonnull RetrieverContext retrieverContext) {
     return applyMCLSideEffect(
         batchItems.stream()
-            .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectSpec()))
+            .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectName()))
             .collect(Collectors.toList()),
         retrieverContext);
   }

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java
@@ -25,7 +25,7 @@ public final Stream<ChangeMCP> apply(
       Collection<ChangeMCP> changeMCPS, @Nonnull RetrieverContext retrieverContext) {
     return applyMCPSideEffect(
         changeMCPS.stream()
-            .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectSpec()))
+            .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectName()))
             .collect(Collectors.toList()),
         retrieverContext);
   }
@@ -41,7 +41,7 @@ public final Stream<MCPItem> postApply(
       Collection<MCLItem> mclItems, @Nonnull RetrieverContext retrieverContext) {
     return postMCPSideEffect(
         mclItems.stream()
-            .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectSpec()))
+            .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectName()))
             .collect(Collectors.toList()),
         retrieverContext);
   }

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MutationHook.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MutationHook.java
@@ -3,6 +3,7 @@
 import com.linkedin.metadata.aspect.ReadItem;
 import com.linkedin.metadata.aspect.RetrieverContext;
 import com.linkedin.metadata.aspect.batch.ChangeMCP;
+import com.linkedin.metadata.aspect.batch.MCPItem;
 import com.linkedin.metadata.aspect.plugins.PluginSpec;
 import com.linkedin.util.Pair;
 import java.util.Collection;
@@ -24,7 +25,7 @@ public final Stream<Pair<ChangeMCP, Boolean>> applyWriteMutation(
       @Nonnull Collection<ChangeMCP> changeMCPS, @Nonnull RetrieverContext retrieverContext) {
     return writeMutation(
         changeMCPS.stream()
-            .filter(i -> shouldApply(i.getChangeType(), i.getEntitySpec(), i.getAspectSpec()))
+            .filter(i -> shouldApply(i.getChangeType(), i.getEntitySpec(), i.getAspectName()))
             .collect(Collectors.toList()),
         retrieverContext);
   }
@@ -34,7 +35,23 @@ public final Stream<Pair<ReadItem, Boolean>> applyReadMutation(
       @Nonnull Collection<ReadItem> items, @Nonnull RetrieverContext retrieverContext) {
     return readMutation(
         items.stream()
-            .filter(i -> isEntityAspectSupported(i.getEntitySpec(), i.getAspectSpec()))
+            .filter(i -> isEntityAspectSupported(i.getEntitySpec(), i.getAspectName()))
+            .collect(Collectors.toList()),
+        retrieverContext);
+  }
+
+  /**
+   * Apply Proposal mutations prior to validation
+   *
+   * @param mcpItems wrapper for MCP
+   * @param retrieverContext retriever context
+   * @return stream of mutated Proposal items
+   */
+  public final Stream<MCPItem> applyProposalMutation(
+      @Nonnull Collection<MCPItem> mcpItems, @Nonnull RetrieverContext retrieverContext) {
+    return proposalMutation(
+        mcpItems.stream()
+            .filter(i -> shouldApply(i.getChangeType(), i.getEntitySpec(), i.getAspectName()))
             .collect(Collectors.toList()),
         retrieverContext);
   }
@@ -48,4 +65,9 @@ protected Stream<Pair<ChangeMCP, Boolean>> writeMutation(
       @Nonnull Collection<ChangeMCP> changeMCPS, @Nonnull RetrieverContext retrieverContext) {
     return changeMCPS.stream().map(i -> Pair.of(i, false));
   }
+
+  protected Stream<MCPItem> proposalMutation(
+      @Nonnull Collection<MCPItem> mcpItems, @Nonnull RetrieverContext retrieverContext) {
+    return Stream.empty();
+  }
 }