Merge branch 'datahub-project:master' into master

acryldata · Mar 18, 2024 · 9064249 · 9064249
2 parents 8606a67 + ad4da57
commit 9064249
Show file tree

Hide file tree

Showing 87 changed files with 3,507 additions and 1,034 deletions.
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -7,13 +7,15 @@ on:
     paths:
       - "metadata-ingestion/**"
       - "metadata-models/**"
+      - "docs/**"
       - "docs-website/**"
   push:
     branches:
       - master
     paths:
       - "metadata-ingestion/**"
       - "metadata-models/**"
+      - "docs/**"
       - "docs-website/**"
   # release:
   #   types: [published, edited]

diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -43,7 +43,9 @@ jobs:
                 "gaurav2733",
                 "dushayntAW",
                 "AvaniSiddhapuraAPT",
-                "akarsh991"
+                "akarsh991",
+                "shubhamjagtap639",
+                "mayurinehate"
               ]'), 
               github.actor
             ) 

diff --git a/build.gradle b/build.gradle
@@ -107,7 +107,7 @@ project.ext.externalDependency = [
     'avro': 'org.apache.avro:avro:1.11.3',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
-    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.9',
+    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.0.3',
     'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.13',
     'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.2',
     'awsRds':'software.amazon.awssdk:rds:2.18.24',

diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
@@ -69,9 +69,7 @@ dependencies {
   runtimeOnly externalDependency.mysqlConnector
   runtimeOnly externalDependency.postgresql
 
-  implementation(externalDependency.awsMskIamAuth) {
-    exclude group: 'software.amazon.awssdk', module: 'third-party-jackson-core'
-  }
+  implementation externalDependency.awsMskIamAuth
 
   annotationProcessor externalDependency.lombok
   annotationProcessor externalDependency.picocli

diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx
@@ -106,6 +106,8 @@ type Props = {
     shouldRefetch?: boolean;
     resetShouldRefetch?: () => void;
     applyView?: boolean;
+    onLineageClick?: () => void;
+    isLineageTab?: boolean;
 };
 
 export const EmbeddedListSearch = ({
@@ -134,6 +136,8 @@ export const EmbeddedListSearch = ({
     shouldRefetch,
     resetShouldRefetch,
     applyView = false,
+    onLineageClick,
+    isLineageTab = false,
 }: Props) => {
     const { shouldRefetchEmbeddedListSearch, setShouldRefetchEmbeddedListSearch } = useEntityContext();
     // Adjust query based on props
@@ -143,7 +147,6 @@ export const EmbeddedListSearch = ({
         unionType,
         filters,
     };
-
     const finalFilters =
         (fixedFilters && mergeFilterSets(fixedFilters, baseFilters)) || generateOrFilters(unionType, filters);
 
@@ -191,6 +194,12 @@ export const EmbeddedListSearch = ({
         fetchPolicy: 'cache-first',
     });
 
+    const [serverError, setServerError] = useState<any>(undefined);
+
+    useEffect(() => {
+        setServerError(error);
+    }, [error]);
+
     useEffect(() => {
         if (shouldRefetch && resetShouldRefetch) {
             refetch({
@@ -282,9 +291,18 @@ export const EmbeddedListSearch = ({
         });
     }
 
+    const isServerOverloadError = [503, 500, 504].includes(serverError?.networkError?.response?.status);
+
+    const onClickLessHops = () => {
+        setServerError(undefined);
+        onChangeFilters(defaultFilters);
+    };
+
+    const ErrorMessage = () => <Message type="error" content="Failed to load results! An unexpected error occurred." />;
+
     return (
         <Container>
-            {error && <Message type="error" content="Failed to load results! An unexpected error occurred." />}
+            {!isLineageTab ? error && <ErrorMessage /> : serverError && !isServerOverloadError && <ErrorMessage />}
             <EmbeddedListSearchHeader
                 onSearch={(q) => onChangeQuery(addFixedQuery(q, fixedQuery as string, emptySearchQuery as string))}
                 placeholderText={placeholderText}
@@ -303,6 +321,10 @@ export const EmbeddedListSearch = ({
             />
             <EmbeddedListSearchResults
                 unionType={unionType}
+                isServerOverloadError={isServerOverloadError}
+                onClickLessHops={onClickLessHops}
+                onLineageClick={onLineageClick}
+                isLineageTab={isLineageTab}
                 loading={loading}
                 searchResponse={data}
                 filters={finalFacets}

diff --git a/...ub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx b/...ub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx
@@ -1,5 +1,5 @@
 import React from 'react';
-import { Pagination, Spin, Typography } from 'antd';
+import { Button, Pagination, Spin, Typography } from 'antd';
 import { LoadingOutlined } from '@ant-design/icons';
 import styled from 'styled-components';
 import { FacetFilterInput, FacetMetadata, SearchResults as SearchResultType } from '../../../../../../types.generated';
@@ -66,6 +66,20 @@ const StyledLoading = styled(LoadingOutlined)`
     padding-bottom: 18px;
 ]`;
 
+const ErrorMessage = styled.div`
+    padding-top: 70px;
+    font-size: 16px;
+    padding-bottom: 40px;
+    width: 100%;
+    text-align: center;
+    flex: 1;
+`;
+
+const StyledLinkButton = styled(Button)`
+    margin: 0 -14px;
+    font-size: 16px;
+`;
+
 interface Props {
     page: number;
     searchResponse?: SearchResultType | null;
@@ -84,6 +98,10 @@ interface Props {
     setNumResultsPerPage: (numResults: number) => void;
     entityAction?: React.FC<EntityActionProps>;
     applyView?: boolean;
+    isServerOverloadError?: any;
+    onClickLessHops?: () => void;
+    onLineageClick?: () => void;
+    isLineageTab?: boolean;
 }
 
 export const EmbeddedListSearchResults = ({
@@ -104,6 +122,10 @@ export const EmbeddedListSearchResults = ({
     setNumResultsPerPage,
     entityAction,
     applyView,
+    isServerOverloadError,
+    onClickLessHops,
+    onLineageClick,
+    isLineageTab = false,
 }: Props) => {
     const pageStart = searchResponse?.start || 0;
     const pageSize = searchResponse?.count || 0;
@@ -131,7 +153,19 @@ export const EmbeddedListSearchResults = ({
                             <Spin indicator={<StyledLoading />} />
                         </LoadingContainer>
                     )}
-                    {!loading && (
+                    {isLineageTab && !loading && isServerOverloadError && (
+                        <ErrorMessage>
+                            Data is too large. Please use
+                            <StyledLinkButton onClick={onLineageClick} type="link">
+                                visualize lineage
+                            </StyledLinkButton>
+                            or see less hops by clicking
+                            <StyledLinkButton onClick={onClickLessHops} type="link">
+                                here
+                            </StyledLinkButton>
+                        </ErrorMessage>
+                    )}
+                    {!loading && !isServerOverloadError && (
                         <EntitySearchResults
                             searchResults={searchResponse?.searchResults || []}
                             additionalPropertiesList={

diff --git a/...ub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx b/...ub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx
@@ -52,6 +52,8 @@ type Props = {
     shouldRefetch?: boolean;
     resetShouldRefetch?: () => void;
     applyView?: boolean;
+    onLineageClick?: () => void;
+    isLineageTab?: boolean;
 };
 
 export const EmbeddedListSearchSection = ({
@@ -69,6 +71,8 @@ export const EmbeddedListSearchSection = ({
     shouldRefetch,
     resetShouldRefetch,
     applyView,
+    onLineageClick,
+    isLineageTab
 }: Props) => {
     const history = useHistory();
     const location = useLocation();
@@ -155,6 +159,8 @@ export const EmbeddedListSearchSection = ({
             shouldRefetch={shouldRefetch}
             resetShouldRefetch={resetShouldRefetch}
             applyView={applyView}
+            onLineageClick={onLineageClick}
+            isLineageTab={isLineageTab}
         />
     );
 };
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx b/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx
@@ -13,6 +13,8 @@ type Props = {
     skipCache?: boolean;
     setSkipCache?: (skipCache: boolean) => void;
     resetShouldRefetch?: () => void;
+    onLineageClick?: () => void;
+    isLineageTab?: boolean;
 };
 
 export const ImpactAnalysis = ({
@@ -24,6 +26,8 @@ export const ImpactAnalysis = ({
     skipCache,
     setSkipCache,
     resetShouldRefetch,
+    onLineageClick,
+    isLineageTab
 }: Props) => {
     const finalStartTimeMillis = startTimeMillis || undefined;
     const finalEndTimeMillis = endTimeMillis || undefined;
@@ -49,6 +53,8 @@ export const ImpactAnalysis = ({
             defaultFilters={[{ field: 'degree', values: ['1'] }]}
             shouldRefetch={shouldRefetch}
             resetShouldRefetch={resetShouldRefetch}
+            onLineageClick={onLineageClick}
+            isLineageTab={isLineageTab}
         />
     );
 };
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Lineage/LineageTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Lineage/LineageTab.tsx
@@ -181,6 +181,8 @@ export const LineageTab = ({
             <LineageTabContext.Provider value={{ isColumnLevelLineage, selectedColumn, lineageDirection }}>
                 <ImpactAnalysis
                     urn={impactAnalysisUrn}
+                    onLineageClick={routeToLineage}
+                    isLineageTab
                     direction={lineageDirection as LineageDirection}
                     startTimeMillis={startTimeMillis}
                     endTimeMillis={endTimeMillis}

diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile
@@ -52,8 +52,8 @@ RUN ls -la
 COPY --from=confluent_base /usr/share/java/cp-base-new/ /usr/share/java/cp-base-new/
 COPY --from=confluent_base /etc/cp-base-new/log4j.properties /etc/cp-base-new/log4j.properties
 
-ADD --chown=kafka:kafka ${GITHUB_REPO_URL}/aws/aws-msk-iam-auth/releases/download/v1.1.6/aws-msk-iam-auth-1.1.6-all.jar /usr/share/java/cp-base-new
-ADD --chown=kafka:kafka ${GITHUB_REPO_URL}/aws/aws-msk-iam-auth/releases/download/v1.1.6/aws-msk-iam-auth-1.1.6-all.jar /opt/kafka/libs
+ADD --chown=kafka:kafka ${GITHUB_REPO_URL}/aws/aws-msk-iam-auth/releases/download/v2.0.3/aws-msk-iam-auth-2.0.3-all.jar /usr/share/java/cp-base-new
+ADD --chown=kafka:kafka ${GITHUB_REPO_URL}/aws/aws-msk-iam-auth/releases/download/v2.0.3/aws-msk-iam-auth-2.0.3-all.jar /opt/kafka/libs
 
 ENV METADATA_AUDIT_EVENT_NAME="MetadataAuditEvent_v4"
 ENV METADATA_CHANGE_EVENT_NAME="MetadataChangeEvent_v4"

diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
@@ -66,7 +66,7 @@ module.exports = {
       },
       items: [
         {
-          to: "docs/",
+          to: "docs/features",
           activeBasePath: "docs",
           label: "Docs",
           position: "right",
@@ -137,12 +137,28 @@ module.exports = {
           dropdownActiveClassDisabled: true,
             dropdownItemsAfter: [
                 {
-                href: "https://docs-website-irpoe2osc-acryldata.vercel.app/docs/",
-                label: "0.11.0",
+                type: 'html',
+                value: '<hr class="dropdown-separator" style="margin: 0.4rem;">',
                 },
                 {
-                href: "https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/",
-                label: "0.10.5",
+                type: 'html',
+                value: '<div class="dropdown__link"><b>Archived versions</b></div>',
+                },
+                {
+                value: `
+                   <a class="dropdown__link" href="https://docs-website-irpoe2osc-acryldata.vercel.app/docs/features">0.11.0
+                   <svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
+                   </a>
+                   `,
+                type: "html",
+                },
+                {
+                value: `
+                   <a class="dropdown__link" href="https://docs-website-1gv2yzn9d-acryldata.vercel.app/docs/features">0.10.5
+                   <svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg>
+                   </a>
+                   `,
+                type: "html",
                 },
             ],
         },

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
@@ -173,6 +173,7 @@ module.exports = {
         },
         {
           "Managed DataHub Release History": [
+            "docs/managed-datahub/release-notes/v_0_2_16",
             "docs/managed-datahub/release-notes/v_0_2_15",
             "docs/managed-datahub/release-notes/v_0_2_14",
             "docs/managed-datahub/release-notes/v_0_2_13",

diff --git a/docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss b/docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss
@@ -1,11 +1,15 @@
+@media (min-width: 997px) and (max-width: 1465px) {
+  .feature {
+    min-height: 20rem !important;
+    max-height: 30rem !important;
+  }
+}
 .feature {
   flex-direction: row;
-  padding: 1.75rem;
   color: var(--ifm-hero-text-color);
   margin: 0rem 2rem 1rem 0rem;
-  min-height: 14rem;
-  max-height: 15rem; 
-  overflow: hidden;
+  min-height: 15rem;
+  max-height: 15rem;
   text-decoration: none !important;
 
   img {
@@ -36,3 +40,4 @@
     border-color: var(--ifm-color-primary);
   }
 }
+
diff --git a/docs-website/src/pages/docs/_components/FeatureCard/index.jsx b/docs-website/src/pages/docs/_components/FeatureCard/index.jsx
@@ -8,7 +8,7 @@ const FeatureCard = ({icon, title, description, to}) => {
 return (
     <div className="col col--4">
       <Link to={useBaseUrl(to)} className={clsx("card", styles.feature)}>
-        <div> 
+        <div className={styles.card_content}>
           {icon}
           <strong>{title}&nbsp;→</strong>
           <span>{description}</span>

diff --git a/docs-website/src/pages/docs/_components/QuickstartCard/index.jsx b/docs-website/src/pages/docs/_components/QuickstartCard/index.jsx
@@ -9,9 +9,13 @@ const QuickstartCard = ({ icon, title, to, color, fontColor }) => {
   return (
     <div className="col col--6">
       <Link to={to} className={clsx("card", styles.feature)} style={{ background: color, color: fontColor}}>
-        <img src={useBaseUrl(`/img/${icon}.svg`)} />
-        <div style={{ margin: "auto 0"}}>
-          <strong>{title}&nbsp;→</strong>
+        <div className={styles.card_content}>
+            <img src={useBaseUrl(`/img/${icon}.svg`)} />
+            <div style={{ margin: "auto 0"}}>
+              <div className={styles.card_title}>
+                {title}&nbsp;→
+              </div>
+            </div>
         </div>
       </Link>
     </div>