Merge branch 'master' into play-oidc-nimbus-upgrade

datahub-project · Oct 26, 2024 · 520af6a · 520af6a
2 parents cb330c6 + 84d86ca
commit 520af6a
Show file tree

Hide file tree

Showing 91 changed files with 49,652 additions and 653 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -70,7 +70,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-          cache: pip
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -44,7 +44,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-          cache: pip
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - name: Install Python dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Build Docs

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -62,16 +62,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
       - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/uv
           key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install package

diff --git a/...raphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java b/...raphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java
@@ -52,7 +52,8 @@ public class ContainerType
           Constants.DEPRECATION_ASPECT_NAME,
           Constants.DATA_PRODUCTS_ASPECT_NAME,
           Constants.STRUCTURED_PROPERTIES_ASPECT_NAME,
-          Constants.FORMS_ASPECT_NAME);
+          Constants.FORMS_ASPECT_NAME,
+          Constants.ACCESS_ASPECT_NAME);
 
   private static final Set<String> FACET_FIELDS = ImmutableSet.of("origin", "platform");
   private static final String ENTITY_NAME = "container";

diff --git a/...e/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java b/...e/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java
@@ -2,6 +2,7 @@
 
 import static com.linkedin.metadata.Constants.*;
 
+import com.linkedin.common.Access;
 import com.linkedin.common.DataPlatformInstance;
 import com.linkedin.common.Deprecation;
 import com.linkedin.common.Forms;
@@ -30,6 +31,7 @@
 import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
 import com.linkedin.datahub.graphql.types.form.FormsMapper;
 import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
+import com.linkedin.datahub.graphql.types.rolemetadata.mappers.AccessMapper;
 import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper;
 import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper;
 import com.linkedin.domain.Domains;
@@ -105,6 +107,11 @@ public static Container map(
               context, new GlossaryTerms(envelopedTerms.getValue().data()), entityUrn));
     }
 
+    final EnvelopedAspect accessAspect = aspects.get(ACCESS_ASPECT_NAME);
+    if (accessAspect != null) {
+      result.setAccess(AccessMapper.map(new Access(accessAspect.getValue().data()), entityUrn));
+    }
+
     final EnvelopedAspect envelopedInstitutionalMemory =
         aspects.get(Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME);
     if (envelopedInstitutionalMemory != null) {

diff --git a/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java
@@ -86,7 +86,7 @@ public class DatasetType
           EMBED_ASPECT_NAME,
           DATA_PRODUCTS_ASPECT_NAME,
           BROWSE_PATHS_V2_ASPECT_NAME,
-          ACCESS_DATASET_ASPECT_NAME,
+          ACCESS_ASPECT_NAME,
           STRUCTURED_PROPERTIES_ASPECT_NAME,
           FORMS_ASPECT_NAME,
           SUB_TYPES_ASPECT_NAME);

diff --git a/...-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/...-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java
@@ -166,7 +166,7 @@ public Dataset apply(
         (dataset, dataMap) ->
             dataset.setBrowsePathV2(BrowsePathsV2Mapper.map(context, new BrowsePathsV2(dataMap))));
     mappingHelper.mapToResult(
-        ACCESS_DATASET_ASPECT_NAME,
+        ACCESS_ASPECT_NAME,
         ((dataset, dataMap) ->
             dataset.setAccess(AccessMapper.map(new Access(dataMap), entityUrn))));
     mappingHelper.mapToResult(

diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -2837,6 +2837,11 @@ type Container implements Entity {
     """
     exists: Boolean
 
+    """
+	The Roles and the properties to access the container
+	"""
+	access: Access
+
     """
     Experimental API.
     For fetching extra entities that do not have custom UI code yet

diff --git a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx
@@ -8,7 +8,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'
 import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection';
 import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection';
 import { getDataForEntityType } from '../shared/containers/profile/utils';
-import { useGetContainerQuery } from '../../../graphql/container.generated';
+import { useGetContainerQuery, GetContainerQuery } from '../../../graphql/container.generated';
 import { ContainerEntitiesTab } from './ContainerEntitiesTab';
 import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
@@ -17,6 +17,8 @@ import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { getDataProduct } from '../shared/utils';
 import EmbeddedProfile from '../shared/embed/EmbeddedProfile';
+import AccessManagement from '../shared/tabs/Dataset/AccessManagement/AccessManagement';
+import { useAppConfig } from '../../useAppConfig';
 
 /**
  * Definition of the DataHub Container entity.
@@ -65,6 +67,8 @@ export class ContainerEntity implements Entity<Container> {
 
     useEntityQuery = useGetContainerQuery;
 
+    appconfig = useAppConfig;
+
     renderProfile = (urn: string) => (
         <EntityProfile
             urn={urn}
@@ -85,6 +89,23 @@ export class ContainerEntity implements Entity<Container> {
                     name: 'Properties',
                     component: PropertiesTab,
                 },
+                {
+                    name: 'Access Management',
+                    component: AccessManagement,
+                    display: {
+                        visible: (_, container: GetContainerQuery) => {
+                            return (
+                                this.appconfig().config.featureFlags.showAccessManagement &&
+                                !!container?.container?.access
+                            );
+                        },
+                        enabled: (_, container: GetContainerQuery) => {
+                            const accessAspect = container?.container?.access;
+                            const rolesList = accessAspect?.roles;
+                            return !!accessAspect && !!rolesList && rolesList.length > 0;
+                        },
+                    },
+                },
             ]}
             sidebarSections={this.getSidebarSections()}
         />

diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx
@@ -3,8 +3,8 @@ import styled from 'styled-components';
 import { Button, Table } from 'antd';
 import { SpinProps } from 'antd/es/spin';
 import { LoadingOutlined } from '@ant-design/icons';
-import { useBaseEntity } from '../../../EntityContext';
-import { GetDatasetQuery, useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated';
+import { useEntityData } from '../../../EntityContext';
+import { useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated';
 import { handleAccessRoles } from './utils';
 import AccessManagerDescription from './AccessManagerDescription';
 
@@ -60,11 +60,12 @@ const AccessButton = styled(Button)`
 `;
 
 export default function AccessManagement() {
-    const baseEntity = useBaseEntity<GetDatasetQuery>();
+    const { entityData } = useEntityData();
+    const entityUrn = (entityData as any)?.urn;
 
     const { data: externalRoles, loading: isLoading } = useGetExternalRolesQuery({
-        variables: { urn: baseEntity?.dataset?.urn as string },
-        skip: !baseEntity?.dataset?.urn,
+        variables: { urn: entityUrn as string },
+        skip: !entityUrn,
     });
 
     const columns = [

diff --git a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx
@@ -17,7 +17,7 @@ const Title = styled.div`
     font-weight: bold;
 `;
 
-const Item = styled.div`
+const Item = styled.pre`
     padding: 6px;
     font-size: 12px;
     border-radius: 2px;

diff --git a/datahub-web-react/src/graphql/container.graphql b/datahub-web-react/src/graphql/container.graphql
@@ -59,6 +59,13 @@ query getContainer($urn: String!) {
         status {
             removed
         }
+        access {
+            roles {
+                role {
+                    urn
+                }
+            }
+        }
         autoRenderAspects: aspects(input: { autoRenderOnly: true }) {
             ...autoRenderAspectFields
         }

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
@@ -101,12 +101,19 @@ module.exports = {
         {
           label: "Automations",
           type: "category",
+          collapsed: false,
           items: [
             {
               label: "Documentation Propagation",
               type: "doc",
               id: "docs/automations/docs-propagation",
             },
+            {
+              label: "BigQuery Metadata Sync",
+              type: "doc",
+              id: "docs/automations/bigquery-metadata-sync",
+              className: "saasOnly",
+            },
             {
               label: "Snowflake Tag Sync",
               type: "doc",

diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock
@@ -1827,7 +1827,7 @@
     "@docusaurus/theme-search-algolia" "2.4.3"
     "@docusaurus/types" "2.4.3"
 
-"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2":
+"@docusaurus/react-loadable@5.5.2":
   version "5.5.2"
   resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce"
   integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==
@@ -6347,9 +6347,9 @@ http-proxy-agent@^7.0.0, http-proxy-agent@^7.0.1:
     debug "^4.3.4"
 
 http-proxy-middleware@^2.0.3:
-  version "2.0.6"
-  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz#e1a4dd6979572c7ab5a4e4b55095d1f32a74963f"
-  integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6"
+  integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==
   dependencies:
     "@types/http-proxy" "^1.17.8"
     http-proxy "^1.18.1"
@@ -9708,6 +9708,14 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1:
   dependencies:
     "@babel/runtime" "^7.10.3"
 
+"react-loadable@npm:@docusaurus/react-loadable@5.5.2":
+  version "5.5.2"
+  resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce"
+  integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==
+  dependencies:
+    "@types/react" "*"
+    prop-types "^15.6.2"
+
 react-markdown@^8.0.6:
   version "8.0.7"
   resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b"