From 30e9cfdce0b350cbc0fd1543acc49881977572c4 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Fri, 19 Jul 2024 10:47:24 -0400 Subject: [PATCH 1/2] fix(connections) Harden adding /gms to connections in backend (#10942) --- metadata-ingestion/src/datahub/cli/cli_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py index 52aa3cb89d1da..bda351a4de6b1 100644 --- a/metadata-ingestion/src/datahub/cli/cli_utils.py +++ b/metadata-ingestion/src/datahub/cli/cli_utils.py @@ -629,6 +629,9 @@ def _ensure_valid_gms_url_acryl_cloud(url: str) -> str: url = url.replace("http://", "https://") if url.endswith("acryl.io"): url = f"{url}/gms" + elif url.endswith("acryl.io/"): + url = f"{url}gms" + return url From eace710aeec8773f3b8eeb48637d7180fb3ab147 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Fri, 19 Jul 2024 13:41:18 -0400 Subject: [PATCH 2/2] feat(siblings) Add flag to prevent combining siblings in the UI (#10952) --- .../graphql/featureflags/FeatureFlags.java | 1 + .../resolvers/config/AppConfigResolver.java | 1 + .../src/main/resources/app.graphql | 5 +++ .../PlatformContent/PlatformContentView.tsx | 11 +++-- .../sidebar/SidebarSiblingsSection.tsx | 15 +++++-- .../src/app/entity/shared/siblingUtils.ts | 4 +- .../lineage/controls/LineageVizToggles.tsx | 44 ++++++++++--------- .../src/app/search/SearchBar.tsx | 11 +++-- .../src/app/search/SearchResults.tsx | 7 ++- .../autoComplete/AutoCompleteEntity.tsx | 7 ++- .../combineSiblingsInSearchResults.test.ts | 4 +- .../utils/combineSiblingsInSearchResults.ts | 5 ++- datahub-web-react/src/app/useAppConfig.ts | 5 +++ datahub-web-react/src/appConfigContext.tsx | 1 + datahub-web-react/src/graphql/app.graphql | 1 + .../src/main/resources/application.yaml | 1 + 16 files changed, 88 insertions(+), 35 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index 2a9af37a06ad9..85a2c09ed79a7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -21,4 +21,5 @@ public class FeatureFlags { private boolean schemaFieldEntityFetchEnabled = false; private boolean businessAttributeEntityEnabled = false; private boolean dataContractsEnabled = false; + private boolean showSeparateSiblings = false; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index caa469003c22e..fb1672d54dc97 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -186,6 +186,7 @@ public CompletableFuture get(final DataFetchingEnvironment environmen .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled()) .setPlatformBrowseV2(_featureFlags.isPlatformBrowseV2()) .setDataContractsEnabled(_featureFlags.isDataContractsEnabled()) + .setShowSeparateSiblings(_featureFlags.isShowSeparateSiblings()) .build(); appConfig.setFeatureFlags(featureFlagsConfig); diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index b3a965981c366..23c2468bae19a 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -497,6 +497,11 @@ type FeatureFlagsConfig { Whether data contracts should be enabled """ dataContractsEnabled: Boolean! + + """ + If turned on, all siblings will be separated with no way to get to a "combined" sibling view + """ + showSeparateSiblings: Boolean! } """ diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx index 52cb656f54eb9..3104dfd911aab 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx @@ -12,6 +12,7 @@ import { StyledTooltip, } from './ParentNodesView'; import ParentEntities from '../../../../../../search/filters/ParentEntities'; +import { useIsShowSeparateSiblingsEnabled } from '../../../../../../useAppConfig'; export const LogoIcon = styled.span` display: flex; @@ -100,6 +101,10 @@ function PlatformContentView(props: Props) { const directParentContainer = parentContainers && parentContainers[0]; const remainingParentContainers = parentContainers && parentContainers.slice(1, parentContainers.length); + const shouldShowSeparateSiblings = useIsShowSeparateSiblingsEnabled(); + const showSiblingPlatformLogos = !shouldShowSeparateSiblings && !!platformLogoUrls; + const showSiblingPlatformNames = !shouldShowSeparateSiblings && !!platformNames; + return ( {typeIcon && {typeIcon}} @@ -110,10 +115,10 @@ function PlatformContentView(props: Props) { {platformName && ( {!platformLogoUrl && !platformLogoUrls && entityLogoComponent} - {!!platformLogoUrl && !platformLogoUrls && ( + {!!platformLogoUrl && !showSiblingPlatformLogos && ( )} - {!!platformLogoUrls && + {showSiblingPlatformLogos && platformLogoUrls.slice(0, 2).map((platformLogoUrlsEntry) => ( <> @@ -122,7 +127,7 @@ function PlatformContentView(props: Props) { )} - {platformNames ? platformNames.join(' & ') : platformName} + {showSiblingPlatformNames ? platformNames.join(' & ') : platformName} {(directParentContainer || instanceId) && } {instanceId && ( diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx index 4ea1ab69e44b3..17e09045196a4 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx @@ -7,6 +7,7 @@ import { CompactEntityNameList } from '../../../../../recommendations/renderer/c import { Dataset, Entity } from '../../../../../../types.generated'; import { SEPARATE_SIBLINGS_URL_PARAM, stripSiblingsFromEntity, useIsSeparateSiblingsMode } from '../../../siblingUtils'; import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; +import { useIsShowSeparateSiblingsEnabled } from '../../../../../useAppConfig'; const EntityListContainer = styled.div` margin-left: -8px; @@ -16,13 +17,15 @@ export const SidebarSiblingsSection = () => { const { entityData } = useEntityData(); const dataNotCombinedWithSiblings = useDataNotCombinedWithSiblings(); + const showSeparateSiblings = useIsShowSeparateSiblingsEnabled(); const isHideSiblingMode = useIsSeparateSiblingsMode(); if (!entityData) { return <>; } - if (isHideSiblingMode) { + // showSeparateSiblings disables the combined view, but with this flag on we show siblings in the sidebar to navigate to them + if (!showSeparateSiblings && isHideSiblingMode) { return (
@@ -36,13 +39,19 @@ export const SidebarSiblingsSection = () => { const siblingEntities = entityData?.siblings?.siblings || []; const entityDataWithoutSiblings = stripSiblingsFromEntity(dataNotCombinedWithSiblings.dataset); - const allSiblingsInGroup = [...siblingEntities, entityDataWithoutSiblings] as Dataset[]; + const allSiblingsInGroup = showSeparateSiblings + ? (siblingEntities as Dataset[]) + : ([...siblingEntities, entityDataWithoutSiblings] as Dataset[]); const allSiblingsInGroupThatExist = allSiblingsInGroup.filter((sibling) => sibling.exists); + if (!allSiblingsInGroupThatExist.length) { + return <>; + } + // you are always going to be in the sibling group, so if the sibling group is just you do not render. // The less than case is likely not neccessary but just there as a safety case for unexpected scenarios - if (allSiblingsInGroupThatExist.length <= 1) { + if (!showSeparateSiblings && allSiblingsInGroupThatExist.length <= 1) { return <>; } diff --git a/datahub-web-react/src/app/entity/shared/siblingUtils.ts b/datahub-web-react/src/app/entity/shared/siblingUtils.ts index d31342a85b7e0..2f50dc99df191 100644 --- a/datahub-web-react/src/app/entity/shared/siblingUtils.ts +++ b/datahub-web-react/src/app/entity/shared/siblingUtils.ts @@ -4,6 +4,7 @@ import { useLocation } from 'react-router-dom'; import * as QueryString from 'query-string'; import { Dataset, Entity, Maybe, SiblingProperties } from '../../../types.generated'; import { GenericEntityProperties } from './types'; +import { useIsShowSeparateSiblingsEnabled } from '../../useAppConfig'; export function stripSiblingsFromEntity(entity: any) { return { @@ -267,8 +268,9 @@ export const SEPARATE_SIBLINGS_URL_PARAM = 'separate_siblings'; // used to determine whether sibling entities should be shown merged or not export function useIsSeparateSiblingsMode() { + const showSeparateSiblings = useIsShowSeparateSiblingsEnabled(); const location = useLocation(); const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); - return params[SEPARATE_SIBLINGS_URL_PARAM] === 'true'; + return showSeparateSiblings || params[SEPARATE_SIBLINGS_URL_PARAM] === 'true'; } diff --git a/datahub-web-react/src/app/lineage/controls/LineageVizToggles.tsx b/datahub-web-react/src/app/lineage/controls/LineageVizToggles.tsx index a55309d73d054..3287b180a864b 100644 --- a/datahub-web-react/src/app/lineage/controls/LineageVizToggles.tsx +++ b/datahub-web-react/src/app/lineage/controls/LineageVizToggles.tsx @@ -8,6 +8,7 @@ import { ANTD_GRAY } from '../../entity/shared/constants'; import { navigateToLineageUrl } from '../utils/navigateToLineageUrl'; import { useIsSeparateSiblingsMode } from '../../entity/shared/siblingUtils'; import { useIsShowColumnsMode } from '../utils/useIsShowColumnsMode'; +import { useIsShowSeparateSiblingsEnabled } from '../../useAppConfig'; const ControlDiv = styled.div` display: flex; @@ -38,6 +39,7 @@ export function LineageVizToggles({ showExpandedTitles, setShowExpandedTitles }: const location = useLocation(); const isHideSiblingMode = useIsSeparateSiblingsMode(); const showColumns = useIsShowColumnsMode(); + const showSeparateSiblings = useIsShowSeparateSiblingsEnabled(); return ( <> @@ -51,26 +53,28 @@ export function LineageVizToggles({ showExpandedTitles, setShowExpandedTitles }: Show Full Titles - - { - navigateToLineageUrl({ - location, - history, - isLineageMode: true, - isHideSiblingMode: !checked, - }); - }} - />{' '} - - Compress Lineage - - - - - + {!showSeparateSiblings && ( + + { + navigateToLineageUrl({ + location, + history, + isLineageMode: true, + isHideSiblingMode: !checked, + }); + }} + />{' '} + + Compress Lineage + + + + + + )} (); const [isDropdownVisible, setIsDropdownVisible] = useState(false); const [isFocused, setIsFocused] = useState(false); + const isShowSeparateSiblingsEnabled = useIsShowSeparateSiblingsEnabled(); + const finalCombineSiblings = isShowSeparateSiblingsEnabled ? false : combineSiblings; useEffect(() => setSelected(initialQuery), [initialQuery]); @@ -223,7 +226,9 @@ export const SearchBar = ({ const autoCompleteEntityOptions = useMemo(() => { return suggestions.map((suggestion: AutoCompleteResultForEntity) => { - const combinedSuggestion = combineSiblingsInAutoComplete(suggestion, { combineSiblings }); + const combinedSuggestion = combineSiblingsInAutoComplete(suggestion, { + combineSiblings: finalCombineSiblings, + }); return { label: , options: combinedSuggestion.combinedEntities.map((combinedEntity) => ({ @@ -232,7 +237,7 @@ export const SearchBar = ({ ), type: combinedEntity.entity.type, @@ -240,7 +245,7 @@ export const SearchBar = ({ })), }; }); - }, [combineSiblings, effectiveQuery, suggestions]); + }, [finalCombineSiblings, effectiveQuery, suggestions]); const previousSelectedQuickFilterValue = usePrevious(selectedQuickFilter?.value); useEffect(() => { diff --git a/datahub-web-react/src/app/search/SearchResults.tsx b/datahub-web-react/src/app/search/SearchResults.tsx index d7ad6d517d8fe..dafe9a20b6ab7 100644 --- a/datahub-web-react/src/app/search/SearchResults.tsx +++ b/datahub-web-react/src/app/search/SearchResults.tsx @@ -29,6 +29,7 @@ import SearchQuerySuggester from './suggestions/SearchQuerySugggester'; import { ANTD_GRAY_V2 } from '../entity/shared/constants'; import { formatNumberWithoutAbbreviation } from '../shared/formatNumber'; import SearchResultsLoadingSection from './SearchResultsLoadingSection'; +import { useIsShowSeparateSiblingsEnabled } from '../useAppConfig'; const SearchResultsWrapper = styled.div<{ v2Styles: boolean }>` display: flex; @@ -173,7 +174,11 @@ export const SearchResults = ({ const totalResults = searchResponse?.total || 0; const lastResultIndex = pageStart + pageSize > totalResults ? totalResults : pageStart + pageSize; const authenticatedUserUrn = useUserContext().user?.urn; - const combinedSiblingSearchResults = combineSiblingsInSearchResults(searchResponse?.searchResults); + const showSeparateSiblings = useIsShowSeparateSiblingsEnabled(); + const combinedSiblingSearchResults = combineSiblingsInSearchResults( + showSeparateSiblings, + searchResponse?.searchResults, + ); const searchResultUrns = combinedSiblingSearchResults.map((result) => result.entity.urn) || []; const selectedEntityUrns = selectedEntities.map((entity) => entity.urn); diff --git a/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx b/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx index 2154837fa5e26..e59fbad316bb2 100644 --- a/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx +++ b/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx @@ -66,8 +66,13 @@ export default function AutoCompleteEntity({ query, entity, siblings, hasParentT const displayName = entityRegistry.getDisplayName(entity.type, entity); const { matchedText, unmatchedText } = getAutoCompleteEntityText(displayName, query); const entities = siblings?.length ? siblings : [entity]; + const platformsToShow = + /* Only show sibling platforms if there are > 0 explicitly included siblings */ + siblings?.length + ? genericEntityProps?.siblingPlatforms + : (genericEntityProps?.platform && [genericEntityProps?.platform]) || undefined; const platforms = - genericEntityProps?.siblingPlatforms + platformsToShow ?.map( (platform) => getPlatformName(entityRegistry.getGenericEntityProperties(EntityType.DataPlatform, platform)) || '', diff --git a/datahub-web-react/src/app/search/utils/combineSiblingsInSearchResults.test.ts b/datahub-web-react/src/app/search/utils/combineSiblingsInSearchResults.test.ts index 4cf61c715b0e9..e3bfb59b3675e 100644 --- a/datahub-web-react/src/app/search/utils/combineSiblingsInSearchResults.test.ts +++ b/datahub-web-react/src/app/search/utils/combineSiblingsInSearchResults.test.ts @@ -491,7 +491,7 @@ const searchResultWithGhostSiblings = [ describe('siblingUtils', () => { describe('combineSiblingsInSearchResults', () => { it('combines search results to deduplicate siblings', () => { - const result = combineSiblingsInSearchResults(searchResultWithSiblings as any); + const result = combineSiblingsInSearchResults(false, searchResultWithSiblings as any); expect(result).toHaveLength(1); expect(result?.[0]?.matchedEntities?.[0]?.urn).toEqual( @@ -507,7 +507,7 @@ describe('siblingUtils', () => { }); it('will not combine an entity with a ghost node', () => { - const result = combineSiblingsInSearchResults(searchResultWithGhostSiblings as any); + const result = combineSiblingsInSearchResults(false, searchResultWithGhostSiblings as any); expect(result).toHaveLength(1); expect(result?.[0]?.matchedEntities?.[0]?.urn).toEqual( diff --git a/datahub-web-react/src/app/search/utils/combineSiblingsInSearchResults.ts b/datahub-web-react/src/app/search/utils/combineSiblingsInSearchResults.ts index 4a5c8da6381b8..ae0470eedeaf2 100644 --- a/datahub-web-react/src/app/search/utils/combineSiblingsInSearchResults.ts +++ b/datahub-web-react/src/app/search/utils/combineSiblingsInSearchResults.ts @@ -9,13 +9,16 @@ type UncombinedSeaerchResults = { export type CombinedSearchResult = CombinedEntity & Pick; export function combineSiblingsInSearchResults( + showSeparateSiblings: boolean, searchResults: Array | undefined = [], ): Array { const combine = createSiblingEntityCombiner(); const combinedSearchResults: Array = []; searchResults.forEach((searchResult) => { - const combinedResult = combine(searchResult.entity); + const combinedResult = showSeparateSiblings + ? { combinedEntity: searchResult.entity, skipped: false } + : combine(searchResult.entity); if (!combinedResult.skipped) { combinedSearchResults.push({ ...searchResult, diff --git a/datahub-web-react/src/app/useAppConfig.ts b/datahub-web-react/src/app/useAppConfig.ts index f167ccad16474..12470acdf64ee 100644 --- a/datahub-web-react/src/app/useAppConfig.ts +++ b/datahub-web-react/src/app/useAppConfig.ts @@ -27,3 +27,8 @@ export function useIsAppConfigContextLoaded() { const appConfig = useAppConfig(); return appConfig.loaded; } + +export function useIsShowSeparateSiblingsEnabled() { + const appConfig = useAppConfig(); + return appConfig.config.featureFlags.showSeparateSiblings; +} diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index 4262f772b006e..54547aaa5642e 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -54,6 +54,7 @@ export const DEFAULT_APP_CONFIG = { platformBrowseV2: false, businessAttributeEntityEnabled: false, dataContractsEnabled: false, + showSeparateSiblings: false, }, }; diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index 662b2f11336fa..bfca27a4ad106 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -69,6 +69,7 @@ query appConfig { platformBrowseV2 businessAttributeEntityEnabled dataContractsEnabled + showSeparateSiblings } } } diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 1d5b7c7904f97..6d66ddaea510a 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -371,6 +371,7 @@ featureFlags: schemaFieldEntityFetchEnabled: ${SCHEMA_FIELD_ENTITY_FETCH_ENABLED:true} # Enables fetching for schema field entities from the database when we hydrate them on schema fields businessAttributeEntityEnabled: ${BUSINESS_ATTRIBUTE_ENTITY_ENABLED:false} # Enables business attribute entity which can be associated with field of dataset dataContractsEnabled: ${DATA_CONTRACTS_ENABLED:true} # Enables the Data Contracts feature (Tab) in the UI + showSeparateSiblings: ${SHOW_SEPARATE_SIBLINGS:false} # If turned on, all siblings will be separated with no way to get to a "combined" sibling view entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true}