Merge branch 'datahub-project:master' into master

.github/workflows/docker-unified.yml

@@ -857,8 +857,14 @@ jobs:
         if: failure()
         run: |
           docker ps -a
-          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log
-          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log
+          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true
+          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
+          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
+          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()

build.gradle

@@ -39,7 +39,7 @@ buildscript {
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
   id 'com.github.johnrengelman.shadow' version '6.1.0'
-  id 'com.palantir.docker' version '0.35.0'
+  id 'com.palantir.docker' version '0.35.0' apply false
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }

datahub-frontend/app/client/KafkaTrackingProducer.java

@@ -1,6 +1,8 @@
 package client;
 
+import com.linkedin.metadata.config.kafka.ProducerConfiguration;
 import com.typesafe.config.Config;
+import config.ConfigurationProvider;
 import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerConfig;
@@ -35,12 +37,12 @@ public class KafkaTrackingProducer {
     private final KafkaProducer<String, String> _producer;
 
     @Inject
-    public KafkaTrackingProducer(@Nonnull Config config, ApplicationLifecycle lifecycle) {
+    public KafkaTrackingProducer(@Nonnull Config config, ApplicationLifecycle lifecycle, final ConfigurationProvider configurationProvider) {
         _isEnabled = !config.hasPath("analytics.enabled") || config.getBoolean("analytics.enabled");
 
         if (_isEnabled) {
             _logger.debug("Analytics tracking is enabled");
-            _producer = createKafkaProducer(config);
+            _producer = createKafkaProducer(config, configurationProvider.getKafka().getProducer());
 
             lifecycle.addStopHook(
                     () -> {
@@ -62,13 +64,15 @@ public void send(ProducerRecord<String, String> record) {
         _producer.send(record);
     }
 
-    private static KafkaProducer createKafkaProducer(Config config) {
+    private static KafkaProducer createKafkaProducer(Config config, ProducerConfiguration producerConfiguration) {
         final Properties props = new Properties();
         props.put(ProducerConfig.CLIENT_ID_CONFIG, "datahub-frontend");
         props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, config.getString("analytics.kafka.delivery.timeout.ms"));
         props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getString("analytics.kafka.bootstrap.server"));
         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // Actor urn.
         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // JSON object.
+        props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, producerConfiguration.getMaxRequestSize());
+        props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, producerConfiguration.getCompressionType());
 
         final String securityProtocolConfig = "analytics.kafka.security.protocol";
         if (config.hasPath(securityProtocolConfig)

datahub-frontend/app/config/ConfigurationProvider.java

@@ -1,6 +1,7 @@
 package config;
 
 import com.linkedin.metadata.config.cache.CacheConfiguration;
+import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import lombok.Data;
 
@@ -11,14 +12,17 @@
 
 /**
  * Minimal sharing between metadata-service and frontend
- * Initially for use of client caching configuration.
  * Does not use the factories module to avoid transitive dependencies.
  */
 @EnableConfigurationProperties
 @PropertySource(value = "application.yml", factory = YamlPropertySourceFactory.class)
 @ConfigurationProperties
 @Data
 public class ConfigurationProvider {
+    /**
+     * Kafka related configs.
+     */
+    private KafkaConfiguration kafka;
 
     /**
      * Configuration for caching

datahub-web-react/src/app/home/HomePageHeader.tsx

@@ -275,6 +275,7 @@ export const HomePageHeader = () => {
                         viewsEnabled={viewsEnabled}
                         combineSiblings
                         showQuickFilters
+                        showViewAllResults
                     />
                     {searchResultsToShow && searchResultsToShow.length > 0 && (
                         <SuggestionsContainer>

datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx

@@ -190,7 +190,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                             <Input
                                 data-testid="cli-version-input"
                                 className="cli-version-input"
-                                placeholder="(e.g. 0.10.5)"
+                                placeholder="(e.g. 0.12.0)"
                                 value={state.config?.version || ''}
                                 onChange={(event) => setVersion(event.target.value)}
                             />

datahub-web-react/src/app/search/SearchBar.tsx

@@ -119,6 +119,7 @@ interface Props {
     setIsSearchBarFocused?: (isSearchBarFocused: boolean) => void;
     onFocus?: () => void;
     onBlur?: () => void;
+    showViewAllResults?: boolean;
 }
 
 const defaultProps = {
@@ -146,6 +147,7 @@ export const SearchBar = ({
     setIsSearchBarFocused,
     onFocus,
     onBlur,
+    showViewAllResults = false,
 }: Props) => {
     const history = useHistory();
     const [searchQuery, setSearchQuery] = useState<string | undefined>(initialQuery);
@@ -203,7 +205,7 @@ export const SearchBar = ({
     const { quickFilters, selectedQuickFilter, setSelectedQuickFilter } = useQuickFiltersContext();
 
     const autoCompleteQueryOptions = useMemo(() => {
-        if (effectiveQuery === '') return [];
+        if (effectiveQuery === '' || !showViewAllResults) return [];
 
         return [
             {
@@ -212,7 +214,7 @@ export const SearchBar = ({
                 type: EXACT_AUTOCOMPLETE_OPTION_TYPE,
             },
         ];
-    }, [effectiveQuery]);
+    }, [effectiveQuery, showViewAllResults]);
 
     const autoCompleteEntityOptions = useMemo(() => {
         return suggestions.map((suggestion: AutoCompleteResultForEntity) => {

datahub-web-react/src/app/search/SearchHeader.tsx

@@ -107,6 +107,7 @@ export const SearchHeader = ({
                     combineSiblings
                     fixAutoComplete
                     showQuickFilters
+                    showViewAllResults
                 />
             </LogoSearchContainer>
             <NavGroup>

datahub-web-react/src/app/search/useGetSearchQueryInputs.ts

@@ -3,7 +3,7 @@ import { useLocation, useParams } from 'react-router';
 import { useMemo } from 'react';
 import { FacetFilterInput, EntityType } from '../../types.generated';
 import { useEntityRegistry } from '../useEntityRegistry';
-import { ENTITY_FILTER_NAME, FILTER_DELIMITER, UnionType } from './utils/constants';
+import { ENTITY_FILTER_NAME, UnionType } from './utils/constants';
 import { useUserContext } from '../context/useUserContext';
 import useFilters from './utils/useFilters';
 import { generateOrFilters } from './utils/generateOrFilters';
@@ -27,12 +27,6 @@ export default function useGetSearchQueryInputs(excludedFilterFields?: Array<str
     const sortInput = useSortInput();
 
     const filters: Array<FacetFilterInput> = useFilters(params);
-    const nonNestedFilters = filters.filter(
-        (f) => !f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
-    );
-    const nestedFilters = filters.filter(
-        (f) => f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
-    );
     const entityFilters: Array<EntityType> = useMemo(
         () =>
             filters
@@ -43,8 +37,8 @@ export default function useGetSearchQueryInputs(excludedFilterFields?: Array<str
     );
 
     const orFilters = useMemo(
-        () => generateOrFilters(unionType, nonNestedFilters, nestedFilters),
-        [nonNestedFilters, nestedFilters, unionType],
+        () => generateOrFilters(unionType, filters, excludedFilterFields),
+        [filters, excludedFilterFields, unionType],
     );
 
     return { entityFilters, query, unionType, filters, orFilters, viewUrn, page, activeType, sortInput };

datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts

@@ -1,7 +1,7 @@
 import {
     DOMAINS_FILTER_NAME,
     ENTITY_SUB_TYPE_FILTER_NAME,
-    ENTITY_TYPE_FILTER_NAME,
+    ENTITY_FILTER_NAME,
     TAGS_FILTER_NAME,
     UnionType,
 } from '../constants';
@@ -10,7 +10,7 @@ import { generateOrFilters } from '../generateOrFilters';
 describe('generateOrFilters', () => {
     it('should generate orFilters with UnionType.AND', () => {
         const filters = [
-            { field: ENTITY_TYPE_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
         ];
         const orFilters = generateOrFilters(UnionType.AND, filters);
@@ -24,7 +24,7 @@ describe('generateOrFilters', () => {
 
     it('should generate orFilters with UnionType.OR', () => {
         const filters = [
-            { field: ENTITY_TYPE_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
         ];
         const orFilters = generateOrFilters(UnionType.OR, filters);
@@ -43,17 +43,23 @@ describe('generateOrFilters', () => {
         const filters = [
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
             { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+            { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] },
         ];
-        const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
-        const orFilters = generateOrFilters(UnionType.AND, filters, nestedFilters);
+        // const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
+        const orFilters = generateOrFilters(UnionType.AND, filters);
 
         expect(orFilters).toMatchObject([
             {
-                and: [...filters, { field: '_entityType', values: ['CONTAINER'] }],
+                and: [
+                    { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+                    { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+                    { field: '_entityType', values: ['CONTAINER'] },
+                ],
             },
             {
                 and: [
-                    ...filters,
+                    { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+                    { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
                     { field: '_entityType', values: ['DATASET'] },
                     { field: 'typeNames', values: ['table'] },
                 ],
@@ -65,9 +71,9 @@ describe('generateOrFilters', () => {
         const filters = [
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
             { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+            { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] },
         ];
-        const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
-        const orFilters = generateOrFilters(UnionType.OR, filters, nestedFilters);
+        const orFilters = generateOrFilters(UnionType.OR, filters);
 
         expect(orFilters).toMatchObject([
             {
@@ -87,4 +93,18 @@ describe('generateOrFilters', () => {
             },
         ]);
     });
+
+    it('should generate orFilters and exclude filters with a provided exclude field', () => {
+        const filters = [
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+        ];
+        const orFilters = generateOrFilters(UnionType.AND, filters, [ENTITY_FILTER_NAME]);
+
+        expect(orFilters).toMatchObject([
+            {
+                and: [{ field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] }],
+            },
+        ]);
+    });
 });

datahub-web-react/src/app/search/utils/generateOrFilters.ts

@@ -26,28 +26,34 @@ function generateInputWithNestedFilters(filters: FacetFilterInput[], nestedFilte
 export function generateOrFilters(
     unionType: UnionType,
     filters: FacetFilterInput[],
-    nestedFilters: FacetFilterInput[] = [],
+    excludedFilterFields: string[] = [],
 ): AndFilterInput[] {
-    if ((filters?.length || 0) === 0 && nestedFilters.length === 0) {
+    if ((filters?.length || 0) === 0) {
         return [];
     }
+    const nonNestedFilters = filters.filter(
+        (f) => !f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
+    );
+    const nestedFilters = filters.filter(
+        (f) => f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
+    );
 
     if (unionType === UnionType.OR) {
         const orFiltersWithNestedFilters = generateInputWithNestedFilters([], nestedFilters);
-        const orFilters = filters.map((filter) => ({
+        const orFilters = nonNestedFilters.map((filter) => ({
             and: [filter],
         }));
         return [...orFilters, ...orFiltersWithNestedFilters];
     }
-    const andFiltersWithNestedFilters = generateInputWithNestedFilters(filters, nestedFilters);
+    const andFiltersWithNestedFilters = generateInputWithNestedFilters(nonNestedFilters, nestedFilters);
 
     if (andFiltersWithNestedFilters.length) {
         return andFiltersWithNestedFilters;
     }
 
     return [
         {
-            and: filters,
+            and: nonNestedFilters,
         },
     ];
 }

docker/broker/env/docker.env

@@ -5,4 +5,6 @@ KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9
 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
 KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
-KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+KAFKA_MESSAGE_MAX_BYTES=5242880
+KAFKA_MAX_MESSAGE_BYTES=5242880

docker/datahub-frontend/Dockerfile

@@ -8,10 +8,12 @@ RUN addgroup -S datahub && adduser -S datahub -G datahub
 # Upgrade Alpine and base packages
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl sqlite \
+    && apk --no-cache add curl sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as prod-install
 
 COPY ./datahub-frontend.zip /

docker/datahub-gms/Dockerfile

@@ -18,7 +18,7 @@ FROM alpine:3 AS base
 ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl bash coreutils gcompat sqlite \
+    && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
@@ -29,6 +29,8 @@ RUN apk --no-cache --update-cache --available upgrade \
     && cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
 COPY --from=binary /go/bin/dockerize /usr/local/bin
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as prod-install
 COPY war.war /datahub/datahub-gms/bin/war.war
 COPY metadata-models/src/main/resources/entity-registry.yml /datahub/datahub-gms/resources/entity-registry.yml

docker/datahub-mae-consumer/Dockerfile

@@ -18,14 +18,16 @@ FROM alpine:3 AS base
 ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl bash coreutils sqlite \
+    && apk --no-cache add curl bash coreutils sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
     && wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
     && cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
 COPY --from=binary /go/bin/dockerize /usr/local/bin
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as prod-install
 COPY mae-consumer-job.jar /datahub/datahub-mae-consumer/bin/
 COPY metadata-models/src/main/resources/entity-registry.yml /datahub/datahub-mae-consumer/resources/entity-registry.yml

docker/datahub-mce-consumer/Dockerfile

@@ -18,7 +18,7 @@ FROM alpine:3 AS base
 ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl bash sqlite \
+    && apk --no-cache add curl bash sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
@@ -33,6 +33,8 @@ COPY docker/datahub-mce-consumer/start.sh /datahub/datahub-mce-consumer/scripts/
 COPY docker/monitoring/client-prometheus-config.yaml /datahub/datahub-mce-consumer/scripts/prometheus-config.yaml
 RUN chmod +x /datahub/datahub-mce-consumer/scripts/start.sh
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as dev-install
 # Dummy stage for development. Assumes code is built on your machine and mounted to this image.
 # See this excellent thread https://github.com/docker/cli/issues/1134