Merge branch 'datahub-project:master' into master

.github/workflows/docker-unified.yml

@@ -1029,6 +1029,7 @@ jobs:
           TEST_STRATEGY: ${{ matrix.test_strategy }}
         run: |
           echo "$DATAHUB_VERSION"
+          ./gradlew --stop
           ./smoke-test/smoke.sh
       - name: Disk Check
         run: df -h . && docker images
@@ -1042,8 +1043,9 @@ jobs:
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: docker logs
+          name: docker-logs-${{ matrix.test_strategy }}
           path: "docker_logs/*.log"
+          retention-days: 5
       - name: Upload screenshots
         uses: actions/upload-artifact@v3
         if: failure()

.github/workflows/pr-labeler.yml

@@ -61,18 +61,14 @@ jobs:
             contains(
               fromJson('[
                 "siladitya2",
-                "sgomezvillamor",
-                "ngamanda",
-                "HarveyLeo",
-                "frsann",
                 "bossenti",
-                "nikolakasev",
                 "PatrickfBraz",
                 "cuong-pham",
                 "sudhakarast",
                 "tkdrahn",
                 "rtekal",
-                "sgm44"
+                "mikeburke24",
+                "DSchmidtDev"
               ]'), 
               github.actor
             ) 

build.gradle

@@ -34,8 +34,8 @@ buildscript {
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
   ext.pegasusVersion = '29.57.0'
   ext.mavenVersion = '3.6.3'
-  ext.springVersion = '6.1.6'
-  ext.springBootVersion = '3.2.6'
+  ext.springVersion = '6.1.13'
+  ext.springBootVersion = '3.2.9'
   ext.springKafkaVersion = '3.1.6'
   ext.openTelemetryVersion = '1.18.0'
   ext.neo4jVersion = '5.14.0'
@@ -222,10 +222,10 @@ project.ext.externalDependency = [
     'playServer': "com.typesafe.play:play-server_2.12:$playVersion",
     'playTest': "com.typesafe.play:play-test_2.12:$playVersion",
     'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion",
-    'pac4j': 'org.pac4j:pac4j-oidc:4.5.7',
+    'pac4j': 'org.pac4j:pac4j-oidc:4.5.8',
     'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
     'postgresql': 'org.postgresql:postgresql:42.3.9',
-    'protobuf': 'com.google.protobuf:protobuf-java:3.19.6',
+    'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
     'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
     'reflections': 'org.reflections:reflections:0.9.9',
@@ -267,7 +267,7 @@ project.ext.externalDependency = [
     'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0',
     'typesafeConfig':'com.typesafe:config:1.4.1',
     'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
-    'zookeeper': 'org.apache.zookeeper:zookeeper:3.7.2',
+    'zookeeper': 'org.apache.zookeeper:zookeeper:3.6.2',
     'wire': 'com.squareup.wire:wire-compiler:3.7.1',
     'charle':  'com.charleskorn.kaml:kaml:0.53.0',
     'common': 'commons-io:commons-io:2.7',

datahub-frontend/app/auth/sso/oidc/OidcConfigs.java

@@ -243,6 +243,9 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe
             Optional.ofNullable(getOptional(configs, OIDC_PREFERRED_JWS_ALGORITHM, null));
       }
 
+      grantType = Optional.ofNullable(getOptional(configs, OIDC_GRANT_TYPE, null));
+      acrValues = Optional.ofNullable(getOptional(configs, OIDC_ACR_VALUES, null));
+
       return this;
     }
 

datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java

@@ -18,7 +18,7 @@ public CustomOidcClient(final OidcConfiguration configuration) {
   protected void clientInit() {
     CommonHelper.assertNotNull("configuration", getConfiguration());
     getConfiguration().init();
-    defaultRedirectionActionBuilder(new OidcRedirectionActionBuilder(getConfiguration(), this));
+    defaultRedirectionActionBuilder(new CustomOidcRedirectionActionBuilder(getConfiguration(), this));
     defaultCredentialsExtractor(new OidcExtractor(getConfiguration(), this));
     defaultAuthenticator(new CustomOidcAuthenticator(this));
     defaultProfileCreator(new OidcProfileCreator<>(getConfiguration(), this));

datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java

@@ -0,0 +1,46 @@
+package auth.sso.oidc.custom;
+
+import java.util.Map;
+import java.util.Optional;
+import org.pac4j.core.context.WebContext;
+import org.pac4j.core.exception.http.RedirectionAction;
+import org.pac4j.core.exception.http.RedirectionActionHelper;
+import org.pac4j.oidc.client.OidcClient;
+import org.pac4j.oidc.config.OidcConfiguration;
+import org.pac4j.oidc.redirect.OidcRedirectionActionBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class CustomOidcRedirectionActionBuilder extends OidcRedirectionActionBuilder {
+
+  private static final Logger logger = LoggerFactory.getLogger(OidcRedirectionActionBuilder.class);
+  public CustomOidcRedirectionActionBuilder(OidcConfiguration configuration, OidcClient client) {
+    super(configuration, client);
+  }
+
+  @Override
+  public Optional<RedirectionAction> getRedirectionAction(WebContext context) {
+    Map<String, String> params = this.buildParams();
+    String computedCallbackUrl = this.client.computeFinalCallbackUrl(context);
+    params.put("redirect_uri", computedCallbackUrl);
+    this.addStateAndNonceParameters(context, params);
+    if (this.configuration.getMaxAge() != null) {
+      params.put("max_age", this.configuration.getMaxAge().toString());
+    }
+
+    String location = this.buildAuthenticationRequestUrl(params);
+
+    logger.debug("Custom parameters: {}", this.configuration.getCustomParams());
+
+    String acrValues = this.configuration.getCustomParam("acr_values");
+
+    if (acrValues != null && !location.contains("acr_values=")) {
+      location += (location.contains("?") ? "&" : "?") + "acr_values=" + acrValues;
+    }
+
+    logger.debug("Authentication request url: {}", location);
+    return Optional.of(RedirectionActionHelper.buildRedirectUrlAction(context, location));
+  }
+
+}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java

@@ -78,7 +78,8 @@ public List<NamedLine> getTimeseriesChart(
       Optional<String> dimension, // Length 1 for now
       Map<String, List<String>> filters,
       Map<String, List<String>> mustNotFilters,
-      Optional<String> uniqueOn) {
+      Optional<String> uniqueOn,
+      String dateRangeField) {
 
     log.debug(
         String.format(
@@ -87,11 +88,11 @@ public List<NamedLine> getTimeseriesChart(
             + String.format("filters: %s, uniqueOn: %s", filters, uniqueOn));
 
     AggregationBuilder filteredAgg =
-        getFilteredAggregation(filters, mustNotFilters, Optional.of(dateRange));
+        getFilteredAggregation(filters, mustNotFilters, Optional.of(dateRange), dateRangeField);
 
     AggregationBuilder dateHistogram =
         AggregationBuilders.dateHistogram(DATE_HISTOGRAM)
-            .field("timestamp")
+            .field(dateRangeField)
             .calendarInterval(new DateHistogramInterval(granularity.name().toLowerCase()));
     uniqueOn.ifPresent(s -> dateHistogram.subAggregation(getUniqueQuery(s)));
 
@@ -128,6 +129,25 @@ public List<NamedLine> getTimeseriesChart(
     }
   }
 
+  public List<NamedLine> getTimeseriesChart(
+      String indexName,
+      DateRange dateRange,
+      DateInterval granularity,
+      Optional<String> dimension, // Length 1 for now
+      Map<String, List<String>> filters,
+      Map<String, List<String>> mustNotFilters,
+      Optional<String> uniqueOn) {
+    return getTimeseriesChart(
+        indexName,
+        dateRange,
+        granularity,
+        dimension,
+        filters,
+        mustNotFilters,
+        uniqueOn,
+        "timestamp");
+  }
+
   private int extractCount(MultiBucketsAggregation.Bucket bucket, boolean didUnique) {
     return didUnique
         ? (int) bucket.getAggregations().<Cardinality>get(UNIQUE).getValue()
@@ -323,20 +343,38 @@ private Filter executeAndExtract(SearchRequest searchRequest) {
     }
   }
 
+  // Make dateRangeField as customizable
   private AggregationBuilder getFilteredAggregation(
       Map<String, List<String>> mustFilters,
       Map<String, List<String>> mustNotFilters,
-      Optional<DateRange> dateRange) {
+      Optional<DateRange> dateRange,
+      String dateRangeField) {
     BoolQueryBuilder filteredQuery = QueryBuilders.boolQuery();
     mustFilters.forEach((key, values) -> filteredQuery.must(QueryBuilders.termsQuery(key, values)));
     mustNotFilters.forEach(
         (key, values) -> filteredQuery.mustNot(QueryBuilders.termsQuery(key, values)));
-    dateRange.ifPresent(range -> filteredQuery.must(dateRangeQuery(range)));
+    dateRange.ifPresent(range -> filteredQuery.must(dateRangeQuery(range, dateRangeField)));
     return AggregationBuilders.filter(FILTERED, filteredQuery);
   }
 
+  private AggregationBuilder getFilteredAggregation(
+      Map<String, List<String>> mustFilters,
+      Map<String, List<String>> mustNotFilters,
+      Optional<DateRange> dateRange) {
+    // Use timestamp as dateRangeField
+    return getFilteredAggregation(mustFilters, mustNotFilters, dateRange, "timestamp");
+  }
+
   private QueryBuilder dateRangeQuery(DateRange dateRange) {
-    return QueryBuilders.rangeQuery("timestamp").gte(dateRange.getStart()).lt(dateRange.getEnd());
+    // Use timestamp as dateRangeField
+    return dateRangeQuery(dateRange, "timestamp");
+  }
+
+  // Make dateRangeField as customizable
+  private QueryBuilder dateRangeQuery(DateRange dateRange, String dateRangeField) {
+    return QueryBuilders.rangeQuery(dateRangeField)
+        .gte(dateRange.getStart())
+        .lt(dateRange.getEnd());
   }
 
   private AggregationBuilder getUniqueQuery(String uniqueOn) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java

@@ -14,6 +14,7 @@
 import com.linkedin.datahub.graphql.generated.ScrollAcrossLineageResults;
 import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.types.common.mappers.LineageFlagsInputMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper;
 import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
 import com.linkedin.datahub.graphql.types.mappers.UrnScrollAcrossLineageResultsMapper;
 import com.linkedin.entity.client.EntityClient;
@@ -89,7 +90,6 @@ public CompletableFuture<ScrollAcrossLineageResults> get(DataFetchingEnvironment
     if (lineageFlags.getEndTimeMillis() == null && endTimeMillis != null) {
       lineageFlags.setEndTimeMillis(endTimeMillis);
     }
-    ;
 
     com.linkedin.metadata.graph.LineageDirection resolvedDirection =
         com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString());
@@ -107,17 +107,13 @@ public CompletableFuture<ScrollAcrossLineageResults> get(DataFetchingEnvironment
                 count);
 
             final SearchFlags searchFlags;
-            final com.linkedin.datahub.graphql.generated.SearchFlags inputFlags =
-                input.getSearchFlags();
+            com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags();
             if (inputFlags != null) {
-              searchFlags =
-                  new SearchFlags()
-                      .setSkipCache(inputFlags.getSkipCache())
-                      .setFulltext(inputFlags.getFulltext())
-                      .setMaxAggValues(inputFlags.getMaxAggValues());
+              searchFlags = SearchFlagsInputMapper.INSTANCE.apply(context, inputFlags);
             } else {
               searchFlags = null;
             }
+
             return UrnScrollAcrossLineageResultsMapper.map(
                 context,
                 _entityClient.scrollAcrossLineage(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java

@@ -1,6 +1,7 @@
 package com.linkedin.datahub.graphql.resolvers.structuredproperties;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
+import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
 
 import com.linkedin.common.urn.Urn;
@@ -21,17 +22,21 @@
 import com.linkedin.structured.PrimitivePropertyValue;
 import com.linkedin.structured.PropertyCardinality;
 import com.linkedin.structured.PropertyValue;
+import com.linkedin.structured.StructuredPropertyDefinition;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 public class UpdateStructuredPropertyResolver
     implements DataFetcher<CompletableFuture<StructuredPropertyEntity>> {
 
   private final EntityClient _entityClient;
 
+  private static final String ALLOWED_TYPES = "allowedTypes";
+
   public UpdateStructuredPropertyResolver(@Nonnull final EntityClient entityClient) {
     _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null");
   }
@@ -52,6 +57,8 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
                   "Unable to update structured property. Please contact your admin.");
             }
             final Urn propertyUrn = UrnUtils.getUrn(input.getUrn());
+            StructuredPropertyDefinition existingDefinition =
+                getExistingStructuredProperty(context, propertyUrn);
             StructuredPropertyDefinitionPatchBuilder builder =
                 new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn);
 
@@ -65,7 +72,7 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
               builder.setImmutable(input.getImmutable());
             }
             if (input.getTypeQualifier() != null) {
-              buildTypeQualifier(input, builder);
+              buildTypeQualifier(input, builder, existingDefinition);
             }
             if (input.getNewAllowedValues() != null) {
               buildAllowedValues(input, builder);
@@ -97,10 +104,16 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
 
   private void buildTypeQualifier(
       @Nonnull final UpdateStructuredPropertyInput input,
-      @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) {
+      @Nonnull final StructuredPropertyDefinitionPatchBuilder builder,
+      @Nullable final StructuredPropertyDefinition existingDefinition) {
     if (input.getTypeQualifier().getNewAllowedTypes() != null) {
       final StringArrayMap typeQualifier = new StringArrayMap();
       StringArray allowedTypes = new StringArray();
+      if (existingDefinition != null
+          && existingDefinition.getTypeQualifier() != null
+          && existingDefinition.getTypeQualifier().get(ALLOWED_TYPES) != null) {
+        allowedTypes.addAll(existingDefinition.getTypeQualifier().get(ALLOWED_TYPES));
+      }
       allowedTypes.addAll(input.getTypeQualifier().getNewAllowedTypes());
       typeQualifier.put("allowedTypes", allowedTypes);
       builder.setTypeQualifier(typeQualifier);
@@ -127,4 +140,18 @@ private void buildAllowedValues(
               builder.addAllowedValue(value);
             });
   }
+
+  private StructuredPropertyDefinition getExistingStructuredProperty(
+      @Nonnull final QueryContext context, @Nonnull final Urn propertyUrn) throws Exception {
+    EntityResponse response =
+        _entityClient.getV2(
+            context.getOperationContext(), STRUCTURED_PROPERTY_ENTITY_NAME, propertyUrn, null);
+
+    if (response != null
+        && response.getAspects().containsKey(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) {
+      return new StructuredPropertyDefinition(
+          response.getAspects().get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME).getValue().data());
+    }
+    return null;
+  }
 }

datahub-graphql-core/src/main/resources/entity.graphql

@@ -9157,6 +9157,10 @@ enum PolicyMatchCondition {
   Whether the field matches the value
   """
   EQUALS
+  """
+  Whether the field value starts with the value
+  """
+  STARTS_WITH
 }
 
 """