Merge pull request #288 from datahub-project/master

Sync Upstream 08-12-24
acryldata · Aug 13, 2024 · a09fb90 · a09fb90
2 parents 129eb66 + 5e9188c
commit a09fb90
Show file tree

Hide file tree

Showing 300 changed files with 7,484 additions and 5,762 deletions.
diff --git a/.github/scripts/docker_helpers.sh b/.github/scripts/docker_helpers.sh
@@ -16,11 +16,11 @@ function get_tag {
 }
 
 function get_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
 }
 
 function get_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
 }
 
 function get_python_docker_release_v {
@@ -32,9 +32,9 @@ function get_unique_tag {
 }
 
 function get_unique_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
 }
 
 function get_unique_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
 }
diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -30,12 +30,10 @@ jobs:
                 "treff7es",
                 "yoonhyejin",
                 "eboneil",
-                "ethan-cartwright",
                 "gabe-lyons",
                 "hsheth2",
                 "jjoyce0510",
                 "maggiehays",
-                "mrjefflewis",
                 "pedro93",
                 "RyanHolstien",
                 "Kunal-kankriya",
@@ -45,7 +43,8 @@ jobs:
                 "kushagra-apptware",
                 "Salman-Apptware",
                 "mayurinehate",
-                "noggi"
+                "noggi",
+                "skrydal"
               ]'), 
               github.actor
             ) 
@@ -60,7 +59,6 @@ jobs:
           ${{ 
             contains(
               fromJson('[
-                "skrydal",
                 "siladitya2",
                 "sgomezvillamor",
                 "ngamanda",

diff --git a/README.md b/README.md
@@ -138,7 +138,6 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Peloton](https://www.onepeloton.com)
 - [PITS Global Data Recovery Services](https://www.pitsdatarecovery.net/)
 - [Razer](https://www.razer.com)
-- [Saxo Bank](https://www.home.saxo)
 - [Showroomprive](https://www.showroomprive.com/)
 - [SpotHero](https://spothero.com)
 - [Stash](https://www.stash.com)

diff --git a/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java b/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java
@@ -41,6 +41,8 @@ public class OidcConfigs extends SsoConfigs {
   public static final String OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS =
       "auth.oidc.extractJwtAccessTokenClaims";
   public static final String OIDC_PREFERRED_JWS_ALGORITHM = "auth.oidc.preferredJwsAlgorithm";
+  public static final String OIDC_GRANT_TYPE = "auth.oidc.grantType";
+  public static final String OIDC_ACR_VALUES = "auth.oidc.acrValues";
 
   /** Default values */
   private static final String DEFAULT_OIDC_USERNAME_CLAIM = "email";
@@ -75,7 +77,9 @@ public class OidcConfigs extends SsoConfigs {
   private final Optional<String> customParamResource;
   private final String readTimeout;
   private final Optional<Boolean> extractJwtAccessTokenClaims;
-  private Optional<String> preferredJwsAlgorithm;
+  private final Optional<String> preferredJwsAlgorithm;
+  private final Optional<String> grantType;
+  private final Optional<String> acrValues;
 
   public OidcConfigs(Builder builder) {
     super(builder);
@@ -98,6 +102,8 @@ public OidcConfigs(Builder builder) {
     this.readTimeout = builder.readTimeout;
     this.extractJwtAccessTokenClaims = builder.extractJwtAccessTokenClaims;
     this.preferredJwsAlgorithm = builder.preferredJwsAlgorithm;
+    this.acrValues = builder.acrValues;
+    this.grantType = builder.grantType;
   }
 
   public static class Builder extends SsoConfigs.Builder<Builder> {
@@ -123,6 +129,8 @@ public static class Builder extends SsoConfigs.Builder<Builder> {
     private String readTimeout = DEFAULT_OIDC_READ_TIMEOUT;
     private Optional<Boolean> extractJwtAccessTokenClaims = Optional.empty();
     private Optional<String> preferredJwsAlgorithm = Optional.empty();
+    private Optional<String> grantType = Optional.empty();
+    private Optional<String> acrValues = Optional.empty();
 
     public Builder from(final com.typesafe.config.Config configs) {
       super.from(configs);
@@ -169,6 +177,8 @@ public Builder from(final com.typesafe.config.Config configs) {
           getOptional(configs, OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).map(Boolean::parseBoolean);
       preferredJwsAlgorithm =
           Optional.ofNullable(getOptional(configs, OIDC_PREFERRED_JWS_ALGORITHM, null));
+      grantType = Optional.ofNullable(getOptional(configs, OIDC_GRANT_TYPE, null));
+      acrValues = Optional.ofNullable(getOptional(configs, OIDC_ACR_VALUES, null));
       return this;
     }
 

diff --git a/datahub-frontend/app/auth/sso/oidc/OidcProvider.java b/datahub-frontend/app/auth/sso/oidc/OidcProvider.java
@@ -3,6 +3,8 @@
 import auth.sso.SsoProvider;
 import auth.sso.oidc.custom.CustomOidcClient;
 import com.google.common.collect.ImmutableMap;
+import java.util.HashMap;
+import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.pac4j.core.client.Client;
 import org.pac4j.core.http.callback.PathParameterCallbackUrlResolver;
@@ -64,9 +66,19 @@ private Client<OidcCredentials> createPac4jClient() {
     _oidcConfigs.getResponseType().ifPresent(oidcConfiguration::setResponseType);
     _oidcConfigs.getResponseMode().ifPresent(oidcConfiguration::setResponseMode);
     _oidcConfigs.getUseNonce().ifPresent(oidcConfiguration::setUseNonce);
+    Map<String, String> customParamsMap = new HashMap<>();
     _oidcConfigs
         .getCustomParamResource()
-        .ifPresent(value -> oidcConfiguration.setCustomParams(ImmutableMap.of("resource", value)));
+        .ifPresent(value -> customParamsMap.put("resource", value));
+    _oidcConfigs
+        .getGrantType()
+        .ifPresent(value -> customParamsMap.put("grant_type", value));
+    _oidcConfigs
+        .getAcrValues()
+        .ifPresent(value -> customParamsMap.put("acr_values", value));
+    if (!customParamsMap.isEmpty()) {
+      oidcConfiguration.setCustomParams(customParamsMap);
+    }
     _oidcConfigs
         .getPreferredJwsAlgorithm()
         .ifPresent(

diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf
@@ -186,6 +186,8 @@ auth.oidc.customParam.resource = ${?AUTH_OIDC_CUSTOM_PARAM_RESOURCE}
 auth.oidc.readTimeout = ${?AUTH_OIDC_READ_TIMEOUT}
 auth.oidc.extractJwtAccessTokenClaims = ${?AUTH_OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS} # Whether to extract claims from JWT access token.  Defaults to false.
 auth.oidc.preferredJwsAlgorithm = ${?AUTH_OIDC_PREFERRED_JWS_ALGORITHM} # Which jws algorithm to use
+auth.oidc.acrValues = ${?AUTH_OIDC_ACR_VALUES}
+auth.oidc.grantType = ${?AUTH_OIDC_GRANT_TYPE}
 
 #
 # By default, the callback URL that should be registered with the identity provider is computed as {$baseUrl}/callback/oidc.

diff --git a/...core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java b/...core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java
@@ -159,9 +159,10 @@ private SearchResult searchForNewUsers(@Nonnull final OperationContext opContext
                                             .setValue(
                                                 String.valueOf(
                                                     trailingMonthDateRange.getStart())))))))),
-        new SortCriterion()
-            .setField(CORP_USER_STATUS_LAST_MODIFIED_FIELD_NAME)
-            .setOrder(SortOrder.DESCENDING),
+        Collections.singletonList(
+            new SortCriterion()
+                .setField(CORP_USER_STATUS_LAST_MODIFIED_FIELD_NAME)
+                .setOrder(SortOrder.DESCENDING)),
         0,
         100);
   }

diff --git a/...in/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java b/...in/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java
@@ -77,7 +77,8 @@ private List<AnalyticsChart> getCharts(MetadataAnalyticsInput input, OperationCo
     }
 
     SearchResult searchResult =
-        _entityClient.searchAcrossEntities(opContext, entities, query, filter, 0, 0, null, null);
+        _entityClient.searchAcrossEntities(
+            opContext, entities, query, filter, 0, 0, Collections.emptyList(), null);
 
     List<AggregationMetadata> aggregationMetadataList =
         searchResult.getMetadata().getAggregations();

diff --git a/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java
@@ -21,5 +21,6 @@ public class FeatureFlags {
   private boolean schemaFieldEntityFetchEnabled = false;
   private boolean businessAttributeEntityEnabled = false;
   private boolean dataContractsEnabled = false;
+  private boolean editableDatasetNameEnabled = false;
   private boolean showSeparateSiblings = false;
 }
diff --git a/...l-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/DebugAccessResolver.java b/...l-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/DebugAccessResolver.java
@@ -33,6 +33,7 @@
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -181,7 +182,7 @@ private Set<Urn> getPoliciesFor(
             Constants.POLICY_ENTITY_NAME,
             "",
             buildFilterToGetPolicies(user, groups, roles),
-            sortCriterion,
+            Collections.singletonList(sortCriterion),
             0,
             10000)
         .getEntities()

diff --git a/...e/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java b/...e/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java
@@ -59,10 +59,11 @@ public CompletableFuture<ListAccessTokenResult> get(DataFetchingEnvironment envi
           if (AuthorizationUtils.canManageTokens(context)
               || isListingSelfTokens(filters, context)) {
             try {
-              final SortCriterion sortCriterion =
-                  new SortCriterion()
-                      .setField(EXPIRES_AT_FIELD_NAME)
-                      .setOrder(SortOrder.DESCENDING);
+              final List<SortCriterion> sortCriteria =
+                  Collections.singletonList(
+                      new SortCriterion()
+                          .setField(EXPIRES_AT_FIELD_NAME)
+                          .setOrder(SortOrder.DESCENDING));
               final SearchResult searchResult =
                   _entityClient.search(
                       context
@@ -74,7 +75,7 @@ public CompletableFuture<ListAccessTokenResult> get(DataFetchingEnvironment envi
                           filters,
                           Collections.emptyList(),
                           context.getOperationContext().getAspectRetriever()),
-                      sortCriterion,
+                      sortCriteria,
                       start,
                       count);
 

diff --git a/...l-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/...l-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java
@@ -186,6 +186,7 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
             .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled())
             .setPlatformBrowseV2(_featureFlags.isPlatformBrowseV2())
             .setDataContractsEnabled(_featureFlags.isDataContractsEnabled())
+            .setEditableDatasetNameEnabled(_featureFlags.isEditableDatasetNameEnabled())
             .setShowSeparateSiblings(_featureFlags.isShowSeparateSiblings())
             .build();
 

diff --git a/...main/java/com/linkedin/datahub/graphql/resolvers/container/ContainerEntitiesResolver.java b/...main/java/com/linkedin/datahub/graphql/resolvers/container/ContainerEntitiesResolver.java
@@ -19,6 +19,7 @@
 import com.linkedin.metadata.query.filter.Filter;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import lombok.extern.slf4j.Slf4j;
@@ -92,7 +93,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
                                         new CriterionArray(ImmutableList.of(filterCriterion))))),
                     start,
                     count,
-                    null,
+                    Collections.emptyList(),
                     null));
 
           } catch (Exception e) {

diff --git a/...e/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java b/...e/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java
@@ -19,6 +19,7 @@
 import com.linkedin.metadata.query.filter.Filter;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.Collections;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
@@ -98,7 +99,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
                                 new ConjunctiveCriterion().setAnd(criteria))),
                     start,
                     count,
-                    null,
+                    Collections.emptyList(),
                     null));
 
           } catch (Exception e) {

diff --git a/...core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/...core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java
@@ -22,6 +22,7 @@
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
@@ -66,9 +67,10 @@ public CompletableFuture<ListDomainsResult> get(final DataFetchingEnvironment en
                     Constants.DOMAIN_ENTITY_NAME,
                     query,
                     filter,
-                    new SortCriterion()
-                        .setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
-                        .setOrder(SortOrder.DESCENDING),
+                    Collections.singletonList(
+                        new SortCriterion()
+                            .setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
+                            .setOrder(SortOrder.DESCENDING)),
                     start,
                     count);
 

diff --git a/...l-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/ListGroupsResolver.java b/...l-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/ListGroupsResolver.java
@@ -21,6 +21,7 @@
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -64,9 +65,10 @@ public CompletableFuture<ListGroupsResult> get(final DataFetchingEnvironment env
                       CORP_GROUP_ENTITY_NAME,
                       query,
                       null,
-                      new SortCriterion()
-                          .setField(CORP_GROUP_CREATED_TIME_INDEX_FIELD_NAME)
-                          .setOrder(SortOrder.DESCENDING),
+                      Collections.singletonList(
+                          new SortCriterion()
+                              .setField(CORP_GROUP_CREATED_TIME_INDEX_FIELD_NAME)
+                              .setOrder(SortOrder.DESCENDING)),
                       start,
                       count);
 

diff --git a/...rc/main/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolver.java b/...rc/main/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolver.java
@@ -21,6 +21,7 @@
 import graphql.schema.DataFetchingEnvironment;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -60,13 +61,13 @@ public CompletableFuture<EntityIncidentsResult> get(DataFetchingEnvironment envi
             // Index!
             // We use the search index so that we can easily sort by the last updated time.
             final Filter filter = buildIncidentsEntityFilter(entityUrn, maybeState);
-            final SortCriterion sortCriterion = buildIncidentsSortCriterion();
+            final List<SortCriterion> sortCriteria = buildIncidentsSortCriteria();
             final SearchResult searchResult =
                 _entityClient.filter(
                     context.getOperationContext(),
                     Constants.INCIDENT_ENTITY_NAME,
                     filter,
-                    sortCriterion,
+                    sortCriteria,
                     start,
                     count);
 
@@ -118,10 +119,10 @@ private Filter buildIncidentsEntityFilter(
     return QueryUtils.newFilter(criterionMap);
   }
 
-  private SortCriterion buildIncidentsSortCriterion() {
+  private List<SortCriterion> buildIncidentsSortCriteria() {
     final SortCriterion sortCriterion = new SortCriterion();
     sortCriterion.setField(CREATED_TIME_SEARCH_INDEX_FIELD_NAME);
     sortCriterion.setOrder(SortOrder.DESCENDING);
-    return sortCriterion;
+    return Collections.singletonList(sortCriterion);
   }
 }
diff --git a/.../datahub/graphql/resolvers/ingest/execution/IngestionSourceExecutionRequestsResolver.java b/.../datahub/graphql/resolvers/ingest/execution/IngestionSourceExecutionRequestsResolver.java
@@ -23,6 +23,7 @@
 import com.linkedin.metadata.search.SearchResult;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -76,9 +77,10 @@ public CompletableFuture<IngestionSourceExecutionRequests> get(
                                 new ConjunctiveCriterion()
                                     .setAnd(
                                         new CriterionArray(ImmutableList.of(filterCriterion))))),
-                    new SortCriterion()
-                        .setField(REQUEST_TIME_MS_FIELD_NAME)
-                        .setOrder(SortOrder.DESCENDING),
+                    Collections.singletonList(
+                        new SortCriterion()
+                            .setField(REQUEST_TIME_MS_FIELD_NAME)
+                            .setOrder(SortOrder.DESCENDING)),
                     start,
                     count);
 

diff --git a/...c/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/ListSecretsResolver.java b/...c/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/ListSecretsResolver.java
@@ -26,6 +26,7 @@
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -73,9 +74,10 @@ public CompletableFuture<ListSecretsResult> get(final DataFetchingEnvironment en
                       Constants.SECRETS_ENTITY_NAME,
                       query,
                       null,
-                      new SortCriterion()
-                          .setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
-                          .setOrder(SortOrder.DESCENDING),
+                      Collections.singletonList(
+                          new SortCriterion()
+                              .setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME)
+                              .setOrder(SortOrder.DESCENDING)),
                       start,
                       count);