Merge branch 'datahub-project:master' into master

.github/workflows/airflow-plugin.yml

@@ -43,16 +43,16 @@ jobs:
             extra_pip_requirements: "apache-airflow~=2.4.3"
             extra_pip_extras: plugin-v2,test-airflow24
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.11"
-            extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt'
+            extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
@@ -73,7 +73,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
         with:

.github/workflows/dagster-plugin.yml

@@ -55,7 +55,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:dagster-plugin:lint :metadata-ingestion-modules:dagster-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
         with:

.github/workflows/gx-plugin.yml

@@ -57,7 +57,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:gx-plugin:lint :metadata-ingestion-modules:gx-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }}
         with:

.github/workflows/metadata-ingestion.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Debug info
         if: always()
         run: |
-          source metadata-ingestion/venv/bin/activate && pip freeze
+          source metadata-ingestion/venv/bin/activate && uv pip freeze
           set -x
           df -hl
           docker image ls

.github/workflows/prefect-plugin.yml

@@ -43,7 +43,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
@@ -55,7 +55,7 @@ jobs:
         run: ./gradlew :metadata-ingestion-modules:prefect-plugin:lint :metadata-ingestion-modules:prefect-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10'}}
         with:

datahub-frontend/app/auth/AuthModule.java

@@ -25,7 +25,7 @@
 import java.util.Collections;
 
 import io.datahubproject.metadata.context.ActorContext;
-import io.datahubproject.metadata.context.AuthorizerContext;
+import io.datahubproject.metadata.context.AuthorizationContext;
 import io.datahubproject.metadata.context.EntityRegistryContext;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.OperationContextConfig;
@@ -183,10 +183,10 @@ protected OperationContext provideOperationContext(
     return OperationContext.builder()
             .operationContextConfig(systemConfig)
             .systemActorContext(systemActorContext)
+            // Authorizer.EMPTY is fine since it doesn't actually apply to system auth
+            .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
             .searchContext(SearchContext.EMPTY)
             .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
-            // Authorizer.EMPTY doesn't actually apply to system auth
-            .authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build())
             .build(systemAuthentication);
   }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java

@@ -11,7 +11,6 @@
 import com.datahub.authorization.ConjunctivePrivilegeGroup;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
 import com.datahub.authorization.EntitySpec;
-import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
@@ -21,7 +20,6 @@
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.List;
-import java.util.Set;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.reflect.ConstructorUtils;
@@ -40,29 +38,25 @@ public class AuthorizationUtils {
 
   public static boolean canManageUsersAndGroups(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorizedEntityType(
-        context.getActorUrn(),
-        context.getAuthorizer(),
+        context.getOperationContext(),
         MANAGE,
         List.of(CORP_USER_ENTITY_NAME, CORP_GROUP_ENTITY_NAME));
   }
 
   public static boolean canManagePolicies(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorizedEntityType(
-        context.getActorUrn(), context.getAuthorizer(), MANAGE, List.of(POLICY_ENTITY_NAME));
+        context.getOperationContext(), MANAGE, List.of(POLICY_ENTITY_NAME));
   }
 
   public static boolean canGeneratePersonalAccessToken(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-            context.getAuthorizer(),
-            context.getActorUrn(),
-            PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE)
-        || AuthUtil.isAuthorized(
-            context.getAuthorizer(), context.getActorUrn(), MANAGE_ACCESS_TOKENS);
+            context.getOperationContext(), PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE)
+        || AuthUtil.isAuthorized(context.getOperationContext(), MANAGE_ACCESS_TOKENS);
   }
 
   public static boolean canManageTokens(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorizedEntityType(
-        context.getActorUrn(), context.getAuthorizer(), MANAGE, List.of(ACCESS_TOKEN_ENTITY_NAME));
+        context.getOperationContext(), MANAGE, List.of(ACCESS_TOKEN_ENTITY_NAME));
   }
 
   /**
@@ -78,13 +72,12 @@ public static boolean canCreateDomains(@Nonnull QueryContext context) {
                 new ConjunctivePrivilegeGroup(
                     ImmutableList.of(PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE.getType()))));
 
-    return AuthUtil.isAuthorized(
-        context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null);
+    return AuthUtil.isAuthorized(context.getOperationContext(), orPrivilegeGroups, null);
   }
 
   public static boolean canManageDomains(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE);
+        context.getOperationContext(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE);
   }
 
   /**
@@ -100,25 +93,22 @@ public static boolean canCreateTags(@Nonnull QueryContext context) {
                 new ConjunctivePrivilegeGroup(
                     ImmutableList.of(PoliciesConfig.MANAGE_TAGS_PRIVILEGE.getType()))));
 
-    return AuthUtil.isAuthorized(
-        context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null);
+    return AuthUtil.isAuthorized(context.getOperationContext(), orPrivilegeGroups, null);
   }
 
   public static boolean canManageTags(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_TAGS_PRIVILEGE);
+        context.getOperationContext(), PoliciesConfig.MANAGE_TAGS_PRIVILEGE);
   }
 
   public static boolean canDeleteEntity(@Nonnull Urn entityUrn, @Nonnull QueryContext context) {
     return AuthUtil.isAuthorizedEntityUrns(
-        context.getAuthorizer(), context.getActorUrn(), DELETE, List.of(entityUrn));
+        context.getOperationContext(), DELETE, List.of(entityUrn));
   }
 
   public static boolean canManageUserCredentials(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
-        PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE);
+        context.getOperationContext(), PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE);
   }
 
   public static boolean canEditGroupMembers(
@@ -130,12 +120,7 @@ public static boolean canEditGroupMembers(
                 new ConjunctivePrivilegeGroup(
                     ImmutableList.of(PoliciesConfig.EDIT_GROUP_MEMBERS_PRIVILEGE.getType()))));
 
-    return isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
-        CORP_GROUP_ENTITY_NAME,
-        groupUrnStr,
-        orPrivilegeGroups);
+    return isAuthorized(context, CORP_GROUP_ENTITY_NAME, groupUrnStr, orPrivilegeGroups);
   }
 
   public static boolean canCreateGlobalAnnouncements(@Nonnull QueryContext context) {
@@ -149,27 +134,21 @@ public static boolean canCreateGlobalAnnouncements(@Nonnull QueryContext context
                     ImmutableList.of(
                         PoliciesConfig.MANAGE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE.getType()))));
 
-    return AuthUtil.isAuthorized(
-        context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null);
+    return AuthUtil.isAuthorized(context.getOperationContext(), orPrivilegeGroups, null);
   }
 
   public static boolean canManageGlobalAnnouncements(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
-        PoliciesConfig.MANAGE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE);
+        context.getOperationContext(), PoliciesConfig.MANAGE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE);
   }
 
   public static boolean canManageGlobalViews(@Nonnull QueryContext context) {
-    return AuthUtil.isAuthorized(
-        context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_GLOBAL_VIEWS);
+    return AuthUtil.isAuthorized(context.getOperationContext(), PoliciesConfig.MANAGE_GLOBAL_VIEWS);
   }
 
   public static boolean canManageOwnershipTypes(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
-        PoliciesConfig.MANAGE_GLOBAL_OWNERSHIP_TYPES);
+        context.getOperationContext(), PoliciesConfig.MANAGE_GLOBAL_OWNERSHIP_TYPES);
   }
 
   public static boolean canEditProperties(@Nonnull Urn targetUrn, @Nonnull QueryContext context) {
@@ -183,11 +162,7 @@ public static boolean canEditProperties(@Nonnull Urn targetUrn, @Nonnull QueryCo
                     ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PROPERTIES_PRIVILEGE.getType()))));
 
     return AuthorizationUtils.isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
-        targetUrn.getEntityType(),
-        targetUrn.toString(),
-        orPrivilegeGroups);
+        context, targetUrn.getEntityType(), targetUrn.toString(), orPrivilegeGroups);
   }
 
   public static boolean canEditEntityQueries(
@@ -202,11 +177,7 @@ public static boolean canEditEntityQueries(
         .allMatch(
             entityUrn ->
                 isAuthorized(
-                    context.getAuthorizer(),
-                    context.getActorUrn(),
-                    entityUrn.getEntityType(),
-                    entityUrn.toString(),
-                    orPrivilegeGroups));
+                    context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups));
   }
 
   public static boolean canCreateQuery(
@@ -251,28 +222,7 @@ public static boolean canView(@Nonnull OperationContext opContext, @Nonnull Urn
         && !opContext.isSystemAuth()
         && VIEW_RESTRICTED_ENTITY_TYPES.contains(urn.getEntityType())) {
 
-      return opContext
-          .getViewAuthorizationContext()
-          .map(
-              viewAuthContext -> {
-
-                // check cache
-                if (viewAuthContext.canView(Set.of(urn))) {
-                  return true;
-                }
-
-                if (!canViewEntity(
-                    opContext.getSessionAuthentication().getActor().toUrnStr(),
-                    opContext.getAuthorizerContext().getAuthorizer(),
-                    urn)) {
-                  return false;
-                }
-
-                // cache viewable urn
-                viewAuthContext.addViewableUrns(Set.of(urn));
-                return true;
-              })
-          .orElse(false);
+      return canViewEntity(opContext, urn);
     }
     return true;
   }
@@ -386,38 +336,32 @@ public static <T> T restrictEntity(@Nonnull Object entity, Class<T> clazz) {
 
   public static boolean canManageStructuredProperties(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
-        PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE);
+        context.getOperationContext(), PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE);
   }
 
   public static boolean canManageForms(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
-        PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE);
+        context.getOperationContext(), PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE);
   }
 
   public static boolean canManageFeatures(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_FEATURES_PRIVILEGE);
+        context.getOperationContext(), PoliciesConfig.MANAGE_FEATURES_PRIVILEGE);
   }
 
   public static boolean isAuthorized(
-      @Nonnull Authorizer authorizer,
-      @Nonnull String actor,
+      @Nonnull QueryContext context,
       @Nonnull String resourceType,
       @Nonnull String resource,
       @Nonnull DisjunctivePrivilegeGroup privilegeGroup) {
     final EntitySpec resourceSpec = new EntitySpec(resourceType, resource);
-    return AuthUtil.isAuthorized(authorizer, actor, privilegeGroup, resourceSpec);
+    return AuthUtil.isAuthorized(context.getOperationContext(), privilegeGroup, resourceSpec);
   }
 
   public static boolean isViewDatasetUsageAuthorized(
       final QueryContext context, final Urn resourceUrn) {
     return AuthUtil.isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
+        context.getOperationContext(),
         PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE,
         new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString()));
   }