From a4f8cfbdc199e90744180a655bfb264eba6f2ba1 Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Mon, 30 Sep 2024 18:00:06 -0700 Subject: [PATCH] RANGER-4948: optimize GDS policy engine to use a single trie-set for all resources across data shares --- .../gds/GdsDataShareEvaluator.java | 166 ++----- .../policyengine/gds/GdsDatasetEvaluator.java | 38 +- .../policyengine/gds/GdsDipEvaluator.java | 21 +- .../policyengine/gds/GdsDshidEvaluator.java | 21 +- .../policyengine/gds/GdsPolicyEngine.java | 415 +++++++----------- .../policyengine/gds/GdsProjectEvaluator.java | 8 +- .../gds/GdsSharedResourceEvaluator.java | 10 +- .../policyengine/gds/TestGdsPolicyEngine.java | 42 +- .../gds/test_gds_policy_engine_hive.json | 2 +- 9 files changed, 262 insertions(+), 461 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java index df4d060184..98f8da493b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java @@ -21,68 +21,44 @@ import org.apache.commons.lang.StringUtils; import org.apache.ranger.plugin.conditionevaluator.RangerConditionEvaluator; -import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; -import org.apache.ranger.plugin.policyengine.RangerPluginContext; import org.apache.ranger.plugin.policyengine.RangerResourceACLs; -import org.apache.ranger.plugin.policyengine.RangerResourceTrie; import org.apache.ranger.plugin.policyevaluator.RangerCustomConditionEvaluator; -import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever; import org.apache.ranger.plugin.util.ServiceGdsInfo.DataShareInfo; -import org.apache.ranger.plugin.util.ServiceGdsInfo.SharedResourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; public class GdsDataShareEvaluator { private static final Logger LOG = LoggerFactory.getLogger(GdsDataShareEvaluator.class); public static final GdsDataShareEvalOrderComparator EVAL_ORDER_COMPARATOR = new GdsDataShareEvalOrderComparator(); - private final DataShareInfo dsh; - private final String name; - private final String zoneName; - private final RangerConditionEvaluator conditionEvaluator; - private final List evaluators; - private final Map> resourceTries; - private final List dsidEvaluators = new ArrayList<>(); + private final DataShareInfo dsh; + private final String name; + private final String zoneName; + private final RangerConditionEvaluator conditionEvaluator; + private final Set evaluators = new TreeSet<>(GdsSharedResourceEvaluator.EVAL_ORDER_COMPARATOR); // keep sorted + private final List dshidEvaluators = new ArrayList<>(); - public GdsDataShareEvaluator(DataShareInfo dsh, List resources, RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) { - LOG.debug("==> GdsDataShareEvaluator({}, {})", dsh, resources); + public GdsDataShareEvaluator(DataShareInfo dsh, RangerServiceDefHelper serviceDefHelper) { + LOG.debug("==> GdsDataShareEvaluator({})", dsh); this.dsh = dsh; this.name = StringUtils.isBlank(dsh.getName()) ? StringUtils.EMPTY : dsh.getName(); this.zoneName = StringUtils.isBlank(dsh.getZoneName()) ? StringUtils.EMPTY : dsh.getZoneName(); this.conditionEvaluator = RangerCustomConditionEvaluator.getInstance().getExpressionEvaluator(dsh.getConditionExpr(), serviceDefHelper.getServiceDef()); - if (resources != null) { - Set resourceKeys = new HashSet<>(); - - evaluators = new ArrayList<>(resources.size()); - resourceTries = new HashMap<>(); - - for (SharedResourceInfo resource : resources) { - GdsSharedResourceEvaluator evaluator = new GdsSharedResourceEvaluator(resource, dsh.getDefaultAccessTypes(), serviceDefHelper, pluginContext); - - evaluators.add(evaluator); - - resourceKeys.addAll(evaluator.getResourceKeys()); - } - - for (String resourceKey : resourceKeys) { - RangerResourceDef resourceDef = serviceDefHelper.getResourceDef(resourceKey); - RangerResourceTrie resourceTrie = new RangerResourceTrie<>(resourceDef, evaluators); - - resourceTries.put(resourceKey, resourceTrie); - } - } else { - evaluators = Collections.emptyList(); - resourceTries = Collections.emptyMap(); - } - - LOG.debug("<== GdsDataShareEvaluator({}, {})", dsh, resources); + LOG.debug("<== GdsDataShareEvaluator({})", dsh); } public Long getId() { @@ -97,81 +73,43 @@ public String getZoneName() { return zoneName; } - public List getSharedResourceEvaluators() { return evaluators; } + public Set getDefaultAccessTypes() { return dsh.getDefaultAccessTypes(); } - public boolean isInDataset(long datasetId) { - boolean ret = false; + public Collection getResourceEvaluators() { return evaluators; } - for (GdsDshidEvaluator dsidEvaluator : dsidEvaluators) { - if (dsidEvaluator.getDatasetId().equals(datasetId)) { - ret = true; + public List getDshidEvaluators() { return dshidEvaluators; } - break; - } - } - - return ret; - } - - public boolean isInProject(long projectId) { - boolean ret = false; - - for (GdsDshidEvaluator dsidEvaluator : dsidEvaluators) { - if (dsidEvaluator.getDatasetEvaluator().isInProject(projectId)) { - ret = true; + public List getResourceEvaluators(RangerAccessRequest request) { + final List ret; + final boolean isAllowed = conditionEvaluator == null || conditionEvaluator.isMatched(request); - break; - } + if (isAllowed) { + ret = evaluators.stream().filter(e -> e.isAllowed(request)).collect(Collectors.toList()); + } else { + ret = Collections.emptyList(); } return ret; } - public void evaluate(RangerAccessRequest request, GdsAccessResult result, Set datasetIds) { - LOG.debug("==> GdsDataShareEvaluator.evaluate({}, {})", request, result); - - Collection evaluators = RangerResourceEvaluatorsRetriever.getEvaluators(resourceTries, request.getResource().getAsMap(), request.getResourceElementMatchingScopes()); - - if (evaluators == null) { - evaluators = Collections.emptyList(); - } else if (evaluators.size() > 1) { - List list = new ArrayList<>(evaluators); - - list.sort(GdsSharedResourceEvaluator.EVAL_ORDER_COMPARATOR); - - evaluators = list; - } - - LOG.debug("GdsDataShareEvaluator.evaluate({}): found {} evaluators", request, evaluators.size()); + public boolean isInDataset(Long datasetId) { + return dshidEvaluators.stream().anyMatch(e -> e.getDatasetId().equals(datasetId) && e.isActive()); + } - if (!evaluators.isEmpty()) { - boolean isAllowed = conditionEvaluator == null || conditionEvaluator.isMatched(request); + public boolean isInProject(Long projectId) { + return dshidEvaluators.stream().anyMatch(e -> e.getDatasetEvaluator().isInProject(projectId) && e.isActive()); + } - if (isAllowed) { - // find if any of the shared resources allow the request - for (GdsSharedResourceEvaluator evaluator : evaluators) { - isAllowed = evaluator.isAllowed(request); + public void collectDatasets(RangerAccessRequest request, GdsAccessResult result, Set datasetIds) { + LOG.debug("==> GdsDataShareEvaluator.collectDatasets({}, {})", request, result); - if (isAllowed) { - break; - } - } + boolean isAllowed = conditionEvaluator == null || conditionEvaluator.isMatched(request); - if (isAllowed) { // now find dsidEvaluators that allow the request and collect their datasetIds - for (GdsDshidEvaluator dsidEvaluator : dsidEvaluators) { - if (!datasetIds.contains(dsidEvaluator.getDatasetId())) { - if (dsidEvaluator.isAllowed(request)) { - datasetIds.add(dsidEvaluator.getDatasetId()); - } - } - } - } - } else { - LOG.debug("GdsDataShareEvaluator.evaluate({}): conditions {} didn't match. Skipped", request, dsh.getConditionExpr()); - } + if (isAllowed) { + dshidEvaluators.stream().filter(e -> !datasetIds.contains(e.getDatasetId()) && e.isAllowed(request) && e.getDatasetEvaluator().isActive()).map(GdsDshidEvaluator::getDatasetId).forEach(datasetIds::add); } - LOG.debug("<== GdsDataShareEvaluator.evaluate({}, {})", request, result); + LOG.debug("<== GdsDataShareEvaluator.collectDatasets({}, {})", request, result); } public void getResourceACLs(RangerAccessRequest request, RangerResourceACLs acls) { @@ -183,35 +121,19 @@ public void getResourceACLs(RangerAccessRequest request, RangerResourceACLs acls boolean isConditional = conditionEvaluator != null; for (GdsSharedResourceEvaluator evaluator : evaluators) { - evaluator.getResourceACLs(request, acls, isConditional, dsidEvaluators); + evaluator.getResourceACLs(request, acls, isConditional, dshidEvaluators); } } LOG.debug("<== GdsDataShareEvaluator.getResourceACLs({}, {})", request, acls); } - void addDshidEvaluator(GdsDshidEvaluator dhidEvaluator) { - dsidEvaluators.add(dhidEvaluator); + void addResourceEvaluator(GdsSharedResourceEvaluator evaluator) { + evaluators.add(evaluator); } - private List getResourceEvaluators(RangerAccessRequest request) { - final List ret; - - Collection evaluators = RangerResourceEvaluatorsRetriever.getEvaluators(resourceTries, request.getResource().getAsMap(), request.getResourceElementMatchingScopes()); - - if (evaluators == null || evaluators.isEmpty()) { - ret = Collections.emptyList(); - } else if (evaluators.size() > 1) { - ret = new ArrayList<>(evaluators); - - ret.sort(GdsSharedResourceEvaluator.EVAL_ORDER_COMPARATOR); - } else { - ret = Collections.singletonList(evaluators.iterator().next()); - } - - LOG.debug("GdsDataShareEvaluator.getResourceEvaluators({}): found {} evaluators", request, ret.size()); - - return ret; + void addDshidEvaluator(GdsDshidEvaluator dhidEvaluator) { + dshidEvaluators.add(dhidEvaluator); } public static class GdsDataShareEvalOrderComparator implements Comparator { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDatasetEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDatasetEvaluator.java index 56397fe744..9c37b67835 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDatasetEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDatasetEvaluator.java @@ -31,7 +31,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Set; public class GdsDatasetEvaluator { private static final Logger LOG = LoggerFactory.getLogger(GdsDatasetEvaluator.class); @@ -85,18 +89,12 @@ public String getName() { return name; } - public boolean isInProject(long projectId) { - boolean ret = false; - - for (GdsDipEvaluator dipEvaluator : dipEvaluators) { - if (dipEvaluator.getProjectId().equals(projectId)) { - ret = true; - - break; - } - } + public boolean isInProject(Long projectId) { + return dipEvaluators.stream().anyMatch(e -> e.getProjectId().equals(projectId) && e.isActive()); + } - return ret; + public boolean isActive() { + return scheduleEvaluator == null || scheduleEvaluator.isApplicable(System.currentTimeMillis()); } public void evaluate(RangerAccessRequest request, GdsAccessResult result, Set projectIds) { @@ -109,9 +107,7 @@ public void evaluate(RangerAccessRequest request, GdsAccessResult result, Set e.evaluate(datasetRequest, datasetResult)); if (!result.getIsAllowed()) { if (datasetResult.getIsAllowed()) { @@ -126,13 +122,7 @@ public void evaluate(RangerAccessRequest request, GdsAccessResult result, Set !projectIds.contains(e.getProjectId()) && e.isAllowed(request) && e.getProjectEvaluator().isActive()).map(GdsDipEvaluator::getProjectId).forEach(projectIds::add); } LOG.debug("<== GdsDatasetEvaluator.evaluate({}, {})", request, result); @@ -174,10 +164,6 @@ void addDipEvaluator(GdsDipEvaluator dipEvaluator) { dipEvaluators.add(dipEvaluator); } - private boolean isActive() { - return scheduleEvaluator == null || scheduleEvaluator.isApplicable(System.currentTimeMillis()); - } - private static class GdsDatasetAccessRequest extends RangerAccessRequestImpl { public GdsDatasetAccessRequest(Long datasetId, RangerServiceDef gdsServiceDef, RangerAccessRequest request) { super.setResource(new RangerDatasetResource(datasetId, gdsServiceDef, request.getResource().getOwnerUser())); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDipEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDipEvaluator.java index ec81b4c3ef..73610fd1f1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDipEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDipEvaluator.java @@ -57,6 +57,16 @@ public Long getProjectId() { public GdsProjectEvaluator getProjectEvaluator() { return projectEvaluator; } + public boolean isActive() { + boolean ret = dip.getStatus() == RangerGds.GdsShareStatus.ACTIVE; + + if (ret && scheduleEvaluator != null) { + ret = scheduleEvaluator.isApplicable(System.currentTimeMillis()); + } + + return ret; + } + public boolean isAllowed(RangerAccessRequest request) { boolean ret = isActive(); @@ -76,15 +86,4 @@ public void getResourceACLs(RangerAccessRequest request, RangerResourceACLs acls LOG.debug("<== GdsDipEvaluator.getResourceACLs({}, {})", request, acls); } - - - private boolean isActive() { - boolean ret = dip.getStatus() == RangerGds.GdsShareStatus.ACTIVE; - - if (ret && scheduleEvaluator != null) { - ret = scheduleEvaluator.isApplicable(System.currentTimeMillis()); - } - - return ret; - } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDshidEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDshidEvaluator.java index c56c55f77e..9d7ba9c9ec 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDshidEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDshidEvaluator.java @@ -59,6 +59,16 @@ public GdsDatasetEvaluator getDatasetEvaluator() { return datasetEvaluator; } + public boolean isActive() { + boolean ret = dshid.getStatus() == RangerGds.GdsShareStatus.ACTIVE; + + if (ret && scheduleEvaluator != null) { + ret = scheduleEvaluator.isApplicable(System.currentTimeMillis()); + } + + return ret; + } + public boolean isAllowed(RangerAccessRequest request) { boolean ret = isActive(); @@ -78,15 +88,4 @@ public void getResourceACLs(RangerAccessRequest request, RangerResourceACLs acls LOG.debug("<== GdsDshidEvaluator.getResourceACLs({}, {})", request, acls); } - - - private boolean isActive() { - boolean ret = dshid.getStatus() == RangerGds.GdsShareStatus.ACTIVE; - - if (ret && scheduleEvaluator != null) { - ret = scheduleEvaluator.isApplicable(System.currentTimeMillis()); - } - - return ret; - } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java index a1593daafe..972e8bf28b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java @@ -19,27 +19,36 @@ package org.apache.ranger.plugin.policyengine.gds; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.plugin.model.RangerGds; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; +import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerPluginContext; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.policyengine.RangerResourceACLs; +import org.apache.ranger.plugin.policyengine.RangerResourceTrie; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; +import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever; import org.apache.ranger.plugin.util.ServiceGdsInfo; -import org.apache.ranger.plugin.util.ServiceGdsInfo.DatasetInfo; -import org.apache.ranger.plugin.util.ServiceGdsInfo.DatasetInProjectInfo; -import org.apache.ranger.plugin.util.ServiceGdsInfo.DataShareInDatasetInfo; -import org.apache.ranger.plugin.util.ServiceGdsInfo.DataShareInfo; -import org.apache.ranger.plugin.util.ServiceGdsInfo.ProjectInfo; -import org.apache.ranger.plugin.util.ServiceGdsInfo.SharedResourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; public class GdsPolicyEngine { @@ -49,11 +58,12 @@ public class GdsPolicyEngine { public static final String RESOURCE_NAME_DATASET_ID = "dataset-id"; public static final String RESOURCE_NAME_PROJECT_ID = "project-id"; - private final ServiceGdsInfo gdsInfo; - private final Set allAccessTypes; - private final Map> zoneDataShares = new HashMap<>(); - private final Map datasets = new HashMap<>(); - private final Map projects = new HashMap<>(); + private final ServiceGdsInfo gdsInfo; + private final Set allAccessTypes; + private final Map projects = new HashMap<>(); + private final Map datasets = new HashMap<>(); + private final Map dataShares = new HashMap<>(); + private final Map zoneResources = new HashMap<>(); public GdsPolicyEngine(ServiceGdsInfo gdsInfo, RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) { LOG.debug("==> RangerGdsPolicyEngine()"); @@ -73,8 +83,8 @@ public ServiceGdsInfo getGdsInfo() { public GdsAccessResult evaluate(RangerAccessRequest request) { LOG.debug("==> RangerGdsPolicyEngine.evaluate({})", request); - GdsAccessResult ret = null; - boolean isAnyAccess = request.isAccessTypeAny(); + final GdsAccessResult ret; + final boolean isAnyAccess = request.isAccessTypeAny(); try { if (isAnyAccess) { @@ -82,20 +92,14 @@ public GdsAccessResult evaluate(RangerAccessRequest request) { RangerAccessRequestUtil.setIsAnyAccessInContext(request.getContext(), Boolean.TRUE); } - List dataShares = getDataShareEvaluators(request); + Collection dataShares = getDataShareEvaluators(request); if (!dataShares.isEmpty()) { ret = new GdsAccessResult(); - if (dataShares.size() > 1) { - dataShares.sort(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR); - } - Set datasetIds = new HashSet<>(); - for (GdsDataShareEvaluator dshEvaluator : dataShares) { - dshEvaluator.evaluate(request, ret, datasetIds); - } + dataShares.forEach(e -> e.collectDatasets(request, ret, datasetIds)); if (!datasetIds.isEmpty()) { Set projectIds = new HashSet<>(); @@ -106,12 +110,13 @@ public GdsAccessResult evaluate(RangerAccessRequest request) { evaluateProjectPolicies(projectIds, request, ret); } } + } else { + ret = null; } } finally { if (isAnyAccess) { RangerAccessRequestUtil.setAllRequestedAccessTypes(request.getContext(), null); RangerAccessRequestUtil.setIsAnyAccessInContext(request.getContext(), Boolean.FALSE); - } } @@ -123,17 +128,7 @@ public GdsAccessResult evaluate(RangerAccessRequest request) { public RangerResourceACLs getResourceACLs(RangerAccessRequest request) { RangerResourceACLs ret = new RangerResourceACLs(); - List dataShares = getDataShareEvaluators(request); - - if (!dataShares.isEmpty()) { - if (dataShares.size() > 1) { - dataShares.sort(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR); - } - - for (GdsDataShareEvaluator dshEvaluator : dataShares) { - dshEvaluator.getResourceACLs(request, ret); - } - } + getDataShareEvaluators(request).forEach(e -> e.getResourceACLs(request, ret)); ret.finalizeAcls(); @@ -189,7 +184,7 @@ public String getProjectName(Long id) { } public Iterator getDatasetResources(long datasetId) { - Set dshEvaluators = new HashSet<>(); + Set dshEvaluators = new TreeSet<>(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR); collectDataSharesForDataset(datasetId, dshEvaluators); @@ -197,7 +192,7 @@ public Iterator getDatasetResources(long datasetId) } public Iterator getProjectResources(long projectId) { - Set dshEvaluators = new HashSet<>(); + Set dshEvaluators = new TreeSet<>(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR); collectDataSharesForProject(projectId, dshEvaluators); @@ -205,22 +200,14 @@ public Iterator getProjectResources(long projectId) } public Iterator getDataShareResources(long dataShareId) { - GdsDataShareEvaluator dshEvaluator = getDataShareEvaluator(dataShareId); + GdsDataShareEvaluator dshEvaluator = dataShares.get(dataShareId); Set dshEvaluators = dshEvaluator == null ? Collections.emptySet() : Collections.singleton(dshEvaluator); return new SharedResourceIter(dshEvaluators); } - public Iterator getResources(List datasetIds, List dataShareIds) { - Set dshEvaluators = new HashSet<>(); - - collectDataShares(null, datasetIds, dataShareIds, dshEvaluators); - - return new SharedResourceIter(dshEvaluators); - } - public Iterator getResources(List projectIds, List datasetIds, List dataShareIds) { - Set dshEvaluators = new HashSet<>(); + Set dshEvaluators = new TreeSet<>(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR); collectDataShares(projectIds, datasetIds, dataShareIds, dshEvaluators); @@ -231,108 +218,112 @@ public Iterator getResources(List projectIds, private void init(RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) { LOG.debug("==> RangerGdsPolicyEngine.init()"); - preprocessGdsServiceDef(gdsInfo.getGdsServiceDef(), serviceDefHelper); - - RangerServiceDef gdsServiceDef = gdsInfo.getGdsServiceDef(); - RangerPolicyEngineOptions options = new RangerPolicyEngineOptions(pluginContext.getConfig().getPolicyEngineOptions(), new RangerServiceDefHelper(gdsServiceDef, false)); - Map> dshResources = new HashMap<>(); - Map dshEvaluators = new HashMap<>(); + preprocess(serviceDefHelper); - if (gdsInfo.getProjects() != null) { - for (ProjectInfo projectInfo : gdsInfo.getProjects()) { - projects.put(projectInfo.getId(), new GdsProjectEvaluator(projectInfo, gdsServiceDef, options)); - } - } + RangerServiceDef gdsServiceDef = gdsInfo.getGdsServiceDef(); + RangerPolicyEngineOptions options = new RangerPolicyEngineOptions(pluginContext.getConfig().getPolicyEngineOptions(), new RangerServiceDefHelper(gdsServiceDef, false)); - if (gdsInfo.getDatasets() != null) { - for (DatasetInfo datasetInfo : gdsInfo.getDatasets()) { - datasets.put(datasetInfo.getId(), new GdsDatasetEvaluator(datasetInfo, gdsServiceDef, options)); - } - } + gdsInfo.getProjects().forEach(project -> projects.put(project.getId(), new GdsProjectEvaluator(project, gdsServiceDef, options))); - // dshResources must be populated before processing dataShares; hence resources should be processed before dataShares - if (gdsInfo.getResources() != null) { - for (SharedResourceInfo resource : gdsInfo.getResources()) { - List resources = dshResources.computeIfAbsent(resource.getDataShareId(), k -> new ArrayList<>()); + gdsInfo.getDatasets().forEach(dataset -> datasets.put(dataset.getId(), new GdsDatasetEvaluator(dataset, gdsServiceDef, options))); - resources.add(resource); - } - } + gdsInfo.getDataShares().forEach(dataShare -> dataShares.put(dataShare.getId(), new GdsDataShareEvaluator(dataShare, serviceDefHelper))); - if (gdsInfo.getDataShares() != null) { - for (DataShareInfo dsh : gdsInfo.getDataShares()) { - GdsDataShareEvaluator dshEvaluator = new GdsDataShareEvaluator(dsh, dshResources.get(dsh.getId()), serviceDefHelper, pluginContext); - List zoneEvaluators = zoneDataShares.computeIfAbsent(dshEvaluator.getZoneName(), k -> new ArrayList<>()); + gdsInfo.getDshids().forEach(dshid -> { + if (dshid.getStatus() == RangerGds.GdsShareStatus.ACTIVE) { + GdsDataShareEvaluator dshEvaluator = dataShares.get(dshid.getDataShareId()); - zoneEvaluators.add(dshEvaluator); - dshEvaluators.put(dsh.getId(), dshEvaluator); - } - } + if (dshEvaluator != null) { + GdsDatasetEvaluator datasetEvaluator = datasets.get(dshid.getDatasetId()); - if (gdsInfo.getDshids() != null) { - for (DataShareInDatasetInfo dshid : gdsInfo.getDshids()) { - if (dshid.getStatus() != RangerGds.GdsShareStatus.ACTIVE) { - LOG.error("RangerGdsPolicyEngine(): dshid is not active {}. Ignored", dshid); + if (datasetEvaluator != null) { + GdsDshidEvaluator dshidEvaluator = new GdsDshidEvaluator(dshid, datasetEvaluator); - continue; + dshEvaluator.addDshidEvaluator(dshidEvaluator); + } else { + LOG.error("RangerGdsPolicyEngine(): invalid datasetId in dshid: {}. Ignored", dshid); + } + } else { + LOG.error("RangerGdsPolicyEngine(): invalid dataShareId in dshid: {}. Ignored", dshid); } + } else { + LOG.error("RangerGdsPolicyEngine(): dshid is not active {}. Ignored", dshid); + } + }); - GdsDataShareEvaluator dshEvaluator = dshEvaluators.get(dshid.getDataShareId()); + gdsInfo.getDips().forEach(dip -> { + if (dip.getStatus() == RangerGds.GdsShareStatus.ACTIVE) { + GdsDatasetEvaluator datasetEvaluator = datasets.get(dip.getDatasetId()); - if (dshEvaluator == null) { - LOG.error("RangerGdsPolicyEngine(): invalid dataShareId in dshid: {}. Ignored", dshid); + if (datasetEvaluator != null) { + GdsProjectEvaluator projectEvaluator = projects.get(dip.getProjectId()); - continue; - } + if (projectEvaluator != null) { + GdsDipEvaluator dipEvaluator = new GdsDipEvaluator(dip, projectEvaluator); - GdsDatasetEvaluator datasetEvaluator = datasets.get(dshid.getDatasetId()); + datasetEvaluator.addDipEvaluator(dipEvaluator); + } else { + LOG.error("RangerGdsPolicyEngine(): invalid projectId in dip: {}. Ignored", dip); + } + } else { + LOG.error("RangerGdsPolicyEngine(): invalid datasetId in dip: {}. Ignored", dip); + } + } else { + LOG.error("RangerGdsPolicyEngine(): dip is not active {}. Ignored", dip); + } + }); - if (datasetEvaluator == null) { - LOG.error("RangerGdsPolicyEngine(): invalid datasetId in dshid: {}. Ignored", dshid); + // purge dataShares that are not part of any dataset + dataShares.values().removeIf(evaluator -> CollectionUtils.isEmpty(evaluator.getDshidEvaluators())); - continue; - } + Map> zoneResEvaluators = new HashMap<>(); - GdsDshidEvaluator dshidEvaluator = new GdsDshidEvaluator(dshid, datasetEvaluator); + gdsInfo.getResources().forEach(resource -> { + GdsDataShareEvaluator dshEvaluator = dataShares.get(resource.getDataShareId()); - dshEvaluator.addDshidEvaluator(dshidEvaluator); - } - } + if (dshEvaluator != null) { + GdsSharedResourceEvaluator evaluator = new GdsSharedResourceEvaluator(resource, dshEvaluator.getDefaultAccessTypes(), serviceDefHelper, pluginContext); - if (gdsInfo.getDips() != null) { - for (DatasetInProjectInfo dip : gdsInfo.getDips()) { - if (dip.getStatus() != RangerGds.GdsShareStatus.ACTIVE) { - LOG.error("RangerGdsPolicyEngine(): dip is not active {}. Ignored", dip); + dshEvaluator.addResourceEvaluator(evaluator); - continue; - } + zoneResEvaluators.computeIfAbsent(dshEvaluator.getZoneName(), k -> new ArrayList<>()).add(evaluator); + } + }); - GdsDatasetEvaluator datasetEvaluator = datasets.get(dip.getDatasetId()); + zoneResEvaluators.forEach((zoneName, evaluators) -> zoneResources.put(zoneName, new GdsZoneResources(zoneName, evaluators, serviceDefHelper, pluginContext))); - if (datasetEvaluator == null) { - LOG.error("RangerGdsPolicyEngine(): invalid datasetId in dip: {}. Ignored", dip); + LOG.debug("<== RangerGdsPolicyEngine.init()"); + } - continue; - } + private void preprocess(RangerServiceDefHelper serviceDefHelper) { + if (gdsInfo.getProjects() == null) { + gdsInfo.setProjects(Collections.emptyList()); + } - GdsProjectEvaluator projectEvaluator = projects.get(dip.getProjectId()); + if (gdsInfo.getDatasets() == null) { + gdsInfo.setDatasets(Collections.emptyList()); + } - if (projectEvaluator == null) { - LOG.error("RangerGdsPolicyEngine(): invalid projectId in dip: {}. Ignored", dip); + if (gdsInfo.getDataShares() == null) { + gdsInfo.setDataShares(Collections.emptyList()); + } else { + gdsInfo.getDataShares().stream().filter(dsh -> dsh.getZoneName() == null).forEach(dsh -> dsh.setZoneName(StringUtils.EMPTY)); + } - continue; - } + if (gdsInfo.getResources() == null) { + gdsInfo.setResources(Collections.emptyList()); + } - GdsDipEvaluator dipEvaluator = new GdsDipEvaluator(dip, projectEvaluator); + if (gdsInfo.getDshids() == null) { + gdsInfo.setDshids(Collections.emptyList()); + } - datasetEvaluator.addDipEvaluator(dipEvaluator); - } + if (gdsInfo.getDips() == null) { + gdsInfo.setDips(Collections.emptyList()); } - LOG.debug("<== RangerGdsPolicyEngine.init()"); - } + RangerServiceDef gdsServiceDef = gdsInfo.getGdsServiceDef(); - private void preprocessGdsServiceDef(RangerServiceDef gdsServiceDef, RangerServiceDefHelper serviceDefHelper) { // populate accessTypes in GDS servicedef with implied accessTypes from the service for (RangerAccessTypeDef gdsAccessTypeDef : gdsServiceDef.getAccessTypes()) { Collection impliedGrants = serviceDefHelper.getImpliedAccessGrants().get(gdsAccessTypeDef.getName()); @@ -345,12 +336,12 @@ private void preprocessGdsServiceDef(RangerServiceDef gdsServiceDef, RangerServi gdsServiceDef.getAccessTypes().addAll(serviceDefHelper.getServiceDef().getAccessTypes()); } - private List getDataShareEvaluators(RangerAccessRequest request) { + private Collection getDataShareEvaluators(RangerAccessRequest request) { LOG.debug("==> RangerGdsPolicyEngine.getDataShareEvaluators({})", request); - List ret = null; + final Collection ret; - if (!zoneDataShares.isEmpty()) { + if (!dataShares.isEmpty()) { Set zoneNames = RangerAccessRequestUtil.getResourceZoneNamesFromContext(request.getContext()); if (zoneNames == null || zoneNames.isEmpty()) { @@ -361,20 +352,10 @@ private List getDataShareEvaluators(RangerAccessRequest r zoneNames = Collections.emptySet(); } - for (String zoneName : zoneNames) { - List zonEvaluators = zoneDataShares.get(zoneName); - - if (zonEvaluators != null && !zonEvaluators.isEmpty()) { - if (ret == null) { - ret = new ArrayList<>(); - } - - ret.addAll(zonEvaluators); - } - } - } + ret = new TreeSet<>(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR); - if (ret == null) { + zoneNames.stream().map(zoneResources::get).filter(Objects::nonNull).forEach(zr -> zr.collectMatchingDataShares(request, ret)); + } else { ret = Collections.emptyList(); } @@ -384,151 +365,75 @@ private List getDataShareEvaluators(RangerAccessRequest r } private void evaluateDatasetPolicies(Set datasetIds, RangerAccessRequest request, GdsAccessResult result, Set projectIds) { - List evaluators = new ArrayList<>(datasetIds.size()); - - for (Long datasetId : datasetIds) { - GdsDatasetEvaluator evaluator = datasets.get(datasetId); - - if (evaluator == null) { - LOG.error("evaluateDatasetPolicies(): invalid datasetId in result: {}. Ignored", datasetId); - - continue; - } - - evaluators.add(evaluator); - } - - if (evaluators.size() > 1) { - evaluators.sort(GdsDatasetEvaluator.EVAL_ORDER_COMPARATOR); - } - - if (!evaluators.isEmpty()) { - for (GdsDatasetEvaluator evaluator : evaluators) { - evaluator.evaluate(request, result, projectIds); - } - } + datasetIds.stream().map(datasets::get).filter(Objects::nonNull).sorted(GdsDatasetEvaluator.EVAL_ORDER_COMPARATOR).forEach(e -> e.evaluate(request, result, projectIds)); } private void evaluateProjectPolicies(Set projectIds, RangerAccessRequest request, GdsAccessResult result) { - List evaluators = new ArrayList<>(projectIds.size()); - - for (Long projectId : projectIds) { - GdsProjectEvaluator evaluator = projects.get(projectId); - - if (evaluator == null) { - LOG.error("evaluateProjectPolicies(): invalid projectId in result: {}. Ignored", projectId); - - continue; - } - - evaluators.add(evaluator); - } - - if (evaluators.size() > 1) { - evaluators.sort(GdsProjectEvaluator.EVAL_ORDER_COMPARATOR); - } - - for (GdsProjectEvaluator evaluator : evaluators) { - evaluator.evaluate(request, result); - } + projectIds.stream().map(projects::get).filter(Objects::nonNull).sorted(GdsProjectEvaluator.EVAL_ORDER_COMPARATOR).forEach(e -> e.evaluate(request, result)); } private GdsDatasetEvaluator getDatasetEvaluator(String dsName) { - GdsDatasetEvaluator ret = null; - - for (GdsDatasetEvaluator evaluator : datasets.values()) { - if (StringUtils.equals(evaluator.getName(), dsName)) { - ret = evaluator; - - break; - } - } - - return ret; + return datasets.values().stream().filter(e -> StringUtils.equals(e.getName(), dsName)).findFirst().orElse(null); } private GdsProjectEvaluator getProjectEvaluator(String projectName) { - GdsProjectEvaluator ret = null; - - for (GdsProjectEvaluator evaluator : projects.values()) { - if (StringUtils.equals(evaluator.getName(), projectName)) { - ret = evaluator; - - break; - } - } - - return ret; - } - - private GdsDataShareEvaluator getDataShareEvaluator(long dataShareId) { - GdsDataShareEvaluator ret = null; - - for (List dshEvaluators : zoneDataShares.values()) { - for (GdsDataShareEvaluator dshEvaluator : dshEvaluators) { - if (dshEvaluator.getId().equals(dataShareId)) { - ret = dshEvaluator; - - break; - } - } - } - - return ret; + return projects.values().stream().filter(e -> StringUtils.equals(e.getName(), projectName)).findFirst().orElse(null); } - private void collectDataSharesForDataset(long datasetId, Set evaluators) { - for (List dshEvaluators : zoneDataShares.values()) { - for (GdsDataShareEvaluator dshEvaluator : dshEvaluators) { - if (dshEvaluator.isInDataset(datasetId)) { - evaluators.add(dshEvaluator); - } - } - } + private void collectDataSharesForDataset(Long datasetId, Set evaluators) { + dataShares.values().stream().filter(e -> e.isInDataset(datasetId)).forEach(evaluators::add); } - private void collectDataSharesForProject(long projectId, Set evaluators) { - for (List dshEvaluators : zoneDataShares.values()) { - for (GdsDataShareEvaluator dshEvaluator : dshEvaluators) { - if (dshEvaluator.isInProject(projectId)) { - evaluators.add(dshEvaluator); - } - } - } + private void collectDataSharesForProject(Long projectId, Set evaluators) { + dataShares.values().stream().filter(e -> e.isInProject(projectId)).forEach(evaluators::add); } private void collectDataShares(List projectIds, List datasetIds, List dataShareIds, Set evaluators) { if (projectIds != null) { - for (Long projectId : projectIds) { - collectDataSharesForProject(projectId, evaluators); - } + projectIds.forEach(projectId -> collectDataSharesForProject(projectId, evaluators)); } if (datasetIds != null) { - for (Long datasetId : datasetIds) { - collectDataSharesForDataset(datasetId, evaluators); - } + datasetIds.forEach(datasetId -> collectDataSharesForDataset(datasetId, evaluators)); } if (dataShareIds != null) { - for (Long dataShareId : dataShareIds) { - GdsDataShareEvaluator evaluator = getDataShareEvaluator(dataShareId); - - if (evaluator != null) { - evaluators.add(evaluator); - } - } + dataShareIds.stream().map(dataShares::get).filter(Objects::nonNull).forEach(evaluators::add); } } private Set getAllAccessTypes(RangerServiceDefHelper serviceDefHelper) { - Set ret = new HashSet<>(); + return serviceDefHelper.getServiceDef().getAccessTypes().stream().map(RangerAccessTypeDef::getName).collect(Collectors.toSet()); + } - for (RangerAccessTypeDef accessTypeDef : serviceDefHelper.getServiceDef().getAccessTypes()) { - ret.add(accessTypeDef.getName()); + private class GdsZoneResources { + private final String zoneName; + private final Map> resourceTries; + + public GdsZoneResources(String zoneName, List evaluators, RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) { + this.zoneName = zoneName; + this.resourceTries = createResourceTries(evaluators, serviceDefHelper, pluginContext); } - return ret; + public String getZoneName() { return zoneName; } + + public void collectMatchingDataShares(RangerAccessRequest request, Collection dshEvaluators) { + Collection evaluators = RangerResourceEvaluatorsRetriever.getEvaluators(resourceTries, request.getResource().getAsMap(), request.getResourceElementMatchingScopes()); + + if (evaluators != null) { + evaluators.stream().filter(e -> e.isAllowed(request)).map(GdsSharedResourceEvaluator::getDataShareId).distinct().map(dataShares::get).filter(Objects::nonNull).forEach(dshEvaluators::add); + } + } + + private Map> createResourceTries(List evaluators, RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) { + Map> ret = new HashMap<>(); + + for (RangerResourceDef resourceDef : serviceDefHelper.getServiceDef().getResources()) { + ret.put(resourceDef.getName(), new RangerResourceTrie<>(resourceDef, evaluators, true, pluginContext)); + } + + return ret; + } } static class SharedResourceIter implements Iterator { @@ -537,16 +442,10 @@ static class SharedResourceIter implements Iterator private GdsSharedResourceEvaluator nextResource = null; SharedResourceIter(Set evaluators) { - if (evaluators == null || evaluators.isEmpty()) { + if (evaluators == null) { dataShareIter = Collections.emptyIterator(); - } else if (evaluators.size() == 1) { - dataShareIter = evaluators.iterator(); } else { - List list = new ArrayList<>(evaluators); - - list.sort(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR); - - dataShareIter = list.iterator(); + dataShareIter = evaluators.iterator(); } setNext(); @@ -573,7 +472,7 @@ private void setNext() { while (dataShareIter.hasNext()) { GdsDataShareEvaluator dataShareEvaluator = dataShareIter.next(); - sharedResourceIter = dataShareEvaluator.getSharedResourceEvaluators().iterator(); + sharedResourceIter = dataShareEvaluator.getResourceEvaluators().iterator(); if (sharedResourceIter.hasNext()) { break; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsProjectEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsProjectEvaluator.java index c9fe1b63c8..89324ddb40 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsProjectEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsProjectEvaluator.java @@ -86,6 +86,10 @@ public String getName() { return name; } + public boolean isActive() { + return scheduleEvaluator == null || scheduleEvaluator.isApplicable(System.currentTimeMillis()); + } + public void evaluate(RangerAccessRequest request, GdsAccessResult result) { LOG.debug("==> GdsDatasetEvaluator.evaluate({}, {})", request, result); @@ -145,10 +149,6 @@ public boolean hasReference(Set users, Set groups, Set r return ret; } - private boolean isActive() { - return scheduleEvaluator == null || scheduleEvaluator.isApplicable(System.currentTimeMillis()); - } - private static class GdsProjectAccessRequest extends RangerAccessRequestImpl { public GdsProjectAccessRequest(Long projectId, RangerServiceDef gdsServiceDef, RangerAccessRequest request) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java index 3091813446..6e36c00cc8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java @@ -42,7 +42,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; public class GdsSharedResourceEvaluator implements RangerResourceEvaluator { private static final Logger LOG = LoggerFactory.getLogger(GdsSharedResourceEvaluator.class); @@ -108,6 +114,8 @@ public boolean isAncestorOf(RangerResourceDef resourceDef) { @Override public boolean isLeaf(String resourceName) { return StringUtils.equals(leafResourceDef.getName(), resourceName); } + public Long getDataShareId() { return resource.getDataShareId(); } + public Collection getResourceKeys() { return resource != null && resource.getResource() != null ? resource.getResource().keySet() : Collections.emptySet(); } diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java index c3884f261f..7bfebbb99e 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java @@ -88,16 +88,14 @@ private void runTests(Reader reader, String testName) { RangerAccessRequestUtil.setResourceZoneNamesInContext(test.request, zoneNames); - if (test.result != null) { - GdsAccessResult result = policyEngine.evaluate(test.request); - - assertEquals(test.name, test.result, result); - } - if (test.acls != null) { RangerResourceACLs acls = policyEngine.getResourceACLs(test.request); assertEquals(test.name, test.acls, acls); + } else { + GdsAccessResult result = policyEngine.evaluate(test.request); + + assertEquals(test.name, test.result, result); } } else if (test.sharedWith != null) { Set users = test.sharedWith.get("users"); @@ -116,34 +114,24 @@ private void runTests(Reader reader, String testName) { assertEquals(test.name, test.projects, projects); } } else if (test.resourceIds != null) { - Set resourceIds = new HashSet<>(); + Iterator iter; if (test.datasetId != null) { - Iterator iter = policyEngine.getDatasetResources(test.datasetId); - - while (iter.hasNext()) { - resourceIds.add(iter.next().getId()); - } + iter = policyEngine.getDatasetResources(test.datasetId); } else if (test.projectId != null) { - Iterator iter = policyEngine.getProjectResources(test.projectId); - - while (iter.hasNext()) { - resourceIds.add(iter.next().getId()); - } + iter = policyEngine.getProjectResources(test.projectId); } else if (test.dataShareId != null) { - Iterator iter = policyEngine.getDataShareResources(test.dataShareId); - - while (iter.hasNext()) { - resourceIds.add(iter.next().getId()); - } + iter = policyEngine.getDataShareResources(test.dataShareId); } else if (test.projectIds != null || test.datasetIds != null || test.dataShareIds != null) { - Iterator iter = policyEngine.getResources(test.projectIds, test.datasetIds, test.dataShareIds); - - while (iter.hasNext()) { - resourceIds.add(iter.next().getId()); - } + iter = policyEngine.getResources(test.projectIds, test.datasetIds, test.dataShareIds); + } else { + iter = Collections.emptyIterator(); } + Set resourceIds = new HashSet<>(); + + iter.forEachRemaining(e -> resourceIds.add(e.getId())); + assertEquals(test.name, test.resourceIds, resourceIds); } } diff --git a/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json b/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json index 37b65b880a..a5e3636b39 100644 --- a/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json +++ b/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json @@ -263,7 +263,7 @@ "resource": { "elements": { "database": "operations", "table": "facilities" } }, "accessType": "update", "user": "ds-user", "userGroups": [] }, - "result": { "datasets": null, "projects": null, "isAllowed": false, "isAudited": false, "policyId": -1 } + "result": null }, {