Skip to content

Commit

Permalink
RANGER-4948: optimize GDS policy engine to use a single trie-set for …
Browse files Browse the repository at this point in the history 
…all resources across data shares
  • Loading branch information
@mneethiraj
mneethiraj committed Oct 4, 2024
1 parent c1aaffb commit a4f8cfb
Show file tree
Hide file tree
Showing 9 changed files with 262 additions and 461 deletions.
166 changes: 44 additions & 122 deletions ...common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,68 +21,44 @@

import org.apache.commons.lang.StringUtils;
import org.apache.ranger.plugin.conditionevaluator.RangerConditionEvaluator;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
import org.apache.ranger.plugin.policyengine.RangerPluginContext;
import org.apache.ranger.plugin.policyengine.RangerResourceACLs;
import org.apache.ranger.plugin.policyengine.RangerResourceTrie;
import org.apache.ranger.plugin.policyevaluator.RangerCustomConditionEvaluator;
import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever;
import org.apache.ranger.plugin.util.ServiceGdsInfo.DataShareInfo;
import org.apache.ranger.plugin.util.ServiceGdsInfo.SharedResourceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;

public class GdsDataShareEvaluator {
private static final Logger LOG = LoggerFactory.getLogger(GdsDataShareEvaluator.class);

public static final GdsDataShareEvalOrderComparator EVAL_ORDER_COMPARATOR = new GdsDataShareEvalOrderComparator();

private final DataShareInfo dsh;
private final String name;
private final String zoneName;
private final RangerConditionEvaluator conditionEvaluator;
private final List<GdsSharedResourceEvaluator> evaluators;
private final Map<String, RangerResourceTrie<GdsSharedResourceEvaluator>> resourceTries;
private final List<GdsDshidEvaluator> dsidEvaluators = new ArrayList<>();
private final DataShareInfo dsh;
private final String name;
private final String zoneName;
private final RangerConditionEvaluator conditionEvaluator;
private final Set<GdsSharedResourceEvaluator> evaluators = new TreeSet<>(GdsSharedResourceEvaluator.EVAL_ORDER_COMPARATOR); // keep sorted
private final List<GdsDshidEvaluator> dshidEvaluators = new ArrayList<>();

public GdsDataShareEvaluator(DataShareInfo dsh, List<SharedResourceInfo> resources, RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) {
LOG.debug("==> GdsDataShareEvaluator({}, {})", dsh, resources);
public GdsDataShareEvaluator(DataShareInfo dsh, RangerServiceDefHelper serviceDefHelper) {
LOG.debug("==> GdsDataShareEvaluator({})", dsh);

this.dsh = dsh;
this.name = StringUtils.isBlank(dsh.getName()) ? StringUtils.EMPTY : dsh.getName();
this.zoneName = StringUtils.isBlank(dsh.getZoneName()) ? StringUtils.EMPTY : dsh.getZoneName();
this.conditionEvaluator = RangerCustomConditionEvaluator.getInstance().getExpressionEvaluator(dsh.getConditionExpr(), serviceDefHelper.getServiceDef());

if (resources != null) {
Set<String> resourceKeys = new HashSet<>();

evaluators = new ArrayList<>(resources.size());
resourceTries = new HashMap<>();

for (SharedResourceInfo resource : resources) {
GdsSharedResourceEvaluator evaluator = new GdsSharedResourceEvaluator(resource, dsh.getDefaultAccessTypes(), serviceDefHelper, pluginContext);

evaluators.add(evaluator);

resourceKeys.addAll(evaluator.getResourceKeys());
}

for (String resourceKey : resourceKeys) {
RangerResourceDef resourceDef = serviceDefHelper.getResourceDef(resourceKey);
RangerResourceTrie<GdsSharedResourceEvaluator> resourceTrie = new RangerResourceTrie<>(resourceDef, evaluators);

resourceTries.put(resourceKey, resourceTrie);
}
} else {
evaluators = Collections.emptyList();
resourceTries = Collections.emptyMap();
}

LOG.debug("<== GdsDataShareEvaluator({}, {})", dsh, resources);
LOG.debug("<== GdsDataShareEvaluator({})", dsh);
}

public Long getId() {
Expand All @@ -97,81 +73,43 @@ public String getZoneName() {
return zoneName;
}

public List<GdsSharedResourceEvaluator> getSharedResourceEvaluators() { return evaluators; }
public Set<String> getDefaultAccessTypes() { return dsh.getDefaultAccessTypes(); }

public boolean isInDataset(long datasetId) {
boolean ret = false;
public Collection<GdsSharedResourceEvaluator> getResourceEvaluators() { return evaluators; }

for (GdsDshidEvaluator dsidEvaluator : dsidEvaluators) {
if (dsidEvaluator.getDatasetId().equals(datasetId)) {
ret = true;
public List<GdsDshidEvaluator> getDshidEvaluators() { return dshidEvaluators; }

break;
}
}

return ret;
}

public boolean isInProject(long projectId) {
boolean ret = false;

for (GdsDshidEvaluator dsidEvaluator : dsidEvaluators) {
if (dsidEvaluator.getDatasetEvaluator().isInProject(projectId)) {
ret = true;
public List<GdsSharedResourceEvaluator> getResourceEvaluators(RangerAccessRequest request) {
final List<GdsSharedResourceEvaluator> ret;
final boolean isAllowed = conditionEvaluator == null || conditionEvaluator.isMatched(request);

break;
}
if (isAllowed) {
ret = evaluators.stream().filter(e -> e.isAllowed(request)).collect(Collectors.toList());
} else {
ret = Collections.emptyList();
}

return ret;
}

public void evaluate(RangerAccessRequest request, GdsAccessResult result, Set<Long> datasetIds) {
LOG.debug("==> GdsDataShareEvaluator.evaluate({}, {})", request, result);

Collection<GdsSharedResourceEvaluator> evaluators = RangerResourceEvaluatorsRetriever.getEvaluators(resourceTries, request.getResource().getAsMap(), request.getResourceElementMatchingScopes());

if (evaluators == null) {
evaluators = Collections.emptyList();
} else if (evaluators.size() > 1) {
List<GdsSharedResourceEvaluator> list = new ArrayList<>(evaluators);

list.sort(GdsSharedResourceEvaluator.EVAL_ORDER_COMPARATOR);

evaluators = list;
}

LOG.debug("GdsDataShareEvaluator.evaluate({}): found {} evaluators", request, evaluators.size());
public boolean isInDataset(Long datasetId) {
return dshidEvaluators.stream().anyMatch(e -> e.getDatasetId().equals(datasetId) && e.isActive());
}

if (!evaluators.isEmpty()) {
boolean isAllowed = conditionEvaluator == null || conditionEvaluator.isMatched(request);
public boolean isInProject(Long projectId) {
return dshidEvaluators.stream().anyMatch(e -> e.getDatasetEvaluator().isInProject(projectId) && e.isActive());
}

if (isAllowed) {
// find if any of the shared resources allow the request
for (GdsSharedResourceEvaluator evaluator : evaluators) {
isAllowed = evaluator.isAllowed(request);
public void collectDatasets(RangerAccessRequest request, GdsAccessResult result, Set<Long> datasetIds) {
LOG.debug("==> GdsDataShareEvaluator.collectDatasets({}, {})", request, result);

if (isAllowed) {
break;
}
}
boolean isAllowed = conditionEvaluator == null || conditionEvaluator.isMatched(request);

if (isAllowed) { // now find dsidEvaluators that allow the request and collect their datasetIds
for (GdsDshidEvaluator dsidEvaluator : dsidEvaluators) {
if (!datasetIds.contains(dsidEvaluator.getDatasetId())) {
if (dsidEvaluator.isAllowed(request)) {
datasetIds.add(dsidEvaluator.getDatasetId());
}
}
}
}
} else {
LOG.debug("GdsDataShareEvaluator.evaluate({}): conditions {} didn't match. Skipped", request, dsh.getConditionExpr());
}
if (isAllowed) {
dshidEvaluators.stream().filter(e -> !datasetIds.contains(e.getDatasetId()) && e.isAllowed(request) && e.getDatasetEvaluator().isActive()).map(GdsDshidEvaluator::getDatasetId).forEach(datasetIds::add);
}

LOG.debug("<== GdsDataShareEvaluator.evaluate({}, {})", request, result);
LOG.debug("<== GdsDataShareEvaluator.collectDatasets({}, {})", request, result);
}

public void getResourceACLs(RangerAccessRequest request, RangerResourceACLs acls) {
Expand All @@ -183,35 +121,19 @@ public void getResourceACLs(RangerAccessRequest request, RangerResourceACLs acls
boolean isConditional = conditionEvaluator != null;

for (GdsSharedResourceEvaluator evaluator : evaluators) {
evaluator.getResourceACLs(request, acls, isConditional, dsidEvaluators);
evaluator.getResourceACLs(request, acls, isConditional, dshidEvaluators);
}
}

LOG.debug("<== GdsDataShareEvaluator.getResourceACLs({}, {})", request, acls);
}

void addDshidEvaluator(GdsDshidEvaluator dhidEvaluator) {
dsidEvaluators.add(dhidEvaluator);
void addResourceEvaluator(GdsSharedResourceEvaluator evaluator) {
evaluators.add(evaluator);
}

private List<GdsSharedResourceEvaluator> getResourceEvaluators(RangerAccessRequest request) {
final List<GdsSharedResourceEvaluator> ret;

Collection<GdsSharedResourceEvaluator> evaluators = RangerResourceEvaluatorsRetriever.getEvaluators(resourceTries, request.getResource().getAsMap(), request.getResourceElementMatchingScopes());

if (evaluators == null || evaluators.isEmpty()) {
ret = Collections.emptyList();
} else if (evaluators.size() > 1) {
ret = new ArrayList<>(evaluators);

ret.sort(GdsSharedResourceEvaluator.EVAL_ORDER_COMPARATOR);
} else {
ret = Collections.singletonList(evaluators.iterator().next());
}

LOG.debug("GdsDataShareEvaluator.getResourceEvaluators({}): found {} evaluators", request, ret.size());

return ret;
void addDshidEvaluator(GdsDshidEvaluator dhidEvaluator) {
dshidEvaluators.add(dhidEvaluator);
}

public static class GdsDataShareEvalOrderComparator implements Comparator<GdsDataShareEvaluator> {
Expand Down
38 changes: 12 additions & 26 deletions ...s-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDatasetEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;

public class GdsDatasetEvaluator {
private static final Logger LOG = LoggerFactory.getLogger(GdsDatasetEvaluator.class);
Expand Down Expand Up @@ -85,18 +89,12 @@ public String getName() {
return name;
}

public boolean isInProject(long projectId) {
boolean ret = false;

for (GdsDipEvaluator dipEvaluator : dipEvaluators) {
if (dipEvaluator.getProjectId().equals(projectId)) {
ret = true;

break;
}
}
public boolean isInProject(Long projectId) {
return dipEvaluators.stream().anyMatch(e -> e.getProjectId().equals(projectId) && e.isActive());
}

return ret;
public boolean isActive() {
return scheduleEvaluator == null || scheduleEvaluator.isApplicable(System.currentTimeMillis());
}

public void evaluate(RangerAccessRequest request, GdsAccessResult result, Set<Long> projectIds) {
Expand All @@ -109,9 +107,7 @@ public void evaluate(RangerAccessRequest request, GdsAccessResult result, Set<Lo
GdsDatasetAccessRequest datasetRequest = new GdsDatasetAccessRequest(getId(), gdsServiceDef, request);
RangerAccessResult datasetResult = datasetRequest.createAccessResult();

for (RangerPolicyEvaluator policyEvaluator : policyEvaluators) {
policyEvaluator.evaluate(datasetRequest, datasetResult);
}
policyEvaluators.forEach(e -> e.evaluate(datasetRequest, datasetResult));

if (!result.getIsAllowed()) {
if (datasetResult.getIsAllowed()) {
Expand All @@ -126,13 +122,7 @@ public void evaluate(RangerAccessRequest request, GdsAccessResult result, Set<Lo
}
}

for (GdsDipEvaluator dipEvaluator : dipEvaluators) {
if (!projectIds.contains(dipEvaluator.getProjectId())) {
if (dipEvaluator.isAllowed(request)) {
projectIds.add(dipEvaluator.getProjectId());
}
}
}
dipEvaluators.stream().filter(e -> !projectIds.contains(e.getProjectId()) && e.isAllowed(request) && e.getProjectEvaluator().isActive()).map(GdsDipEvaluator::getProjectId).forEach(projectIds::add);
}

LOG.debug("<== GdsDatasetEvaluator.evaluate({}, {})", request, result);
Expand Down Expand Up @@ -174,10 +164,6 @@ void addDipEvaluator(GdsDipEvaluator dipEvaluator) {
dipEvaluators.add(dipEvaluator);
}

private boolean isActive() {
return scheduleEvaluator == null || scheduleEvaluator.isApplicable(System.currentTimeMillis());
}

private static class GdsDatasetAccessRequest extends RangerAccessRequestImpl {
public GdsDatasetAccessRequest(Long datasetId, RangerServiceDef gdsServiceDef, RangerAccessRequest request) {
super.setResource(new RangerDatasetResource(datasetId, gdsServiceDef, request.getResource().getOwnerUser()));
Expand Down
21 changes: 10 additions & 11 deletions agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDipEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,16 @@ public Long getProjectId() {

public GdsProjectEvaluator getProjectEvaluator() { return projectEvaluator; }

public boolean isActive() {
boolean ret = dip.getStatus() == RangerGds.GdsShareStatus.ACTIVE;

if (ret && scheduleEvaluator != null) {
ret = scheduleEvaluator.isApplicable(System.currentTimeMillis());
}

return ret;
}

public boolean isAllowed(RangerAccessRequest request) {
boolean ret = isActive();

Expand All @@ -76,15 +86,4 @@ public void getResourceACLs(RangerAccessRequest request, RangerResourceACLs acls

LOG.debug("<== GdsDipEvaluator.getResourceACLs({}, {})", request, acls);
}


private boolean isActive() {
boolean ret = dip.getStatus() == RangerGds.GdsShareStatus.ACTIVE;

if (ret && scheduleEvaluator != null) {
ret = scheduleEvaluator.isApplicable(System.currentTimeMillis());
}

return ret;
}
}
21 changes: 10 additions & 11 deletions agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDshidEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,16 @@ public GdsDatasetEvaluator getDatasetEvaluator() {
return datasetEvaluator;
}

public boolean isActive() {
boolean ret = dshid.getStatus() == RangerGds.GdsShareStatus.ACTIVE;

if (ret && scheduleEvaluator != null) {
ret = scheduleEvaluator.isApplicable(System.currentTimeMillis());
}

return ret;
}

public boolean isAllowed(RangerAccessRequest request) {
boolean ret = isActive();

Expand All @@ -78,15 +88,4 @@ public void getResourceACLs(RangerAccessRequest request, RangerResourceACLs acls

LOG.debug("<== GdsDshidEvaluator.getResourceACLs({}, {})", request, acls);
}


private boolean isActive() {
boolean ret = dshid.getStatus() == RangerGds.GdsShareStatus.ACTIVE;

if (ret && scheduleEvaluator != null) {
ret = scheduleEvaluator.isApplicable(System.currentTimeMillis());
}

return ret;
}
}
Loading

0 comments on commit a4f8cfb

Please sign in to comment.