diff --git a/docs/.gitbook/assets/image (7).png b/docs/.gitbook/assets/image (7).png
new file mode 100644
index 00000000000..6de63522f61
Binary files /dev/null and b/docs/.gitbook/assets/image (7).png differ
diff --git a/docs/.gitbook/assets/image (8).png b/docs/.gitbook/assets/image (8).png
new file mode 100644
index 00000000000..6de63522f61
Binary files /dev/null and b/docs/.gitbook/assets/image (8).png differ
diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml
index 4952898d350..42714e60244 100644
--- a/opencga-analysis/pom.xml
+++ b/opencga-analysis/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java
index 0bb3dc4c14a..6c80531c1d8 100644
--- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java
+++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java
@@ -28,6 +28,7 @@
import org.opencb.biodata.models.variant.avro.VariantType;
import org.opencb.biodata.models.variant.metadata.SampleVariantStats;
import org.opencb.biodata.models.variant.metadata.VariantMetadata;
+import org.opencb.biodata.models.variant.stats.VariantStats;
import org.opencb.biodata.tools.variant.converters.ga4gh.Ga4ghVariantConverter;
import org.opencb.biodata.tools.variant.converters.ga4gh.factories.AvroGa4GhVariantFactory;
import org.opencb.biodata.tools.variant.converters.ga4gh.factories.ProtoGa4GhVariantFactory;
@@ -73,10 +74,7 @@
import org.opencb.opencga.storage.core.utils.CellBaseUtils;
import org.opencb.opencga.storage.core.variant.BeaconResponse;
import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
-import org.opencb.opencga.storage.core.variant.adaptors.VariantField;
-import org.opencb.opencga.storage.core.variant.adaptors.VariantIterable;
-import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException;
-import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
+import org.opencb.opencga.storage.core.variant.adaptors.*;
import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator;
import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat;
import org.opencb.opencga.storage.core.variant.query.ParsedQuery;
@@ -96,8 +94,7 @@
import static org.opencb.opencga.analysis.variant.manager.operations.VariantFileIndexerOperationManager.FILE_GET_QUERY_OPTIONS;
import static org.opencb.opencga.core.api.ParamConstants.ACL_PARAM;
import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*;
-import static org.opencb.opencga.storage.core.variant.adaptors.sample.VariantSampleDataManager.SAMPLE_BATCH_SIZE;
-import static org.opencb.opencga.storage.core.variant.adaptors.sample.VariantSampleDataManager.SAMPLE_BATCH_SIZE_DEFAULT;
+import static org.opencb.opencga.storage.core.variant.adaptors.sample.VariantSampleDataManager.*;
import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.*;
public class VariantStorageManager extends StorageManager implements AutoCloseable {
@@ -630,7 +627,12 @@ public DataResult getSampleData(String variant, String study, QueryOpti
int skip = options.getInt(SKIP, 0);
// Make initial batchLimit shorter
int batchLimit = Math.min(SAMPLE_BATCH_SIZE_DEFAULT, limit * 3 + skip);
+ // but not too short
+ batchLimit = Math.max(100, batchLimit);
int batchSkip = 0;
+ int numReadSamples = 0;
+ int numValidSamples = 0;
+ boolean exactNumSamples = false;
String studyFqn = query.getString(STUDY.key());
options.putAll(query);
@@ -664,6 +666,9 @@ public DataResult getSampleData(String variant, String study, QueryOpti
.map(SampleEntry::getSampleId)
.collect(Collectors.toList());
moreResults = samplesInResult.size() == batchLimit;
+ // The count of samples is exact if there is no more results
+ exactNumSamples = !moreResults;
+ numReadSamples += samplesInResult.size();
StopWatch checkPermissionsStopWatch = StopWatch.createStarted();
String userId = catalogManager.getUserManager().getUserId(token);
@@ -680,6 +685,7 @@ public DataResult getSampleData(String variant, String study, QueryOpti
auditAttributes.put("checkSamplePermissionsTimeMillis",
checkPermissionsStopWatch.getTime(TimeUnit.MILLISECONDS)
+ auditAttributes.getInt("checkSamplePermissionsTimeMillis", 0));
+ numValidSamples += validSamples.size();
List samplesToReturn;
if (skip > validSamples.size()) {
samplesToReturn = Collections.emptyList();
@@ -713,25 +719,33 @@ public DataResult getSampleData(String variant, String study, QueryOpti
variantResult.getStudies().get(0).setSamples(sampleEntries);
variantResult.getStudies().get(0).setFiles(fileEntries);
- ObjectMap attributes = new ObjectMap();
-// VariantStats stats = variantResult.getStudies().get(0).getStats(StudyEntry.DEFAULT_COHORT);
-// if (stats != null) {
-// List genotypesFilter = new ArrayList<>(options.getAsStringList(GENOTYPE.key()));
-// if (genotypesFilter.isEmpty()) {
-// genotypesFilter.add(GenotypeClass.MAIN_ALT.name());
-// }
-// int expectedSamplesCount = 0;
-// List gtsInVariant = new ArrayList<>(stats.getGenotypeCount().keySet());
-// List genotypesToReturn = GenotypeClass.filter(genotypesFilter, gtsInVariant);
-// for (String gt : genotypesToReturn) {
-// expectedSamplesCount += stats.getGenotypeCount().getOrDefault(gt, 0);
-// }
-// }
-// result.getAttributes().put(NUM_SAMPLES.key(), samplesToReturn.size());
-// result.getAttributes().put(NUM_TOTAL_SAMPLES.key(), validSamples.size());
-
- return new DataResult<>(((int) stopWatch.getTime(TimeUnit.MILLISECONDS)),
- new ArrayList<>(), 1, Collections.singletonList(variantResult), 1, attributes);
+ VariantQueryResult result = new VariantQueryResult<>(
+ ((int) stopWatch.getTime(TimeUnit.MILLISECONDS)),
+ 1, 1, new ArrayList<>(), Collections.singletonList(variantResult), null, null)
+ .setNumSamples(sampleEntries.size());
+ if (exactNumSamples) {
+ result.setNumTotalSamples(numValidSamples).setApproximateCount(false);
+ } else {
+ VariantStats stats = variantResult.getStudies().get(0).getStats(StudyEntry.DEFAULT_COHORT);
+ if (stats != null) {
+ List genotypesFilter = new ArrayList<>(options.getAsStringList(GENOTYPE.key()));
+ if (genotypesFilter.isEmpty()) {
+ genotypesFilter.add(GenotypeClass.MAIN_ALT.name());
+ }
+ int expectedSamplesCount = 0;
+ List gtsInVariant = new ArrayList<>(stats.getGenotypeCount().keySet());
+ List genotypesToReturn = GenotypeClass.filter(genotypesFilter, gtsInVariant);
+ for (String gt : genotypesToReturn) {
+ expectedSamplesCount += stats.getGenotypeCount().getOrDefault(gt, 0);
+ }
+
+ int numTotalSamples = ((int) (expectedSamplesCount * (((float) numValidSamples) / numReadSamples)));
+ result.setNumTotalSamples(numTotalSamples);
+ result.setApproximateCountSamplingSize(numReadSamples);
+ }
+ result.setApproximateCount(true);
+ }
+ return result;
});
}
}
diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml
index aeb4a73d662..2abc622abd5 100644
--- a/opencga-app/pom.xml
+++ b/opencga-app/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/catalog/FileCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/catalog/FileCommandExecutor.java
index 339f8bef97c..f80ac347f1c 100644
--- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/catalog/FileCommandExecutor.java
+++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/catalog/FileCommandExecutor.java
@@ -19,6 +19,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
+import org.opencb.biodata.models.clinical.interpretation.Software;
import org.opencb.commons.datastore.core.FacetField;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.QueryOptions;
@@ -307,7 +308,15 @@ private RestResponse update() throws ClientException {
.setBioformat(commandOptions.bioformat)
.setDescription(commandOptions.description)
.setTags(commandOptions.tags)
- .setSampleIds(commandOptions.sampleIds);
+ .setSampleIds(commandOptions.sampleIds)
+ .setSoftware(new Software(
+ commandOptions.softwareName,
+ commandOptions.softwareVersion,
+ commandOptions.softwareRepository,
+ commandOptions.softwareCommit,
+ commandOptions.softwareWebsite,
+ commandOptions.softwareParams.isEmpty() ? null : commandOptions.softwareParams
+ ));
ObjectMap params = new ObjectMap();
params.putIfNotNull(FileDBAdaptor.QueryParams.STUDY.key(), commandOptions.study);
diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java
index e81f9de899e..ee512f55835 100644
--- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java
+++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java
@@ -37,7 +37,6 @@
import org.opencb.opencga.core.models.study.VariableSet;
import org.opencb.opencga.core.models.user.User;
import org.opencb.opencga.core.response.RestResponse;
-import org.opencb.opencga.core.tools.result.ToolStep;
import java.text.SimpleDateFormat;
import java.time.Instant;
@@ -45,8 +44,8 @@
import java.util.stream.Collectors;
import static org.opencb.opencga.core.common.IOUtils.humanReadableByteCount;
-import static org.opencb.opencga.core.models.common.Enums.ExecutionStatus.DONE;
import static org.opencb.opencga.core.models.common.Enums.ExecutionStatus.RUNNING;
+import static org.opencb.opencga.core.models.common.Status.READY;
/**
* Created by pfurio on 28/11/16.
@@ -501,12 +500,25 @@ private void printRecursiveTree(List fileTreeList, StringBuilder sb, S
FileTree fileTree = iterator.next();
File file = fileTree.getFile();
- sb.append(String.format("%s %s [%s, %s, %s]\n",
- indent.isEmpty() ? "" : indent + (iterator.hasNext() ? "├──" : "└──"),
- file.getType() == File.Type.FILE ? file.getName() : file.getName() + "/",
- file.getName(),
- file.getInternal() != null && file.getInternal().getStatus() != null ? file.getInternal().getStatus().getName() : "",
- humanReadableByteCount(file.getSize(), false)));
+ if (!indent.isEmpty()) {
+ sb.append(indent);
+ sb.append(iterator.hasNext() ? "├──" : "└──");
+ sb.append(" ");
+ }
+ if (file.getType() == File.Type.FILE) {
+ sb.append(file.getName());
+ sb.append(" [");
+ if (file.getInternal() != null
+ && file.getInternal().getStatus() != null
+ && !READY.equals(file.getInternal().getStatus().getName())) {
+ sb.append(file.getInternal().getStatus().getName()).append(", ");
+ }
+ sb.append(humanReadableByteCount(file.getSize(), false)).append("]");
+
+ } else {
+ sb.append(file.getName()).append("/");
+ }
+ sb.append("\n");
if (file.getType() == File.Type.DIRECTORY) {
printRecursiveTree(fileTree.getChildren(), sb, indent + (iterator.hasNext()? "│ " : " "));
diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FileCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FileCommandOptions.java
index b4ff2a7f118..11d4403a36f 100644
--- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FileCommandOptions.java
+++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FileCommandOptions.java
@@ -16,10 +16,7 @@
package org.opencb.opencga.app.cli.main.options;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
+import com.beust.jcommander.*;
import org.opencb.opencga.app.cli.GeneralCliOptions;
import org.opencb.opencga.app.cli.GeneralCliOptions.CommonCommandOptions;
import org.opencb.opencga.app.cli.GeneralCliOptions.DataModelOptions;
@@ -30,7 +27,9 @@
import org.opencb.opencga.core.api.ParamConstants;
import org.opencb.opencga.core.models.file.File;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Created by sgallego on 6/14/16.
@@ -317,6 +316,24 @@ public class UpdateCommandOptions extends BaseFileCommand {
@Parameter(names = {"--tags"}, description = "Comma separated list of sample names or ids", arity = 1)
public List tags;
+ @Parameter(names = {"--software-name"}, description = "Software name", arity = 1)
+ public String softwareName;
+
+ @Parameter(names = {"--software-version"}, description = "Software version", arity = 1)
+ public String softwareVersion;
+
+ @Parameter(names = {"--software-repository"}, description = "Software repository", arity = 1)
+ public String softwareRepository;
+
+ @Parameter(names = {"--software-commit"}, description = "Software commit", arity = 1)
+ public String softwareCommit;
+
+ @Parameter(names = {"--software-website"}, description = "Software website", arity = 1)
+ public String softwareWebsite;
+
+ @DynamicParameter(names = {"--software-params"}, description = "Software params")
+ public Map softwareParams = new HashMap<>();
+
}
// @Parameters(commandNames = {"relink"}, commandDescription = "Change location of linked or STAGED file.")
diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml
index ee6759fc878..c1bc43bb7ee 100644
--- a/opencga-catalog/pom.xml
+++ b/opencga-catalog/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java
index fbd5ce79e60..66b8a468cac 100644
--- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java
+++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java
@@ -360,7 +360,7 @@ private List getUserInfoFromLDAP(String host, List userList,
}
private String getMail(Attributes attributes) throws NamingException {
- return (String) attributes.get("mail").get(0);
+ return attributes.get("mail") != null ? (String) attributes.get("mail").get(0) : "";
}
private String getUID(Attributes attributes) throws NamingException {
diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java
index c29f1b7cf10..b1da74503e1 100644
--- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java
+++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java
@@ -416,9 +416,6 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis
// Validate there is only one sample per member
for (Individual member : family.getMembers()) {
- if (member.getSamples() == null || member.getSamples().isEmpty()) {
- throw new CatalogException("Missing sample for member '" + member.getId() + "'");
- }
if (member.getSamples().size() > 1) {
throw new CatalogException("More than one sample found for member '" + member.getId() + "'.");
}
@@ -435,6 +432,10 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis
if (proband == null) {
throw new CatalogException("Proband '" + clinicalAnalysis.getProband().getId() + "' not found in family");
}
+ if (proband.getSamples() == null || proband.getSamples().isEmpty()) {
+ throw new CatalogException("Missing samples for proband '" + proband.getId() + "'");
+ }
+
clinicalAnalysis.setProband(proband);
// Validate the proband has a disorder
diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java
index 2fcb4e6f82a..3a09b0cdbc2 100644
--- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java
+++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java
@@ -931,11 +931,19 @@ private OpenCGAResult update(Study study, Family family, FamilyUpdatePar
if (updateParams != null && ListUtils.isNotEmpty(updateParams.getPhenotypes())) {
tmpFamily.setPhenotypes(updateParams.getPhenotypes());
+ } else if (ListUtils.isNotEmpty(updateParams.getMembers())) {
+ // Recalculate phenotypes if list of members has changed
+ tmpFamily.setPhenotypes(Collections.emptyList());
+ parameters.put(FamilyDBAdaptor.QueryParams.PHENOTYPES.key(), tmpFamily.getPhenotypes());
} else {
tmpFamily.setPhenotypes(family.getPhenotypes());
}
if (updateParams != null && ListUtils.isNotEmpty(updateParams.getDisorders())) {
tmpFamily.setDisorders(updateParams.getDisorders());
+ } else if (ListUtils.isNotEmpty(updateParams.getMembers())) {
+ // Recalculate disorders if list of members has changed
+ tmpFamily.setDisorders(Collections.emptyList());
+ parameters.put(FamilyDBAdaptor.QueryParams.DISORDERS.key(), tmpFamily.getDisorders());
} else {
tmpFamily.setDisorders(family.getDisorders());
}
diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java
index e29aa58f124..3f1933f4ff6 100644
--- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java
+++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java
@@ -1348,15 +1348,23 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat
parameters.put(IndividualDBAdaptor.QueryParams.SAMPLES.key(), sampleList);
}
- if (updateParams != null && StringUtils.isNotEmpty(updateParams.getFather())) {
- OpenCGAResult queryResult = internalGet(studyUid, updateParams.getFather(), INCLUDE_INDIVIDUAL_IDS, userId);
+ if (updateParams != null && updateParams.getFather() != null) {
+ if (StringUtils.isNotEmpty(updateParams.getFather())) {
+ OpenCGAResult queryResult = internalGet(studyUid, updateParams.getFather(), INCLUDE_INDIVIDUAL_IDS, userId);
+ parameters.put(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), queryResult.first().getUid());
+ } else {
+ parameters.put(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), -1L);
+ }
parameters.remove(IndividualDBAdaptor.QueryParams.FATHER.key());
- parameters.put(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), queryResult.first().getUid());
}
- if (updateParams != null && StringUtils.isNotEmpty(updateParams.getMother())) {
- OpenCGAResult queryResult = internalGet(studyUid, updateParams.getMother(), INCLUDE_INDIVIDUAL_IDS, userId);
+ if (updateParams != null && updateParams.getMother() != null) {
+ if (StringUtils.isNotEmpty(updateParams.getMother())) {
+ OpenCGAResult queryResult = internalGet(studyUid, updateParams.getMother(), INCLUDE_INDIVIDUAL_IDS, userId);
+ parameters.put(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), queryResult.first().getUid());
+ } else {
+ parameters.put(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), -1L);
+ }
parameters.remove(IndividualDBAdaptor.QueryParams.MOTHER.key());
- parameters.put(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), queryResult.first().getUid());
}
checkUpdateAnnotations(study, individual, parameters, options, VariableSet.AnnotableDataModels.INDIVIDUAL, individualDBAdaptor,
diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java
index 0a9ccf72408..c085b5791e2 100644
--- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java
+++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java
@@ -382,8 +382,7 @@ public List getJobInputFilesFromParams(String study, Job job, String token
inputFiles.add(file);
} catch (CatalogException e) {
throw new CatalogException("Cannot find file '" + entry.getValue() + "' "
- + "from job param '" + entry.getKey() + "'; (study = " + study + ", token = " + token
- + ") :" + e.getMessage(), e);
+ + "from job param '" + entry.getKey() + "'; (study = " + study + ") :" + e.getMessage(), e);
}
}
} else if (entry.getValue() instanceof Map) {
diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java
index fbf7463d82a..a7859834fde 100644
--- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java
+++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java
@@ -161,7 +161,9 @@ public OpenCGAResult create(User user, String password, @Nullable String t
ParamUtils.checkObj(user, "User");
ParamUtils.checkValidUserId(user.getId());
ParamUtils.checkParameter(user.getName(), "name");
- checkEmail(user.getEmail());
+ if (StringUtils.isNotEmpty(user.getEmail())) {
+ checkEmail(user.getEmail());
+ }
ParamUtils.checkObj(user.getAccount(), "account");
user.setQuota(ParamUtils.defaultObject(user.getQuota(), UserQuota::new));
user.setFilters(ParamUtils.defaultObject(user.getFilters(), LinkedList::new));
diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java
index 0f5146195d4..f5e8dc9eddd 100644
--- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java
+++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java
@@ -1789,6 +1789,32 @@ public void testUpdateIndividualParents() throws CatalogException {
assertEquals(1, individual.getFather().getVersion());
}
+ @Test
+ public void testRemoveIndividualParents() throws CatalogException {
+ IndividualManager individualManager = catalogManager.getIndividualManager();
+ individualManager.create(studyFqn, new Individual().setId("child"), QueryOptions.empty(), token);
+ individualManager.create(studyFqn, new Individual().setId("father"), QueryOptions.empty(), token);
+ individualManager.create(studyFqn, new Individual().setId("mother"), QueryOptions.empty(), token);
+
+ DataResult individualDataResult = individualManager.update(studyFqn, "child",
+ new IndividualUpdateParams().setFather("father").setMother("mother"), QueryOptions.empty(), token);
+ assertEquals(1, individualDataResult.getNumUpdated());
+
+ Individual individual = individualManager.get(studyFqn, "child", QueryOptions.empty(), token).first();
+
+ assertEquals("mother", individual.getMother().getId());
+ assertEquals(1, individual.getMother().getVersion());
+
+ assertEquals("father", individual.getFather().getId());
+ assertEquals(1, individual.getFather().getVersion());
+
+ individualManager.update(studyFqn, "child", new IndividualUpdateParams().setFather("").setMother(""), QueryOptions.empty(), token);
+ individual = individualManager.get(studyFqn, "child", QueryOptions.empty(), token).first();
+
+ assertNull(individual.getMother().getId());
+ assertNull(individual.getFather().getId());
+ }
+
@Test
public void testIndividualRelatives() throws CatalogException {
IndividualManager individualManager = catalogManager.getIndividualManager();
diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java
index 2a36cf5fea8..16431d64662 100644
--- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java
+++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java
@@ -98,7 +98,7 @@ public void setUpCatalogManager(CatalogManager catalogManager) throws IOExceptio
public void tearDown() throws Exception {
}
- private DataResult createDummyFamily() throws CatalogException {
+ private Family getDummyFamily() {
Disorder disease1 = new Disorder("dis1", "Disease 1", "HPO", null, "", null);
Disorder disease2 = new Disorder("dis2", "Disease 2", "HPO", null, "", null);
@@ -140,9 +140,13 @@ private DataResult createDummyFamily() throws CatalogException {
))
.setParentalConsanguinity(true);
- Family family = new Family("family", "family", null, Arrays.asList(disease1, disease2),
+ return new Family("family", "family", null, Arrays.asList(disease1, disease2),
Arrays.asList(relChild1, relChild2, relChild3, relFather, relMother), "", -1,
Collections.emptyList(), Collections.emptyMap());
+ }
+
+ private DataResult createDummyFamily() throws CatalogException {
+ Family family = getDummyFamily();
return familyManager.create(STUDY, family, QueryOptions.empty(), sessionIdUser);
}
@@ -244,6 +248,43 @@ public void createClinicalWithComments() throws CatalogException {
Long.parseLong(clinical.first().getComments().get(1).getDate()));
}
+ @Test
+ public void createClinicalWithMissingSamplesInFamily() throws CatalogException {
+ Family family = getDummyFamily();
+ for (Individual member : family.getMembers()) {
+ if (!member.getId().equals("child1")) {
+ member.setSamples(Collections.emptyList());
+ }
+ }
+ familyManager.create(STUDY, family, QueryOptions.empty(), sessionIdUser);
+
+ // And only add sample to proband
+ for (Individual member : family.getMembers()) {
+ if (member.getId().equals("child1")) {
+ member.setSamples(Collections.singletonList(new Sample().setId("sample2")));
+ }
+ }
+
+ ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis()
+ .setId("analysis").setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY)
+ .setDueDate("20180510100000")
+ .setProband(new Individual().setId("child1"));
+ clinicalAnalysis.setFamily(family);
+ DataResult clinicalAnalysisDataResult = catalogManager.getClinicalAnalysisManager().create(STUDY,
+ clinicalAnalysis, QueryOptions.empty(), sessionIdUser);
+
+ assertEquals("child1", clinicalAnalysisDataResult.first().getFamily().getMembers().get(0).getId());
+ assertEquals("father", clinicalAnalysisDataResult.first().getFamily().getMembers().get(1).getId());
+ assertEquals("mother", clinicalAnalysisDataResult.first().getFamily().getMembers().get(2).getId());
+ assertEquals("child2", clinicalAnalysisDataResult.first().getFamily().getMembers().get(3).getId());
+ assertEquals("child3", clinicalAnalysisDataResult.first().getFamily().getMembers().get(4).getId());
+ assertEquals("sample2", clinicalAnalysisDataResult.first().getFamily().getMembers().get(0).getSamples().get(0).getId());
+ assertTrue(clinicalAnalysisDataResult.first().getFamily().getMembers().get(1).getSamples().isEmpty());
+ assertTrue(clinicalAnalysisDataResult.first().getFamily().getMembers().get(2).getSamples().isEmpty());
+ assertTrue(clinicalAnalysisDataResult.first().getFamily().getMembers().get(3).getSamples().isEmpty());
+ assertTrue(clinicalAnalysisDataResult.first().getFamily().getMembers().get(4).getSamples().isEmpty());
+ }
+
@Test
public void updateClinicalComments() throws CatalogException {
Individual individual = new Individual()
diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml
index 1313b24c3b8..1827031bf89 100644
--- a/opencga-client/pom.xml
+++ b/opencga-client/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-client/src/main/python/pyopencga/opencga_client.py b/opencga-client/src/main/python/pyopencga/opencga_client.py
index 6e5e563337c..09b25ae4820 100644
--- a/opencga-client/src/main/python/pyopencga/opencga_client.py
+++ b/opencga-client/src/main/python/pyopencga/opencga_client.py
@@ -147,13 +147,13 @@ def wait_for_job(self, response=None, study_id=None, job_id=None, retry_seconds=
retry_seconds = retry_seconds if retry_seconds >= 10 else 10
while True:
- job_info = self.jobs.info(study=study_id, jobs=job_id, include='internal.status').get_result(0)
+ job_info = self.jobs.info(study=study_id, jobs=job_id).get_result(0)
if job_info['internal']['status']['name'] in ['ERROR', 'ABORTED']:
raise ValueError('{} ({}): {}'.format(
job_info['status']['name'], job_info['status']['date'],
job_info['status']['message']
))
- elif job_info['status']['name'] in ['DONE']:
+ elif job_info['internal']['status']['name'] in ['DONE']:
break
time.sleep(retry_seconds)
diff --git a/opencga-client/src/main/python/pyopencga/rest_response.py b/opencga-client/src/main/python/pyopencga/rest_response.py
index 0909957477a..df87212e394 100644
--- a/opencga-client/src/main/python/pyopencga/rest_response.py
+++ b/opencga-client/src/main/python/pyopencga/rest_response.py
@@ -1,4 +1,5 @@
import sys
+import pandas as pd
from pyopencga.commons import deprecated
@@ -85,8 +86,11 @@ def _get_param_value(result, field):
else:
return result2
- def print_results(self, fields=None, response_pos=None, limit=None, separator='\t', metadata=True, outfile=None):
+ def print_results(self, fields=None, response_pos=None, limit=None, separator='\t', title=None,
+ metadata=True, outfile=None):
outfhand = sys.stdout if outfile is None else open(outfile, 'w')
+ if title is not None:
+ outfhand.write(title + '\n' + '-'*(len(title)+5) + '\n')
if metadata:
for event_type in ['INFO', 'WARNING', 'ERROR']:
for event in self.get_response_events(event_type):
@@ -118,6 +122,11 @@ def print_results(self, fields=None, response_pos=None, limit=None, separator='\
values = [self._get_param_value(result, field) for field in fields]
outfhand.write(separator.join(map(str, values)) + '\n')
+ def to_data_frame(self, response_pos=None):
+ responses = [self.get_response(response_pos)] if response_pos is not None else self.get_responses()
+ return pd.concat([pd.json_normalize(result) for response in responses
+ for result in response['results']]).reset_index(drop=True)
+
def get_response_events(self, event_type=None):
"""
Return response events by name
diff --git a/opencga-client/src/main/python/requirements.txt b/opencga-client/src/main/python/requirements.txt
index b0055c6118a..5a8aa59e862 100644
--- a/opencga-client/src/main/python/requirements.txt
+++ b/opencga-client/src/main/python/requirements.txt
@@ -2,3 +2,4 @@ requests
pip
pathlib
pyyaml
+pandas
diff --git a/opencga-client/src/main/python/setup.py b/opencga-client/src/main/python/setup.py
index 1cfd882215b..00663de7702 100644
--- a/opencga-client/src/main/python/setup.py
+++ b/opencga-client/src/main/python/setup.py
@@ -37,7 +37,8 @@
'requests >= 2.7',
'pip >= 7.1.2',
'pathlib >= 1.0.1',
- 'pyyaml >= 3.12'
+ 'pyyaml >= 3.12',
+ 'pandas >= 1.1.5'
],
project_urls={
'Documentation': 'http://docs.opencb.org/display/opencga/Python',
diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml
index 035af3e825f..32fb7336b2d 100644
--- a/opencga-core/pom.xml
+++ b/opencga-core/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java
index 4786e3af637..f1e128ab854 100644
--- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java
+++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java
@@ -453,4 +453,5 @@ public class ParamConstants {
public static final String COMMAND_PARAMETER = "command";
public static final String COMMAND_PARAMETER_DESCRIPTION = "Command name to execute in this tool.";
+ public static final String FIELD_PARAM = "field";
}
diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/response/VariantQueryResult.java b/opencga-core/src/main/java/org/opencb/opencga/core/response/VariantQueryResult.java
index db726b53de9..d3210420db8 100644
--- a/opencga-core/src/main/java/org/opencb/opencga/core/response/VariantQueryResult.java
+++ b/opencga-core/src/main/java/org/opencb/opencga/core/response/VariantQueryResult.java
@@ -48,6 +48,10 @@ public class VariantQueryResult extends OpenCGAResult {
public VariantQueryResult() {
}
+ public VariantQueryResult(long time, int numResults, long numMatches, List events, List result) {
+ this(time, numResults, numMatches, events, result, null, null, null, null, null);
+ }
+
public VariantQueryResult(long time, int numResults, long numMatches, List events, List result,
Map> samples, String source) {
this(time, numResults, numMatches, events, result, samples, source, null, null, null);
@@ -57,14 +61,24 @@ public VariantQueryResult(long time, int numResults, long numMatches, List> samples, String source, Boolean approximateCount,
Integer approximateCountSamplingSize, Integer numTotalSamples) {
super((int) time, events, numResults, result, numMatches);
- setSamples(samples);
- setSource(source);
- setApproximateCount(approximateCount);
- setApproximateCountSamplingSize(approximateCountSamplingSize);
+ if(samples != null) {
+ setSamples(samples);
+ }
+ if(source != null) {
+ setSource(source);
+ }
+ if(approximateCount != null) {
+ setApproximateCount(approximateCount);
+ }
+ if(approximateCountSamplingSize != null) {
+ setApproximateCountSamplingSize(approximateCountSamplingSize);
+ }
if (samples != null) {
setNumSamples(samples.values().stream().mapToInt(List::size).sum());
}
- setNumTotalSamples(numTotalSamples);
+ if (numTotalSamples != null) {
+ setNumTotalSamples(numTotalSamples);
+ }
}
public VariantQueryResult(DataResult dataResult) {
@@ -91,9 +105,11 @@ public VariantQueryResult(DataResult dataResult, Map> sa
}
if (samples != null) {
this.setNumSamples(samples.values().stream().mapToInt(List::size).sum());
+ this.setNumTotalSamples(getNumSamples());
+ }
+ if (source != null) {
+ this.setSource(source);
}
- this.setNumTotalSamples(getNumSamples());
- this.setSource(source);
}
public Map> getSamples() {
diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml
index 3aa46828859..4fdc079ecd1 100644
--- a/opencga-master/pom.xml
+++ b/opencga-master/pom.xml
@@ -22,7 +22,7 @@
opencga
org.opencb.opencga
- 2.0.1
+ 2.0.2
..
diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java
index 08d4603d268..534ecd2d97a 100644
--- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java
+++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java
@@ -63,6 +63,7 @@
import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
import org.opencb.opencga.catalog.db.api.JobDBAdaptor;
import org.opencb.opencga.catalog.db.api.StudyDBAdaptor;
+import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException;
import org.opencb.opencga.catalog.exceptions.CatalogDBException;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.exceptions.CatalogIOException;
@@ -77,7 +78,6 @@
import org.opencb.opencga.core.common.JacksonUtils;
import org.opencb.opencga.core.common.TimeUtils;
import org.opencb.opencga.core.config.Execution;
-import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.file.File;
import org.opencb.opencga.core.models.file.FileAclEntry;
@@ -458,55 +458,44 @@ protected int checkPendingJob(Job job) {
Tool tool;
try {
tool = new ToolFactory().getTool(job.getTool().getId());
- } catch (ToolException e) {
+ } catch (Exception e) {
logger.error(e.getMessage(), e);
return abortJob(job, "Tool " + job.getTool().getId() + " not found");
}
+ try {
+ checkToolExecutionPermission(job);
+ } catch (Exception e) {
+ return abortJob(job, e);
+ }
+
PrivateJobUpdateParams updateParams = new PrivateJobUpdateParams();
updateParams.setTool(new ToolInfo(tool.id(), tool.description(), tool.scope(), tool.type(), tool.resource()));
if (tool.scope() == Tool.Scope.PROJECT) {
- String projectFqn = job.getStudy().getId().substring(0, job.getStudy().getId().indexOf(":"));
- OpenCGAResult studyResult;
+ String projectFqn = job.getStudy().getId().substring(0, job.getStudy().getId().indexOf(ParamConstants.PROJECT_STUDY_SEPARATOR));
try {
- studyResult = catalogManager.getStudyManager().search(projectFqn, new Query(),
+ List studyFqnSet = catalogManager.getStudyManager().search(projectFqn, new Query(),
new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.GROUPS.key(),
- StudyDBAdaptor.QueryParams.FQN.key())), token);
+ StudyDBAdaptor.QueryParams.FQN.key())), token)
+ .getResults()
+ .stream()
+ .map(Study::getFqn)
+ .filter(fqn -> !fqn.equals(job.getStudy().getId()))
+ .distinct()
+ .collect(Collectors.toList());
+
+ updateParams.setStudy(new JobStudyParam(job.getStudy().getId(), studyFqnSet));
} catch (CatalogException e) {
- logger.error(e.getMessage(), e);
- return abortJob(job, e.getMessage());
+ return abortJob(job, e);
}
- // Validate user is owner or admin
- if (!job.getStudy().getId().startsWith(job.getUserId() + "@")) {
- // It is not the owner, so we check if it is the admin
- for (Study study : studyResult.getResults()) {
- for (Group group : study.getGroups()) {
- if (group.getId().equals("@admins")) {
- // If the user does not belong to the admins group
- if (!group.getUserIds().contains(job.getUserId())) {
- return abortJob(job, "User '" + job.getUserId() + "' is not owner or admin of study '" + study.getFqn()
- + "'. The tool '" + job.getTool().getId()
- + "' can only be executed by the project owners or admins");
- }
- break;
- }
- }
- }
- }
-
- // Obtain other study fqns
- Set studyFqnSet = studyResult.getResults().stream()
- .map(Study::getFqn)
- .filter(fqn -> !fqn.equals(job.getStudy().getId()))
- .collect(Collectors.toSet());
- updateParams.setStudy(new JobStudyParam(job.getStudy().getId(), new ArrayList<>(studyFqnSet)));
}
String userToken;
try {
userToken = catalogManager.getUserManager().getNonExpiringToken(job.getUserId(), token);
} catch (CatalogException e) {
+ logger.error(e.getMessage(), e);
return abortJob(job, "Internal error. Could not obtain token for user '" + job.getUserId() + "'");
}
@@ -516,8 +505,7 @@ protected int checkPendingJob(Job job) {
List inputFiles = catalogManager.getJobManager().getJobInputFilesFromParams(job.getStudy().getId(), job, token);
updateParams.setInput(inputFiles);
} catch (CatalogException e) {
- logger.error(e.getMessage(), e);
- return abortJob(job, e.getMessage());
+ return abortJob(job, e);
}
}
@@ -581,6 +569,66 @@ protected int checkPendingJob(Job job) {
return 1;
}
+ protected void checkToolExecutionPermission(Job job) throws Exception {
+ Tool tool = new ToolFactory().getTool(job.getTool().getId());
+
+ if (tool.scope().equals(Tool.Scope.GLOBAL)) {
+ if (!job.getUserId().equals(ParamConstants.OPENCGA_USER_ID)) {
+ throw new CatalogAuthorizationException("Only user '" + ParamConstants.OPENCGA_USER_ID + "' "
+ + "can run tools with scope '" + Tool.Scope.GLOBAL + "'");
+ }
+ } else {
+ if (job.getStudy().getId().startsWith(job.getUserId() + ParamConstants.USER_PROJECT_SEPARATOR)) {
+ // If the user is the owner of the project, accept all.
+ return;
+ }
+
+ // Validate user is owner or belongs to the right group
+ String requiredGroup;
+ if (tool.type().equals(Tool.Type.OPERATION)) {
+ requiredGroup = ParamConstants.ADMINS_GROUP;
+ } else {
+ requiredGroup = ParamConstants.MEMBERS_GROUP;
+ }
+
+ List studiesToValidate;
+ if (tool.scope() == Tool.Scope.PROJECT) {
+ String projectFqn = job.getStudy().getId().substring(0, job.getStudy().getId()
+ .indexOf(ParamConstants.PROJECT_STUDY_SEPARATOR));
+ studiesToValidate = catalogManager.getStudyManager().search(projectFqn, new Query(),
+ new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.GROUPS.key(),
+ StudyDBAdaptor.QueryParams.FQN.key())), token).getResults();
+ } else {
+ studiesToValidate = catalogManager.getStudyManager().get(job.getStudy().getId(),
+ new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.GROUPS.key(),
+ StudyDBAdaptor.QueryParams.FQN.key())), token).getResults();
+ }
+
+ List missingStudies = new LinkedList<>();
+ // It is not the owner, so we check if it is the right group
+ for (Study study : studiesToValidate) {
+ for (Group group : study.getGroups()) {
+ if (group.getId().equals(requiredGroup)) {
+ // If the user does not belong to the admins group
+ if (!group.getUserIds().contains(job.getUserId())) {
+ missingStudies.add(study.getFqn());
+ }
+ break;
+ }
+ }
+ }
+
+ if (!missingStudies.isEmpty()) {
+ throw new CatalogAuthorizationException("User '" + job.getUserId() + "' is not member of "
+ + requiredGroup + " of studies '" + missingStudies
+ + "'. The tool '" + job.getTool().getId()
+ + "' can only be executed by the project owners or members of " + requiredGroup);
+ }
+
+ }
+
+ }
+
private String getQueue(Tool tool) {
String queue = "default";
Execution execution = catalogManager.getConfiguration().getAnalysis().getExecution();
@@ -797,6 +845,11 @@ private boolean canBeQueued(String toolId, int maxJobs) {
}
}
+ private int abortJob(Job job, Exception e) {
+ logger.error(e.getMessage(), e);
+ return abortJob(job, e.getMessage());
+ }
+
private int abortJob(Job job, String description) {
logger.info("Aborting job: {} - Reason: '{}'", job.getId(), description);
return setStatus(job, new Enums.ExecutionStatus(Enums.ExecutionStatus.ABORTED, description));
diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml
index 2da765a9313..9935accb971 100644
--- a/opencga-server/pom.xml
+++ b/opencga-server/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java
index bcb2e54ab5a..aaa54698e2b 100644
--- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java
+++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java
@@ -454,12 +454,6 @@ private void parseParams() throws VersionException {
queryOptions.put(QueryOptions.LIMIT, limit);
query.remove("sid");
-// Exceptions
- if (query.containsKey("status")) {
- query.put("status.name", query.get("status"));
- query.remove("status");
- }
-
// Remove deprecated fields
query.remove("variableSet");
query.remove("annotationsetName");
diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java
index 8d80bcf7bcc..6d85464577d 100644
--- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java
+++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java
@@ -631,11 +631,16 @@ public Response sampleQuery(
public Response sampleAggregationStats(@ApiParam(value =
"List of facet fields separated by semicolons, e.g.: studies;type."
+ " For nested faceted fields use >>, e.g.: chromosome>>type ."
- + " Accepted values: chromosome, type, genotype, consequenceType, biotype, clinicalSignificance, dp, qual, filter") @QueryParam("fields") String fields) {
+ + " Accepted values: chromosome, type, genotype, consequenceType, biotype, clinicalSignificance, dp, qual, filter") @QueryParam(ParamConstants.FIELD_PARAM) String field) {
return run(() -> {
// Get all query options
QueryOptions queryOptions = new QueryOptions(uriInfo.getQueryParameters(), true);
- queryOptions.put(QueryOptions.FACET, fields);
+ if (StringUtils.isEmpty(field)) {
+ String fields = queryOptions.getString("fields");
+ queryOptions.put(QueryOptions.FACET, fields);
+ } else {
+ queryOptions.put(QueryOptions.FACET, field);
+ }
Query query = getVariantQuery(queryOptions);
return variantManager.facet(query, queryOptions, token);
});
@@ -816,11 +821,16 @@ public Response cohortStatsDelete(@ApiParam(value = ParamConstants.STUDY_PARAM)
// WARN: Only available in Solr
@ApiImplicitParam(name = "trait", value = ANNOT_TRAIT_DESCR, dataType = "string", paramType = "query"),
})
- public Response getAggregationStats(@ApiParam(value = "List of facet fields separated by semicolons, e.g.: studies;type. For nested faceted fields use >>, e.g.: chromosome>>type;percentile(gerp)") @QueryParam("fields") String fields) {
+ public Response getAggregationStats(@ApiParam(value = "List of facet fields separated by semicolons, e.g.: studies;type. For nested faceted fields use >>, e.g.: chromosome>>type;percentile(gerp)") @QueryParam(ParamConstants.FIELD_PARAM) String field) {
return run(() -> {
// Get all query options
QueryOptions queryOptions = new QueryOptions(uriInfo.getQueryParameters(), true);
- queryOptions.put(QueryOptions.FACET, fields);
+ if (StringUtils.isEmpty(field)) {
+ String fields = queryOptions.getString("fields");
+ queryOptions.put(QueryOptions.FACET, fields);
+ } else {
+ queryOptions.put(QueryOptions.FACET, field);
+ }
Query query = getVariantQuery(queryOptions);
return variantManager.facet(query, queryOptions, token);
diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml
index 590fa707b9f..39a4b5d9e20 100644
--- a/opencga-storage/opencga-storage-app/pom.xml
+++ b/opencga-storage/opencga-storage-app/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml
index 70ddad7f4da..cbb73581d15 100644
--- a/opencga-storage/opencga-storage-benchmark/pom.xml
+++ b/opencga-storage/opencga-storage-benchmark/pom.xml
@@ -21,7 +21,7 @@
opencga-storage
org.opencb.opencga
- 2.0.1
+ 2.0.2
4.0.0
diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml
index 8ea65b29072..638cfa8c7e7 100644
--- a/opencga-storage/opencga-storage-core/pom.xml
+++ b/opencga-storage/opencga-storage-core/pom.xml
@@ -25,7 +25,7 @@
org.opencb.opencga
opencga-storage
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java
index 5c37a6c8ebd..9542c904cfb 100644
--- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java
+++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java
@@ -100,7 +100,7 @@ private Object getOrIterator(String study, String proband, String father, String
boolean iterator) {
// Prepare query and options
int skip = getSkip(inputOptions);
- int limit = getLimit(inputOptions);
+ int limit = inputOptions.containsKey(QueryOptions.LIMIT) ? getLimit(inputOptions) : (Integer.MAX_VALUE - skip);
int samplingSize = getSamplingSize(inputOptions, DEFAULT_SAMPLING_SIZE, iterator);
QueryOptions options = buildQueryOptions(inputOptions);
// Always sort results for compound heterozygous
diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantAggregationExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantAggregationExecutor.java
index a26cbbd7dac..b3ee87e3c92 100644
--- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantAggregationExecutor.java
+++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantAggregationExecutor.java
@@ -1,5 +1,6 @@
package org.opencb.opencga.storage.core.variant.query.executors;
+import org.apache.solr.common.StringUtils;
import org.opencb.commons.datastore.core.FacetField;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
@@ -57,7 +58,9 @@ public final VariantQueryResult aggregation(Query query, QueryOption
}
String facet = options.getString(QueryOptions.FACET);
-
+ if (StringUtils.isEmpty(facet)) {
+ throw new VariantQueryException("Empty facet query!");
+ }
try {
return aggregation(query, options, facet);
} catch (Exception e) {
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml
index 084dc9ae133..0167014f7dd 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml
@@ -23,7 +23,7 @@
org.opencb.opencga
opencga-storage-hadoop
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/stats/VariantStatsHBaseMapReduceAnalysisExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/stats/VariantStatsHBaseMapReduceAnalysisExecutor.java
index dfa34939b10..55d4d7ce7ea 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/stats/VariantStatsHBaseMapReduceAnalysisExecutor.java
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/stats/VariantStatsHBaseMapReduceAnalysisExecutor.java
@@ -68,7 +68,7 @@ public void run() throws ToolException {
studyId,
null,
params
- ), engine.getOptions(), "Calculate sample variant stats");
+ ), "Calculate variant stats");
} catch (VariantQueryException | StorageEngineException e) {
throw new ToolExecutorException(e);
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/HBaseVariantStatsCalculator.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/HBaseVariantStatsCalculator.java
index 277dee7f55c..04d24b0ceaa 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/HBaseVariantStatsCalculator.java
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/HBaseVariantStatsCalculator.java
@@ -107,12 +107,14 @@ private final class HBaseToGenotypeCountConverter extends HBaseToStudyEntryConve
private final Set fileIds;
private final Map> samplesInFile;
private final boolean statsMultiAllelic;
+ private final int studyId;
private String defaultGenotype;
private HBaseToGenotypeCountConverter(VariantStorageMetadataManager metadataManager,
boolean statsMultiAllelic, String unknownGenotype) {
super(metadataManager, null);
sampleIdsSet = new HashSet<>(sampleIds);
+ studyId = sm.getId();
this.statsMultiAllelic = statsMultiAllelic;
if (excludeFiles(this.statsMultiAllelic, unknownGenotype, Aggregation.NONE)) {
fileIds = Collections.emptySet();
@@ -153,14 +155,14 @@ public boolean apply(Variant variant, VariantRow result, VariantStatsPartial par
AtomicBoolean withStudy = new AtomicBoolean(false);
result.walker()
.onStudy(studyId -> {
- if (studyId == sm.getId()) {
+ if (studyId == this.studyId) {
withStudy.set(true);
}
})
.onSample(sample -> {
int sampleId = sample.getSampleId();
// Exclude other samples
- if (sampleIdsSet.contains(sampleId)) {
+ if (sample.getStudyId() == studyId && sampleIdsSet.contains(sampleId)) {
processedSamples.add(sampleId);
String gt = sample.getGT();
@@ -174,7 +176,7 @@ public boolean apply(Variant variant, VariantRow result, VariantStatsPartial par
})
.onFile(file -> {
int fileId = file.getFileId();
- if (fileIds.contains(fileId)) {
+ if (file.getStudyId() == studyId && fileIds.contains(fileId)) {
filesInThisVariant.add(fileId);
String secAlt = file.getString(FILE_SEC_ALTS_IDX);
@@ -199,7 +201,11 @@ public boolean apply(Variant variant, VariantRow result, VariantStatsPartial par
}
}
})
- .onFillMissing((studyId, value) -> fillMissingColumnValue.set(value))
+ .onFillMissing((studyId, value) -> {
+ if (studyId == this.studyId) {
+ fillMissingColumnValue.set(value);
+ }
+ })
.walk();
if (!withStudy.get()) {
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-cdh5.13/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-cdh5.13/pom.xml
index db3640b6bbe..90057de2756 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-cdh5.13/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-cdh5.13/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage-hadoop-deps
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.31/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.31/pom.xml
index a55d74d11f1..4608acfa490 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.31/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.31/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage-hadoop-deps
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.8/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.8/pom.xml
index a9d2b77b377..bbe8d356420 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.8/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.8/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage-hadoop-deps
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml
index 5f418297b8b..6bc6ea229f6 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage-hadoop-deps
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.5/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.5/pom.xml
index cb4d40f632a..b59cafddddd 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.5/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.5/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage-hadoop-deps
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml
index 782e2f9f650..8c58176e4e5 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage-hadoop-deps
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml
index 0cec6ff494a..21c8735cd5a 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage-hadoop-deps
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml
index 373ce328cb0..1cfc425c1c2 100644
--- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml
@@ -50,7 +50,7 @@
org.opencb.opencga
opencga-storage-hadoop
- 2.0.1
+ 2.0.2
../pom.xml
@@ -66,7 +66,7 @@
opencga-storage-hadoop-deps
- 2.0.1
+ 2.0.2
pom
diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml
index 90713489705..613500a1e38 100644
--- a/opencga-storage/opencga-storage-hadoop/pom.xml
+++ b/opencga-storage/opencga-storage-hadoop/pom.xml
@@ -30,12 +30,12 @@
org.opencb.opencga
opencga-storage
- 2.0.1
+ 2.0.2
../pom.xml
opencga-storage-hadoop
- 2.0.1
+ 2.0.2
pom
diff --git a/opencga-storage/opencga-storage-mongodb/pom.xml b/opencga-storage/opencga-storage-mongodb/pom.xml
index 9f397837c1c..6313dae1803 100644
--- a/opencga-storage/opencga-storage-mongodb/pom.xml
+++ b/opencga-storage/opencga-storage-mongodb/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml
index 14e8d1dd43b..adaabdebae3 100644
--- a/opencga-storage/opencga-storage-server/pom.xml
+++ b/opencga-storage/opencga-storage-server/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga-storage
- 2.0.1
+ 2.0.2
../pom.xml
diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml
index d9614bf947e..a7ee809f731 100644
--- a/opencga-storage/pom.xml
+++ b/opencga-storage/pom.xml
@@ -22,13 +22,13 @@
org.opencb.opencga
opencga
- 2.0.1
+ 2.0.2
../pom.xml
opencga-storage
- 2.0.1
+ 2.0.2
pom
diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml
index 4af804bb7c0..70a2a655b03 100644
--- a/opencga-test/pom.xml
+++ b/opencga-test/pom.xml
@@ -24,7 +24,7 @@
org.opencb.opencga
opencga-test
- 2.0.1
+ 2.0.2
pom
diff --git a/pom.xml b/pom.xml
index 158deadad6b..7d790107efa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
org.opencb.opencga
opencga
- 2.0.1
+ 2.0.2
pom
OpenCGA
@@ -41,12 +41,12 @@
- 2.0.1
- ${opencga.version}
+ 2.0.2
+ 2.0.2
2.1.0-beta
4.8.0
2.0.5
- 4.0.5
+ 4.0.6
0.2.0
2.10.1