Skip to content

Commit

Permalink
Merge branch 'release-2.0.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
@imedina
imedina committed Apr 21, 2021
2 parents aef72b7 + 7defda6 commit caf04bf
Show file tree
Hide file tree
Showing 52 changed files with 377 additions and 146 deletions.
Binary file added docs/.gitbook/assets/image (7).png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docs/.gitbook/assets/image (8).png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion opencga-analysis/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>2.0.1</version>
<version>2.0.2</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
64 changes: 39 additions & 25 deletions ...ysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.opencb.biodata.models.variant.avro.VariantType;
import org.opencb.biodata.models.variant.metadata.SampleVariantStats;
import org.opencb.biodata.models.variant.metadata.VariantMetadata;
import org.opencb.biodata.models.variant.stats.VariantStats;
import org.opencb.biodata.tools.variant.converters.ga4gh.Ga4ghVariantConverter;
import org.opencb.biodata.tools.variant.converters.ga4gh.factories.AvroGa4GhVariantFactory;
import org.opencb.biodata.tools.variant.converters.ga4gh.factories.ProtoGa4GhVariantFactory;
Expand Down Expand Up @@ -73,10 +74,7 @@
import org.opencb.opencga.storage.core.utils.CellBaseUtils;
import org.opencb.opencga.storage.core.variant.BeaconResponse;
import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
import org.opencb.opencga.storage.core.variant.adaptors.VariantField;
import org.opencb.opencga.storage.core.variant.adaptors.VariantIterable;
import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException;
import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
import org.opencb.opencga.storage.core.variant.adaptors.*;
import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator;
import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat;
import org.opencb.opencga.storage.core.variant.query.ParsedQuery;
Expand All @@ -96,8 +94,7 @@
import static org.opencb.opencga.analysis.variant.manager.operations.VariantFileIndexerOperationManager.FILE_GET_QUERY_OPTIONS;
import static org.opencb.opencga.core.api.ParamConstants.ACL_PARAM;
import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*;
import static org.opencb.opencga.storage.core.variant.adaptors.sample.VariantSampleDataManager.SAMPLE_BATCH_SIZE;
import static org.opencb.opencga.storage.core.variant.adaptors.sample.VariantSampleDataManager.SAMPLE_BATCH_SIZE_DEFAULT;
import static org.opencb.opencga.storage.core.variant.adaptors.sample.VariantSampleDataManager.*;
import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.*;

public class VariantStorageManager extends StorageManager implements AutoCloseable {
Expand Down Expand Up @@ -630,7 +627,12 @@ public DataResult<Variant> getSampleData(String variant, String study, QueryOpti
int skip = options.getInt(SKIP, 0);
// Make initial batchLimit shorter
int batchLimit = Math.min(SAMPLE_BATCH_SIZE_DEFAULT, limit * 3 + skip);
// but not too short
batchLimit = Math.max(100, batchLimit);
int batchSkip = 0;
int numReadSamples = 0;
int numValidSamples = 0;
boolean exactNumSamples = false;

String studyFqn = query.getString(STUDY.key());
options.putAll(query);
Expand Down Expand Up @@ -664,6 +666,9 @@ public DataResult<Variant> getSampleData(String variant, String study, QueryOpti
.map(SampleEntry::getSampleId)
.collect(Collectors.toList());
moreResults = samplesInResult.size() == batchLimit;
// The count of samples is exact if there is no more results
exactNumSamples = !moreResults;
numReadSamples += samplesInResult.size();

StopWatch checkPermissionsStopWatch = StopWatch.createStarted();
String userId = catalogManager.getUserManager().getUserId(token);
Expand All @@ -680,6 +685,7 @@ public DataResult<Variant> getSampleData(String variant, String study, QueryOpti
auditAttributes.put("checkSamplePermissionsTimeMillis",
checkPermissionsStopWatch.getTime(TimeUnit.MILLISECONDS)
+ auditAttributes.getInt("checkSamplePermissionsTimeMillis", 0));
numValidSamples += validSamples.size();
List<String> samplesToReturn;
if (skip > validSamples.size()) {
samplesToReturn = Collections.emptyList();
Expand Down Expand Up @@ -713,25 +719,33 @@ public DataResult<Variant> getSampleData(String variant, String study, QueryOpti
variantResult.getStudies().get(0).setSamples(sampleEntries);
variantResult.getStudies().get(0).setFiles(fileEntries);

ObjectMap attributes = new ObjectMap();
// VariantStats stats = variantResult.getStudies().get(0).getStats(StudyEntry.DEFAULT_COHORT);
// if (stats != null) {
// List<String> genotypesFilter = new ArrayList<>(options.getAsStringList(GENOTYPE.key()));
// if (genotypesFilter.isEmpty()) {
// genotypesFilter.add(GenotypeClass.MAIN_ALT.name());
// }
// int expectedSamplesCount = 0;
// List<String> gtsInVariant = new ArrayList<>(stats.getGenotypeCount().keySet());
// List<String> genotypesToReturn = GenotypeClass.filter(genotypesFilter, gtsInVariant);
// for (String gt : genotypesToReturn) {
// expectedSamplesCount += stats.getGenotypeCount().getOrDefault(gt, 0);
// }
// }
// result.getAttributes().put(NUM_SAMPLES.key(), samplesToReturn.size());
// result.getAttributes().put(NUM_TOTAL_SAMPLES.key(), validSamples.size());

return new DataResult<>(((int) stopWatch.getTime(TimeUnit.MILLISECONDS)),
new ArrayList<>(), 1, Collections.singletonList(variantResult), 1, attributes);
VariantQueryResult<Variant> result = new VariantQueryResult<>(
((int) stopWatch.getTime(TimeUnit.MILLISECONDS)),
1, 1, new ArrayList<>(), Collections.singletonList(variantResult), null, null)
.setNumSamples(sampleEntries.size());
if (exactNumSamples) {
result.setNumTotalSamples(numValidSamples).setApproximateCount(false);
} else {
VariantStats stats = variantResult.getStudies().get(0).getStats(StudyEntry.DEFAULT_COHORT);
if (stats != null) {
List<String> genotypesFilter = new ArrayList<>(options.getAsStringList(GENOTYPE.key()));
if (genotypesFilter.isEmpty()) {
genotypesFilter.add(GenotypeClass.MAIN_ALT.name());
}
int expectedSamplesCount = 0;
List<String> gtsInVariant = new ArrayList<>(stats.getGenotypeCount().keySet());
List<String> genotypesToReturn = GenotypeClass.filter(genotypesFilter, gtsInVariant);
for (String gt : genotypesToReturn) {
expectedSamplesCount += stats.getGenotypeCount().getOrDefault(gt, 0);
}

int numTotalSamples = ((int) (expectedSamplesCount * (((float) numValidSamples) / numReadSamples)));
result.setNumTotalSamples(numTotalSamples);
result.setApproximateCountSamplingSize(numReadSamples);
}
result.setApproximateCount(true);
}
return result;
});
}
}
Expand Down
2 changes: 1 addition & 1 deletion opencga-app/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>2.0.1</version>
<version>2.0.2</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
11 changes: 10 additions & 1 deletion .../src/main/java/org/opencb/opencga/app/cli/main/executors/catalog/FileCommandExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.opencb.biodata.models.clinical.interpretation.Software;
import org.opencb.commons.datastore.core.FacetField;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.QueryOptions;
Expand Down Expand Up @@ -307,7 +308,15 @@ private RestResponse<File> update() throws ClientException {
.setBioformat(commandOptions.bioformat)
.setDescription(commandOptions.description)
.setTags(commandOptions.tags)
.setSampleIds(commandOptions.sampleIds);
.setSampleIds(commandOptions.sampleIds)
.setSoftware(new Software(
commandOptions.softwareName,
commandOptions.softwareVersion,
commandOptions.softwareRepository,
commandOptions.softwareCommit,
commandOptions.softwareWebsite,
commandOptions.softwareParams.isEmpty() ? null : commandOptions.softwareParams
));

ObjectMap params = new ObjectMap();
params.putIfNotNull(FileDBAdaptor.QueryParams.STUDY.key(), commandOptions.study);
Expand Down
28 changes: 20 additions & 8 deletions opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,15 @@
import org.opencb.opencga.core.models.study.VariableSet;
import org.opencb.opencga.core.models.user.User;
import org.opencb.opencga.core.response.RestResponse;
import org.opencb.opencga.core.tools.result.ToolStep;

import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;

import static org.opencb.opencga.core.common.IOUtils.humanReadableByteCount;
import static org.opencb.opencga.core.models.common.Enums.ExecutionStatus.DONE;
import static org.opencb.opencga.core.models.common.Enums.ExecutionStatus.RUNNING;
import static org.opencb.opencga.core.models.common.Status.READY;

/**
* Created by pfurio on 28/11/16.
Expand Down Expand Up @@ -501,12 +500,25 @@ private void printRecursiveTree(List<FileTree> fileTreeList, StringBuilder sb, S
FileTree fileTree = iterator.next();
File file = fileTree.getFile();

sb.append(String.format("%s %s [%s, %s, %s]\n",
indent.isEmpty() ? "" : indent + (iterator.hasNext() ? "├──" : "└──"),
file.getType() == File.Type.FILE ? file.getName() : file.getName() + "/",
file.getName(),
file.getInternal() != null && file.getInternal().getStatus() != null ? file.getInternal().getStatus().getName() : "",
humanReadableByteCount(file.getSize(), false)));
if (!indent.isEmpty()) {
sb.append(indent);
sb.append(iterator.hasNext() ? "├──" : "└──");
sb.append(" ");
}
if (file.getType() == File.Type.FILE) {
sb.append(file.getName());
sb.append(" [");
if (file.getInternal() != null
&& file.getInternal().getStatus() != null
&& !READY.equals(file.getInternal().getStatus().getName())) {
sb.append(file.getInternal().getStatus().getName()).append(", ");
}
sb.append(humanReadableByteCount(file.getSize(), false)).append("]");

} else {
sb.append(file.getName()).append("/");
}
sb.append("\n");

if (file.getType() == File.Type.DIRECTORY) {
printRecursiveTree(fileTree.getChildren(), sb, indent + (iterator.hasNext()? "│ " : " "));
Expand Down
25 changes: 21 additions & 4 deletions opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FileCommandOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@

package org.opencb.opencga.app.cli.main.options;

import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import com.beust.jcommander.*;
import org.opencb.opencga.app.cli.GeneralCliOptions;
import org.opencb.opencga.app.cli.GeneralCliOptions.CommonCommandOptions;
import org.opencb.opencga.app.cli.GeneralCliOptions.DataModelOptions;
Expand All @@ -30,7 +27,9 @@
import org.opencb.opencga.core.api.ParamConstants;
import org.opencb.opencga.core.models.file.File;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* Created by sgallego on 6/14/16.
Expand Down Expand Up @@ -317,6 +316,24 @@ public class UpdateCommandOptions extends BaseFileCommand {
@Parameter(names = {"--tags"}, description = "Comma separated list of sample names or ids", arity = 1)
public List<String> tags;

@Parameter(names = {"--software-name"}, description = "Software name", arity = 1)
public String softwareName;

@Parameter(names = {"--software-version"}, description = "Software version", arity = 1)
public String softwareVersion;

@Parameter(names = {"--software-repository"}, description = "Software repository", arity = 1)
public String softwareRepository;

@Parameter(names = {"--software-commit"}, description = "Software commit", arity = 1)
public String softwareCommit;

@Parameter(names = {"--software-website"}, description = "Software website", arity = 1)
public String softwareWebsite;

@DynamicParameter(names = {"--software-params"}, description = "Software params")
public Map<String, String> softwareParams = new HashMap<>();

}

// @Parameters(commandNames = {"relink"}, commandDescription = "Change location of linked or STAGED file.")
Expand Down
2 changes: 1 addition & 1 deletion opencga-catalog/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>2.0.1</version>
<version>2.0.2</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion ...c/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ private List<Attributes> getUserInfoFromLDAP(String host, List<String> userList,
}

private String getMail(Attributes attributes) throws NamingException {
return (String) attributes.get("mail").get(0);
return attributes.get("mail") != null ? (String) attributes.get("mail").get(0) : "";
}

private String getUID(Attributes attributes) throws NamingException {
Expand Down
7 changes: 4 additions & 3 deletions ...ga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,9 +416,6 @@ public OpenCGAResult<ClinicalAnalysis> create(String studyStr, ClinicalAnalysis

// Validate there is only one sample per member
for (Individual member : family.getMembers()) {
if (member.getSamples() == null || member.getSamples().isEmpty()) {
throw new CatalogException("Missing sample for member '" + member.getId() + "'");
}
if (member.getSamples().size() > 1) {
throw new CatalogException("More than one sample found for member '" + member.getId() + "'.");
}
Expand All @@ -435,6 +432,10 @@ public OpenCGAResult<ClinicalAnalysis> create(String studyStr, ClinicalAnalysis
if (proband == null) {
throw new CatalogException("Proband '" + clinicalAnalysis.getProband().getId() + "' not found in family");
}
if (proband.getSamples() == null || proband.getSamples().isEmpty()) {
throw new CatalogException("Missing samples for proband '" + proband.getId() + "'");
}

clinicalAnalysis.setProband(proband);

// Validate the proband has a disorder
Expand Down
8 changes: 8 additions & 0 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -931,11 +931,19 @@ private OpenCGAResult<Family> update(Study study, Family family, FamilyUpdatePar

if (updateParams != null && ListUtils.isNotEmpty(updateParams.getPhenotypes())) {
tmpFamily.setPhenotypes(updateParams.getPhenotypes());
} else if (ListUtils.isNotEmpty(updateParams.getMembers())) {
// Recalculate phenotypes if list of members has changed
tmpFamily.setPhenotypes(Collections.emptyList());
parameters.put(FamilyDBAdaptor.QueryParams.PHENOTYPES.key(), tmpFamily.getPhenotypes());
} else {
tmpFamily.setPhenotypes(family.getPhenotypes());
}
if (updateParams != null && ListUtils.isNotEmpty(updateParams.getDisorders())) {
tmpFamily.setDisorders(updateParams.getDisorders());
} else if (ListUtils.isNotEmpty(updateParams.getMembers())) {
// Recalculate disorders if list of members has changed
tmpFamily.setDisorders(Collections.emptyList());
parameters.put(FamilyDBAdaptor.QueryParams.DISORDERS.key(), tmpFamily.getDisorders());
} else {
tmpFamily.setDisorders(family.getDisorders());
}
Expand Down
20 changes: 14 additions & 6 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1348,15 +1348,23 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat
parameters.put(IndividualDBAdaptor.QueryParams.SAMPLES.key(), sampleList);
}

if (updateParams != null && StringUtils.isNotEmpty(updateParams.getFather())) {
OpenCGAResult<Individual> queryResult = internalGet(studyUid, updateParams.getFather(), INCLUDE_INDIVIDUAL_IDS, userId);
if (updateParams != null && updateParams.getFather() != null) {
if (StringUtils.isNotEmpty(updateParams.getFather())) {
OpenCGAResult<Individual> queryResult = internalGet(studyUid, updateParams.getFather(), INCLUDE_INDIVIDUAL_IDS, userId);
parameters.put(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), queryResult.first().getUid());
} else {
parameters.put(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), -1L);
}
parameters.remove(IndividualDBAdaptor.QueryParams.FATHER.key());
parameters.put(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), queryResult.first().getUid());
}
if (updateParams != null && StringUtils.isNotEmpty(updateParams.getMother())) {
OpenCGAResult<Individual> queryResult = internalGet(studyUid, updateParams.getMother(), INCLUDE_INDIVIDUAL_IDS, userId);
if (updateParams != null && updateParams.getMother() != null) {
if (StringUtils.isNotEmpty(updateParams.getMother())) {
OpenCGAResult<Individual> queryResult = internalGet(studyUid, updateParams.getMother(), INCLUDE_INDIVIDUAL_IDS, userId);
parameters.put(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), queryResult.first().getUid());
} else {
parameters.put(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), -1L);
}
parameters.remove(IndividualDBAdaptor.QueryParams.MOTHER.key());
parameters.put(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), queryResult.first().getUid());
}

checkUpdateAnnotations(study, individual, parameters, options, VariableSet.AnnotableDataModels.INDIVIDUAL, individualDBAdaptor,
Expand Down
3 changes: 1 addition & 2 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,8 +382,7 @@ public List<File> getJobInputFilesFromParams(String study, Job job, String token
inputFiles.add(file);
} catch (CatalogException e) {
throw new CatalogException("Cannot find file '" + entry.getValue() + "' "
+ "from job param '" + entry.getKey() + "'; (study = " + study + ", token = " + token
+ ") :" + e.getMessage(), e);
+ "from job param '" + entry.getKey() + "'; (study = " + study + ") :" + e.getMessage(), e);
}
}
} else if (entry.getValue() instanceof Map) {
Expand Down
Loading

0 comments on commit caf04bf

Please sign in to comment.