diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java index bfef1fbca25..c608f5e2c1a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java @@ -77,7 +77,7 @@ public RestResponse login(CustomUsersCommandOptions.Logi String token = session.getSession().getToken(); String errorMsg = "Missing password. "; if (StringUtils.isNotEmpty(token)) { - errorMsg += "Active token detected "; + errorMsg += "Active token detected. Please logout first."; } CommandLineUtils.error(errorMsg); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/processors/CommandProcessor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/processors/CommandProcessor.java index ef06e2c8944..8c43c340ab6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/processors/CommandProcessor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/processors/CommandProcessor.java @@ -136,9 +136,13 @@ private boolean checkAutoRefresh(OpencgaCommandExecutor commandExecutor) { public void loadSessionStudies(OpencgaCommandExecutor commandExecutor) { Session session = commandExecutor.getSessionManager().getSession(); - logger.debug("Loading session studies using token: " - + session.getToken()); OpenCGAClient openCGAClient = commandExecutor.getOpenCGAClient(); + logger.debug("openCGAClient Token: " + openCGAClient.getToken()); + if(StringUtils.isEmpty(openCGAClient.getToken())) { + openCGAClient.setToken(session.getToken()); + } + logger.debug("Loading session studies using token: " + + openCGAClient.getToken()); try { // Query the server to retrieve the studies of user projects RestResponse res = openCGAClient.getProjectClient().search(new ObjectMap()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java index aba2f366b77..e6a885fbafa 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java @@ -1,5 +1,6 @@ package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; +import org.apache.commons.lang3.tuple.Pair; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -67,6 +68,8 @@ private void checkStudy(VariantStorageEngine engine, String study) throws Storag if (fileSets.isEmpty()) { logger.info("No concurrent file loadings found in study '{}'", study); return; + } else { + logger.info("Found {} sets of files with shared samples in study '{}'", fileSets.size(), study); } Map fileTasks = new HashMap<>(); @@ -85,10 +88,12 @@ private void checkStudy(VariantStorageEngine engine, String study) throws Storag } } - Set> fileSetsToInvalidate = new HashSet<>(); - Set affectedFiles = new HashSet<>(); - Set affectedSamples = new HashSet<>(); for (Set fileSet : fileSets) { + Set affectedFiles = new HashSet<>(); + Set affectedSamples = new HashSet<>(); + Set invalidFiles = new HashSet<>(); + Set invalidSampleIndexes = new HashSet<>(); + // Check if any task from this file set overlaps in time List tasks = new ArrayList<>(); for (Integer fileId : fileSet) { @@ -97,8 +102,11 @@ private void checkStudy(VariantStorageEngine engine, String study) throws Storag tasks.add(task); } } - if (tasks.size() > 1) { - logger.info("Found {} tasks loading files {}", tasks.size(), fileSet); + if (tasks.size() <= 1) { + continue; + } else { + logger.info("--------------------"); + logger.info("Found {} tasks loading files {} in study {}", tasks.size(), fileSet, study); for (int i = 0; i < tasks.size(); i++) { TaskMetadata task1 = tasks.get(i); Date task1start = task1.getStatus().firstKey(); @@ -108,8 +116,7 @@ private void checkStudy(VariantStorageEngine engine, String study) throws Storag Date task2start = task2.getStatus().firstKey(); Date task2end = task2.getStatus().lastKey(); if (task1start.before(task2end) && task1end.after(task2start)) { - fileSetsToInvalidate.add(fileSet); - affectedFiles.addAll(task1.getFileIds()); + affectedFiles.addAll(fileSet); List task1Files = task1.getFileIds().stream().map(fileId -> "'" + metadataManager.getFileName(studyId, fileId) + "'(" + fileId + ")").collect(Collectors.toList()); List task2Files = task2.getFileIds().stream().map(fileId -> "'" + metadataManager.getFileName(studyId, fileId) + "'(" + fileId + ")").collect(Collectors.toList()); @@ -131,8 +138,6 @@ private void checkStudy(VariantStorageEngine engine, String study) throws Storag } } - Set invalidFiles = new HashSet<>(); - List invalidSampleIndexes = new ArrayList<>(); for (Integer sampleId : affectedSamples) { String sampleName = metadataManager.getSampleName(studyId, sampleId); SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sampleId); @@ -145,7 +150,7 @@ private void checkStudy(VariantStorageEngine engine, String study) throws Storag metadataManager.getFileName(studyId, file), file); } } - } else if (sampleMetadata.getSampleIndexStatus(Optional.of(sampleMetadata.getSampleIndexVersion()).orElse(-1)) == TaskMetadata.Status.READY) { + } else if (sampleMetadata.getSampleIndexStatus(Optional.ofNullable(sampleMetadata.getSampleIndexVersion()).orElse(-1)) == TaskMetadata.Status.READY) { for (Integer fileId : sampleMetadata.getFiles()) { if (affectedFiles.contains(fileId)) { FileMetadata fileMetadata = metadataManager.getFileMetadata(studyId, fileId); @@ -195,6 +200,8 @@ private void checkStudy(VariantStorageEngine engine, String study) throws Storag } } } else { + logger.info("Sample '{}'({}) sample index is not in READY status. Invalidate to ensure rebuild", sampleName, sampleId); + logger.info(" - Invalidating sample index for sample '{}'({})", sampleName, sampleId); invalidSampleIndexes.add(sampleId); } } @@ -210,9 +217,19 @@ private void checkStudy(VariantStorageEngine engine, String study) throws Storag invalidSamples.addAll(metadataManager.getSampleIdsFromFileId(studyId, fileId)); } - logger.info("Affected files: {}", invalidFiles); - logger.info("Affected samples: {}", invalidSamples); - logger.info("Affected sample indexes: {}", invalidSampleIndexes); + logger.info("Study '{}'", study); + List> invalidFilesPairs = invalidFiles.stream() + .map(fileId -> Pair.of(metadataManager.getFileName(studyId, fileId), fileId)) + .collect(Collectors.toList()); + logger.info("Affected files: {}", invalidFilesPairs); + List> invalidSamplesPairs = invalidSamples.stream() + .map(sampleId -> Pair.of(metadataManager.getSampleName(studyId, sampleId), sampleId)) + .collect(Collectors.toList()); + logger.info("Affected samples: {}", invalidSamplesPairs); + List> invalidSampleIndexesPairs = invalidSampleIndexes.stream() + .map(sampleId -> Pair.of(metadataManager.getSampleName(studyId, sampleId), sampleId)) + .collect(Collectors.toList()); + logger.info("Affected sample indexes: {}", invalidSampleIndexesPairs); } } else { ObjectMap event = new ObjectMap() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java index 6f6f5cbf18d..e7ea95c807d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java @@ -9,20 +9,24 @@ import org.bson.conversions.Bson; import org.opencb.opencga.catalog.db.api.CohortDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Migration(id = "syncCohortsAndSamplesMigration" , - description = "Sync array of samples from cohort with array of cohortIds from Sample", + description = "Sync array of samples from cohort with array of cohortIds from Sample.", version = "2.12.6", domain = Migration.MigrationDomain.CATALOG, language = Migration.MigrationLanguage.JAVA, - date = 20240621 + date = 20240621, + patch = 2 // TASK-6998 ) public class SyncCohortsAndSamplesMigration extends MigrationTool { @@ -31,8 +35,18 @@ protected void run() throws Exception { MongoCollection sampleCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION); MongoCollection sampleArchiveCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION); + // Fill map study uid - fqn + Map uidFqnMap = new HashMap<>(); + Bson studyProjection = Projections.include(StudyDBAdaptor.QueryParams.UID.key(), StudyDBAdaptor.QueryParams.FQN.key()); + queryMongo(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, new Document(), studyProjection, study -> { + long studyUid = study.get(StudyDBAdaptor.QueryParams.UID.key(), Number.class).longValue(); + String studyFqn = study.getString(StudyDBAdaptor.QueryParams.FQN.key()); + uidFqnMap.put(studyUid, studyFqn); + }); + queryMongo(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, new Document(), - Projections.include(CohortDBAdaptor.QueryParams.ID.key(), CohortDBAdaptor.QueryParams.SAMPLES.key()), + Projections.include(CohortDBAdaptor.QueryParams.ID.key(), CohortDBAdaptor.QueryParams.SAMPLES.key(), + CohortDBAdaptor.QueryParams.STUDY_UID.key()), cohortDoc -> { String cohortId = cohortDoc.getString(CohortDBAdaptor.QueryParams.ID.key()); List samples = cohortDoc.getList(CohortDBAdaptor.QueryParams.SAMPLES.key(), Document.class); @@ -50,8 +64,11 @@ protected void run() throws Exception { long addedMissingCohort = sampleCollection.updateMany(query, update).getModifiedCount(); sampleArchiveCollection.updateMany(query, update); + long studyUid = cohortDoc.get(CohortDBAdaptor.QueryParams.STUDY_UID.key(), Number.class).longValue(); + // Ensure there aren't any samples pointing to this cohort that are not in the samples array query = Filters.and( + Filters.eq(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid), Filters.nin(SampleDBAdaptor.QueryParams.UID.key(), sampleUids), Filters.eq(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), cohortId), Filters.eq(MongoDBAdaptor.LAST_OF_VERSION, true) @@ -61,10 +78,10 @@ protected void run() throws Exception { sampleArchiveCollection.updateMany(query, update); if (addedMissingCohort > 0 || removedNonAssociatedCohort > 0) { - logger.info("Fixed cohort '{}' references. " + logger.info("Fixed cohort '{}' references from study '{}'. " + "Added missing reference to {} samples. " + "Removed non-associated reference from {} samples.", - cohortId, addedMissingCohort, removedNonAssociatedCohort); + cohortId, uidFqnMap.get(studyUid), addedMissingCohort, removedNonAssociatedCohort); } } });