Merge pull request #2174 from opencb/TASK-1245

TASK-1245 - Error installing catalog in dirty database

opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java

@@ -160,7 +160,7 @@ private void status() throws CatalogException, JsonProcessingException {
         System.out.println(JacksonUtils.getDefaultObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(result));
     }
 
-    private void install() throws CatalogException, URISyntaxException {
+    private void install() throws CatalogException {
         AdminCliOptionsParser.InstallCatalogCommandOptions commandOptions = catalogCommandOptions.installCatalogCommandOptions;
 
         validateConfiguration(commandOptions);
@@ -176,43 +176,20 @@ private void install() throws CatalogException, URISyntaxException {
             throw new CatalogException("No admin password found. Please, insert your password.");
         }
 
-        CatalogManager catalogManager = new CatalogManager(configuration);
-        if (catalogManager.existsCatalogDB()) {
-            if (commandOptions.force) {
-                // The password of the old db should match the one to be used in the new installation. Otherwise, they can obtain the same
-                // results calling first to "catalog delete" and then "catalog install"
-                String token = catalogManager.getUserManager().loginAsAdmin(commandOptions.commonOptions.adminPassword).getToken();
-                catalogManager.deleteCatalogDB(token);
-            } else {
-                // Check admin password ...
-                try {
-                    catalogManager.getUserManager().loginAsAdmin(commandOptions.commonOptions.adminPassword);
-                    logger.warn("A database called " + catalogManager.getCatalogDatabase() + " already exists");
-                    return;
-                } catch (CatalogException e) {
-                    throw new CatalogException("A database called " + catalogManager.getCatalogDatabase() + " with a different admin"
-                            + " password already exists. If you are aware of that installation, please delete it first.");
-                }
-            }
+        try (CatalogManager catalogManager = new CatalogManager(configuration)) {
+            catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), commandOptions.commonOptions.adminPassword,
+                    commandOptions.email, commandOptions.organization, commandOptions.force, true);
         }
-
-        logger.info("\nInstalling database {} in {}\n", catalogManager.getCatalogDatabase(),
-                configuration.getCatalog().getDatabase().getHosts());
-
-        catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), commandOptions.commonOptions.adminPassword,
-                commandOptions.email, commandOptions.organization);
     }
 
-    private void delete() throws CatalogException, URISyntaxException {
+    private void delete() throws CatalogException {
         validateConfiguration(catalogCommandOptions.deleteCatalogCommandOptions);
 
-        CatalogManager catalogManager = new CatalogManager(configuration);
-        String token = catalogManager.getUserManager()
-                .loginAsAdmin(catalogCommandOptions.deleteCatalogCommandOptions.commonOptions.adminPassword).getToken();
-
-        logger.info("\nDeleting database {} from {}\n", catalogManager.getCatalogDatabase(), configuration.getCatalog().getDatabase()
-                .getHosts());
-        catalogManager.deleteCatalogDB(token);
+        try (CatalogManager catalogManager = new CatalogManager(configuration)) {
+            logger.info("\nDeleting database {} from {}\n", catalogManager.getCatalogDatabase(), configuration.getCatalog().getDatabase()
+                    .getHosts());
+            catalogManager.deleteCatalogDB(catalogCommandOptions.deleteCatalogCommandOptions.commonOptions.adminPassword);
+        }
     }
 
     private void index() throws CatalogException {

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java

@@ -46,10 +46,10 @@ public interface DBAdaptorFactory extends AutoCloseable {
     /**
      * Creates the indexes needed to make queries faster.
      *
-     * @param uniqueIndexesOnly boolean indicating whether to index unique indexes only.
+     * @param wholeIndexes boolean indicating whether to generate all indexes or only the unique ones.
      * @throws CatalogDBException if there is any problem creating the indexes.
      */
-    void createIndexes(boolean uniqueIndexesOnly) throws CatalogDBException;
+    void createIndexes(boolean wholeIndexes) throws CatalogDBException;
 
     String getCatalogDatabase(String prefix);
 

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java

@@ -81,7 +81,7 @@ public long getNewAutoIncrementId(ClientSession clientSession, String field) { /
         return result.getResults().get(0).getLong(field);
     }
 
-    public void createIndexes(boolean uniqueIndexesOnly) {
+    public void createIndexes(boolean wholeIndexes) {
         InputStream resourceAsStream = getClass().getResourceAsStream("/catalog-indexes.txt");
         ObjectMapper objectMapper = getDefaultObjectMapper();
         BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(resourceAsStream));
@@ -100,7 +100,7 @@ public void createIndexes(boolean uniqueIndexesOnly) {
                     myIndexes.put("fields", new ObjectMap((Map) hashMap.get("fields")));
                     myIndexes.put("options", new ObjectMap((Map) hashMap.getOrDefault("options", Collections.emptyMap())));
 
-                    if (!uniqueIndexesOnly || myIndexes.get("options").getBoolean("unique")) {
+                    if (wholeIndexes || myIndexes.get("options").getBoolean("unique")) {
                         indexes.get(collection).add(myIndexes);
                     }
                 }

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java

@@ -204,8 +204,8 @@ public void installCatalogDB(Configuration configuration) throws CatalogExceptio
     }
 
     @Override
-    public void createIndexes(boolean uniqueIndexesOnly) throws CatalogDBException {
-        metaDBAdaptor.createIndexes(uniqueIndexesOnly);
+    public void createIndexes(boolean wholeIndexes) {
+        metaDBAdaptor.createIndexes(wholeIndexes);
     }
 
     @Override

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java

@@ -17,11 +17,8 @@
 package org.opencb.opencga.catalog.managers;
 
 import org.apache.commons.lang3.StringUtils;
-import org.opencb.commons.datastore.core.DataStoreServerAddress;
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.commons.datastore.core.QueryOptions;
-import org.opencb.commons.datastore.mongodb.MongoDataStore;
-import org.opencb.commons.datastore.mongodb.MongoDataStoreManager;
 import org.opencb.opencga.catalog.auth.authentication.JwtManager;
 import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager;
 import org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager;
@@ -43,14 +40,13 @@
 import org.opencb.opencga.core.models.study.Study;
 import org.opencb.opencga.core.models.user.Account;
 import org.opencb.opencga.core.models.user.User;
+import org.opencb.opencga.core.response.OpenCGAResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.LinkedList;
-import java.util.List;
 
 import static org.opencb.opencga.catalog.managers.AbstractManager.OPENCGA;
 import static org.opencb.opencga.core.api.ParamConstants.ADMIN_PROJECT;
@@ -175,14 +171,38 @@ public boolean existsCatalogDB() {
         return catalogDBAdaptorFactory.isCatalogDBReady();
     }
 
-    public void installCatalogDB(String secretKey, String password, String email, String organization) throws CatalogException {
+    public void installCatalogDB(String secretKey, String password, String email, String organization, boolean force, boolean wholeIndexes)
+            throws CatalogException {
+        if (existsCatalogDB()) {
+            if (force) {
+                // The password of the old db should match the one to be used in the new installation. Otherwise, they can obtain the same
+                // results calling first to "catalog delete" and then "catalog install"
+                deleteCatalogDB(password);
+            } else {
+                // Check admin password ...
+                try {
+                    userManager.loginAsAdmin(password);
+                    logger.warn("A database called " + getCatalogDatabase() + " already exists");
+                    return;
+                } catch (CatalogException e) {
+                    throw new CatalogException("A database called " + getCatalogDatabase() + " with a different admin"
+                            + " password already exists. If you are aware of that installation, please delete it first.");
+                }
+            }
+        }
 
-        installCatalogDB(secretKey, password, email, organization, false);
+        try {
+            logger.info("Installing database {} in {}", getCatalogDatabase(), configuration.getCatalog().getDatabase().getHosts());
+            installCatalogDB(secretKey, password, email, organization);
+            String token = userManager.loginAsAdmin(password).getToken();
+            installIndexes(token, wholeIndexes);
+        } catch (Exception e) {
+            clearCatalog();
+            throw e;
+        }
     }
 
-    public void installCatalogDB(String secretKey, String password, String email, String organization, boolean test)
-            throws CatalogException {
-
+    private void installCatalogDB(String secretKey, String password, String email, String organization) throws CatalogException {
         if (existsCatalogDB()) {
             throw new CatalogException("Nothing to install. There already exists a catalog database");
         }
@@ -209,50 +229,48 @@ public void installCatalogDB(String secretKey, String password, String email, St
 
         // Skip old available migrations
         migrationManager.skipPendingMigrations(token);
-
-        installIndexes(token, test);
     }
 
     public void installIndexes(String token) throws CatalogException {
         installIndexes(token, false);
     }
 
-    public void installIndexes(String token, boolean test) throws CatalogException {
+    public void installIndexes(String token, boolean wholeIndexes) throws CatalogException {
         if (!OPENCGA.equals(userManager.getUserId(token))) {
             throw new CatalogAuthorizationException("Only the admin can install new indexes");
         }
-        catalogDBAdaptorFactory.createIndexes(test);
-    }
-
-    public void deleteCatalogDB(String token) throws CatalogException, URISyntaxException {
-        String userId = userManager.getUserId(token);
-        if (!authorizationManager.isInstallationAdministrator(userId)) {
-            throw new CatalogException("Only the admin can delete the database");
+        catalogDBAdaptorFactory.createIndexes(wholeIndexes);
+    }
+
+    public void deleteCatalogDB(String password) throws CatalogException {
+        try {
+            userManager.loginAsAdmin(password);
+        } catch (CatalogException e) {
+            // Validate that the admin user exists.
+            OpenCGAResult<User> result = catalogDBAdaptorFactory.getCatalogUserDBAdaptor().get(OPENCGA, QueryOptions.empty());
+            if (result.getNumResults() == 1) {
+                // Admin user exists so we have to fail. Password must be incorrect.
+                throw e;
+            } else {
+                logger.error("Password could not be validated. Database seems corrupted. Deleting...");
+            }
         }
 
-        catalogDBAdaptorFactory.deleteCatalogDB();
         clearCatalog();
     }
 
-    private void clearCatalog() throws URISyntaxException {
-        List<DataStoreServerAddress> dataStoreServerAddresses = new LinkedList<>();
-        for (String hostPort : configuration.getCatalog().getDatabase().getHosts()) {
-            if (hostPort.contains(":")) {
-                String[] split = hostPort.split(":");
-                Integer port = Integer.valueOf(split[1]);
-                dataStoreServerAddresses.add(new DataStoreServerAddress(split[0], port));
-            } else {
-                dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort, 27017));
-            }
+    private void clearCatalog() throws CatalogException {
+        // Clear DB
+        catalogDBAdaptorFactory.deleteCatalogDB();
+        catalogDBAdaptorFactory.close();
+
+        // Clear workspace folder
+        Path rootdir;
+        try {
+            rootdir = Paths.get(UriUtils.createDirectoryUri(configuration.getWorkspace()));
+        } catch (URISyntaxException e) {
+            throw new CatalogException("Could not create uri for " + configuration.getWorkspace(), e);
         }
-        MongoDataStoreManager mongoManager = new MongoDataStoreManager(dataStoreServerAddresses);
-//        MongoDataStore db = mongoManager.get(catalogConfiguration.getDatabase().getDatabase());
-        MongoDataStore db = mongoManager.get(getCatalogDatabase());
-        db.getDb().drop();
-//        mongoManager.close(catalogConfiguration.getDatabase().getDatabase());
-        mongoManager.close(getCatalogDatabase());
-
-        Path rootdir = Paths.get(UriUtils.createDirectoryUri(configuration.getWorkspace()));
         deleteFolderTree(rootdir.toFile());
     }
 

opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java

@@ -25,7 +25,6 @@
 import org.opencb.opencga.core.models.user.Account;
 
 import java.io.IOException;
-import java.net.URISyntaxException;
 import java.util.*;
 
 /**
@@ -44,20 +43,11 @@ private CatalogDemo() {
      * @param adminPassword Administrator password.
      * @param force Used in the case where a database already exists with the same name. When force = true, it will override it.
      * @throws CatalogException when there is already a database with the same name and force is false.
-     * @throws URISyntaxException when there is a problem parsing the URI read from the configuration file.
      */
     public static void createDemoDatabase(CatalogManager catalogManager, String adminPassword, boolean force)
-            throws CatalogException, URISyntaxException {
-        if (catalogManager.existsCatalogDB()) {
-            if (force) {
-                String token = catalogManager.getUserManager().loginAsAdmin(adminPassword).getToken();
-                catalogManager.deleteCatalogDB(token);
-            } else {
-                throw new CatalogException("A database called " + catalogManager.getCatalogDatabase() + " already exists");
-            }
-        }
-        catalogManager.installCatalogDB(catalogManager.getConfiguration().getAdmin().getSecretKey(), adminPassword,
-                "opencga@admin.com", "");
+            throws CatalogException {
+        catalogManager.installCatalogDB(catalogManager.getConfiguration().getAdmin().getSecretKey(), adminPassword, "opencga@admin.com",
+                "", force, true);
         try {
             populateDatabase(catalogManager);
         } catch (IOException e) {

opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java

@@ -132,7 +132,8 @@ public void before() throws Exception {
         CatalogManagerExternalResource.clearCatalog(configuration);
 
         catalogManager = new CatalogManager(configuration);
-        catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), "4dMiNiStR4t0R.", "opencga@admin.com", "", true);
+        catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, false);
+
         fileManager = catalogManager.getFileManager();
 
         catalogManager.getUserManager().create(ownerUser, ownerUser, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, null);

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java

@@ -77,7 +77,7 @@ public void before() throws Exception {
         }
         configuration.getAdmin().setSecretKey(PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH));
         catalogManager = new CatalogManager(configuration);
-        catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true);
+        catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, false);
         catalogManager.close();
         // FIXME!! Should not need to create again the catalogManager
         //  Have to create again the CatalogManager, as it has a random "secretKey" inside

opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java

@@ -72,7 +72,7 @@ public void before() throws CatalogException, IOException, URISyntaxException {
 
         CatalogManagerExternalResource.clearCatalog(configuration);
         catalogManager = new CatalogManager(configuration);
-        catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true);
+        catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, false);
 
         //Create USER
         catalogManager.getUserManager().create("user", "name", "mi@mail.com", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, null);

opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java

@@ -67,11 +67,11 @@ public static void beforeClass() throws IOException, CatalogException, URISyntax
         configuration.getAdmin().setAlgorithm("HS256");
         catalogManager = new CatalogManager(configuration);
         if (catalogManager.existsCatalogDB()) {
-            String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken();
-            catalogManager.deleteCatalogDB(token);
+            catalogManager.deleteCatalogDB(TestParamConstants.ADMIN_PASSWORD);
         }
         catalogManager.installCatalogDB(PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH),
-                TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true);
+                TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, false);
+
         loader = new CatalogSampleAnnotationsLoader(catalogManager);
 
         String pedFileName = "20130606_g1k.ped";

opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java

@@ -267,7 +267,7 @@ public Response install(
             @ApiParam(value = "JSON containing the mandatory parameters", required = true) InstallationParams installParams) {
         try {
             catalogManager.installCatalogDB(installParams.getSecretKey(), installParams.getPassword(), installParams.getEmail(),
-                    installParams.getOrganization());
+                    installParams.getOrganization(), false, true);
             return createOkResponse(DataResult.empty());
         } catch (Exception e) {
             return createErrorResponse(e);