TASK-5564 - Update data sources for CellBase 6.2

.github/workflows/task.yml

@@ -21,5 +21,5 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: test
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag ${{ github.ref_name }}
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,builder --tag ${{ github.ref_name }}
     secrets: inherit

cellbase-app/app/cloud/docker/cellbase-builder/Dockerfile

@@ -11,7 +11,7 @@ LABEL org.label-schema.vendor="OpenCB" \
 ## We need to be root to install dependencies
 USER root
 RUN apt-get update -y && \
-    apt-get install -y git default-mysql-client libjson-perl libdbi-perl libdbd-mysql-perl libdbd-mysql-perl libtry-tiny-perl && \
+    apt-get install -y git default-mysql-client libjson-perl libdbi-perl libdbd-mysql-perl libdbd-mysql-perl libtry-tiny-perl libxml-simple-perl liblog-log4perl-perl libxml-parser-perl libxml-dom-perl && \
     mkdir /opt/ensembl && chown cellbase:cellbase /opt/ensembl && \
     rm -rf /var/lib/apt/lists/*
 
@@ -26,6 +26,10 @@ RUN cd /opt/ensembl && \
     git clone https://github.com/Ensembl/ensembl-variation.git && \
     git clone https://github.com/Ensembl/ensembl-funcgen.git && \
     git clone https://github.com/Ensembl/ensembl-compara.git && \
-    git clone https://github.com/Ensembl/ensembl-io.git
+    git clone https://github.com/Ensembl/ensembl-io.git && \
+    git clone --branch cvs/release-0_7 https://github.com/biomart/biomart-perl
 
-ENV PERL5LIB=$PERL5LIB:/opt/ensembl/bioperl-live:/opt/ensembl/ensembl/modules:/opt/ensembl/ensembl-variation/modules:/opt/ensembl/ensembl-funcgen/modules:/opt/ensembl/ensembl-compara/modules:/opt/ensembl/lib/perl/5.18.2:/opt/cellbase/scripts/ensembl-scripts
+## Give writting permissions to allow the script ensembl_canonical.pl to create sub-folder for cache purposes
+RUN chmod -R 777 /opt/cellbase/scripts/ensembl-scripts/
+
+ENV PERL5LIB=$PERL5LIB:/opt/ensembl/bioperl-live:/opt/ensembl/ensembl/modules:/opt/ensembl/ensembl-variation/modules:/opt/ensembl/ensembl-funcgen/modules:/opt/ensembl/ensembl-compara/modules:/opt/ensembl/lib/perl/5.18.2:/opt/cellbase/scripts/ensembl-scripts:/opt/ensembl/biomart-perl/lib

cellbase-app/app/scripts/ensembl-scripts/DB_CONFIG.pm

@@ -134,16 +134,16 @@ our $ENSEMBL_GENOMES_PORT = "4157";
 our $ENSEMBL_GENOMES_USER = "anonymous";
 
 ## Vertebrates
-our $HOMO_SAPIENS_CORE = "homo_sapiens_core_110_38";
-our $HOMO_SAPIENS_VARIATION = "homo_sapiens_variation_110_38";
-our $HOMO_SAPIENS_FUNCTIONAL = "homo_sapiens_funcgen_110_38";
-our $HOMO_SAPIENS_COMPARA = "homo_sapiens_compara_110_38";
+our $HOMO_SAPIENS_CORE = "homo_sapiens_core_111_38";
+our $HOMO_SAPIENS_VARIATION = "homo_sapiens_variation_111_38";
+our $HOMO_SAPIENS_FUNCTIONAL = "homo_sapiens_funcgen_111_38";
+our $HOMO_SAPIENS_COMPARA = "homo_sapiens_compara_111_38";
 #our $HOMO_SAPIENS_CORE = "homo_sapiens_core_78_38";
 #our $HOMO_SAPIENS_VARIATION = "homo_sapiens_variation_78_38";
 #our $HOMO_SAPIENS_FUNCTIONAL = "homo_sapiens_funcgen_78_38";
-our $MUS_MUSCULUS_CORE = "mus_musculus_core_78_38";
-our $MUS_MUSCULUS_VARIATION = "mus_musculus_variation_78_38";
-our $MUS_MUSCULUS_FUNCTIONAL = "mus_musculus_funcgen_78_38";
+our $MUS_MUSCULUS_CORE = "mus_musculus_core_111_39";
+our $MUS_MUSCULUS_VARIATION = "mus_musculus_variation_111_39";
+our $MUS_MUSCULUS_FUNCTIONAL = "mus_musculus_funcgen_111_39";
 our $RATTUS_NORVEGICUS_CORE = "rattus_norvegicus_core_78_5";
 our $RATTUS_NORVEGICUS_VARIATION = "rattus_norvegicus_variation_78_5";
 our $RATTUS_NORVEGICUS_FUNCTIONAL = "rattus_norvegicus_funcgen_78_5";

cellbase-app/app/scripts/ensembl-scripts/ensembl_canonical.pl

@@ -0,0 +1,61 @@
+#!/usr/bin/env perl
+
+use strict;
+use Getopt::Long;
+use Data::Dumper;
+use JSON;
+use DB_CONFIG;
+
+use BioMart::Initializer;
+use BioMart::Query;
+use BioMart::QueryRunner;
+
+## Default values
+my $species = 'hsapiens';
+my $outdir = "./";
+
+## Parsing command line
+GetOptions ('species=s' => \$species, 'outdir=s' => \$outdir);
+
+
+my $confFile = "/opt/cellbase/scripts/ensembl-scripts/martURLLocation.xml";
+
+# NB: change action to 'clean' if you wish to start a fresh configuration
+# and to 'cached' if you want to skip configuration step on subsequent runs from the same registry
+my $action='clean';
+my $initializer = BioMart::Initializer->new('registryFile'=>$confFile, 'action'=>$action);
+my $registry = $initializer->getRegistry;
+
+my $query = BioMart::Query->new('registry'=>$registry,'virtualSchemaName'=>'default');
+
+$query->setDataset($species."_gene_ensembl");
+
+$query->addAttribute("ensembl_gene_id");
+$query->addAttribute("ensembl_transcript_id");
+$query->addAttribute("transcript_is_canonical");
+
+$query->formatter("TSV");
+
+# Open the file for writing
+open(my $fh, '>', "$outdir/ensembl_canonical.txt") or die "Cannot open ensembl_canonical.txt file: $!";
+
+# Save the original stdout
+my $original_stdout = *STDOUT;
+open(STDOUT, '>&', $fh) or die "Can't redirect STDOUT: $!";
+
+my $query_runner = BioMart::QueryRunner->new();
+
+# to obtain unique rows only
+$query_runner->uniqueRowsOnly(1);
+$query_runner->execute($query);
+#$query_runner->printHeader();
+#print ENSEMBL_CANONICAL $query_runner->printResults();
+# Call printResults which prints to STDOUT (now redirected to the file)
+$query_runner->printResults();
+#$query_runner->printFooter();
+
+# Restore the original stdout
+open(STDOUT, '>&', $original_stdout) or die "Can't restore STDOUT: $!";
+
+# Close the filehandle
+close($fh) or die "Failed to close file: $!";

cellbase-app/app/scripts/ensembl-scripts/gene_extra_info.pl

@@ -16,7 +16,9 @@
 ####################################################################
 ## Parsing command line options	####################################
 ####################################################################
-# USAGE: ./gene_extra_info.pl --species "Homo sapiens" --outdir ../../appl_db/ird_v1/hsa ...
+##docker run -it --mount type=bind,source=/tmp,target=/tmp opencb/cellbase-builder:6.2.0-SNAPSHOT /opt/cellbase/scripts/ensembl-scripts/gene_extra_info.pl -s "Mus musculus" -o /tmp
+
+# USAGE: ./gene_extra_info.pl --species "Homo sapiens" --assembly "GRCh38" --outdir ../../appl_db/ird_v1/hsa ...
 
 ## Parsing command line
 GetOptions ('species=s' => \$species, 'assembly=s' => \$assembly, 'outdir=s' => \$outdir, 'phylo=s' => \$phylo,
@@ -50,8 +52,8 @@
 
 if ($phylo eq "" || $phylo eq "vertebrate") {
 	print ("In vertebrates section\n");
-	if ($species eq "Homo sapiens" && $assembly eq "GRCh38") {
-		print ("Human selected, assembly ".$assembly." selected, connecting to port ".$ENSEMBL_PORT."\n");
+	if ($species eq "Homo sapiens" || $species eq "Mus musculus") {
+		print ($species." selected, assembly ".$assembly." selected, connecting to port ".$ENSEMBL_PORT."\n");
 		Bio::EnsEMBL::Registry->load_registry_from_db(
 			-host     => $ENSEMBL_HOST,
 			-user     => $ENSEMBL_USER,

cellbase-app/app/scripts/ensembl-scripts/genome_info.pl

@@ -17,7 +17,9 @@
 ####################################################################
 ## Parsing command line options ####################################
 ####################################################################
-# USAGE: ./genome_info.pl --species "Homo sapiens" --outfile ../../appl_db/ird_v1/hsa ...
+##docker run -it --mount type=bind,source=/tmp,target=/tmp opencb/cellbase-builder:6.2.0-SNAPSHOT /opt/cellbase/scripts/ensembl-scripts/genome_info.pl --species "Mus musculus" --assembly GRCm39 --outfile /tmp
+
+# USAGE: ./genome_info.pl --species "Homo sapiens" --assembly GRCh38 --outfile ../../appl_db/ird_v1/hsa ...
 
 ## Parsing command line
 GetOptions ('species=s' => \$species, 'assembly=s' => \$assembly, 'o|outfile=s' => \$outfile, 'phylo=s' => \$phylo,
@@ -29,7 +31,6 @@
 
 if ($outfile eq "") {
     $outfile = "/ensembl-data/genome_info.json";
-    # $outfile = "/ensembl-data/$species.json";
 }
 
 ####################################################################
@@ -42,17 +43,13 @@
 # Bio::EnsEMBL::Registry->load_all("$ENSEMBL_REGISTRY");
 if($phylo eq "" || $phylo eq "vertebrate") {
     print ("In vertebrates section\n");
-    if ($species eq "Homo sapiens" && $assembly eq "GRCh38") {
-        print ("Human selected, assembly ".$assembly." selected, connecting to port ".$ENSEMBL_PORT."\n");
-        Bio::EnsEMBL::Registry->load_registry_from_db(
-            -host     => $ENSEMBL_HOST,
-            -user     => $ENSEMBL_USER,
-            -port     => $ENSEMBL_PORT,
-            -verbose  => $verbose
-        );
-    } else {
-        print ("Human selected, assembly ".$assembly." no supported\n");
-    }
+    print ("Species: ".$species.", assembly ".$assembly.", connecting to: ".$ENSEMBL_HOST.":".$ENSEMBL_PORT."\n");
+    Bio::EnsEMBL::Registry->load_registry_from_db(
+        -host     => $ENSEMBL_HOST,
+        -user     => $ENSEMBL_USER,
+        -port     => $ENSEMBL_PORT,
+        -verbose  => $verbose
+    );
 } else {
     print ("In no-vertebrates section\n");
     Bio::EnsEMBL::Registry->load_registry_from_db(
@@ -64,7 +61,6 @@
 
 my $slice_adaptor = Bio::EnsEMBL::Registry->get_adaptor($species, "core", "Slice");
 my $karyotype_adaptor = Bio::EnsEMBL::Registry->get_adaptor($species, "core", "KaryotypeBand");
-# my $gene_adaptor = Bio::EnsEMBL::Registry->get_adaptor($species, "core", "Gene");
 ####################################################################
 
 my %info_stats = ();
@@ -81,12 +77,10 @@
 	$chromosome{'start'} = int($chrom->start());
 	$chromosome{'end'} = int($chrom->end());
 	$chromosome{'size'} = int($chrom->seq_region_length());
-#	$chromosome{'numberGenes'} = scalar @{$chrom->get_all_Genes()};
 	$chromosome{'isCircular'} = $chrom->is_circular();
 
 	my @cytobands = ();
 	foreach my $cyto(@{$karyotype_adaptor->fetch_all_by_chr_name($chrom->seq_region_name)}) {
-#		print $cytoband->name."\n";
         my %cytoband = ();
         $cytoband{'name'} = $cyto->name();
         $cytoband{'start'} = int($cyto->start());
@@ -96,7 +90,7 @@
 		push(@cytobands, \%cytoband);
 	}
 
-	## check if any cytoband has been added
+	## Check if any cytoband has been added
 	## If not a unique cytoband covering all chromosome is added.
 	if(@cytobands == 0) {
 		my %cytoband = ();
@@ -110,7 +104,6 @@
 	$chromosome{'cytobands'} = \@cytobands;
 
 	push(@chromosomes, \%chromosome);
-#    push(@chrom_ids, $chrom->seq_region_name);
 }
 $info_stats{'chromosomes'} = \@chromosomes;
 
@@ -124,7 +117,6 @@
         $supercontig{'start'} = int($supercon->start());
         $supercontig{'end'} = int($supercon->end());
         $supercontig{'size'} = int($supercon->seq_region_length());
-#        $supercontig{'numberGenes'} = scalar @{$supercon->get_all_Genes()};
         $supercontig{'isCircular'} = $supercon->is_circular();
 
         ## Adding an unique cytoband covering all chromosome is added.
@@ -151,7 +143,7 @@
 
 sub print_parameters {
     print "Parameters: ";
-    print "species: $species, outfile: $outfile, ";
+    print "species: $species, assembly: $assembly, outfile: $outfile, ";
     print "ensembl-registry: $ENSEMBL_REGISTRY, ";
     print "ensembl-host: $ENSEMBL_HOST, ensembl-port: $ENSEMBL_PORT, ";
     print "ensembl-user: $ENSEMBL_USER, verbose: $verbose, help: $help";

cellbase-app/app/scripts/ensembl-scripts/martURLLocation.xml

@@ -0,0 +1,19 @@
+<!--
+  ~ Copyright 2015-2020 OpenCB
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<MartRegistry>
+    <MartURLLocation database="ensembl_mart_111" default="1" displayName="Ensembl Genes 111" host="www.ensembl.org" includeDatasets="" martUser="" name="ENSEMBL_MART_ENSEMBL" path="/biomart/martservice" port="80" serverVirtualSchema="default" visible="1" />
+</MartRegistry>

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CliOptionsParser.java

@@ -66,8 +66,8 @@ public class CommonCommandOptions {
                 description = "Set the logging level, accepted values are: debug, info, warn, error and fatal")
         public String logLevel = "info";
 
-        @Parameter(names = {"-C", "--config"}, arity = 1,
-                description = "Path to CellBase configuration.yml file")
+        @Deprecated
+        @Parameter(names = {"-C", "--config"}, arity = 1, hidden = true,  description = "Path to CellBase configuration.yml file")
         public String conf;
     }
 

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java

@@ -35,18 +35,12 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
-/**
- * Created by imedina on 03/02/15.
- */
+
 public abstract class CommandExecutor {
 
     protected String logLevel;
-//    protected boolean verbose;
     protected String conf;
 
-    @Deprecated
-    protected String configFile;
-
     protected String appHome;
 
     protected CellBaseConfiguration configuration;
@@ -55,35 +49,13 @@ public abstract class CommandExecutor {
     protected Logger logger;
 
     public CommandExecutor() {
-
     }
 
     public CommandExecutor(String logLevel, String conf) {
         this.logLevel = logLevel;
         this.conf = conf;
 
-        /**
-         * System property 'app.home' is set up by cellbase.sh. If by any reason this is null
-         * then CELLBASE_HOME environment variable is used instead.
-         */
-        this.appHome = System.getProperty("app.home", System.getenv("CELLBASE_HOME"));
-
-        if (StringUtils.isEmpty(conf)) {
-            this.conf = this.appHome + "/conf";
-        }
-
-        if (logLevel != null && !logLevel.isEmpty()) {
-            // We must call to this method
-            setLogLevel(logLevel);
-        }
-    }
-
-    public CommandExecutor(String logLevel, boolean verbose, String conf) {
-        this.logLevel = logLevel;
-//        this.verbose = verbose;
-        this.conf = conf;
-
-        /**
+        /*
          * System property 'app.home' is set up by cellbase.sh. If by any reason this is null
          * then CELLBASE_HOME environment variable is used instead.
          */
@@ -124,41 +96,30 @@ public void setLogLevel(String logLevel) {
         this.logLevel = logLevel;
     }
 
-//    public boolean isVerbose() {
-//        return verbose;
-//    }
-//
-//    public void setVerbose(boolean verbose) {
-//        this.verbose = verbose;
-//    }
-
-    public String getConfigFile() {
-        return configFile;
-    }
-
-    public void setConfigFile(String configFile) {
-        this.configFile = configFile;
-    }
-
     public Logger getLogger() {
         return logger;
     }
 
-    /*
+    /**
      * This method attempts to first data configuration from CLI parameter, if not present then uses
      * the configuration from installation directory, if not exists then loads JAR configuration.json or yml.
+     *
+     * @throws URISyntaxException If any URI problem occurs
+     * @throws IOException If any IO problem occurs
      */
     public void loadCellBaseConfiguration() throws URISyntaxException, IOException {
         Path confPath = Paths.get(this.conf);
         FileUtils.checkDirectory(confPath);
 
         if (Files.exists(confPath.resolve("configuration.json"))) {
             logger.debug("Loading configuration from '{}'", confPath.resolve("configuration.json").toAbsolutePath());
-            this.configuration = CellBaseConfiguration.load(new FileInputStream(confPath.resolve("configuration.json").toFile()),
-                    CellBaseConfiguration.ConfigurationFileFormat.JSON);
+            this.configuration = CellBaseConfiguration
+                    .load(Files.newInputStream(confPath.resolve("configuration.json").toFile().toPath()),
+                            CellBaseConfiguration.ConfigurationFileFormat.JSON);
         } else if (Files.exists(Paths.get(this.appHome + "/conf/configuration.yml"))) {
             logger.debug("Loading configuration from '{}'", this.appHome + "/conf/configuration.yml");
-            this.configuration = CellBaseConfiguration.load(new FileInputStream(new File(this.appHome + "/conf/configuration.yml")));
+            this.configuration = CellBaseConfiguration
+                    .load(Files.newInputStream(new File(this.appHome + "/conf/configuration.yml").toPath()));
         } else {
             InputStream inputStream = CellBaseConfiguration.class.getClassLoader().getResourceAsStream("conf/configuration.json");
             String configurationFilePath = "conf/configuration.json";
@@ -198,10 +159,4 @@ public void loadClientConfiguration() throws IOException {
             }
         }
     }
-
-    protected void makeDir(Path folderPath) throws IOException {
-        if (!Files.exists(folderPath)) {
-            Files.createDirectories(folderPath);
-        }
-    }
 }