From f953ea327e19c86643b2ab7d53a83799b7438dca Mon Sep 17 00:00:00 2001 From: Swaathik Date: Mon, 17 Oct 2016 11:42:12 +0100 Subject: [PATCH 01/24] lib: MongoDbAdaptors use the centralized count, distinct and execute methods --- .../lib/impl/ClinicalMongoDBAdaptor.java | 6 +- .../cellbase/lib/impl/GeneMongoDBAdaptor.java | 9 ++- .../lib/impl/GenomeMongoDBAdaptor.java | 6 +- .../cellbase/lib/impl/MongoDBAdaptor.java | 59 ++++++++++++------- .../lib/impl/ProteinMongoDBAdaptor.java | 8 +-- ...oteinProteinInteractionMongoDBAdaptor.java | 9 ++- .../lib/impl/RegulationMongoDBAdaptor.java | 9 ++- .../lib/impl/TranscriptMongoDBAdaptor.java | 3 +- .../lib/impl/VariantMongoDBAdaptor.java | 8 +-- .../cellbase/lib/impl/XRefMongoDBAdaptor.java | 4 +- 10 files changed, 70 insertions(+), 51 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java index 5b14e8f67a..c9053bcb39 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java @@ -88,13 +88,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bson = parseQuery(query); - return mongoDBCollection.count(bson); + return count(bson, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bson = parseQuery(query); - return mongoDBCollection.distinct(field, bson); + return distinct(field, bson, mongoDBCollection); } @Override @@ -111,7 +111,7 @@ public QueryResult get(Query query, QueryOptions options) { public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); QueryOptions parsedOptions = parseQueryOptions(options); - return mongoDBCollection.find(bson, parsedOptions); + return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java index c8ea9b731b..b0e28608c1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java @@ -75,13 +75,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.count(bsonDocument); + return count(bsonDocument, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.distinct(field, bsonDocument); + return distinct(field, bsonDocument, mongoDBCollection); } @Override @@ -92,15 +92,14 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - options = addPrivateExcludeOptions(options); - return mongoDBCollection.find(bson, null, Gene.class, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, Gene.class); } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java index c7c323921b..7993838e6a 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java @@ -260,13 +260,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bson = parseQuery(query); - return mongoDBCollection.count(bson); + return count(bson, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bson = parseQuery(query); - return mongoDBCollection.distinct(field, bson); + return distinct(field, bson, mongoDBCollection); } @Override @@ -282,7 +282,7 @@ public QueryResult get(Query query, QueryOptions options) { @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index 9f909f4700..94a7704524 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -389,39 +389,58 @@ public QueryResult getIntervalFrequencies(Bson query, Region region, int interva return queryResult; } + protected QueryResult count(Bson query, MongoDBCollection mongoDBCollection) { +// QueryResult result = cacheManager.get(); + QueryResult result = null; + if (result == null) { + result = mongoDBCollection.count(query); +// cacheManager.set(result); + } + return result; + } - - - - protected QueryResult executeDistinct(Object id, String fields, Document query) { -// long dbTimeStart, dbTimeEnd; -// dbTimeStart = System.currentTimeMillis(); - QueryResult queryResult = mongoDBCollection.distinct(fields, query); -// List dbObjectList = new LinkedList<>(); -// while (cursor.hasNext()) { -// dbObjectList.add(cursor.next()); -// } -// dbTimeEnd = System.currentTimeMillis(); - // setting queryResult fields - queryResult.setId(id.toString()); -// queryResult.setDbTime(Long.valueOf(dbTimeEnd - dbTimeStart).intValue()); -// queryResult.setNumResults(dbObjectList.size()); - return queryResult; + protected QueryResult distinct(String field, Bson query, MongoDBCollection mongoDBCollection) { +// QueryResult result = cacheManager.get(); + QueryResult result = null; + if (result == null) { + result = mongoDBCollection.distinct(field, query); +// cacheManager.set(result); + } + return result; + } + + protected QueryResult executeBsonQuery(Bson query, Bson projection, QueryOptions options, + MongoDBCollection mongoDBCollection, Class clazz) { + QueryResult result = null; + if (options.getBoolean("cache")) { +// result = cacheManager.get(); +// result = null; + if (result == null) { + options = addPrivateExcludeOptions(options); + result = mongoDBCollection.find(query, projection, clazz, options); +// cacheManager.set(result); + } + } else { + options = addPrivateExcludeOptions(options); + result = mongoDBCollection.find(query, projection, clazz, options); + } + return result; } + @Deprecated protected QueryResult executeQuery(Object id, Document query, QueryOptions options) { return executeQueryList2(Arrays.asList(id), Arrays.asList(query), options, mongoDBCollection).get(0); } - + @Deprecated protected QueryResult executeQuery(Object id, Document query, QueryOptions options, MongoDBCollection mongoDBCollection2) { return executeQueryList2(Arrays.asList(id), Arrays.asList(query), options, mongoDBCollection2).get(0); } - + @Deprecated protected List executeQueryList2(List ids, List queries, QueryOptions options) { return executeQueryList2(ids, queries, options, mongoDBCollection); } - + @Deprecated protected List executeQueryList2(List ids, List queries, QueryOptions options, MongoDBCollection mongoDBCollection2) { List queryResults = new ArrayList<>(ids.size()); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java index bbda1344b6..c7c5c173f0 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java @@ -307,13 +307,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson document = parseQuery(query); - return mongoDBCollection.count(document); + return count(document, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson document = parseQuery(query); - return mongoDBCollection.distinct(field, document); + return distinct(field, document, mongoDBCollection); } @Override @@ -324,13 +324,13 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return mongoDBCollection.find(bson, null, Entry.class, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, Entry.class); } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java index e28fc60b8d..483d934aa4 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java @@ -66,12 +66,14 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { - return mongoDBCollection.count(parseQuery(query)); + Bson document = parseQuery(query); + return count(document, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { - return mongoDBCollection.distinct(field, parseQuery(query)); + Bson document = parseQuery(query); + return distinct(field, document, mongoDBCollection); } @Override @@ -86,7 +88,8 @@ public QueryResult get(Query query, QueryOptions options) { @Override public QueryResult nativeGet(Query query, QueryOptions options) { - return mongoDBCollection.find(parseQuery(query), options); + Bson bson = parseQuery(query); + return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java index c594cf835f..88eaef87d7 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java @@ -91,13 +91,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.count(bsonDocument); + return count(bsonDocument, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.distinct(field, bsonDocument); + return distinct(field, bsonDocument, mongoDBCollection); } @Override @@ -108,14 +108,13 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - options = addPrivateExcludeOptions(options); - return mongoDBCollection.find(bson, null, RegulatoryFeature.class, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, RegulatoryFeature.class); } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java index c278634e7f..faf67961a4 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java @@ -91,8 +91,7 @@ public QueryResult count(Query query) { @Override public QueryResult distinct(Query query, String field) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.distinct(field, bsonDocument); - + return distinct(field, bsonDocument, mongoDBCollection); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java index 0471412772..57bcb1f442 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java @@ -108,13 +108,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson document = parseQuery(query); - return mongoDBCollection.count(document); + return count(document, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson document = parseQuery(query); - return mongoDBCollection.distinct(field, document); + return distinct(field, document, mongoDBCollection); } @Override @@ -134,7 +134,7 @@ public QueryResult get(Query query, QueryOptions options) { // options = addPrivateExcludeOptions(options); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return mongoDBCollection.find(bson, null, Variant.class, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, Variant.class); } // FIXME: patch to exclude annotation.additionalAttributes from the results - to remove as soon as the variation @@ -158,7 +158,7 @@ public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); // options.put(MongoDBCollection.SKIP_COUNT, true); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java index 45fc10dda3..c60aa465fc 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java @@ -70,13 +70,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bson = parseQuery(query); - return mongoDBCollection.count(bson); + return count(bson, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bson = parseQuery(query); - return mongoDBCollection.distinct(field, bson); + return distinct(field, bson, mongoDBCollection); } @Override From 1b8784bffa7dd3b461391ac81454c03366c43b7f Mon Sep 17 00:00:00 2001 From: imedina Date: Thu, 20 Oct 2016 15:11:07 +0100 Subject: [PATCH 02/24] core: add cache section to configuration file, also a test has been added, closes #290 --- cellbase-core/pom.xml | 4 + .../cellbase/core/config/CacheProperties.java | 81 ++ .../core/config/CellBaseConfiguration.java | 72 +- .../config/CellBaseConfigurationTest.java | 22 + .../resources/cellBaseProperties_test.json | 136 --- .../test/resources/configuration_test.json | 907 ++++++++++++++++++ 6 files changed, 1065 insertions(+), 157 deletions(-) create mode 100644 cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java create mode 100644 cellbase-core/src/test/java/org/opencb/cellbase/core/config/CellBaseConfigurationTest.java delete mode 100644 cellbase-core/src/test/resources/cellBaseProperties_test.json create mode 100644 cellbase-core/src/test/resources/configuration_test.json diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 73bd9567e5..4039300c57 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -36,6 +36,10 @@ com.fasterxml.jackson.core jackson-databind + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + org.glassfish.jersey.core jersey-client diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java new file mode 100644 index 0000000000..f2b25ba34c --- /dev/null +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java @@ -0,0 +1,81 @@ +package org.opencb.cellbase.core.config; + +/** + * Created by imedina on 20/10/16. + */ +public class CacheProperties { + + /** + * This field contain the host and port, ie. host[:port] + */ + private String host; + private boolean active; + + /** + * Accepted values are: JSON, Kryo + */ + private String serialization; + + private int slowThreshold; + + public static final boolean DEFAULT_ACTVE = true; + public static final String DEFAULT_SERIALIZATION = "JSON"; + + public CacheProperties() { + this("", DEFAULT_ACTVE, DEFAULT_SERIALIZATION, 50); + } + + public CacheProperties(String host, boolean active, String serialization, int slowThreshold) { + this.host = host; + this.active = active; + this.serialization = serialization; + this.slowThreshold = slowThreshold; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CacheProperties{"); + sb.append("host='").append(host).append('\''); + sb.append(", active=").append(active); + sb.append(", serialization='").append(serialization).append('\''); + sb.append(", slowThreshold=").append(slowThreshold); + sb.append('}'); + return sb.toString(); + } + + public String getHost() { + return host; + } + + public CacheProperties setHost(String host) { + this.host = host; + return this; + } + + public boolean isActive() { + return active; + } + + public CacheProperties setActive(boolean active) { + this.active = active; + return this; + } + + public String getSerialization() { + return serialization; + } + + public CacheProperties setSerialization(String serialization) { + this.serialization = serialization; + return this; + } + + public int getSlowThreshold() { + return slowThreshold; + } + + public CacheProperties setSlowThreshold(int slowThreshold) { + this.slowThreshold = slowThreshold; + return this; + } +} diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CellBaseConfiguration.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CellBaseConfiguration.java index 33eb843972..0326191945 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CellBaseConfiguration.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CellBaseConfiguration.java @@ -33,6 +33,7 @@ public class CellBaseConfiguration { private String wiki; private String defaultOutdir; private Databases databases; + private CacheProperties cache; private DownloadProperties download; private SpeciesProperties species; @@ -42,28 +43,66 @@ public static CellBaseConfiguration load(InputStream configurationInputStream) t return jsonMapper.readValue(configurationInputStream, CellBaseConfiguration.class); } + public List getAllSpecies() { + List allSpecies = new ArrayList<>(); + allSpecies.addAll(species.getVertebrates()); + allSpecies.addAll(species.getMetazoa()); + allSpecies.addAll(species.getFungi()); + allSpecies.addAll(species.getProtist()); + allSpecies.addAll(species.getPlants()); + + return allSpecies; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CellBaseConfiguration{"); + sb.append("version='").append(version).append('\''); + sb.append(", apiVersion='").append(apiVersion).append('\''); + sb.append(", wiki='").append(wiki).append('\''); + sb.append(", defaultOutdir='").append(defaultOutdir).append('\''); + sb.append(", databases=").append(databases); + sb.append(", cache=").append(cache); + sb.append(", download=").append(download); + sb.append(", species=").append(species); + sb.append('}'); + return sb.toString(); + } + public String getVersion() { return version; } - public void setVersion(String version) { + public CellBaseConfiguration setVersion(String version) { this.version = version; + return this; } public String getApiVersion() { return apiVersion; } - public void setApiVersion(String apiVersion) { + public CellBaseConfiguration setApiVersion(String apiVersion) { this.apiVersion = apiVersion; + return this; } public String getWiki() { return wiki; } - public void setWiki(String wiki) { + public CellBaseConfiguration setWiki(String wiki) { this.wiki = wiki; + return this; + } + + public String getDefaultOutdir() { + return defaultOutdir; + } + + public CellBaseConfiguration setDefaultOutdir(String defaultOutdir) { + this.defaultOutdir = defaultOutdir; + return this; } public Databases getDatabases() { @@ -75,39 +114,30 @@ public CellBaseConfiguration setDatabases(Databases databases) { return this; } - public String getDefaultOutdir() { - return defaultOutdir; + public CacheProperties getCache() { + return cache; } - public void setDefaultOutdir(String defaultOutdir) { - this.defaultOutdir = defaultOutdir; + public CellBaseConfiguration setCache(CacheProperties cache) { + this.cache = cache; + return this; } public DownloadProperties getDownload() { return download; } - public void setDownload(DownloadProperties download) { + public CellBaseConfiguration setDownload(DownloadProperties download) { this.download = download; + return this; } public SpeciesProperties getSpecies() { return species; } - public void setSpecies(SpeciesProperties species) { + public CellBaseConfiguration setSpecies(SpeciesProperties species) { this.species = species; + return this; } - - public List getAllSpecies() { - List allSpecies = new ArrayList<>(); - allSpecies.addAll(species.getVertebrates()); - allSpecies.addAll(species.getMetazoa()); - allSpecies.addAll(species.getFungi()); - allSpecies.addAll(species.getProtist()); - allSpecies.addAll(species.getPlants()); - - return allSpecies; - } - } diff --git a/cellbase-core/src/test/java/org/opencb/cellbase/core/config/CellBaseConfigurationTest.java b/cellbase-core/src/test/java/org/opencb/cellbase/core/config/CellBaseConfigurationTest.java new file mode 100644 index 0000000000..2ba6d495ce --- /dev/null +++ b/cellbase-core/src/test/java/org/opencb/cellbase/core/config/CellBaseConfigurationTest.java @@ -0,0 +1,22 @@ +package org.opencb.cellbase.core.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.Test; + +/** + * Created by imedina on 20/10/16. + */ +public class CellBaseConfigurationTest { + + + @Test + public void load() throws Exception { + + CellBaseConfiguration cellBaseConfiguration = CellBaseConfiguration + .load(getClass().getResourceAsStream("/configuration_test.json")); + + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + System.out.println(objectMapper.writer().withDefaultPrettyPrinter().writeValueAsString(cellBaseConfiguration)); + } +} \ No newline at end of file diff --git a/cellbase-core/src/test/resources/cellBaseProperties_test.json b/cellbase-core/src/test/resources/cellBaseProperties_test.json deleted file mode 100644 index 5b85beff66..0000000000 --- a/cellbase-core/src/test/resources/cellBaseProperties_test.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "version":"v3", - "apiVersion":"v3.1", - "wiki":"https://github.com/opencb/cellbase/wiki", - "databases": { - "mongodb": { - "host":"${CELLBASE.DB.HOST}", - "user":"${CELLBASE.DB.USER}", - "password":"${CELLBASE.DB.PASSWORD}" - }, - "neo4j": { - "host":"${CELLBASE.DB.HOST}", - "user":"${CELLBASE.DB.USER}", - "password":"${CELLBASE.DB.PASSWORD}" - } - }, - "defaultOutdir":"/tmp", - "download":{ - "ensembl":{ - "database":{ - "host":"ensembldb.ensembl.org:3306", - "user":"anonymous", - "password":"" - }, - "url":{ - "host":"ftp://ftp.ensembl.org/pub" - } - }, - "ensemblGenomes":{ - "database":{ - "host":"mysql-eg-publicsql.ebi.ac.uk:4157", - "user":"anonymous", - "password":"" - }, - "url":{ - "host":"ftp://ftp.ensembl.org/pub" - } - } - }, - "species":{ - "vertebrates":[ - { - "id":"hsapiens", - "scientificName":"Homo sapiens", - "assemblies":[ - { - "ensemblVersion":"75_37", - "name":"GRCh37.p13" - }, - { - "ensemblVersion":"78_38", - "name":"GRCh38" - } - ] - }, - { - "id":"mmusculus", - "scientificName":"Mus musculus", - "assemblies":[ - { - "ensemblVersion":"78_38", - "name":"GRCm38.p3" - } - ] - }, - { - "id":"cfamiliaris", - "scientificName":"Canis familiaris", - "assemblies":[ - { - "ensemblVersion":"78_31", - "name":"CanFam3.1" - } - ] - } - ], - "metazoa":[ - { - "id":"dmelanogaster", - "scientificName":"Drosophila melanogaster", - "assemblies":[ - { - "ensemblVersion":"24_77_546", - "name":"BDGP5" - } - ] - } - ], - "fungi":[ - { - "id":"scerevisiae", - "scientificName":"Saccharomyces cerevisiae", - "assemblies":[ - { - "ensemblVersion":"24_77_4", - "name":"R64-1-1" - } - ] - } - ], - "protist":[ - { - "id":"pfalciparum", - "scientificName":"Plasmodium falciparum", - "assemblies":[ - { - "ensemblVersion":"24_77_3", - "name":"ASM276v1" - } - ] - } - ], - "plants":[ - { - "id":"athaliana", - "scientificName":"Arabidopsis thaliana", - "assemblies":[ - { - "ensemblVersion":"24_77_10", - "name":"TAIR10" - } - ] - }, - { - "id": "alyrata", - "scientificName": "Arabidopsis lyrata", - "assemblies": [ - { - "ensemblVersion": "24_77_10", - "name": "v.1.0" - } - ] - } - ] - } -} \ No newline at end of file diff --git a/cellbase-core/src/test/resources/configuration_test.json b/cellbase-core/src/test/resources/configuration_test.json new file mode 100644 index 0000000000..a04d17e1be --- /dev/null +++ b/cellbase-core/src/test/resources/configuration_test.json @@ -0,0 +1,907 @@ +{ + "version": "v4", + "apiVersion": "${cellbase.version}", + "wiki": "https://github.com/opencb/cellbase/wiki", + "defaultOutdir": "/tmp", + "databases": { + "mongodb": { + "host": "${CELLBASE.DB.MONGODB.HOST}", + "user": "${CELLBASE.DB.USER}", + "password": "${CELLBASE.DB.PASSWORD}", + "options": { + "authenticationDatabase": "${CELLBASE.DB.MONGODB.AUTHENTICATIONDATABASE}", + "readPreference": "${CELLBASE.DB.MONGODB.READPREFERENCE}", + "replicaSet": "${CELLBASE.DB.MONGODB.REPLICASET}" + } + }, + "neo4j": { + "hsapiens": { + "host": "${CELLBASE.DB.NEO4J.HOST}", + "user": "${CELLBASE.DB.USER}", + "password": "${CELLBASE.DB.PASSWORD}" + }, + "mmusculus": { + "host": "${CELLBASE.DB.NEO4J.HOST}", + "user": "${CELLBASE.DB.USER}", + "password": "${CELLBASE.DB.PASSWORD}" + } + } + }, + "cache": { + "host": "localhost:6379", + "active": true + }, + "download": { + "ensembl": { + "database": { + "host": "ensembldb.ensembl.org:3306", + "user": "anonymous", + "password": "" + }, + "libs": "${CELLBASE.ENSEMBL.LIBS}", + "url": { + "host": "ftp://ftp.ensembl.org/pub" + } + }, + "ensemblGenomes":{ + "database":{ + "host": "mysql-eg-publicsql.ebi.ac.uk:4157", + "user": "anonymous", + "password": "" + }, + "libs": "${CELLBASE.ENSEMBL.LIBS}", + "url":{ + "host":"ftp://ftp.ensemblgenomes.org/pub" + } + }, + "geneUniprotXref": { + "host": "ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/" + }, + "geneExpressionAtlas": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/gxa/allgenes_updown_in_organism_part_2.0.14.tab.gz" + }, + "mirbase": { + "host": "ftp://mirbase.org/pub/mirbase/CURRENT/" + }, + "mirbaseReadme": { + "host": "ftp://mirbase.org/pub/mirbase/CURRENT/README" + }, + "targetScan": { + "host": "http://hgdownload.cse.ucsc.edu/goldenPath/" + }, + "miRTarBase": { + "host": "http://mirtarbase.mbc.nctu.edu.tw/cache/download/4.5/" + }, + "uniprot": { + "host": "ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.xml.gz" + }, + "uniprotRelNotes": { + "host": "ftp://ftp.uniprot.org/pub/databases/uniprot/relnotes.txt" + }, + "intact": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/intact/current/psimitab/intact.txt" + }, + "interpro": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/interpro/current/protein2ipr.dat.gz" + }, + "interproRelNotes": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/interpro/current/release_notes.txt" + }, + "conservation": { + "host": "ftp://hgdownload.cse.ucsc.edu/goldenPath/" + }, + "gerp": { + "host": "http://mendel.stanford.edu/SidowLab/downloads/gerp/hg19.GERP_scores.tar.gz" + }, + "clinvar": { + "host": "ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2016-10.xml.gz" + }, + "clinvarSummary": { + "host": "ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz" + }, + "clinvarEfoTerms": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/eva/ClinVar/ClinVar_Traits_EFO_Names_260615.csv" + }, + "gwasCatalog": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/2016/09/28/gwas-catalog-associations.tsv" + }, + "hpo": { + "host": "http://compbio.charite.de/hudson/job/hpo.annotations.monthly/lastStableBuild/artifact/annotation/ALL_SOURCES_ALL_FREQUENCIES_diseases_to_genes_to_phenotypes.txt" + }, + "disgenet": { + "host": "http://www.disgenet.org/ds/DisGeNET/results/all_gene_disease_associations.tsv.gz" + }, + "disgenetReadme": { + "host": "http://www.disgenet.org/ds/DisGeNET/results/readme.txt" + }, + "dgidb": { + "host": "http://dgidb.genome.wustl.edu/downloads/interactions.tsv" + }, + "cadd": { + "host": "http://krishna.gs.washington.edu/download/CADD/v1.3/whole_genome_SNVs.tsv.gz" + }, + "reactome": { + "host": "http://www.reactome.org/download/current/biopax.zip" + } + }, + "species": { + "vertebrates": [ + { + "id": "hsapiens", + "scientificName": "Homo sapiens", + "assemblies": [ + { + "ensemblVersion":"82_37", + "name":"GRCh37" + }, + { + "ensemblVersion":"86_38", + "name":"GRCh38" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "gene_disease_association", + "variation", + "variation_functional_score", + "regulation", + "protein", + "conservation", + "clinical" + ] + }, + { + "id":"mmusculus", + "scientificName":"Mus musculus", + "assemblies":[ + { + "ensemblVersion":"82_38", + "name":"GRCm38" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "regulation", + "protein", + "conservation" + ] + }, + { + "id":"drerio", + "scientificName":"Danio rerio", + "assemblies":[ + { + "ensemblVersion":"82_10", + "name":"GRCz10" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"rnorvegicus", + "scientificName":"Rattus norvegicus", + "assemblies":[ + { + "ensemblVersion":"82_6", + "name":"Rnor_6.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"ptroglodytes", + "scientificName":"Pan troglodytes", + "assemblies":[ + { + "ensemblVersion":"82_214", + "name":"CHIMP2.1.4" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"ggorilla", + "scientificName":"Gorilla gorilla", + "assemblies":[ + { + "ensemblVersion":"82_31", + "name":"gorGor3.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"pabelii", + "scientificName":"Pongo abelii", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"PPYG2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"mmulatta", + "scientificName":"Macaca mulatta", + "assemblies":[ + { + "ensemblVersion":"82_10", + "name":"MMUL_1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"csabaeus", + "scientificName":"Chlorocebus sabaeus", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"ChlSab1.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"sscrofa", + "scientificName":"Sus scrofa", + "assemblies":[ + { + "ensemblVersion":"82_102", + "name":"Sscrofa10.2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"cfamiliaris", + "scientificName":"Canis familiaris", + "assemblies":[ + { + "ensemblVersion":"82_31", + "name":"CanFam3.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"ecaballus", + "scientificName":"Equus caballus", + "assemblies":[ + { + "ensemblVersion":"82_2", + "name":"EquCab2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"ocuniculus", + "scientificName":"Oryctolagus cuniculus", + "assemblies":[ + { + "ensemblVersion":"82_2", + "name":"OryCun2.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"ggallus", + "scientificName":"Gallus gallus", + "assemblies":[ + { + "ensemblVersion":"82_4", + "name":"Galgal4" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"btaurus", + "scientificName":"Bos taurus", + "assemblies":[ + { + "ensemblVersion":"82_31", + "name":"UMD3.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"fcatus", + "scientificName":"Felis catus", + "assemblies":[ + { + "ensemblVersion":"82_62", + "name":"Felis_catus_6.2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"cintestinalis", + "scientificName":"Ciona intestinalis", + "assemblies":[ + { + "ensemblVersion":"82_3", + "name":"KH" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"oaries", + "scientificName":"Ovis aries", + "assemblies":[ + { + "ensemblVersion":"82_31", + "name":"Oar_v3.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"olatipes", + "scientificName":"Oryzias latipes", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"HdrR" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"ttruncatus", + "scientificName":"Tursiops truncatus", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"turTru1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"lafricana", + "scientificName":"Loxodonta africana", + "assemblies":[ + { + "ensemblVersion":"82_3", + "name":"Loxafr3.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"aplatyrhynchos", + "scientificName":"Anas platyrhynchos", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"BGI_duck_1.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ], + "metazoa":[ + { + "id":"celegans", + "scientificName":"Caenorhabditis elegans", + "assemblies":[ + { + "ensemblVersion":"29_82_245", + "name":"WBcel235" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"dmelanogaster", + "scientificName":"Drosophila melanogaster", + "assemblies":[ + { + "ensemblVersion":"29_82_6", + "name":"BDGP6" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"dsimulans", + "scientificName":"Drosophila simulans", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"GCA_000259055.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"agambiae", + "scientificName":"Anopheles gambiae", + "assemblies":[ + { + "ensemblVersion":"29_82_4", + "name":"AgamP4" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"bmori", + "scientificName":"Bombyx mori", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"GCA_000151625.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ], + "fungi":[ + { + "id":"scerevisiae", + "scientificName":"Saccharomyces cerevisiae", + "assemblies":[ + { + "ensemblVersion":"29_82_4", + "name":"R64-1-1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"spombe", + "scientificName":"Schizosaccharomyces pombe", + "assemblies":[ + { + "ensemblVersion":"29_82_2", + "name":"ASM294v2" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"afumigatus", + "scientificName":"Aspergillus fumigatus", + "assemblies":[ + { + "ensemblVersion":"29_82_2", + "name":"CADRE" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"aniger", + "scientificName":"Aspergillus niger", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"CADRE" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"anidulans", + "scientificName":"Aspergillus nidulans", + "assemblies":[ + { + "ensemblVersion":"29_82_6", + "name":"ASM1142v1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"moryzae", + "scientificName":"Magnaporthe oryzae", + "assemblies":[ + { + "ensemblVersion":"29_82_8", + "name":"MG8" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ], + "protist":[ + { + "id":"pfalciparum", + "scientificName":"Plasmodium falciparum", + "assemblies":[ + { + "ensemblVersion":"29_82_3", + "name":"ASM276v1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"lmajor", + "scientificName":"Leishmania major", + "assemblies":[ + { + "ensemblVersion":"29_82_2", + "name":"ASM272v2" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"ddiscoideum", + "scientificName":"Dictyostelium discoideum", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"dictybase.01" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"glamblia", + "scientificName":"Giardia lamblia", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"GCA_000002435.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"pultimum", + "scientificName":"Pythium ultimum", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"pug" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ], + "plants":[ + { + "id":"athaliana", + "scientificName":"Arabidopsis thaliana", + "assemblies":[ + { + "ensemblVersion":"29_82_10", + "name":"TAIR10" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"alyrata", + "scientificName":"Arabidopsis lyrata", + "assemblies":[ + { + "ensemblVersion":"29_82_10", + "name":"v.1.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"osativa", + "scientificName":"Oryza sativa", + "assemblies":[ + { + "ensemblVersion":"29_82_7", + "name":"IRGSP-1.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"gmax", + "scientificName":"Glycine max", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"V1.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"vvinifera", + "scientificName":"Vitis vinifera", + "assemblies":[ + { + "ensemblVersion":"29_82_3", + "name":"IGGP_12x" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"zmays", + "scientificName":"Zea mays", + "assemblies":[ + { + "ensemblVersion":"29_82_6", + "name":"AGPv3" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"hvulgare", + "scientificName":"Hordeum vulgare", + "assemblies":[ + { + "ensemblVersion":"29_82_2", + "name":"082214v1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"sbicolor", + "scientificName":"Sorghum bicolor", + "assemblies":[ + { + "ensemblVersion":"29_82_14", + "name":"Sorbi1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"slycopersicum", + "scientificName":"Solanum lycopersicum", + "assemblies":[ + { + "ensemblVersion":"29_82_250", + "name":"GCA_000188115.2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"stuberosum", + "scientificName":"Solanum tuberosum", + "assemblies":[ + { + "ensemblVersion":"29_82_4", + "name":"3.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ] + } +} From 9aec33f2eefd59132baea3a3cb3012188c2185af Mon Sep 17 00:00:00 2001 From: imedina Date: Thu, 20 Oct 2016 15:34:41 +0100 Subject: [PATCH 03/24] core: initial work in new cache, CacheManager class created --- .../cellbase/core/cache/CacheManager.java | 30 +++++++++++++++++++ .../cellbase/core/config/CacheProperties.java | 4 +-- .../cellbase/lib/impl/MongoDBAdaptor.java | 11 ++++++- 3 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java new file mode 100644 index 0000000000..5154c2dcde --- /dev/null +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -0,0 +1,30 @@ +package org.opencb.cellbase.core.cache; + +import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.commons.datastore.core.QueryResult; + +/** + * Created by imedina on 20/10/16. + */ +public class CacheManager { + + + public CacheManager() { + } + + public CacheManager(CellBaseConfiguration configuration) { + + + } + + + public QueryResult get(String key, Class clazz) { + + return null; + } + + public void set(QueryResult queryResult) { + + } + +} diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java index f2b25ba34c..135fac2cdf 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java @@ -6,13 +6,13 @@ public class CacheProperties { /** - * This field contain the host and port, ie. host[:port] + * This field contain the host and port, ie. host[:port]. */ private String host; private boolean active; /** - * Accepted values are: JSON, Kryo + * Accepted values are: JSON, Kryo. */ private String serialization; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index 94a7704524..19c2cd89a0 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -19,10 +19,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.mongodb.QueryBuilder; import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.*; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import org.bson.*; import org.bson.conversions.Bson; import org.opencb.biodata.models.core.Region; +import org.opencb.cellbase.core.cache.CacheManager; import org.opencb.cellbase.core.common.IntervalFeatureFrequency; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -45,6 +49,8 @@ enum QueryValueType {INTEGER, STRING} protected MongoDataStore mongoDataStore; protected MongoDBCollection mongoDBCollection; + private CacheManager cacheManager; + protected Logger logger = LoggerFactory.getLogger(this.getClass()); protected ObjectMapper objectMapper; @@ -428,6 +434,9 @@ protected QueryResult executeBsonQuery(Bson query, Bson projection, Query return result; } + + + @Deprecated protected QueryResult executeQuery(Object id, Document query, QueryOptions options) { return executeQueryList2(Arrays.asList(id), Arrays.asList(query), options, mongoDBCollection).get(0); From 011b7d5948f4b965b0a92bfe4d2752bb53dfeed0 Mon Sep 17 00:00:00 2001 From: Swaathik Date: Thu, 20 Oct 2016 17:48:17 +0100 Subject: [PATCH 04/24] core: CellbaseConfiguration passed to all MongodbAdaptors --- cellbase-core/pom.xml | 10 ++ .../cellbase/core/cache/CacheManager.java | 43 +++++++- .../src/main/resources/configuration.json | 21 +--- cellbase-grpc/pom.xml | 102 ------------------ .../lib/impl/ClinicalMongoDBAdaptor.java | 6 +- .../lib/impl/ConservationMongoDBAdaptor.java | 6 +- .../cellbase/lib/impl/GeneMongoDBAdaptor.java | 6 +- .../lib/impl/GenomeMongoDBAdaptor.java | 6 +- .../cellbase/lib/impl/MetaMongoDBAdaptor.java | 6 +- .../cellbase/lib/impl/MongoDBAdaptor.java | 12 ++- .../lib/impl/MongoDBAdaptorFactory.java | 25 +++-- .../lib/impl/ProteinMongoDBAdaptor.java | 6 +- ...oteinProteinInteractionMongoDBAdaptor.java | 6 +- .../lib/impl/RegulationMongoDBAdaptor.java | 6 +- .../lib/impl/TranscriptMongoDBAdaptor.java | 6 +- .../lib/impl/VariantMongoDBAdaptor.java | 6 +- .../cellbase/lib/impl/XRefMongoDBAdaptor.java | 6 +- 17 files changed, 118 insertions(+), 161 deletions(-) delete mode 100644 cellbase-grpc/pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 4039300c57..21acbfeda5 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -44,6 +44,16 @@ org.glassfish.jersey.core jersey-client + + org.redisson + redisson + 2.2.0 + + + com.esotericsoftware.kryo + kryo + 2.22 + junit junit diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 5154c2dcde..5a1ca4d2d9 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -1,12 +1,25 @@ package org.opencb.cellbase.core.cache; import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; +import org.redisson.Config; +import org.redisson.Redisson; + +import org.redisson.RedissonClient; +import org.redisson.codec.JsonJacksonCodec; +import org.redisson.codec.KryoCodec; + /** * Created by imedina on 20/10/16. */ public class CacheManager { + private CellBaseConfiguration cellBaseConfiguration; + private Config redissonConfig; + private RedissonClient redissonClient; + public CacheManager() { @@ -14,17 +27,43 @@ public CacheManager() { public CacheManager(CellBaseConfiguration configuration) { + if (configuration != null) { + String codec = configuration.getCache().getSerialization(); + redissonConfig = new Config(); + redissonConfig.useSingleServer().setAddress(configuration.getCache().getHost()); + + if ("Kryo".equals(codec)) { + redissonConfig.setCodec(new KryoCodec()); + } + else if("JSON".equals(codec)) { + redissonConfig.setCodec(new JsonJacksonCodec()); + } + this.redissonClient = Redisson.create(redissonConfig); + } } public QueryResult get(String key, Class clazz) { - + //redissonClient.getMap() + //if empty return null else type cast object return null; } - public void set(QueryResult queryResult) { + public void set(String Key, Query query, QueryResult queryResult) { +// redissonClient.getMap(key); + // insert the object into map with key and query and queryResult + } + + public String createKey(String species, String collection, Query query, QueryOptions queryOptions){ + String key = null; + // cellBase + // version We get from CellBaseConfiguration + // CB:version:species:collection + // Sort query and queryOptions + // get SHA1 + return key; } } diff --git a/cellbase-core/src/main/resources/configuration.json b/cellbase-core/src/main/resources/configuration.json index b9ec534e2a..2c22997992 100644 --- a/cellbase-core/src/main/resources/configuration.json +++ b/cellbase-core/src/main/resources/configuration.json @@ -3,8 +3,6 @@ "apiVersion": "${cellbase.version}", "wiki": "https://github.com/opencb/cellbase/wiki", "databases": { -// "user": "${CELLBASE.DB.USER}", -// "password": "${CELLBASE.DB.PASSWORD}", "mongodb": { "host": "${CELLBASE.DB.MONGODB.HOST}", "user": "${CELLBASE.DB.USER}", @@ -26,24 +24,7 @@ "user": "${CELLBASE.DB.USER}", "password": "${CELLBASE.DB.PASSWORD}" } - // { - // "id": "", - // "host": "${CELLBASE.DB.NEO4J.HOST}", - // "user": "${CELLBASE.DB.NEO4J.USER}", - // "password": "${CELLBASE.DB.NEO4J.PASSWORD}" - // }, - // { - // "id": "", - // "host": "${CELLBASE.DB.NEO4J.HOST}", - // "user": "${CELLBASE.DB.NEO4J.USER}", - // "password": "${CELLBASE.DB.NEO4J.PASSWORD}" - // }, - // { - // "id": "", - // "host": "${CELLBASE.DB.NEO4J.HOST}", - // "user": "${CELLBASE.DB.NEO4J.USER}", - // "password": "${CELLBASE.DB.NEO4J.PASSWORD}" - // }, + } }, "defaultOutdir": "/tmp", diff --git a/cellbase-grpc/pom.xml b/cellbase-grpc/pom.xml deleted file mode 100644 index 0bfdfeb55a..0000000000 --- a/cellbase-grpc/pom.xml +++ /dev/null @@ -1,102 +0,0 @@ - - - - - - cellbase - org.opencb.cellbase - 4.0.1 - - 4.0.0 - - cellbase-grpc - - - - - org.opencb.cellbase - cellbase-core - - - org.opencb.cellbase - cellbase-mongodb - - - org.opencb.biodata - biodata-models - - - com.google.protobuf - protobuf-java - - - io.grpc - grpc-all - 0.13.1 - - - junit - junit - test - - - - - - - - - - - - - - kr.motd.maven - os-maven-plugin - 1.4.0.Final - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.5.0 - - - com.google.protobuf:protoc:3.0.0-beta-2:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:0.13.1:exe:${os.detected.classifier} - - - - - - compile - compile-custom - - - - - - - \ No newline at end of file diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java index c9053bcb39..5e1c77970a 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java @@ -27,6 +27,7 @@ import org.opencb.biodata.models.variant.avro.VariantTraitAssociation; import org.opencb.cellbase.core.api.ClinicalDBAdaptor; import org.opencb.cellbase.core.common.clinical.ClinicalVariant; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; @@ -41,8 +42,9 @@ */ public class ClinicalMongoDBAdaptor extends MongoDBAdaptor implements ClinicalDBAdaptor { - public ClinicalMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ClinicalMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("clinical"); logger.debug("ClinicalMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java index caf6751e76..5de6f90d43 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java @@ -25,6 +25,7 @@ import org.opencb.biodata.models.variant.avro.Score; import org.opencb.cellbase.core.api.ConservationDBAdaptor; import org.opencb.biodata.models.core.GenomicScoreRegion; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -40,8 +41,9 @@ @Deprecated public class ConservationMongoDBAdaptor extends MongoDBAdaptor implements ConservationDBAdaptor { - public ConservationMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ConservationMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("conservation"); logger.debug("ConservationMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java index 98fd0c9e7e..90db4172d4 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java @@ -25,6 +25,7 @@ import org.opencb.biodata.models.core.Gene; import org.opencb.biodata.models.core.Region; import org.opencb.cellbase.core.api.GeneDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -40,8 +41,9 @@ */ public class GeneMongoDBAdaptor extends MongoDBAdaptor implements GeneDBAdaptor { - public GeneMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public GeneMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("gene"); logger.debug("GeneMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java index 54f1f8ca18..fb96a90c3e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java @@ -25,6 +25,7 @@ import org.opencb.cellbase.core.common.DNASequenceUtils; import org.opencb.biodata.models.core.GenomeSequenceFeature; import org.opencb.biodata.models.core.GenomicScoreRegion; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -43,8 +44,9 @@ public class GenomeMongoDBAdaptor extends MongoDBAdaptor implements GenomeDBAdap private MongoDBCollection genomeInfoMongoDBCollection; private MongoDBCollection conservationMongoDBCollection; - public GenomeMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public GenomeMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); genomeInfoMongoDBCollection = mongoDataStore.getCollection("genome_info"); mongoDBCollection = mongoDataStore.getCollection("genome_sequence"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java index dffc18a7c4..09f75768b1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java @@ -3,6 +3,7 @@ import org.bson.BsonDocument; import org.bson.Document; import org.opencb.cellbase.core.api.CellBaseDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; @@ -17,8 +18,9 @@ */ public class MetaMongoDBAdaptor extends MongoDBAdaptor implements CellBaseDBAdaptor { - public MetaMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public MetaMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("metadata"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index 19c2cd89a0..b1bd502436 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -28,6 +28,7 @@ import org.opencb.biodata.models.core.Region; import org.opencb.cellbase.core.cache.CacheManager; import org.opencb.cellbase.core.common.IntervalFeatureFrequency; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; @@ -48,21 +49,23 @@ enum QueryValueType {INTEGER, STRING} protected MongoDataStore mongoDataStore; protected MongoDBCollection mongoDBCollection; - + protected CellBaseConfiguration cellBaseConfiguration; private CacheManager cacheManager; protected Logger logger = LoggerFactory.getLogger(this.getClass()); protected ObjectMapper objectMapper; - public MongoDBAdaptor(MongoDataStore mongoDataStore) { - this("", "", mongoDataStore); + public MongoDBAdaptor(MongoDataStore mongoDataStore, CellBaseConfiguration cellBaseConfiguration) { + this("", "", mongoDataStore, cellBaseConfiguration); } - public MongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { + public MongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { this.species = species; this.assembly = assembly; this.mongoDataStore = mongoDataStore; + this.cellBaseConfiguration = cellBaseConfiguration; logger = LoggerFactory.getLogger(this.getClass().toString()); objectMapper = new ObjectMapper(); @@ -420,6 +423,7 @@ protected QueryResult executeBsonQuery(Bson query, Bson projection, Query MongoDBCollection mongoDBCollection, Class clazz) { QueryResult result = null; if (options.getBoolean("cache")) { +// cacheManger.createKey(this.species, mongoDBCollection, query, queryOptions); // result = cacheManager.get(); // result = null; if (result == null) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java index 127f37443f..8cdc4bf8ae 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java @@ -35,10 +35,12 @@ public class MongoDBAdaptorFactory extends DBAdaptorFactory { * MongoDataStoreManager acts as singleton by keeping a reference to all databases connections created. */ private MongoDataStoreManager mongoDataStoreManager; + private CellBaseConfiguration cellBaseConfiguration; // private static Map mongoDatastoreFactory; public MongoDBAdaptorFactory(CellBaseConfiguration cellBaseConfiguration) { super(cellBaseConfiguration); + this.cellBaseConfiguration = cellBaseConfiguration; init(); } @@ -151,7 +153,7 @@ public GenomeDBAdaptor getGenomeDBAdaptor(String species) { @Override public GenomeDBAdaptor getGenomeDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new GenomeMongoDBAdaptor(species, assembly, mongoDatastore); + return new GenomeMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } @Override @@ -162,7 +164,7 @@ public CellBaseDBAdaptor getMetaDBAdaptor(String species) { @Override public CellBaseDBAdaptor getMetaDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new MetaMongoDBAdaptor(species, assembly, mongoDatastore); + return new MetaMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } @Override @@ -173,7 +175,8 @@ public GeneDBAdaptor getGeneDBAdaptor(String species) { @Override public GeneDBAdaptor getGeneDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - GeneMongoDBAdaptor geneMongoDBAdaptor = new GeneMongoDBAdaptor(species, assembly, mongoDatastore); + GeneMongoDBAdaptor geneMongoDBAdaptor = new GeneMongoDBAdaptor(species, assembly, mongoDatastore, + cellBaseConfiguration); // geneMongoDBAdaptor.setClinicalDBAdaptor(getClinicalDBAdaptor(species, assembly)); return geneMongoDBAdaptor; } @@ -187,7 +190,7 @@ public TranscriptDBAdaptor getTranscriptDBAdaptor(String species) { @Override public TranscriptDBAdaptor getTranscriptDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new TranscriptMongoDBAdaptor(species, assembly, mongoDatastore); + return new TranscriptMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } @@ -199,7 +202,7 @@ public ConservationDBAdaptor getConservationDBAdaptor(String species) { @Override public ConservationDBAdaptor getConservationDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ConservationMongoDBAdaptor(species, assembly, mongoDatastore); + return new ConservationMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } @@ -211,7 +214,7 @@ public XRefDBAdaptor getXRefDBAdaptor(String species) { @Override public XRefDBAdaptor getXRefDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new XRefMongoDBAdaptor(species, assembly, mongoDatastore); + return new XRefMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } @@ -223,7 +226,7 @@ public VariantDBAdaptor getVariationDBAdaptor(String species) { @Override public VariantDBAdaptor getVariationDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new VariantMongoDBAdaptor(species, assembly, mongoDatastore); + return new VariantMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } // @Override @@ -258,7 +261,7 @@ public ClinicalDBAdaptor getClinicalDBAdaptor(String species) { @Override public ClinicalDBAdaptor getClinicalDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ClinicalMongoDBAdaptor(species, assembly, mongoDatastore); + return new ClinicalMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } @@ -296,7 +299,7 @@ public ProteinDBAdaptor getProteinDBAdaptor(String species) { @Override public ProteinDBAdaptor getProteinDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ProteinMongoDBAdaptor(species, assembly, mongoDatastore); + return new ProteinMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } @@ -308,7 +311,7 @@ public ProteinProteinInteractionDBAdaptor getProteinProteinInteractionDBAdaptor( @Override public ProteinProteinInteractionDBAdaptor getProteinProteinInteractionDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ProteinProteinInteractionMongoDBAdaptor(species, assembly, mongoDatastore); + return new ProteinProteinInteractionMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } @@ -320,7 +323,7 @@ public RegulationDBAdaptor getRegulationDBAdaptor(String species) { @Override public RegulationDBAdaptor getRegulationDBAdaptor(String species, String assembly) { MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new RegulationMongoDBAdaptor(species, assembly, mongoDatastore); + return new RegulationMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); } // // @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java index c7c5c173f0..de796402c1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java @@ -26,6 +26,7 @@ import org.opencb.biodata.models.variant.avro.ProteinVariantAnnotation; import org.opencb.biodata.models.variant.avro.Score; import org.opencb.cellbase.core.api.ProteinDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.variant.annotation.VariantAnnotationUtils; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -71,8 +72,9 @@ public class ProteinMongoDBAdaptor extends MongoDBAdaptor implements ProteinDBAd } - public ProteinMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ProteinMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("protein"); proteinSubstitutionMongoDBCollection = mongoDataStore.getCollection("protein_functional_prediction"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java index 483d934aa4..f94fb837db 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java @@ -21,6 +21,7 @@ import org.bson.conversions.Bson; import org.opencb.biodata.models.protein.Interaction; import org.opencb.cellbase.core.api.ProteinProteinInteractionDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; @@ -37,8 +38,9 @@ public class ProteinProteinInteractionMongoDBAdaptor extends MongoDBAdaptor implements ProteinProteinInteractionDBAdaptor { - public ProteinProteinInteractionMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ProteinProteinInteractionMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("variation"); logger.debug("ProteinProteinInteractionMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java index 89fb8cf486..7352dab1a7 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java @@ -22,6 +22,7 @@ import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.core.RegulatoryFeature; import org.opencb.cellbase.core.api.RegulationDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -38,8 +39,9 @@ */ public class RegulationMongoDBAdaptor extends MongoDBAdaptor implements RegulationDBAdaptor { - public RegulationMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public RegulationMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("regulatory_region"); logger.debug("RegulationMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java index faf67961a4..6daa174e39 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java @@ -10,6 +10,7 @@ import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.core.Transcript; import org.opencb.cellbase.core.api.TranscriptDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -24,8 +25,9 @@ */ public class TranscriptMongoDBAdaptor extends MongoDBAdaptor implements TranscriptDBAdaptor { - public TranscriptMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public TranscriptMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("gene"); logger.debug("TranscriptMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java index 2b7092dda9..77c6285d96 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java @@ -28,6 +28,7 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.Score; import org.opencb.cellbase.core.api.VariantDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.cellbase.lib.VariantMongoIterator; import org.opencb.commons.datastore.core.Query; @@ -53,8 +54,9 @@ public class VariantMongoDBAdaptor extends MongoDBAdaptor implements VariantDBAd private MongoDBCollection caddDBCollection; - public VariantMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public VariantMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("variation"); caddDBCollection = mongoDataStore.getCollection("variation_functional_score"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java index c60aa465fc..fdb959cca4 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java @@ -23,6 +23,7 @@ import org.bson.conversions.Bson; import org.opencb.biodata.models.core.Xref; import org.opencb.cellbase.core.api.XRefDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; @@ -40,8 +41,9 @@ */ public class XRefMongoDBAdaptor extends MongoDBAdaptor implements XRefDBAdaptor { - public XRefMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public XRefMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, mongoDataStore, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("gene"); logger.debug("XRefMongoDBAdaptor: in 'constructor'"); From 99708948540f3e23fbb9882e45c4bf984e9a8d6e Mon Sep 17 00:00:00 2001 From: Swaathik Date: Fri, 21 Oct 2016 09:51:49 +0100 Subject: [PATCH 05/24] core: Checkstyle corrections made --- .../org/opencb/cellbase/core/cache/CacheManager.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 5a1ca4d2d9..9048530f35 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -34,8 +34,7 @@ public CacheManager(CellBaseConfiguration configuration) { if ("Kryo".equals(codec)) { redissonConfig.setCodec(new KryoCodec()); - } - else if("JSON".equals(codec)) { + } else if ("JSON".equals(codec)) { redissonConfig.setCodec(new JsonJacksonCodec()); } this.redissonClient = Redisson.create(redissonConfig); @@ -50,13 +49,13 @@ public QueryResult get(String key, Class clazz) { return null; } - public void set(String Key, Query query, QueryResult queryResult) { + public void set(String key, Query query, QueryResult queryResult) { // redissonClient.getMap(key); // insert the object into map with key and query and queryResult } - public String createKey(String species, String collection, Query query, QueryOptions queryOptions){ - String key = null; + public String createKey(String species, String collection, Query query, QueryOptions queryOptions) { + String key = "test"; // cellBase // version We get from CellBaseConfiguration // CB:version:species:collection From 828b3fd5557339d628f9bbd313e000ae58f74a9e Mon Sep 17 00:00:00 2001 From: Swaathik Date: Fri, 21 Oct 2016 11:04:00 +0100 Subject: [PATCH 06/24] lib: CreateMongoDBDatastore has been moved to MongoDBAdaptor --- .../cellbase/core/api/DBAdaptorFactory.java | 5 +- .../cellbase/core/cache/CacheManager.java | 25 ++-- .../lib/impl/ClinicalMongoDBAdaptor.java | 5 +- .../lib/impl/ConservationMongoDBAdaptor.java | 5 +- .../cellbase/lib/impl/GeneMongoDBAdaptor.java | 6 +- .../lib/impl/GenomeMongoDBAdaptor.java | 10 +- .../cellbase/lib/impl/MetaMongoDBAdaptor.java | 6 +- .../cellbase/lib/impl/MongoDBAdaptor.java | 141 ++++++++++++++++-- .../lib/impl/MongoDBAdaptorFactory.java | 134 ++--------------- .../lib/impl/ProteinMongoDBAdaptor.java | 5 +- ...oteinProteinInteractionMongoDBAdaptor.java | 5 +- .../lib/impl/RegulationMongoDBAdaptor.java | 5 +- .../lib/impl/TranscriptMongoDBAdaptor.java | 5 +- .../lib/impl/VariantMongoDBAdaptor.java | 5 +- .../cellbase/lib/impl/XRefMongoDBAdaptor.java | 5 +- 15 files changed, 178 insertions(+), 189 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java index 88ebfd3a61..03e95ec98e 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java @@ -17,7 +17,6 @@ package org.opencb.cellbase.core.api; import org.opencb.cellbase.core.config.CellBaseConfiguration; -import org.opencb.cellbase.core.config.Species; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,7 +98,7 @@ public void setConfiguration(CellBaseConfiguration cellBaseConfiguration) { public abstract ConservationDBAdaptor getConservationDBAdaptor(String species, String assembly); - protected Species getSpecies(String speciesName) { +/* protected Species getSpecies(String speciesName) { Species species = null; for (Species sp : cellBaseConfiguration.getAllSpecies()) { if (speciesName.equalsIgnoreCase(sp.getId()) || speciesName.equalsIgnoreCase(sp.getScientificName())) { @@ -122,6 +121,6 @@ protected String getAssembly(Species species, String assemblyName) { } } return assembly; - } + }*/ } diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 9048530f35..a2d22ebfcb 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -6,7 +6,6 @@ import org.opencb.commons.datastore.core.QueryResult; import org.redisson.Config; import org.redisson.Redisson; - import org.redisson.RedissonClient; import org.redisson.codec.JsonJacksonCodec; import org.redisson.codec.KryoCodec; @@ -21,7 +20,6 @@ public class CacheManager { private RedissonClient redissonClient; - public CacheManager() { } @@ -32,9 +30,9 @@ public CacheManager(CellBaseConfiguration configuration) { redissonConfig = new Config(); redissonConfig.useSingleServer().setAddress(configuration.getCache().getHost()); - if ("Kryo".equals(codec)) { + if ("Kryo".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new KryoCodec()); - } else if ("JSON".equals(codec)) { + } else if ("JSON".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new JsonJacksonCodec()); } this.redissonClient = Redisson.create(redissonConfig); @@ -54,15 +52,18 @@ public void set(String key, Query query, QueryResult queryResult) { // insert the object into map with key and query and queryResult } - public String createKey(String species, String collection, Query query, QueryOptions queryOptions) { - String key = "test"; - // cellBase - // version We get from CellBaseConfiguration - // CB:version:species:collection - // Sort query and queryOptions + public String createKey(String species, String collection, String subcategory, Query query, QueryOptions queryOptions) { + String key = ""; + if (queryOptions.getBoolean("cache", false)) { + key = "set"; + // cellBase + // version We get from CellBaseConfiguration + // CB:version:species:collection + // Sort query and queryOptions + + // get SHA1 - // get SHA1 + } return key; } - } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java index 5e1c77970a..6622d552a3 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java @@ -31,7 +31,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -42,9 +41,9 @@ */ public class ClinicalMongoDBAdaptor extends MongoDBAdaptor implements ClinicalDBAdaptor { - public ClinicalMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + public ClinicalMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("clinical"); logger.debug("ClinicalMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java index 5de6f90d43..a6c17f8704 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java @@ -30,7 +30,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -41,9 +40,9 @@ @Deprecated public class ConservationMongoDBAdaptor extends MongoDBAdaptor implements ConservationDBAdaptor { - public ConservationMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + public ConservationMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("conservation"); logger.debug("ConservationMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java index 90db4172d4..8c2e080971 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java @@ -31,7 +31,6 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -41,9 +40,8 @@ */ public class GeneMongoDBAdaptor extends MongoDBAdaptor implements GeneDBAdaptor { - public GeneMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, - CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + public GeneMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("gene"); logger.debug("GeneMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java index fb96a90c3e..ecc053dc87 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java @@ -20,18 +20,17 @@ import com.mongodb.client.model.Filters; import org.bson.Document; import org.bson.conversions.Bson; +import org.opencb.biodata.models.core.GenomeSequenceFeature; +import org.opencb.biodata.models.core.GenomicScoreRegion; import org.opencb.biodata.models.core.Region; import org.opencb.cellbase.core.api.GenomeDBAdaptor; import org.opencb.cellbase.core.common.DNASequenceUtils; -import org.opencb.biodata.models.core.GenomeSequenceFeature; -import org.opencb.biodata.models.core.GenomicScoreRegion; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -44,9 +43,8 @@ public class GenomeMongoDBAdaptor extends MongoDBAdaptor implements GenomeDBAdap private MongoDBCollection genomeInfoMongoDBCollection; private MongoDBCollection conservationMongoDBCollection; - public GenomeMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, - CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + public GenomeMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); genomeInfoMongoDBCollection = mongoDataStore.getCollection("genome_info"); mongoDBCollection = mongoDataStore.getCollection("genome_sequence"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java index 09f75768b1..8fb4391193 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java @@ -7,7 +7,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.Iterator; import java.util.List; @@ -18,9 +17,8 @@ */ public class MetaMongoDBAdaptor extends MongoDBAdaptor implements CellBaseDBAdaptor { - public MetaMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, - CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + public MetaMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("metadata"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index b1bd502436..96ac819461 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -29,11 +29,16 @@ import org.opencb.cellbase.core.cache.CacheManager; import org.opencb.cellbase.core.common.IntervalFeatureFrequency; import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.cellbase.core.config.DatabaseCredentials; +import org.opencb.cellbase.core.config.Species; +import org.opencb.commons.datastore.core.DataStoreServerAddress; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; import org.opencb.commons.datastore.mongodb.MongoDataStore; +import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +52,7 @@ enum QueryValueType {INTEGER, STRING} protected String species; protected String assembly; + protected MongoDataStoreManager mongoDataStoreManager; protected MongoDataStore mongoDataStore; protected MongoDBCollection mongoDBCollection; protected CellBaseConfiguration cellBaseConfiguration; @@ -56,21 +62,21 @@ enum QueryValueType {INTEGER, STRING} protected ObjectMapper objectMapper; - public MongoDBAdaptor(MongoDataStore mongoDataStore, CellBaseConfiguration cellBaseConfiguration) { - this("", "", mongoDataStore, cellBaseConfiguration); + public MongoDBAdaptor(CellBaseConfiguration cellBaseConfiguration) { + this("", "", cellBaseConfiguration); } - public MongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, - CellBaseConfiguration cellBaseConfiguration) { + public MongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { this.species = species; this.assembly = assembly; - this.mongoDataStore = mongoDataStore; this.cellBaseConfiguration = cellBaseConfiguration; logger = LoggerFactory.getLogger(this.getClass().toString()); objectMapper = new ObjectMapper(); initSpeciesAssembly(species, assembly); + init(); + this.mongoDataStore = createMongoDBDatastore(species, assembly); // jsonObjectMapper = new ObjectMapper(); } @@ -83,6 +89,119 @@ private void initSpeciesAssembly(String species, String assembly) { } } + private void init() { + if (mongoDataStoreManager == null) { +// String[] hosts = cellBaseConfiguration.getDatabases().get("mongodb").getHost().split(","); + String[] hosts = cellBaseConfiguration.getDatabases().getMongodb().getHost().split(","); + List dataStoreServerAddresses = new ArrayList<>(hosts.length); + for (String host : hosts) { + String[] hostPort = host.split(":"); + if (hostPort.length == 1) { + dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort[0], 27017)); + } else { + dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort[0], Integer.parseInt(hostPort[1]))); + } + } + mongoDataStoreManager = new MongoDataStoreManager(dataStoreServerAddresses); + logger.debug("MongoDBAdaptorFactory constructor, this should be only be printed once"); + } + +// logger = LoggerFactory.getLogger(this.getClass()); + } + + private MongoDataStore createMongoDBDatastore(String species, String assembly) { + /** + Database name has the following pattern in lower case and with no '.' in the name: + cellbase_speciesId_assembly_cellbaseVersion + Example: + cellbase_hsapiens_grch37_v3 + **/ + +// DatabaseProperties mongodbCredentials = cellBaseConfiguration.getDatabases().get("mongodb"); + DatabaseCredentials mongodbCredentials = cellBaseConfiguration.getDatabases().getMongodb(); + + // We need to look for the species object in the configuration + Species speciesObject = getSpecies(species); + if (speciesObject != null) { + species = speciesObject.getId(); + assembly = getAssembly(speciesObject, assembly).toLowerCase(); + + if (species != null && !species.isEmpty() && assembly != null && !assembly.isEmpty()) { + + // Database name is built following the above pattern + String database = "cellbase" + "_" + species + "_" + assembly.replaceAll("\\.", "").replaceAll("-", "") + .replaceAll("_", "") + "_" + cellBaseConfiguration.getVersion(); + logger.debug("Database for the species is '{}'", database); + + MongoDBConfiguration mongoDBConfiguration; + MongoDBConfiguration.Builder builder; + // For authenticated databases + if (!mongodbCredentials.getUser().isEmpty() + && !mongodbCredentials.getPassword().isEmpty()) { + // MongoDB could authenticate against different databases + if (mongodbCredentials.getOptions().containsKey("authenticationDatabase")) { + builder = MongoDBConfiguration.builder() + .add("username", mongodbCredentials.getUser()) + .add("password", mongodbCredentials.getPassword()) + .add("readPreference", mongodbCredentials.getOptions().get("readPreference")) + .add("authenticationDatabase", mongodbCredentials.getOptions() + .get("authenticationDatabase")); + } else { + builder = MongoDBConfiguration.builder() + .add("username", mongodbCredentials.getUser()) + .add("password", mongodbCredentials.getPassword()) + .add("readPreference", mongodbCredentials.getOptions().get("readPreference")); + } + + String replicaSet = mongodbCredentials.getOptions().get("replicaSet"); + if (replicaSet != null && !replicaSet.isEmpty() && !replicaSet.contains("CELLBASE.DB.MONGODB.REPLICASET")) { + builder.add("replicaSet", mongodbCredentials.getOptions().get("replicaSet")); + } + mongoDBConfiguration = builder.build(); + } else { + mongoDBConfiguration = MongoDBConfiguration.builder().init().build(); + } + + // A MongoDataStore to this host and database is returned + MongoDataStore mongoDatastore = mongoDataStoreManager.get(database, mongoDBConfiguration); + + // we return the MongoDataStore object + return mongoDatastore; + } else { + logger.error("Species name or assembly are not valid, species '{}', assembly '{}'", species, assembly); + return null; + } + } else { + logger.error("Species name is not valid: '{}'", species); + return null; + } + } + + protected Species getSpecies(String speciesName) { + Species species = null; + for (Species sp : cellBaseConfiguration.getAllSpecies()) { + if (speciesName.equalsIgnoreCase(sp.getId()) || speciesName.equalsIgnoreCase(sp.getScientificName())) { + species = sp; + break; + } + } + return species; + } + + protected String getAssembly(Species species, String assemblyName) { + String assembly = null; + if (assemblyName == null || assemblyName.isEmpty()) { + assembly = species.getAssemblies().get(0).getName(); + } else { + for (Species.Assembly assembly1 : species.getAssemblies()) { + if (assemblyName.equalsIgnoreCase(assembly1.getName())) { + assembly = assembly1.getName(); + } + } + } + return assembly; + } + protected QueryOptions addPrivateExcludeOptions(QueryOptions options) { if (options != null) { if (options.get("exclude") == null) { @@ -270,7 +389,6 @@ protected QueryResult groupBy(Bson query, List groupByField, String feat } - public QueryResult getIntervalFrequencies(Bson query, Region region, int intervalSize, QueryOptions options) { // MONGO QUERY TO IMPLEMENT // db.variation.aggregate({$match: {$and: [{chromosome: "1"}, {start: {$gt: 251391, $lt: 2701391}}]}}, {$group: @@ -439,20 +557,21 @@ protected QueryResult executeBsonQuery(Bson query, Bson projection, Query } - - @Deprecated protected QueryResult executeQuery(Object id, Document query, QueryOptions options) { return executeQueryList2(Arrays.asList(id), Arrays.asList(query), options, mongoDBCollection).get(0); } + @Deprecated protected QueryResult executeQuery(Object id, Document query, QueryOptions options, MongoDBCollection mongoDBCollection2) { return executeQueryList2(Arrays.asList(id), Arrays.asList(query), options, mongoDBCollection2).get(0); } + @Deprecated protected List executeQueryList2(List ids, List queries, QueryOptions options) { return executeQueryList2(ids, queries, options, mongoDBCollection); } + @Deprecated protected List executeQueryList2(List ids, List queries, QueryOptions options, MongoDBCollection mongoDBCollection2) { @@ -555,9 +674,6 @@ private int getChunkEnd(int id, int chunkSize) { } - - - public QueryResult next(String chromosome, int position, QueryOptions options, MongoDBCollection mongoDBCollection) { QueryBuilder builder; if (options.getString("strand") == null || options.getString("strand").equals("") @@ -724,9 +840,6 @@ protected Document getReturnFields(QueryOptions options) { // } - - - /* * For histograms */ diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java index 8cdc4bf8ae..aae77ee06f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java @@ -19,119 +19,21 @@ import org.bson.Document; import org.opencb.cellbase.core.api.*; import org.opencb.cellbase.core.config.CellBaseConfiguration; -import org.opencb.cellbase.core.config.DatabaseCredentials; -import org.opencb.cellbase.core.config.Species; -import org.opencb.commons.datastore.core.DataStoreServerAddress; -import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; -import org.opencb.commons.datastore.mongodb.MongoDataStore; -import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; - -import java.util.ArrayList; -import java.util.List; public class MongoDBAdaptorFactory extends DBAdaptorFactory { /** * MongoDataStoreManager acts as singleton by keeping a reference to all databases connections created. */ - private MongoDataStoreManager mongoDataStoreManager; + private CellBaseConfiguration cellBaseConfiguration; // private static Map mongoDatastoreFactory; public MongoDBAdaptorFactory(CellBaseConfiguration cellBaseConfiguration) { super(cellBaseConfiguration); this.cellBaseConfiguration = cellBaseConfiguration; - - init(); } - private void init() { - if (mongoDataStoreManager == null) { -// String[] hosts = cellBaseConfiguration.getDatabases().get("mongodb").getHost().split(","); - String[] hosts = cellBaseConfiguration.getDatabases().getMongodb().getHost().split(","); - List dataStoreServerAddresses = new ArrayList<>(hosts.length); - for (String host : hosts) { - String[] hostPort = host.split(":"); - if (hostPort.length == 1) { - dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort[0], 27017)); - } else { - dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort[0], Integer.parseInt(hostPort[1]))); - } - } - mongoDataStoreManager = new MongoDataStoreManager(dataStoreServerAddresses); - logger.debug("MongoDBAdaptorFactory constructor, this should be only be printed once"); - } - -// logger = LoggerFactory.getLogger(this.getClass()); - } - - private MongoDataStore createMongoDBDatastore(String species, String assembly) { - /** - Database name has the following pattern in lower case and with no '.' in the name: - cellbase_speciesId_assembly_cellbaseVersion - Example: - cellbase_hsapiens_grch37_v3 - **/ - -// DatabaseProperties mongodbCredentials = cellBaseConfiguration.getDatabases().get("mongodb"); - DatabaseCredentials mongodbCredentials = cellBaseConfiguration.getDatabases().getMongodb(); - - // We need to look for the species object in the configuration - Species speciesObject = getSpecies(species); - if (speciesObject != null) { - species = speciesObject.getId(); - assembly = getAssembly(speciesObject, assembly).toLowerCase(); - - if (species != null && !species.isEmpty() && assembly != null && !assembly.isEmpty()) { - - // Database name is built following the above pattern - String database = "cellbase" + "_" + species + "_" + assembly.replaceAll("\\.", "").replaceAll("-", "") - .replaceAll("_", "") + "_" + cellBaseConfiguration.getVersion(); - logger.debug("Database for the species is '{}'", database); - - MongoDBConfiguration mongoDBConfiguration; - MongoDBConfiguration.Builder builder; - // For authenticated databases - if (!mongodbCredentials.getUser().isEmpty() - && !mongodbCredentials.getPassword().isEmpty()) { - // MongoDB could authenticate against different databases - if (mongodbCredentials.getOptions().containsKey("authenticationDatabase")) { - builder = MongoDBConfiguration.builder() - .add("username", mongodbCredentials.getUser()) - .add("password", mongodbCredentials.getPassword()) - .add("readPreference", mongodbCredentials.getOptions().get("readPreference")) - .add("authenticationDatabase", mongodbCredentials.getOptions() - .get("authenticationDatabase")); - } else { - builder = MongoDBConfiguration.builder() - .add("username", mongodbCredentials.getUser()) - .add("password", mongodbCredentials.getPassword()) - .add("readPreference", mongodbCredentials.getOptions().get("readPreference")); - } - - String replicaSet = mongodbCredentials.getOptions().get("replicaSet"); - if (replicaSet != null && !replicaSet.isEmpty() && !replicaSet.contains("CELLBASE.DB.MONGODB.REPLICASET")) { - builder.add("replicaSet", mongodbCredentials.getOptions().get("replicaSet")); - } - mongoDBConfiguration = builder.build(); - } else { - mongoDBConfiguration = MongoDBConfiguration.builder().init().build(); - } - - // A MongoDataStore to this host and database is returned - MongoDataStore mongoDatastore = mongoDataStoreManager.get(database, mongoDBConfiguration); - - // we return the MongoDataStore object - return mongoDatastore; - } else { - logger.error("Species name or assembly are not valid, species '{}', assembly '{}'", species, assembly); - return null; - } - } else { - logger.error("Species name is not valid: '{}'", species); - return null; - } - } @Override @@ -152,8 +54,7 @@ public GenomeDBAdaptor getGenomeDBAdaptor(String species) { @Override public GenomeDBAdaptor getGenomeDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new GenomeMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new GenomeMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @Override @@ -163,8 +64,7 @@ public CellBaseDBAdaptor getMetaDBAdaptor(String species) { @Override public CellBaseDBAdaptor getMetaDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new MetaMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new MetaMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @Override @@ -174,9 +74,7 @@ public GeneDBAdaptor getGeneDBAdaptor(String species) { @Override public GeneDBAdaptor getGeneDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - GeneMongoDBAdaptor geneMongoDBAdaptor = new GeneMongoDBAdaptor(species, assembly, mongoDatastore, - cellBaseConfiguration); + GeneMongoDBAdaptor geneMongoDBAdaptor = new GeneMongoDBAdaptor(species, assembly, cellBaseConfiguration); // geneMongoDBAdaptor.setClinicalDBAdaptor(getClinicalDBAdaptor(species, assembly)); return geneMongoDBAdaptor; } @@ -189,8 +87,7 @@ public TranscriptDBAdaptor getTranscriptDBAdaptor(String species) { @Override public TranscriptDBAdaptor getTranscriptDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new TranscriptMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new TranscriptMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -201,8 +98,7 @@ public ConservationDBAdaptor getConservationDBAdaptor(String species) { @Override public ConservationDBAdaptor getConservationDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ConservationMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new ConservationMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -213,8 +109,7 @@ public XRefDBAdaptor getXRefDBAdaptor(String species) { @Override public XRefDBAdaptor getXRefDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new XRefMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new XRefMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -225,8 +120,7 @@ public VariantDBAdaptor getVariationDBAdaptor(String species) { @Override public VariantDBAdaptor getVariationDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new VariantMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new VariantMongoDBAdaptor(species, assembly, cellBaseConfiguration); } // @Override @@ -260,8 +154,7 @@ public ClinicalDBAdaptor getClinicalDBAdaptor(String species) { @Override public ClinicalDBAdaptor getClinicalDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ClinicalMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new ClinicalMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -298,8 +191,7 @@ public ProteinDBAdaptor getProteinDBAdaptor(String species) { @Override public ProteinDBAdaptor getProteinDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ProteinMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new ProteinMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -310,8 +202,7 @@ public ProteinProteinInteractionDBAdaptor getProteinProteinInteractionDBAdaptor( @Override public ProteinProteinInteractionDBAdaptor getProteinProteinInteractionDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ProteinProteinInteractionMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new ProteinProteinInteractionMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -322,8 +213,7 @@ public RegulationDBAdaptor getRegulationDBAdaptor(String species) { @Override public RegulationDBAdaptor getRegulationDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new RegulationMongoDBAdaptor(species, assembly, mongoDatastore, cellBaseConfiguration); + return new RegulationMongoDBAdaptor(species, assembly, cellBaseConfiguration); } // // @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java index de796402c1..8a04dc168c 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java @@ -32,7 +32,6 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -72,9 +71,9 @@ public class ProteinMongoDBAdaptor extends MongoDBAdaptor implements ProteinDBAd } - public ProteinMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + public ProteinMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("protein"); proteinSubstitutionMongoDBCollection = mongoDataStore.getCollection("protein_functional_prediction"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java index f94fb837db..695753b204 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java @@ -25,7 +25,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.ArrayList; import java.util.Iterator; @@ -38,9 +37,9 @@ public class ProteinProteinInteractionMongoDBAdaptor extends MongoDBAdaptor implements ProteinProteinInteractionDBAdaptor { - public ProteinProteinInteractionMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + public ProteinProteinInteractionMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("variation"); logger.debug("ProteinProteinInteractionMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java index 7352dab1a7..8f85ffe130 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java @@ -27,7 +27,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.ArrayList; import java.util.Iterator; @@ -39,9 +38,9 @@ */ public class RegulationMongoDBAdaptor extends MongoDBAdaptor implements RegulationDBAdaptor { - public RegulationMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + public RegulationMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("regulatory_region"); logger.debug("RegulationMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java index 6daa174e39..6c17b6b161 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java @@ -15,7 +15,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -25,9 +24,9 @@ */ public class TranscriptMongoDBAdaptor extends MongoDBAdaptor implements TranscriptDBAdaptor { - public TranscriptMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + public TranscriptMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("gene"); logger.debug("TranscriptMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java index 77c6285d96..cb9bd74ed1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java @@ -35,7 +35,6 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -54,9 +53,9 @@ public class VariantMongoDBAdaptor extends MongoDBAdaptor implements VariantDBAd private MongoDBCollection caddDBCollection; - public VariantMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + public VariantMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("variation"); caddDBCollection = mongoDataStore.getCollection("variation_functional_score"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java index fdb959cca4..a7716ff4a8 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java @@ -27,7 +27,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.ArrayList; import java.util.Arrays; @@ -41,9 +40,9 @@ */ public class XRefMongoDBAdaptor extends MongoDBAdaptor implements XRefDBAdaptor { - public XRefMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore, + public XRefMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { - super(species, assembly, mongoDataStore, cellBaseConfiguration); + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("gene"); logger.debug("XRefMongoDBAdaptor: in 'constructor'"); From fc0b199521853d89fe9d99e703f6deab4a7a4711 Mon Sep 17 00:00:00 2001 From: Swaathik Date: Fri, 21 Oct 2016 12:17:43 +0100 Subject: [PATCH 07/24] core: cacheManager improved --- cellbase-core/pom.xml | 5 ++++ .../cellbase/core/cache/CacheManager.java | 27 ++++++++++++++++--- .../lib/impl/ClinicalMongoDBAdaptor.java | 2 +- .../cellbase/lib/impl/GeneMongoDBAdaptor.java | 4 +-- .../lib/impl/GenomeMongoDBAdaptor.java | 2 +- .../cellbase/lib/impl/MongoDBAdaptor.java | 12 +++++---- .../lib/impl/ProteinMongoDBAdaptor.java | 4 +-- ...oteinProteinInteractionMongoDBAdaptor.java | 2 +- .../lib/impl/RegulationMongoDBAdaptor.java | 4 +-- .../lib/impl/VariantMongoDBAdaptor.java | 4 +-- 10 files changed, 46 insertions(+), 20 deletions(-) diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 21acbfeda5..611a54088c 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -54,6 +54,11 @@ kryo 2.22 + + commons-codec + commons-codec + 1.10 + junit junit diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index a2d22ebfcb..618acf6e31 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -1,5 +1,6 @@ package org.opencb.cellbase.core.cache; +import org.apache.commons.codec.digest.DigestUtils; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -10,6 +11,8 @@ import org.redisson.codec.JsonJacksonCodec; import org.redisson.codec.KryoCodec; +import java.util.*; + /** * Created by imedina on 20/10/16. @@ -52,10 +55,26 @@ public void set(String key, Query query, QueryResult queryResult) { // insert the object into map with key and query and queryResult } - public String createKey(String species, String collection, String subcategory, Query query, QueryOptions queryOptions) { - String key = ""; + public String createKey(String species, String subcategory, Query query, QueryOptions queryOptions) { + StringBuilder key = new StringBuilder(); if (queryOptions.getBoolean("cache", false)) { - key = "set"; + key.append("cb:").append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") + .append(subcategory); + + SortedMap> map = new TreeMap>(); + // TODO: remove cache from options + for (String item: query.keySet()) { + map.put(item.toLowerCase(), new TreeSet(Arrays.asList(query.get(item)))); + } + + for (String item: queryOptions.keySet()) { + map.put(item.toLowerCase(), new TreeSet(Arrays.asList(queryOptions.get(item)))); + } + + String sha1 = DigestUtils.sha1Hex(map.toString()); + + key.append(":").append(sha1); + // cellBase // version We get from CellBaseConfiguration // CB:version:species:collection @@ -64,6 +83,6 @@ public String createKey(String species, String collection, String subcategory, Q // get SHA1 } - return key; + return key.toString(); } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java index 6622d552a3..9f42a2a636 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java @@ -112,7 +112,7 @@ public QueryResult get(Query query, QueryOptions options) { public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); QueryOptions parsedOptions = parseQueryOptions(options); - return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java index 8c2e080971..7bad002953 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java @@ -92,14 +92,14 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return executeBsonQuery(bson, null, options, mongoDBCollection, Gene.class); + return executeBsonQuery(bson, null, query , options, mongoDBCollection, Gene.class); } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java index ecc053dc87..80f5b2f52f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java @@ -287,7 +287,7 @@ public QueryResult get(Query query, QueryOptions options) { @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index 96ac819461..25efaa097d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -77,7 +77,7 @@ public MongoDBAdaptor(String species, String assembly, CellBaseConfiguration cel initSpeciesAssembly(species, assembly); init(); this.mongoDataStore = createMongoDBDatastore(species, assembly); -// jsonObjectMapper = new ObjectMapper(); + this.cacheManager = new CacheManager(cellBaseConfiguration); } private void initSpeciesAssembly(String species, String assembly) { @@ -537,21 +537,23 @@ protected QueryResult distinct(String field, Bson query, MongoDBCollection mongo return result; } - protected QueryResult executeBsonQuery(Bson query, Bson projection, QueryOptions options, + protected QueryResult executeBsonQuery(Bson bsonQuery, Bson projection, Query query, QueryOptions options, MongoDBCollection mongoDBCollection, Class clazz) { QueryResult result = null; if (options.getBoolean("cache")) { -// cacheManger.createKey(this.species, mongoDBCollection, query, queryOptions); + + String key = cacheManager.createKey(this.species, "gene", query, options); + // if( !key.isEmpty()){ // result = cacheManager.get(); // result = null; if (result == null) { options = addPrivateExcludeOptions(options); - result = mongoDBCollection.find(query, projection, clazz, options); + result = mongoDBCollection.find(bsonQuery, projection, clazz, options); // cacheManager.set(result); } } else { options = addPrivateExcludeOptions(options); - result = mongoDBCollection.find(query, projection, clazz, options); + result = mongoDBCollection.find(bsonQuery, projection, clazz, options); } return result; } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java index 8a04dc168c..91d07dfe7b 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java @@ -325,13 +325,13 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return executeBsonQuery(bson, null, options, mongoDBCollection, Entry.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Entry.class); } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java index 695753b204..4d607544d1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java @@ -90,7 +90,7 @@ public QueryResult get(Query query, QueryOptions options) { @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java index 8f85ffe130..02572569b8 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java @@ -109,13 +109,13 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return executeBsonQuery(bson, null, options, mongoDBCollection, RegulatoryFeature.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, RegulatoryFeature.class); } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java index cb9bd74ed1..4c1703e2de 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java @@ -135,7 +135,7 @@ public QueryResult get(Query query, QueryOptions options) { // options = addPrivateExcludeOptions(options); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return executeBsonQuery(bson, null, options, mongoDBCollection, Variant.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Variant.class); } // FIXME: patch to exclude annotation.additionalAttributes from the results - to remove as soon as the variation @@ -159,7 +159,7 @@ public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); // options.put(MongoDBCollection.SKIP_COUNT, true); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return executeBsonQuery(bson, null, options, mongoDBCollection, Document.class); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override From c1464817ae791915e7dc278ebe7719064045369e Mon Sep 17 00:00:00 2001 From: wbari Date: Fri, 21 Oct 2016 14:56:08 +0100 Subject: [PATCH 08/24] core: implemented and improved cacheManager --- .../cellbase/core/cache/CacheManager.java | 33 ++++++++++++------- .../src/main/resources/configuration.json | 4 +++ .../cellbase/lib/impl/MongoDBAdaptor.java | 23 +++++++------ 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 618acf6e31..1494260615 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -10,6 +10,7 @@ import org.redisson.RedissonClient; import org.redisson.codec.JsonJacksonCodec; import org.redisson.codec.KryoCodec; +import org.redisson.core.RMap; import java.util.*; @@ -29,6 +30,7 @@ public CacheManager() { public CacheManager(CellBaseConfiguration configuration) { if (configuration != null) { + this.cellBaseConfiguration = configuration; String codec = configuration.getCache().getSerialization(); redissonConfig = new Config(); redissonConfig.useSingleServer().setAddress(configuration.getCache().getHost()); @@ -45,24 +47,36 @@ public CacheManager(CellBaseConfiguration configuration) { public QueryResult get(String key, Class clazz) { - //redissonClient.getMap() + + RMap> map = redissonClient.getMap(key); + Map> result = new HashMap>(); + Set set = new HashSet(); + set.add(0); + result = map.getAll(set); + if (!result.isEmpty()) { + Object queryResult= result.get(0).get("result"); + return (QueryResult) queryResult; + } //if empty return null else type cast object return null; } public void set(String key, Query query, QueryResult queryResult) { -// redissonClient.getMap(key); - // insert the object into map with key and query and queryResult + RMap> map =redissonClient.getMap(key); + Map record = new HashMap(); + record.put("query", query); + record.put("result", queryResult); + map.fastPut(0, record); } public String createKey(String species, String subcategory, Query query, QueryOptions queryOptions) { - StringBuilder key = new StringBuilder(); - if (queryOptions.getBoolean("cache", false)) { - key.append("cb:").append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") + StringBuilder key = new StringBuilder("cb:"); + key.append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") .append(subcategory); + SortedMap> map = new TreeMap>(); - SortedMap> map = new TreeMap>(); // TODO: remove cache from options + for (String item: query.keySet()) { map.put(item.toLowerCase(), new TreeSet(Arrays.asList(query.get(item)))); } @@ -72,17 +86,12 @@ public String createKey(String species, String subcategory, Query query, QueryOp } String sha1 = DigestUtils.sha1Hex(map.toString()); - key.append(":").append(sha1); - // cellBase // version We get from CellBaseConfiguration // CB:version:species:collection // Sort query and queryOptions - // get SHA1 - - } return key.toString(); } } diff --git a/cellbase-core/src/main/resources/configuration.json b/cellbase-core/src/main/resources/configuration.json index 2c22997992..0d90822e71 100644 --- a/cellbase-core/src/main/resources/configuration.json +++ b/cellbase-core/src/main/resources/configuration.json @@ -28,6 +28,10 @@ } }, "defaultOutdir": "/tmp", + "cache": { + "host": "localhost:6379", + "active": true + }, "download": { "ensembl": { "database": { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index 25efaa097d..8d59f0a53e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -540,22 +540,21 @@ protected QueryResult distinct(String field, Bson query, MongoDBCollection mongo protected QueryResult executeBsonQuery(Bson bsonQuery, Bson projection, Query query, QueryOptions options, MongoDBCollection mongoDBCollection, Class clazz) { QueryResult result = null; - if (options.getBoolean("cache")) { - + if (options.getBoolean("cache", true)) { String key = cacheManager.createKey(this.species, "gene", query, options); - // if( !key.isEmpty()){ -// result = cacheManager.get(); -// result = null; - if (result == null) { + result = cacheManager.get(key, clazz); + if (result != null) { + return result; + } else { + options = addPrivateExcludeOptions(options); + result = mongoDBCollection.find(bsonQuery, projection, clazz, options); + cacheManager.set(key, query, result); + } + } else { options = addPrivateExcludeOptions(options); result = mongoDBCollection.find(bsonQuery, projection, clazz, options); -// cacheManager.set(result); } - } else { - options = addPrivateExcludeOptions(options); - result = mongoDBCollection.find(bsonQuery, projection, clazz, options); - } - return result; + return result; } From adc8c61e68b2a99dd64c237c6481477655d3bace Mon Sep 17 00:00:00 2001 From: wbari Date: Fri, 21 Oct 2016 16:47:24 +0100 Subject: [PATCH 09/24] core: modified cacheManager and webservices for cache option --- cellbase-core/pom.xml | 2 +- .../opencb/cellbase/core/cache/CacheManager.java | 15 +++++++++++++-- .../src/main/resources/configuration.json | 4 +++- .../src/test/resources/configuration_test.json | 3 ++- .../cellbase/server/ws/GenericRestWSServer.java | 5 +++++ .../cellbase/server/ws/feature/GeneWSServer.java | 7 ++++++- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 611a54088c..03076044d5 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -52,7 +52,7 @@ com.esotericsoftware.kryo kryo - 2.22 + 2.23.0 commons-codec diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 1494260615..10e8b5e133 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -10,6 +10,7 @@ import org.redisson.RedissonClient; import org.redisson.codec.JsonJacksonCodec; import org.redisson.codec.KryoCodec; +import org.redisson.codec.SerializationCodec; import org.redisson.core.RMap; import java.util.*; @@ -39,7 +40,10 @@ public CacheManager(CellBaseConfiguration configuration) { redissonConfig.setCodec(new KryoCodec()); } else if ("JSON".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new JsonJacksonCodec()); + } else if ("JDK".equalsIgnoreCase(codec)) { + redissonConfig.setCodec(new SerializationCodec()); } + this.redissonClient = Redisson.create(redissonConfig); } @@ -48,6 +52,7 @@ public CacheManager(CellBaseConfiguration configuration) { public QueryResult get(String key, Class clazz) { + long start = System.currentTimeMillis(); RMap> map = redissonClient.getMap(key); Map> result = new HashMap>(); Set set = new HashSet(); @@ -55,7 +60,10 @@ public QueryResult get(String key, Class clazz) { result = map.getAll(set); if (!result.isEmpty()) { Object queryResult= result.get(0).get("result"); - return (QueryResult) queryResult; + QueryResult queryResult1 = (QueryResult) queryResult; + queryResult1.setDbTime((int) (System.currentTimeMillis() - start)); + queryResult1.setWarningMsg("Cached!! :-)"); + return queryResult1; } //if empty return null else type cast object return null; @@ -81,8 +89,11 @@ public String createKey(String species, String subcategory, Query query, QueryOp map.put(item.toLowerCase(), new TreeSet(Arrays.asList(query.get(item)))); } + + queryOptions.remove("cache"); for (String item: queryOptions.keySet()) { - map.put(item.toLowerCase(), new TreeSet(Arrays.asList(queryOptions.get(item)))); +// List objects = Arrays.asList(queryOptions.getAsStringList(item)); + map.put(item.toLowerCase(), new TreeSet(queryOptions.getAsStringList(item))); } String sha1 = DigestUtils.sha1Hex(map.toString()); diff --git a/cellbase-core/src/main/resources/configuration.json b/cellbase-core/src/main/resources/configuration.json index 0d90822e71..cf67ac4d05 100644 --- a/cellbase-core/src/main/resources/configuration.json +++ b/cellbase-core/src/main/resources/configuration.json @@ -30,8 +30,10 @@ "defaultOutdir": "/tmp", "cache": { "host": "localhost:6379", - "active": true + "active": true, + "serialization": "JDK" }, + "download": { "ensembl": { "database": { diff --git a/cellbase-core/src/test/resources/configuration_test.json b/cellbase-core/src/test/resources/configuration_test.json index a04d17e1be..afb2dfeca7 100644 --- a/cellbase-core/src/test/resources/configuration_test.json +++ b/cellbase-core/src/test/resources/configuration_test.json @@ -29,7 +29,8 @@ }, "cache": { "host": "localhost:6379", - "active": true + "active": true, + "serialization": "kryo" }, "download": { "ensembl": { diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/GenericRestWSServer.java index a6562fa14d..c6045a5bd1 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/GenericRestWSServer.java @@ -258,6 +258,11 @@ public void parseQueryParams() { queryOptions.put("limit", (limit > 0) ? Math.min(limit, LIMIT_MAX) : LIMIT_DEFAULT); queryOptions.put("skip", (skip >= 0) ? skip : -1); queryOptions.put("count", (count != null && !count.equals("")) && Boolean.parseBoolean(count)); + + + queryOptions.put("cache", (multivaluedMap.get("cache") != null) + ? Boolean.parseBoolean(multivaluedMap.get("cache").get(0)) + : false); // outputFormat = (outputFormat != null && !outputFormat.equals("")) ? outputFormat : "json"; // Add all the others QueryParams from the URL diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/feature/GeneWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/feature/GeneWSServer.java index 4ef7adefc0..1dbe4b2f3c 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/feature/GeneWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/feature/GeneWSServer.java @@ -466,7 +466,12 @@ public Response getAllIDs() { value = "Comma separated list of gene names for which drug data is available, " + "e.g.: BRCA2,TTN." + " Exact text matches will be returned", - required = false, dataType = "list of strings", paramType = "query") + required = false, dataType = "list of strings", paramType = "query"), + @ApiImplicitParam(name = "cache", + value = "Comma separated list of gene names for which drug data is available, " + + "e.g.: BRCA2,TTN." + + " Exact text matches will be returned", + required = false, defaultValue = "false", dataType = "boolean", paramType = "query") }) public Response getByEnsemblId(@PathParam("geneId") @ApiParam(name = "geneId", From 81077d481973c095c4321772f218bfb4298a167f Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 24 Oct 2016 13:58:10 +0100 Subject: [PATCH 10/24] core: fixed pom for Kryo --- .../cellbase/core/cache/CacheManager.java | 44 +++++++------------ .../src/main/resources/configuration.json | 2 +- pom.xml | 1 - 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 10e8b5e133..b77c17eb13 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -1,5 +1,6 @@ package org.opencb.cellbase.core.cache; +import com.google.common.collect.Sets; import org.apache.commons.codec.digest.DigestUtils; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; @@ -10,7 +11,6 @@ import org.redisson.RedissonClient; import org.redisson.codec.JsonJacksonCodec; import org.redisson.codec.KryoCodec; -import org.redisson.codec.SerializationCodec; import org.redisson.core.RMap; import java.util.*; @@ -23,6 +23,7 @@ public class CacheManager { private CellBaseConfiguration cellBaseConfiguration; private Config redissonConfig; private RedissonClient redissonClient; + private String codec; public CacheManager() { @@ -30,18 +31,16 @@ public CacheManager() { public CacheManager(CellBaseConfiguration configuration) { - if (configuration != null) { + if (configuration != null && configuration.getCache() != null) { this.cellBaseConfiguration = configuration; - String codec = configuration.getCache().getSerialization(); redissonConfig = new Config(); redissonConfig.useSingleServer().setAddress(configuration.getCache().getHost()); + codec = configuration.getCache().getSerialization(); if ("Kryo".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new KryoCodec()); } else if ("JSON".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new JsonJacksonCodec()); - } else if ("JDK".equalsIgnoreCase(codec)) { - redissonConfig.setCodec(new SerializationCodec()); } this.redissonClient = Redisson.create(redissonConfig); @@ -55,22 +54,21 @@ public QueryResult get(String key, Class clazz) { long start = System.currentTimeMillis(); RMap> map = redissonClient.getMap(key); Map> result = new HashMap>(); - Set set = new HashSet(); - set.add(0); + Set set = new HashSet(Arrays.asList(0)); + result = map.getAll(set); + if (!result.isEmpty()) { Object queryResult= result.get(0).get("result"); QueryResult queryResult1 = (QueryResult) queryResult; queryResult1.setDbTime((int) (System.currentTimeMillis() - start)); - queryResult1.setWarningMsg("Cached!! :-)"); return queryResult1; } - //if empty return null else type cast object return null; } public void set(String key, Query query, QueryResult queryResult) { - RMap> map =redissonClient.getMap(key); + RMap> map = redissonClient.getMap(key); Map record = new HashMap(); record.put("query", query); record.put("result", queryResult); @@ -78,31 +76,21 @@ public void set(String key, Query query, QueryResult queryResult) { } public String createKey(String species, String subcategory, Query query, QueryOptions queryOptions) { + + queryOptions.remove("cache"); StringBuilder key = new StringBuilder("cb:"); key.append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") .append(subcategory); SortedMap> map = new TreeMap>(); - // TODO: remove cache from options - - for (String item: query.keySet()) { - map.put(item.toLowerCase(), new TreeSet(Arrays.asList(query.get(item)))); - } - + for (String item: Sets.union(query.keySet(), queryOptions.keySet())) { + map.put(item.toLowerCase(), new TreeSet(query.getAsStringList(item))); + } - queryOptions.remove("cache"); - for (String item: queryOptions.keySet()) { -// List objects = Arrays.asList(queryOptions.getAsStringList(item)); - map.put(item.toLowerCase(), new TreeSet(queryOptions.getAsStringList(item))); - } + String sha1 = DigestUtils.sha1Hex(map.toString()); + key.append(":").append(sha1); + queryOptions.add("cache", "true"); - String sha1 = DigestUtils.sha1Hex(map.toString()); - key.append(":").append(sha1); - // cellBase - // version We get from CellBaseConfiguration - // CB:version:species:collection - // Sort query and queryOptions - // get SHA1 return key.toString(); } } diff --git a/cellbase-core/src/main/resources/configuration.json b/cellbase-core/src/main/resources/configuration.json index cf67ac4d05..cdb16b2cca 100644 --- a/cellbase-core/src/main/resources/configuration.json +++ b/cellbase-core/src/main/resources/configuration.json @@ -31,7 +31,7 @@ "cache": { "host": "localhost:6379", "active": true, - "serialization": "JDK" + "serialization": "kryo" }, "download": { diff --git a/pom.xml b/pom.xml index b083d4db1e..2c81a1a9ba 100644 --- a/pom.xml +++ b/pom.xml @@ -181,7 +181,6 @@ org.objenesis objenesis 2.1 - test junit From f1393caca65d89ce4970d8a87090810eab50a0bd Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 24 Oct 2016 15:15:23 +0100 Subject: [PATCH 11/24] core: slowThreshold check in cacheManager --- .../opencb/cellbase/core/cache/CacheManager.java | 16 ++++++++-------- .../src/main/resources/configuration.json | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index b77c17eb13..04d4b259ed 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -23,8 +23,6 @@ public class CacheManager { private CellBaseConfiguration cellBaseConfiguration; private Config redissonConfig; private RedissonClient redissonClient; - private String codec; - public CacheManager() { } @@ -35,7 +33,7 @@ public CacheManager(CellBaseConfiguration configuration) { this.cellBaseConfiguration = configuration; redissonConfig = new Config(); redissonConfig.useSingleServer().setAddress(configuration.getCache().getHost()); - codec = configuration.getCache().getSerialization(); + String codec = configuration.getCache().getSerialization(); if ("Kryo".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new KryoCodec()); @@ -68,11 +66,13 @@ public QueryResult get(String key, Class clazz) { } public void set(String key, Query query, QueryResult queryResult) { - RMap> map = redissonClient.getMap(key); - Map record = new HashMap(); - record.put("query", query); - record.put("result", queryResult); - map.fastPut(0, record); + if (queryResult.getDbTime() > cellBaseConfiguration.getCache().getSlowThreshold()) { + RMap> map = redissonClient.getMap(key); + Map record = new HashMap(); + record.put("query", query); + record.put("result", queryResult); + map.fastPut(0, record); + } } public String createKey(String species, String subcategory, Query query, QueryOptions queryOptions) { diff --git a/cellbase-core/src/main/resources/configuration.json b/cellbase-core/src/main/resources/configuration.json index cdb16b2cca..2e87ffb82c 100644 --- a/cellbase-core/src/main/resources/configuration.json +++ b/cellbase-core/src/main/resources/configuration.json @@ -31,7 +31,8 @@ "cache": { "host": "localhost:6379", "active": true, - "serialization": "kryo" + "serialization": "kryo", + "slowThreshold": 50 }, "download": { From e33b10d31e0b7b9d2af101f881903011df2b54d7 Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 24 Oct 2016 15:21:35 +0100 Subject: [PATCH 12/24] core: removed union from different sets --- .../java/org/opencb/cellbase/core/cache/CacheManager.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 04d4b259ed..71fa064c5a 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -83,10 +83,14 @@ public String createKey(String species, String subcategory, Query query, QueryOp .append(subcategory); SortedMap> map = new TreeMap>(); - for (String item: Sets.union(query.keySet(), queryOptions.keySet())) { + for (String item: query.keySet()) { map.put(item.toLowerCase(), new TreeSet(query.getAsStringList(item))); } + for (String item: queryOptions.keySet()) { + map.put(item.toLowerCase(), new TreeSet(queryOptions.getAsStringList(item))); + } + String sha1 = DigestUtils.sha1Hex(map.toString()); key.append(":").append(sha1); queryOptions.add("cache", "true"); From f73167dd2017f62614f3b59dbf9eb8e2aec2c3bc Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 24 Oct 2016 15:27:01 +0100 Subject: [PATCH 13/24] core : removed unsed imports --- .../main/java/org/opencb/cellbase/core/cache/CacheManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 71fa064c5a..c51178e85b 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -1,6 +1,5 @@ package org.opencb.cellbase.core.cache; -import com.google.common.collect.Sets; import org.apache.commons.codec.digest.DigestUtils; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; From 18361dc37d64f7612a41558ceb5abefb4024f37d Mon Sep 17 00:00:00 2001 From: wbari Date: Tue, 25 Oct 2016 12:18:40 +0100 Subject: [PATCH 14/24] core : added ability to cope Redis failure and fallback to DB --- .../cellbase/core/cache/CacheManager.java | 78 ++++++++++++++----- .../cellbase/lib/impl/MongoDBAdaptor.java | 7 +- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index c51178e85b..c8f3198cc8 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -1,27 +1,32 @@ package org.opencb.cellbase.core.cache; -import org.apache.commons.codec.digest.DigestUtils; -import org.opencb.cellbase.core.config.CellBaseConfiguration; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.datastore.core.QueryResult; +import java.util.*; +import java.util.regex.Pattern; import org.redisson.Config; import org.redisson.Redisson; +import org.redisson.core.RMap; +import org.redisson.core.RKeys; import org.redisson.RedissonClient; -import org.redisson.codec.JsonJacksonCodec; import org.redisson.codec.KryoCodec; -import org.redisson.core.RMap; - -import java.util.*; +import org.redisson.codec.JsonJacksonCodec; +import org.redisson.client.RedisConnectionException; +import org.opencb.commons.datastore.core.Query; +import org.apache.commons.codec.digest.DigestUtils; +import org.opencb.commons.datastore.core.QueryResult; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.cellbase.core.config.CellBaseConfiguration; /** * Created by imedina on 20/10/16. */ public class CacheManager { + + private final String dataBase = "cb:"; private CellBaseConfiguration cellBaseConfiguration; private Config redissonConfig; private RedissonClient redissonClient; + private boolean redisState; public CacheManager() { } @@ -33,51 +38,65 @@ public CacheManager(CellBaseConfiguration configuration) { redissonConfig = new Config(); redissonConfig.useSingleServer().setAddress(configuration.getCache().getHost()); String codec = configuration.getCache().getSerialization(); + redisState = true; if ("Kryo".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new KryoCodec()); } else if ("JSON".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new JsonJacksonCodec()); } - - this.redissonClient = Redisson.create(redissonConfig); } - } public QueryResult get(String key, Class clazz) { long start = System.currentTimeMillis(); + redissonClient = Redisson.create(redissonConfig); RMap> map = redissonClient.getMap(key); Map> result = new HashMap>(); + QueryResult queryResult = new QueryResult(); Set set = new HashSet(Arrays.asList(0)); - result = map.getAll(set); + try { + result = map.getAll(set); + } catch (RedisConnectionException e) { + redisState = false; + queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)"); + return queryResult; + } if (!result.isEmpty()) { - Object queryResult= result.get(0).get("result"); - QueryResult queryResult1 = (QueryResult) queryResult; - queryResult1.setDbTime((int) (System.currentTimeMillis() - start)); - return queryResult1; + Object resultMap= result.get(0).get("result"); + queryResult =(QueryResult) resultMap; + queryResult.setDbTime((int) (System.currentTimeMillis() - start)); } - return null; + redissonClient.shutdown(); + return queryResult; } public void set(String key, Query query, QueryResult queryResult) { + if (queryResult.getDbTime() > cellBaseConfiguration.getCache().getSlowThreshold()) { + redissonClient = Redisson.create(redissonConfig); RMap> map = redissonClient.getMap(key); Map record = new HashMap(); record.put("query", query); record.put("result", queryResult); - map.fastPut(0, record); + try { + map.fastPut(0, record); + } catch (RedisConnectionException e) { + redisState = false; + queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)"); + } + redissonClient.shutdown(); } } public String createKey(String species, String subcategory, Query query, QueryOptions queryOptions) { queryOptions.remove("cache"); - StringBuilder key = new StringBuilder("cb:"); + StringBuilder key = new StringBuilder(dataBase); key.append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") .append(subcategory); SortedMap> map = new TreeMap>(); @@ -96,4 +115,23 @@ public String createKey(String species, String subcategory, Query query, QueryOp return key.toString(); } + + public boolean isActive() { + return cellBaseConfiguration.getCache().isActive() && redisState; + } + + public void clear() { + redissonClient = Redisson.create(redissonConfig); + RKeys redisKeys = redissonClient.getKeys(); + redisKeys.deleteByPattern(dataBase + "*"); + redissonClient.shutdown(); + } + + public void clear(Pattern pattern) { + redissonClient = Redisson.create(redissonConfig); + RKeys redisKeys = redissonClient.getKeys(); + redisKeys.deleteByPattern(pattern.toString()); + redissonClient.shutdown(); + } + } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index 8d59f0a53e..d4db459cc0 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -540,12 +540,15 @@ protected QueryResult distinct(String field, Bson query, MongoDBCollection mongo protected QueryResult executeBsonQuery(Bson bsonQuery, Bson projection, Query query, QueryOptions options, MongoDBCollection mongoDBCollection, Class clazz) { QueryResult result = null; - if (options.getBoolean("cache", true)) { + + if (options.getBoolean("cache") && cacheManager.isActive()) { + //TODO: replace harcoded sub category String key = cacheManager.createKey(this.species, "gene", query, options); result = cacheManager.get(key, clazz); - if (result != null) { + if (result.getResult().size() != 0) { return result; } else { + options.replace("cache", true, false); options = addPrivateExcludeOptions(options); result = mongoDBCollection.find(bsonQuery, projection, clazz, options); cacheManager.set(key, query, result); From b0d450b40d875099cd5ad533fad76dd977ada98b Mon Sep 17 00:00:00 2001 From: wbari Date: Tue, 25 Oct 2016 12:33:46 +0100 Subject: [PATCH 15/24] core: rename final variable --- .../java/org/opencb/cellbase/core/cache/CacheManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index c8f3198cc8..9e8b80fcb2 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -22,7 +22,7 @@ */ public class CacheManager { - private final String dataBase = "cb:"; + private final String DATABASE = "cb:"; private CellBaseConfiguration cellBaseConfiguration; private Config redissonConfig; private RedissonClient redissonClient; @@ -96,7 +96,7 @@ public void set(String key, Query query, QueryResult queryResult) { public String createKey(String species, String subcategory, Query query, QueryOptions queryOptions) { queryOptions.remove("cache"); - StringBuilder key = new StringBuilder(dataBase); + StringBuilder key = new StringBuilder(DATABASE); key.append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") .append(subcategory); SortedMap> map = new TreeMap>(); @@ -123,7 +123,7 @@ public boolean isActive() { public void clear() { redissonClient = Redisson.create(redissonConfig); RKeys redisKeys = redissonClient.getKeys(); - redisKeys.deleteByPattern(dataBase + "*"); + redisKeys.deleteByPattern(DATABASE + "*"); redissonClient.shutdown(); } From c2e994d6587ed4c4ebce3ef1034f5bd94561a59b Mon Sep 17 00:00:00 2001 From: wbari Date: Thu, 7 Sep 2017 17:21:24 +0100 Subject: [PATCH 16/24] cache: fixed merge confilit --- cellbase-core/src/main/resources/configuration.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/cellbase-core/src/main/resources/configuration.json b/cellbase-core/src/main/resources/configuration.json index 4dcd61dba8..70b5b016d7 100644 --- a/cellbase-core/src/main/resources/configuration.json +++ b/cellbase-core/src/main/resources/configuration.json @@ -25,10 +25,7 @@ "user": "${CELLBASE.DB.USER}", "password": "${CELLBASE.DB.PASSWORD}" } -<<<<<<< HEAD -======= ->>>>>>> develop } }, "defaultOutdir": "/tmp", From 5621921885e67372c7fa8dfeeb5a7bfff8ba12ab Mon Sep 17 00:00:00 2001 From: wbari Date: Thu, 14 Sep 2017 15:44:54 +0100 Subject: [PATCH 17/24] core: fixed clinicalDBAdaptor merge issues --- .../opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java index a7f8061c85..5670964810 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java @@ -29,9 +29,7 @@ public class ClinicalMongoDBAdaptor extends MongoDBAdaptor implements ClinicalDB public ClinicalMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { super(species, assembly, cellBaseConfiguration); - mongoDBCollection = mongoDataStore.getCollection("clinical"); - - + mongoDBCollection = mongoDataStore.getCollection("clinical_variants"); logger.debug("ClinicalMongoDBAdaptor: in 'constructor'"); } @@ -97,7 +95,7 @@ public QueryResult get(Query query, QueryOptions options) { QueryOptions parsedOptions = parseQueryOptions(options, query); parsedOptions = addPrivateExcludeOptions(parsedOptions, PRIVATE_CLINICAL_FIELDS); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()).toJson()); - return mongoDBCollection.find(bson, null, Variant.class, parsedOptions); + return executeBsonQuery(bson, null, query, parsedOptions, mongoDBCollection, Variant.class); } @Override @@ -106,7 +104,7 @@ public QueryResult nativeGet(Query query, QueryOptions options) { QueryOptions parsedOptions = parseQueryOptions(options, query); parsedOptions = addPrivateExcludeOptions(parsedOptions, PRIVATE_CLINICAL_FIELDS); logger.info("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()).toJson()); - return mongoDBCollection.find(bson, parsedOptions); + return executeBsonQuery(bson, null, query, parsedOptions, mongoDBCollection, Document.class); } @Override From f4bd7565d00bdeb58177a6662f603ea69fdc592c Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 25 Sep 2017 14:24:41 +0100 Subject: [PATCH 18/24] cache: upgraded redisson version and adopted code --- cellbase-core/pom.xml | 2 +- .../java/org/opencb/cellbase/core/cache/CacheManager.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 0ad09c10e7..b0e08ec49d 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -34,7 +34,7 @@ org.redisson redisson - 2.2.0 + 3.5.3 com.esotericsoftware.kryo diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 9e8b80fcb2..48e8f1d044 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -2,11 +2,11 @@ import java.util.*; import java.util.regex.Pattern; -import org.redisson.Config; import org.redisson.Redisson; -import org.redisson.core.RMap; -import org.redisson.core.RKeys; -import org.redisson.RedissonClient; +import org.redisson.api.RKeys; +import org.redisson.api.RMap; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; import org.redisson.codec.KryoCodec; import org.redisson.codec.JsonJacksonCodec; import org.redisson.client.RedisConnectionException; From b420ab4bed799399feb4d12c50d4ac0c3b8b52f3 Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 25 Sep 2017 15:01:16 +0100 Subject: [PATCH 19/24] cache: fixed upgraded version issues --- .../opencb/cellbase/core/cache/CacheManager.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 48e8f1d044..404ded37e8 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -2,6 +2,7 @@ import java.util.*; import java.util.regex.Pattern; + import org.redisson.Redisson; import org.redisson.api.RKeys; import org.redisson.api.RMap; @@ -36,7 +37,7 @@ public CacheManager(CellBaseConfiguration configuration) { if (configuration != null && configuration.getCache() != null) { this.cellBaseConfiguration = configuration; redissonConfig = new Config(); - redissonConfig.useSingleServer().setAddress(configuration.getCache().getHost()); + redissonConfig.useSingleServer().setAddress("redis://" + configuration.getCache().getHost()); String codec = configuration.getCache().getSerialization(); redisState = true; @@ -67,8 +68,8 @@ public QueryResult get(String key, Class clazz) { } if (!result.isEmpty()) { - Object resultMap= result.get(0).get("result"); - queryResult =(QueryResult) resultMap; + Object resultMap = result.get(0).get("result"); + queryResult = (QueryResult) resultMap; queryResult.setDbTime((int) (System.currentTimeMillis() - start)); } redissonClient.shutdown(); @@ -98,14 +99,14 @@ public String createKey(String species, String subcategory, Query query, QueryOp queryOptions.remove("cache"); StringBuilder key = new StringBuilder(DATABASE); key.append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") - .append(subcategory); + .append(subcategory); SortedMap> map = new TreeMap>(); - for (String item: query.keySet()) { + for (String item : query.keySet()) { map.put(item.toLowerCase(), new TreeSet(query.getAsStringList(item))); } - for (String item: queryOptions.keySet()) { + for (String item : queryOptions.keySet()) { map.put(item.toLowerCase(), new TreeSet(queryOptions.getAsStringList(item))); } From 1af7b26cf6ca63c22665d6aadd87b005785265a5 Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 25 Sep 2017 15:37:20 +0100 Subject: [PATCH 20/24] cache: more improvements and checks in cacheManager --- .../cellbase/core/cache/CacheManager.java | 81 ++++++++++++------- .../cellbase/core/config/CacheProperties.java | 1 + 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 404ded37e8..9c18dca897 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -3,6 +3,8 @@ import java.util.*; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.opencb.cellbase.core.config.CacheProperties; import org.redisson.Redisson; import org.redisson.api.RKeys; import org.redisson.api.RMap; @@ -34,13 +36,24 @@ public CacheManager() { public CacheManager(CellBaseConfiguration configuration) { + CacheProperties cache; + if (configuration != null && configuration.getCache() != null) { this.cellBaseConfiguration = configuration; + cache = configuration.getCache(); redissonConfig = new Config(); - redissonConfig.useSingleServer().setAddress("redis://" + configuration.getCache().getHost()); - String codec = configuration.getCache().getSerialization(); redisState = true; + String host = (StringUtils.isNotEmpty(cache.getHost())) + ? cache.getHost() + : CacheProperties.DEFAULT_HOST; + redissonConfig.useSingleServer().setAddress("redis://" + host); + + String codec = (StringUtils.isNotEmpty(cache.getSerialization())) + ? cache.getSerialization() + : CacheProperties.DEFAULT_SERIALIZATION; + + if ("Kryo".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new KryoCodec()); } else if ("JSON".equalsIgnoreCase(codec)) { @@ -52,34 +65,36 @@ public CacheManager(CellBaseConfiguration configuration) { public QueryResult get(String key, Class clazz) { - long start = System.currentTimeMillis(); - redissonClient = Redisson.create(redissonConfig); - RMap> map = redissonClient.getMap(key); - Map> result = new HashMap>(); QueryResult queryResult = new QueryResult(); - Set set = new HashSet(Arrays.asList(0)); - - try { - result = map.getAll(set); - } catch (RedisConnectionException e) { - redisState = false; - queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)"); - return queryResult; - } + if (isActive()) { + long start = System.currentTimeMillis(); + redissonClient = getRedissonClient(); + RMap> map = redissonClient.getMap(key); + Map> result = new HashMap>(); - if (!result.isEmpty()) { - Object resultMap = result.get(0).get("result"); - queryResult = (QueryResult) resultMap; - queryResult.setDbTime((int) (System.currentTimeMillis() - start)); + Set set = new HashSet(Arrays.asList(0)); + + try { + result = map.getAll(set); + } catch (RedisConnectionException e) { + redisState = false; + queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)"); + return queryResult; + } + + if (!result.isEmpty()) { + Object resultMap = result.get(0).get("result"); + queryResult = (QueryResult) resultMap; + queryResult.setDbTime((int) (System.currentTimeMillis() - start)); + } } - redissonClient.shutdown(); return queryResult; } public void set(String key, Query query, QueryResult queryResult) { - if (queryResult.getDbTime() > cellBaseConfiguration.getCache().getSlowThreshold()) { - redissonClient = Redisson.create(redissonConfig); + if (isActive() && queryResult.getDbTime() > cellBaseConfiguration.getCache().getSlowThreshold()) { + redissonClient = getRedissonClient(); RMap> map = redissonClient.getMap(key); Map record = new HashMap(); record.put("query", query); @@ -90,7 +105,6 @@ public void set(String key, Query query, QueryResult queryResult) { redisState = false; queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)"); } - redissonClient.shutdown(); } } @@ -122,17 +136,26 @@ public boolean isActive() { } public void clear() { - redissonClient = Redisson.create(redissonConfig); - RKeys redisKeys = redissonClient.getKeys(); + RKeys redisKeys = getRedissonClient().getKeys(); redisKeys.deleteByPattern(DATABASE + "*"); - redissonClient.shutdown(); } public void clear(Pattern pattern) { - redissonClient = Redisson.create(redissonConfig); - RKeys redisKeys = redissonClient.getKeys(); + RKeys redisKeys = getRedissonClient().getKeys(); redisKeys.deleteByPattern(pattern.toString()); - redissonClient.shutdown(); } + public void close() { + if (redissonClient != null) { + redissonClient.shutdown(); + redissonClient = null; + } + } + + private synchronized RedissonClient getRedissonClient() { + if (redissonClient == null) { + redissonClient = Redisson.create(redissonConfig); + } + return redissonClient; + } } diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java index 135fac2cdf..d7668ecd9f 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java @@ -20,6 +20,7 @@ public class CacheProperties { public static final boolean DEFAULT_ACTVE = true; public static final String DEFAULT_SERIALIZATION = "JSON"; + public static final String DEFAULT_HOST = "localhost:6379"; public CacheProperties() { this("", DEFAULT_ACTVE, DEFAULT_SERIALIZATION, 50); From a6c96ad43974eeb925bfd19ab1c0c100e46db40d Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 25 Sep 2017 16:12:46 +0100 Subject: [PATCH 21/24] cache: stylecheck and add new class members --- .../cellbase/core/cache/CacheManager.java | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index 9c18dca897..bb6a995db2 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -29,6 +29,7 @@ public class CacheManager { private CellBaseConfiguration cellBaseConfiguration; private Config redissonConfig; private RedissonClient redissonClient; + private CacheProperties cache; private boolean redisState; public CacheManager() { @@ -36,24 +37,16 @@ public CacheManager() { public CacheManager(CellBaseConfiguration configuration) { - CacheProperties cache; - if (configuration != null && configuration.getCache() != null) { this.cellBaseConfiguration = configuration; cache = configuration.getCache(); redissonConfig = new Config(); redisState = true; - String host = (StringUtils.isNotEmpty(cache.getHost())) - ? cache.getHost() - : CacheProperties.DEFAULT_HOST; + ? cache.getHost() : CacheProperties.DEFAULT_HOST; redissonConfig.useSingleServer().setAddress("redis://" + host); - String codec = (StringUtils.isNotEmpty(cache.getSerialization())) - ? cache.getSerialization() - : CacheProperties.DEFAULT_SERIALIZATION; - - + ? cache.getSerialization() : CacheProperties.DEFAULT_SERIALIZATION; if ("Kryo".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new KryoCodec()); } else if ("JSON".equalsIgnoreCase(codec)) { @@ -62,7 +55,6 @@ public CacheManager(CellBaseConfiguration configuration) { } } - public QueryResult get(String key, Class clazz) { QueryResult queryResult = new QueryResult(); @@ -71,9 +63,7 @@ public QueryResult get(String key, Class clazz) { redissonClient = getRedissonClient(); RMap> map = redissonClient.getMap(key); Map> result = new HashMap>(); - Set set = new HashSet(Arrays.asList(0)); - try { result = map.getAll(set); } catch (RedisConnectionException e) { @@ -81,7 +71,6 @@ public QueryResult get(String key, Class clazz) { queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)"); return queryResult; } - if (!result.isEmpty()) { Object resultMap = result.get(0).get("result"); queryResult = (QueryResult) resultMap; @@ -93,7 +82,7 @@ public QueryResult get(String key, Class clazz) { public void set(String key, Query query, QueryResult queryResult) { - if (isActive() && queryResult.getDbTime() > cellBaseConfiguration.getCache().getSlowThreshold()) { + if (isActive() && queryResult.getDbTime() > cache.getSlowThreshold()) { redissonClient = getRedissonClient(); RMap> map = redissonClient.getMap(key); Map record = new HashMap(); @@ -115,24 +104,20 @@ public String createKey(String species, String subcategory, Query query, QueryOp key.append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") .append(subcategory); SortedMap> map = new TreeMap>(); - for (String item : query.keySet()) { map.put(item.toLowerCase(), new TreeSet(query.getAsStringList(item))); } - for (String item : queryOptions.keySet()) { map.put(item.toLowerCase(), new TreeSet(queryOptions.getAsStringList(item))); } - String sha1 = DigestUtils.sha1Hex(map.toString()); key.append(":").append(sha1); queryOptions.add("cache", "true"); - return key.toString(); } public boolean isActive() { - return cellBaseConfiguration.getCache().isActive() && redisState; + return cache.isActive() && redisState; } public void clear() { From edba074ebdc5900076d59f406cac03fa32d89e9d Mon Sep 17 00:00:00 2001 From: wbari Date: Mon, 25 Sep 2017 16:34:05 +0100 Subject: [PATCH 22/24] cache: upgraded kryo version --- cellbase-core/pom.xml | 2 +- .../java/org/opencb/cellbase/core/cache/CacheManager.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index b0e08ec49d..83e89593ba 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -39,7 +39,7 @@ com.esotericsoftware.kryo kryo - 2.23.0 + 2.24.0 commons-codec diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index bb6a995db2..f360ad26df 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -43,10 +43,10 @@ public CacheManager(CellBaseConfiguration configuration) { redissonConfig = new Config(); redisState = true; String host = (StringUtils.isNotEmpty(cache.getHost())) - ? cache.getHost() : CacheProperties.DEFAULT_HOST; + ? cache.getHost() : cache.DEFAULT_HOST; redissonConfig.useSingleServer().setAddress("redis://" + host); String codec = (StringUtils.isNotEmpty(cache.getSerialization())) - ? cache.getSerialization() : CacheProperties.DEFAULT_SERIALIZATION; + ? cache.getSerialization() : cache.DEFAULT_SERIALIZATION; if ("Kryo".equalsIgnoreCase(codec)) { redissonConfig.setCodec(new KryoCodec()); } else if ("JSON".equalsIgnoreCase(codec)) { From 7e598f729a0e29c67e8ebe07c191342ca1613d2e Mon Sep 17 00:00:00 2001 From: wbari Date: Thu, 28 Sep 2017 13:28:28 +0100 Subject: [PATCH 23/24] cache: fixed static instance issue --- .../cellbase/core/cache/CacheManager.java | 36 +++++++++---------- .../cellbase/lib/impl/MongoDBAdaptor.java | 4 +-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index f360ad26df..a099553127 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -1,23 +1,23 @@ package org.opencb.cellbase.core.cache; -import java.util.*; -import java.util.regex.Pattern; - +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.core.config.CacheProperties; +import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.core.QueryResult; import org.redisson.Redisson; import org.redisson.api.RKeys; import org.redisson.api.RMap; import org.redisson.api.RedissonClient; -import org.redisson.config.Config; -import org.redisson.codec.KryoCodec; -import org.redisson.codec.JsonJacksonCodec; import org.redisson.client.RedisConnectionException; -import org.opencb.commons.datastore.core.Query; -import org.apache.commons.codec.digest.DigestUtils; -import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.redisson.codec.JsonJacksonCodec; +import org.redisson.codec.KryoCodec; +import org.redisson.config.Config; + +import java.util.*; +import java.util.regex.Pattern; /** @@ -28,7 +28,7 @@ public class CacheManager { private final String DATABASE = "cb:"; private CellBaseConfiguration cellBaseConfiguration; private Config redissonConfig; - private RedissonClient redissonClient; + private static RedissonClient redissonClient; private CacheProperties cache; private boolean redisState; @@ -60,10 +60,10 @@ public QueryResult get(String key, Class clazz) { QueryResult queryResult = new QueryResult(); if (isActive()) { long start = System.currentTimeMillis(); - redissonClient = getRedissonClient(); - RMap> map = redissonClient.getMap(key); + RMap> map = getRedissonClient().getMap(key); Map> result = new HashMap>(); Set set = new HashSet(Arrays.asList(0)); + try { result = map.getAll(set); } catch (RedisConnectionException e) { @@ -72,8 +72,9 @@ public QueryResult get(String key, Class clazz) { return queryResult; } if (!result.isEmpty()) { - Object resultMap = result.get(0).get("result"); - queryResult = (QueryResult) resultMap; + queryResult = (QueryResult) result.get(0).get("result"); + // we are getting two objects, second one is query, not used at the moment + queryResult.setWarningMsg("Data is originated from Redis Cache !!!"); queryResult.setDbTime((int) (System.currentTimeMillis() - start)); } } @@ -83,8 +84,7 @@ public QueryResult get(String key, Class clazz) { public void set(String key, Query query, QueryResult queryResult) { if (isActive() && queryResult.getDbTime() > cache.getSlowThreshold()) { - redissonClient = getRedissonClient(); - RMap> map = redissonClient.getMap(key); + RMap> map = getRedissonClient().getMap(key); Map record = new HashMap(); record.put("query", query); record.put("result", queryResult); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index 6a14db6939..0aed448cc5 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -56,11 +56,11 @@ enum QueryValueType {INTEGER, STRING} protected String species; protected String assembly; - protected MongoDataStoreManager mongoDataStoreManager; + protected static MongoDataStoreManager mongoDataStoreManager; protected MongoDataStore mongoDataStore; protected MongoDBCollection mongoDBCollection; protected CellBaseConfiguration cellBaseConfiguration; - private CacheManager cacheManager; + private static CacheManager cacheManager; protected Logger logger = LoggerFactory.getLogger(this.getClass()); From c0568fc505930255f63fb4ce6952792f284435a9 Mon Sep 17 00:00:00 2001 From: wbari Date: Wed, 4 Oct 2017 10:41:32 +0100 Subject: [PATCH 24/24] cache: fixed todos and removed subcategory from cache key --- .../cellbase/core/cache/CacheManager.java | 5 ++--- .../cellbase/lib/impl/GeneMongoDBAdaptor.java | 19 ++++++++++--------- .../cellbase/lib/impl/MongoDBAdaptor.java | 3 +-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java index a099553127..3f54ed85bb 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -97,12 +97,11 @@ public void set(String key, Query query, QueryResult queryResult) { } } - public String createKey(String species, String subcategory, Query query, QueryOptions queryOptions) { + public String createKey(String species, Query query, QueryOptions queryOptions) { queryOptions.remove("cache"); StringBuilder key = new StringBuilder(DATABASE); - key.append(cellBaseConfiguration.getVersion()).append(":").append(species).append(":") - .append(subcategory); + key.append(cellBaseConfiguration.getVersion()).append(":").append(species); SortedMap> map = new TreeMap>(); for (String item : query.keySet()) { map.put(item.toLowerCase(), new TreeSet(query.getAsStringList(item))); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java index 652ec1857c..23acbc95be 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java @@ -16,6 +16,10 @@ package org.opencb.cellbase.lib.impl; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.mongodb.MongoClient; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; @@ -34,9 +38,11 @@ import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import java.io.IOException; import java.util.*; import java.util.function.Consumer; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Created by imedina on 25/11/15. @@ -99,13 +105,10 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return executeBsonQuery(bson, null, query, options, mongoDBCollection, Gene.class); - -/*TODO: Wasim Check with Javier - options = addPrivateExcludeOptions(options); if (postDBFilteringParametersEnabled(query)) { - QueryResult nativeQueryResult = postDBFiltering(query, mongoDBCollection.find(bson, options)); + QueryResult nativeQueryResult = postDBFiltering(query, + executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class)); QueryResult queryResult = new QueryResult<>(nativeQueryResult.getId(), nativeQueryResult.getDbTime(), nativeQueryResult.getNumResults(), nativeQueryResult.getNumTotalResults(), nativeQueryResult.getWarningMsg(), @@ -128,11 +131,9 @@ public QueryResult get(Query query, QueryOptions options) { }).collect(Collectors.toList())); return queryResult; } else { - logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return mongoDBCollection.find(bson, null, Gene.class, options); + logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()).toJson()); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Gene.class); } -*/ - } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index 0aed448cc5..ea7dc5d295 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -579,8 +579,7 @@ protected QueryResult executeBsonQuery(Bson bsonQuery, Bson projection, Q QueryResult result = null; if (options.getBoolean("cache") && cacheManager.isActive()) { - //TODO: replace harcoded sub category - String key = cacheManager.createKey(this.species, "gene", query, options); + String key = cacheManager.createKey(this.species, query, options); result = cacheManager.get(key, clazz); if (result.getResult().size() != 0) { return result;