Feature cache

cellbase-core/pom.xml

@@ -23,7 +23,6 @@
             <groupId>org.opencb.biodata</groupId>
             <artifactId>biodata-tools</artifactId>
         </dependency>
-
         <dependency>
             <groupId>org.glassfish.jersey.core</groupId>
             <artifactId>jersey-client</artifactId>
@@ -32,7 +31,21 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
         </dependency>
-
+        <dependency>
+            <groupId>org.redisson</groupId>
+            <artifactId>redisson</artifactId>
+            <version>3.5.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.esotericsoftware.kryo</groupId>
+            <artifactId>kryo</artifactId>
+            <version>2.24.0</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.10</version>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
@@ -125,9 +138,11 @@
                       protobuf-java directly, you will be transitively depending on the
                       protobuf-java version that grpc depends on.
                     -->
-                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+                    </protocArtifact>
                     <pluginId>grpc-java</pluginId>
-                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
+                    </pluginArtifact>
                     <!--<protoSourceRoot>src/main/resources/</protoSourceRoot>-->
                 </configuration>
                 <executions>

cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java

@@ -17,7 +17,6 @@
 package org.opencb.cellbase.core.api;
 
 import org.opencb.cellbase.core.config.CellBaseConfiguration;
-import org.opencb.cellbase.core.config.Species;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -105,7 +104,7 @@ public void setConfiguration(CellBaseConfiguration cellBaseConfiguration) {
     public abstract ConservationDBAdaptor getConservationDBAdaptor(String species, String assembly);
 
 
-    protected Species getSpecies(String speciesName) {
+/*    protected Species getSpecies(String speciesName) {
         Species species = null;
         for (Species sp : cellBaseConfiguration.getAllSpecies()) {
             if (speciesName.equalsIgnoreCase(sp.getId()) || speciesName.equalsIgnoreCase(sp.getScientificName())) {
@@ -128,6 +127,6 @@ protected String getAssembly(Species species, String assemblyName) {
             }
         }
         return assembly;
-    }
+    }*/
 
 }

cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java

@@ -0,0 +1,145 @@
+package org.opencb.cellbase.core.cache;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.opencb.cellbase.core.config.CacheProperties;
+import org.opencb.cellbase.core.config.CellBaseConfiguration;
+import org.opencb.commons.datastore.core.Query;
+import org.opencb.commons.datastore.core.QueryOptions;
+import org.opencb.commons.datastore.core.QueryResult;
+import org.redisson.Redisson;
+import org.redisson.api.RKeys;
+import org.redisson.api.RMap;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.RedisConnectionException;
+import org.redisson.codec.JsonJacksonCodec;
+import org.redisson.codec.KryoCodec;
+import org.redisson.config.Config;
+
+import java.util.*;
+import java.util.regex.Pattern;
+
+
+/**
+ * Created by imedina on 20/10/16.
+ */
+public class CacheManager {
+
+    private final String DATABASE = "cb:";
+    private CellBaseConfiguration cellBaseConfiguration;
+    private Config redissonConfig;
+    private static RedissonClient redissonClient;
+    private CacheProperties cache;
+    private boolean redisState;
+
+    public CacheManager() {
+    }
+
+    public CacheManager(CellBaseConfiguration configuration) {
+
+        if (configuration != null && configuration.getCache() != null) {
+            this.cellBaseConfiguration = configuration;
+            cache = configuration.getCache();
+            redissonConfig = new Config();
+            redisState = true;
+            String host = (StringUtils.isNotEmpty(cache.getHost()))
+                    ? cache.getHost() : cache.DEFAULT_HOST;
+            redissonConfig.useSingleServer().setAddress("redis://" + host);
+            String codec = (StringUtils.isNotEmpty(cache.getSerialization()))
+                    ? cache.getSerialization() : cache.DEFAULT_SERIALIZATION;
+            if ("Kryo".equalsIgnoreCase(codec)) {
+                redissonConfig.setCodec(new KryoCodec());
+            } else if ("JSON".equalsIgnoreCase(codec)) {
+                redissonConfig.setCodec(new JsonJacksonCodec());
+            }
+        }
+    }
+
+    public <T> QueryResult<T> get(String key, Class<T> clazz) {
+
+        QueryResult<T> queryResult = new QueryResult<T>();
+        if (isActive()) {
+            long start = System.currentTimeMillis();
+            RMap<Integer, Map<String, Object>> map = getRedissonClient().getMap(key);
+            Map<Integer, Map<String, Object>> result = new HashMap<Integer, Map<String, Object>>();
+            Set<Integer> set = new HashSet<Integer>(Arrays.asList(0));
+
+            try {
+                result = map.getAll(set);
+            } catch (RedisConnectionException e) {
+                redisState = false;
+                queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)");
+                return queryResult;
+            }
+            if (!result.isEmpty()) {
+                queryResult = (QueryResult<T>) result.get(0).get("result");
+                // we are getting two objects, second one is query, not used at the moment
+                queryResult.setWarningMsg("Data is originated from Redis Cache !!!");
+                queryResult.setDbTime((int) (System.currentTimeMillis() - start));
+            }
+        }
+        return queryResult;
+    }
+
+    public void set(String key, Query query, QueryResult queryResult) {
+
+        if (isActive() && queryResult.getDbTime() > cache.getSlowThreshold()) {
+            RMap<Integer, Map<String, Object>> map = getRedissonClient().getMap(key);
+            Map<String, Object> record = new HashMap<String, Object>();
+            record.put("query", query);
+            record.put("result", queryResult);
+            try {
+                map.fastPut(0, record);
+            } catch (RedisConnectionException e) {
+                redisState = false;
+                queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)");
+            }
+        }
+    }
+
+    public String createKey(String species, Query query, QueryOptions queryOptions) {
+
+        queryOptions.remove("cache");
+        StringBuilder key = new StringBuilder(DATABASE);
+        key.append(cellBaseConfiguration.getVersion()).append(":").append(species);
+        SortedMap<String, SortedSet<Object>> map = new TreeMap<String, SortedSet<Object>>();
+        for (String item : query.keySet()) {
+            map.put(item.toLowerCase(), new TreeSet<Object>(query.getAsStringList(item)));
+        }
+        for (String item : queryOptions.keySet()) {
+            map.put(item.toLowerCase(), new TreeSet<Object>(queryOptions.getAsStringList(item)));
+        }
+        String sha1 = DigestUtils.sha1Hex(map.toString());
+        key.append(":").append(sha1);
+        queryOptions.add("cache", "true");
+        return key.toString();
+    }
+
+    public boolean isActive() {
+        return cache.isActive() && redisState;
+    }
+
+    public void clear() {
+        RKeys redisKeys = getRedissonClient().getKeys();
+        redisKeys.deleteByPattern(DATABASE + "*");
+    }
+
+    public void clear(Pattern pattern) {
+        RKeys redisKeys = getRedissonClient().getKeys();
+        redisKeys.deleteByPattern(pattern.toString());
+    }
+
+    public void close() {
+        if (redissonClient != null) {
+            redissonClient.shutdown();
+            redissonClient = null;
+        }
+    }
+
+    private synchronized RedissonClient getRedissonClient() {
+        if (redissonClient == null) {
+            redissonClient = Redisson.create(redissonConfig);
+        }
+        return redissonClient;
+    }
+}

cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java

@@ -0,0 +1,82 @@
+package org.opencb.cellbase.core.config;
+
+/**
+ * Created by imedina on 20/10/16.
+ */
+public class CacheProperties {
+
+    /**
+     * This field contain the host and port, ie. host[:port].
+     */
+    private String host;
+    private boolean active;
+
+    /**
+     * Accepted values are: JSON, Kryo.
+     */
+    private String serialization;
+
+    private int slowThreshold;
+
+    public static final boolean DEFAULT_ACTVE = true;
+    public static final String DEFAULT_SERIALIZATION = "JSON";
+    public static final String DEFAULT_HOST = "localhost:6379";
+
+    public CacheProperties() {
+        this("", DEFAULT_ACTVE, DEFAULT_SERIALIZATION, 50);
+    }
+
+    public CacheProperties(String host, boolean active, String serialization, int slowThreshold) {
+        this.host = host;
+        this.active = active;
+        this.serialization = serialization;
+        this.slowThreshold = slowThreshold;
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder("CacheProperties{");
+        sb.append("host='").append(host).append('\'');
+        sb.append(", active=").append(active);
+        sb.append(", serialization='").append(serialization).append('\'');
+        sb.append(", slowThreshold=").append(slowThreshold);
+        sb.append('}');
+        return sb.toString();
+    }
+
+    public String getHost() {
+        return host;
+    }
+
+    public CacheProperties setHost(String host) {
+        this.host = host;
+        return this;
+    }
+
+    public boolean isActive() {
+        return active;
+    }
+
+    public CacheProperties setActive(boolean active) {
+        this.active = active;
+        return this;
+    }
+
+    public String getSerialization() {
+        return serialization;
+    }
+
+    public CacheProperties setSerialization(String serialization) {
+        this.serialization = serialization;
+        return this;
+    }
+
+    public int getSlowThreshold() {
+        return slowThreshold;
+    }
+
+    public CacheProperties setSlowThreshold(int slowThreshold) {
+        this.slowThreshold = slowThreshold;
+        return this;
+    }
+}