Merge branch 'signum-network:develop' into develop

conf/mainnet/node.sqlite.properties

@@ -12,7 +12,21 @@ DB.Optimize = on
 ## WAL may occupy up to twice more disc space than others while running the node, but allows read concurrency and usually better performance (see more here: https://www.sqlite.org/wal.html)
 ## It's highly recommended to use WAL mode during syncing, to dramatically reduce I/O operations and though faster sync times.
 ## Info: MEMORY journal mode is not supported
-DB.SqliteJournalMode = WAL
+# DB.SqliteJournalMode = WAL
+
+## Sqlite synchronous
+## https://www.sqlite.org/pragma.html#pragma_synchronous
+## Lowering this parameter can increase performance, but decrease reliability.
+## FULL (default) is the safest, but NORMAL should be also fine for running a local node.
+## Setting this to OFF could lead to database corruption on opertaing system crash or power loss!
+# DB.SqliteSynchronous = NORMAL
+
+## Sqlite cache_size
+## https://www.sqlite.org/pragma.html#pragma_cache_size
+## Positive value sets the number of cache pages to use (default page size is 4096 bytes).
+## Negative value sets the number of cache pages, so it would use approximately N * 1024 bytes of memory. 
+## Increasing this the number of cache pages can increase performance at the cost of memory usage.
+# DB.SqliteCacheSize = -2000
 
 # Database connection JDBC url
 DB.Url=jdbc:sqlite:file:./db/signum.sqlite.db

conf/node-default.properties

@@ -30,10 +30,10 @@
 #### DATABASE BACKEND SELECTION ####
 
 ## We are using the H2 file based as default with no username/password.
-
 # DB.Url=jdbc:h2:file:./db/signum-v2;DB_CLOSE_ON_EXIT=FALSE
-# DB.Username=
-# DB.Password=
+
+## If you want to use Sqlite:
+# DB.Url=jdbc:sqlite:file:./db/signum.sqlite.db
 
 ## If you want to use MariaDB:
 

openapi/signum-api.json

@@ -2,7 +2,7 @@
   "openapi": "3.0.3",
   "info": {
     "title": "Signum Node API",
-    "version": "3.8.0",
+    "version": "3.8.2",
     "description": "This is the API documentation of the Signum Node"
   },
   "paths": {

src/brs/Signum.java

@@ -76,7 +76,7 @@
  */
 public final class Signum {
 
-    public static final Version VERSION = Version.parse("v3.8.1");
+    public static final Version VERSION = Version.parse("v3.8.2");
     public static final String APPLICATION = "BRS";
 
     public static final String CONF_FOLDER = "./conf";

src/brs/at/ATProcessorCache.java

@@ -279,7 +279,6 @@ public int findTransactionHeight(Long transactionId, int height, Long atID, long
       throw new CacheMissException();
     }
 
-    // TODO: there must be another way how this works. This is very fragile
     int count = 0;
     Collection<Transaction> transactions = atContext.transactions;
     for (Transaction t : transactions) {

src/brs/db/sql/SqlBlockchainStore.java

@@ -2,7 +2,6 @@
 
 import brs.*;
 import brs.Block;
-import brs.Constants;
 import brs.Transaction;
 import brs.Attachment.CommitmentAdd;
 import brs.Attachment.CommitmentRemove;
@@ -182,19 +181,10 @@ public Collection<Transaction> getTransactions(Account account, int numberOfConf
       );
 
       if (includeIndirectIncoming) {
-
-        int blockTimeStampHeight = getHeightForBlockTimeStamp(blockTimestamp);
-        SelectLimitPercentStep<Record1<Long>> indirectIncomings = ctx
-          .select(INDIRECT_INCOMING.TRANSACTION_ID)
-          .from(INDIRECT_INCOMING)
-          .where(INDIRECT_INCOMING.ACCOUNT_ID.eq(account.getId()))
-          .and(INDIRECT_INCOMING.HEIGHT.le(Math.max(blockTimeStampHeight, height)))
-          .orderBy(INDIRECT_INCOMING.DB_ID.desc())
-          .limit(Constants.MAX_API_RETURNED_ITEMS * 2); // cap this to keep the query scalable
-
         select = select.unionAll(ctx.selectFrom(TRANSACTION)
           .where(conditions)
-          .and(TRANSACTION.ID.in(indirectIncomings)));
+          .and(TRANSACTION.ID.in(ctx.select(INDIRECT_INCOMING.TRANSACTION_ID).from(INDIRECT_INCOMING)
+            .where(INDIRECT_INCOMING.ACCOUNT_ID.eq(account.getId())))));
       }
 
       SelectQuery<TransactionRecord> selectQuery = select
@@ -215,23 +205,6 @@ private static int getHeightForNumberOfConfirmations(int numberOfConfirmations)
     return height;
   }
 
-  private static int getHeightForBlockTimeStamp(int blockTimestamp) {
-    if(blockTimestamp > 0){
-      Record1<Integer> height = Db.useDSLContext(ctx -> {
-        return ctx.select(TRANSACTION.HEIGHT)
-            .from(TRANSACTION)
-            .where(TRANSACTION.BLOCK_TIMESTAMP.le(blockTimestamp))
-            .orderBy(TRANSACTION.BLOCK_TIMESTAMP.desc())
-            .limit(1)
-            .fetchOne();
-        }
-      );
-
-      return height != null ? height.value1(): 0;
-    }
-    return 0;
-  }
-
   @Override
   public Collection<Transaction> getTransactions(Long senderId, Long recipientId, int numberOfConfirmations, byte type, byte subtype, int blockTimestamp, int from, int to, boolean includeIndirectIncoming, boolean bidirectional) {
     int height = getHeightForNumberOfConfirmations(numberOfConfirmations);
@@ -272,38 +245,27 @@ public Collection<Transaction> getTransactions(Long senderId, Long recipientId,
       if (includeIndirectIncoming) {
         // makes only sense if for recipient. Sender is implicitely included.
         if (!bidirectional && hasRecipient) {
-          int blockTimeStampHeight = getHeightForBlockTimeStamp(blockTimestamp);
-          SelectLimitPercentStep<Record1<Long>> indirectIncomingsForRecipient = ctx
-            .select(INDIRECT_INCOMING.TRANSACTION_ID)
-            .from(INDIRECT_INCOMING)
-            .where(INDIRECT_INCOMING.ACCOUNT_ID.eq(recipientId))
-            .and(INDIRECT_INCOMING.HEIGHT.le(Math.max(blockTimeStampHeight, height)))
-            .orderBy(INDIRECT_INCOMING.DB_ID.desc())
-            .limit(Constants.MAX_API_RETURNED_ITEMS * 2); // cap this to keep the query scalable
-
-
           select = select.unionAll(ctx
             .selectFrom(TRANSACTION)
             .where(conditions)
-            .and(TRANSACTION.ID.in(indirectIncomingsForRecipient))
-          );
+            .and(TRANSACTION.ID.in(ctx
+                .select(INDIRECT_INCOMING.TRANSACTION_ID)
+                .from(INDIRECT_INCOMING)
+                .where(INDIRECT_INCOMING.ACCOUNT_ID.eq(recipientId))
+              )
+            ));
         }
 
         if (bidirectional) {
-          int blockTimeStampHeight = getHeightForBlockTimeStamp(blockTimestamp);
-          SelectLimitPercentStep<Record1<Long>> indirectIncomingsForBidirectional = ctx
-            .select(INDIRECT_INCOMING.TRANSACTION_ID)
-            .from(INDIRECT_INCOMING)
-            .where(hasRecipient ? INDIRECT_INCOMING.ACCOUNT_ID.eq(recipientId) : null)
-            .or(hasSender ? INDIRECT_INCOMING.ACCOUNT_ID.eq(senderId) : null)
-            .and(INDIRECT_INCOMING.HEIGHT.le(Math.max(blockTimeStampHeight, height)))
-            .orderBy(INDIRECT_INCOMING.DB_ID.desc())
-            .limit(Constants.MAX_API_RETURNED_ITEMS * 2); // cap this to keep the query scalable
-
           select = select.unionAll(ctx
             .selectFrom(TRANSACTION)
             .where(conditions)
-            .and(TRANSACTION.ID.in(indirectIncomingsForBidirectional)
+            .and(TRANSACTION.ID.in(ctx
+                .select(INDIRECT_INCOMING.TRANSACTION_ID)
+                .from(INDIRECT_INCOMING)
+                .where(hasRecipient ? INDIRECT_INCOMING.ACCOUNT_ID.eq(recipientId) : null)
+                .or(hasSender ? INDIRECT_INCOMING.ACCOUNT_ID.eq(senderId) : null)
+              )
             )
           );
         }

src/brs/db/sql/dialects/DatabaseInstanceSqlite.java

@@ -27,6 +27,23 @@ private String getJournalMode(){
     return "WAL";
   }
 
+  private String getSynchronousMode(){
+    String synchronous = propertyService.getString(Props.DB_SQLITE_SYNCHRONOUS).toUpperCase();
+    switch (synchronous) {
+      case "FULL":
+        return "FULL";
+      case "OFF":
+        logger.warn("SQLite synchronous mode set to: OFF. This could result in a database corruption, when the operating system crashes or the computer loses power!");
+        return "OFF";
+      default:
+        return "NORMAL";
+    }
+  }
+
+  private int getCacheSize(){
+    return propertyService.getInt(Props.DB_SQLITE_CACHE_SIZE);
+  }
+
   @Override
   protected HikariConfig configureImpl(HikariConfig config) {
     config.setMaximumPoolSize(10);
@@ -35,6 +52,8 @@ protected HikariConfig configureImpl(HikariConfig config) {
     config.addDataSourceProperty("busy_timeout", "30000");
     config.addDataSourceProperty("wal_autocheckpoint", "500");
     config.addDataSourceProperty("journal_mode", getJournalMode());
+    config.addDataSourceProperty("synchronous", getSynchronousMode());
+    config.addDataSourceProperty("cache_size", getCacheSize());
     return config;
   }
 

src/brs/props/Props.java

@@ -122,6 +122,8 @@ public class Props {
   public static final Prop<Boolean> DB_H2_DEFRAG_ON_SHUTDOWN = new Prop<>("Db.H2.DefragOnShutdown", true);
   public static final Prop<Boolean> DB_OPTIMIZE = new Prop<>("DB.Optimize", true);
   public static final Prop<String> DB_SQLITE_JOURNAL_MODE = new Prop<>("DB.SqliteJournalMode", "WAL");
+  public static final Prop<String> DB_SQLITE_SYNCHRONOUS = new Prop<>("DB.SqliteSynchronous", "NORMAL");
+  public static final Prop<Integer> DB_SQLITE_CACHE_SIZE = new Prop<>("DB.SqliteCacheSize", -2000);
 
   public static final Prop<Integer> BRS_BLOCK_CACHE_MB = new Prop<>("node.blockCacheMB", 40);
   public static final Prop<Integer> BRS_AT_PROCESSOR_CACHE_BLOCK_COUNT = new Prop<>("node.atProcessorCacheBlockCount", 1000);