feat: v1.5.x

common/protocol/package.json

@@ -19,14 +19,9 @@
     "doc": "typedoc"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.370.0",
     "@bundlr-network/client": "^0.8.9",
-    "@cosmjs/proto-signing": "^0.27.1",
-    "@cosmjs/stargate": "^0.27.1",
-    "@kyvejs/sdk": "1.1.1",
-    "@kyvejs/types": "1.2.0",
-    "@types/cli-progress": "^3.9.2",
-    "@types/jsonfile": "^6.0.1",
+    "@kyvejs/sdk": "1.2.0",
+    "@kyvejs/types": "1.3.0",
     "arweave": "^1.10.17",
     "axios": "^0.27.2",
     "bignumber.js": "^9.1.2",
@@ -35,17 +30,16 @@
     "fs-extra": "^10.0.1",
     "jsonfile": "^6.1.0",
     "jszip": "^3.10.1",
-    "level": "^8.0.0",
     "prando": "^6.0.1",
     "prom-client": "^14.1.0",
-    "protobufjs": "^6.11.4",
     "seedrandom": "^3.0.5",
     "semver": "^7.5.3",
     "tslog": "^3.2.2",
     "unique-names-generator": "^4.6.0",
     "uuid": "^9.0.0"
   },
   "devDependencies": {
+    "@types/jsonfile": "^6.0.1",
     "@types/clone": "^2.1.1",
     "@types/diff": "^5.0.7",
     "@types/fs-extra": "^9.0.13",

common/protocol/src/index.ts

@@ -274,6 +274,10 @@ export class Validator {
         "Specify the home directory of the node where logs and the cache should save their data. [default current directory]",
         "./"
       )
+      .option(
+        "--skip-data-availability-check",
+        "Skip data availability check and join pool instantly without waiting for the data source. WARNING: Only use this if you know what you are doing since this can lead to timeout slashes"
+      )
       .action((options) => {
         this.start(options);
       });
@@ -326,9 +330,13 @@ export class Validator {
       process.exit(1);
     }
 
-    // until data is not available we wait and idle
-    while (!(await this.isDataAvailable())) {
-      await sleep(IDLE_TIME);
+    // by default we check if the first data items are available
+    // to protect the node operator from timeout slashes due to
+    // misconfiguration of the data source
+    if (!options.skipDataAvailabilityCheck) {
+      while (!(await this.isDataAvailable())) {
+        await sleep(IDLE_TIME);
+      }
     }
 
     await this.setupValidator();
@@ -356,3 +364,8 @@ export * from "./types";
 
 // export utils
 export * from "./utils";
+
+// add this so we can JSON.stringify bignumbers
+(BigInt.prototype as any).toJSON = function () {
+  return this.toString();
+};

common/protocol/src/methods/checks/isDataAvailable.ts

@@ -11,14 +11,16 @@ import { Validator, standardizeError } from "../..";
  * @return {Promise<boolean>}
  */
 export async function isDataAvailable(this: Validator): Promise<boolean> {
+  let nextKey = "";
+
   try {
     // log debug method
     if (this.pool.data!.current_key) {
       this.logger.debug(`this.runtime.nextKey(${this.pool.data!.current_key})`);
     }
 
     // get the next key to node has to fetch
-    const nextKey = this.pool.data!.current_key
+    nextKey = this.pool.data!.current_key
       ? await this.runtime.nextKey(this, this.pool.data!.current_key)
       : this.pool.data!.start_key;
 
@@ -45,8 +47,10 @@ export async function isDataAvailable(this: Validator): Promise<boolean> {
 
     return true;
   } catch (err) {
-    this.logger.warn(`Data not available for next key. Retrying in 60s ...`);
-    this.logger.debug(standardizeError(err));
+    this.logger.warn(
+      `Data not available for next key ${nextKey}. Retrying in 60s ...`
+    );
+    this.logger.warn(standardizeError(err));
 
     return false;
   }

common/protocol/src/methods/checks/isPoolActive.ts

@@ -37,6 +37,9 @@ export function isPoolActive(this: Validator): boolean {
     case PoolStatus.POOL_STATUS_UNSPECIFIED:
       this.logger.info("Pool status is currently unspecified. Idling ...");
       return false;
+    case PoolStatus.POOL_STATUS_END_KEY_REACHED:
+      this.logger.info("End key reached. Idling ...");
+      return false;
     default:
       this.logger.info("Pool status is currently unknown. Idling ...");
       return false;

common/protocol/src/methods/main/runCache.ts

@@ -119,6 +119,15 @@ export async function runCache(this: Validator): Promise<void> {
       this.m.cache_index_tail.set(Math.max(0, currentIndex - 1));
 
       for (let i = currentIndex; i < targetIndex; i++) {
+        // if the end key is not empty and we have reached the end key of the pool
+        // we do not sync past this key
+        if (this.pool.data?.end_key && this.pool.data.end_key === key) {
+          this.logger.info(
+            `Reached pool end key "${key}", the node will not continue collecting data past this key.`
+          );
+          break;
+        }
+
         // check if data item was already collected. If it was
         // already collected we don't need to retrieve it again
         this.logger.debug(`this.cacheProvider.exists(${i.toString()})`);
@@ -223,11 +232,6 @@ export async function runCache(this: Validator): Promise<void> {
         key = nextKey;
       }
 
-      // indicate that current caching round is done
-      this.logger.debug(
-        `Finished caching from index ${currentIndex} to ${targetIndex}. Waiting for next round ...`
-      );
-
       // wait until a new bundle proposal is available. We don't need
       // to sync the pool here because the pool state already gets
       // synced in the other main function "runNode" so we only listen

common/protocol/src/methods/main/runNode.ts

@@ -1,3 +1,4 @@
+import { PoolStatus } from "@kyvejs/types/lcd/kyve/pool/v1beta1/pool";
 import { Validator } from "../..";
 import { IDLE_TIME, sleep } from "../../utils";
 
@@ -33,6 +34,11 @@ export async function runNode(this: Validator): Promise<void> {
       process.exit(1);
     }
 
+    if (this.pool.status === PoolStatus.POOL_STATUS_END_KEY_REACHED) {
+      this.logger.info(`Reached pool end key. Shutting down node ...`);
+      process.exit(0);
+    }
+
     // log warnings if storage provider balance is low
     await this.isStorageBalanceLow();
 

common/protocol/src/methods/validate/validateBundleProposal.ts

@@ -234,6 +234,19 @@ export async function validateBundleProposal(
       // validate each data item in bundle with custom runtime validation
       for (let i = 0; i < proposedBundle.length; i++) {
         if (valid) {
+          // if the pool has an end key and we find out that a data item
+          // has the end key and it is not the last data item in the bundle
+          // we consider the bundle invalid
+          if (this.pool.data?.end_key) {
+            if (
+              i < proposedBundle.length - 1 &&
+              proposedBundle[i].key === this.pool.data?.end_key
+            ) {
+              valid = false;
+              break;
+            }
+          }
+
           this.logger.debug(
             `this.runtime.validateDataItem($THIS, $PROPOSED_DATA_ITEM, $VALIDATION_DATA_ITEM)`
           );