Persist identity and peers when using local storage (#547)

* Persist identity and peers when using local storage Local storage is great because it's faster, and we can download tarballs in minutes at every boot and have tolerable downtimes. But at each boot, the node regenerates its identity. We already have a mechanism to pre-populate identities in the node yaml, but it's cumbersome. Instead, with this PR, we create a mini-pvc of 1Gi to store node identities, whenever local storage is in use. This way, local storage nodes always boot quickly, except the first time they boot, and there is no configuration burden. It is also possible to scale up the number of nodes without having to prepare identities. I tested private chain and ghostnet node, in both local and non-local storage. It behaves as intended. I also tried to pre-populate an identity in the values file in local storage and confirmed that it is not broken. * completely remove chmod 777 * fix error in snapshot-importer: check file existence * fix snapshot importer as well * fix tests * Update charts/tezos/scripts/upgrade-storage.sh Co-authored-by: Aryeh Harris <harryttd@users.noreply.github.com> * fix tests, add comment for persistent dir per review * typo * fix perms issue in the absence of 777 --------- Co-authored-by: Aryeh Harris <harryttd@users.noreply.github.com>
oxheadalpha · Mar 27, 2023 · 090f695 · 090f695
1 parent 35c9e6a
commit 090f695
Show file tree

Hide file tree

Showing 9 changed files with 47 additions and 21 deletions.
diff --git a/charts/tezos/scripts/snapshot-importer.sh b/charts/tezos/scripts/snapshot-importer.sh
@@ -12,7 +12,7 @@ if [ ! -f ${snapshot_file} ]; then
     exit 0
 fi
 
-if [ -d ${node_data_dir}/context ]; then
+if [ -e ${node_data_dir}/context/store.dict ]; then
     echo "Blockchain has already been imported. If a tarball"
     echo "instead of a regular tezos snapshot was used, it was"
     echo "imported in the snapshot-downloader container."

diff --git a/charts/tezos/scripts/upgrade-storage.sh b/charts/tezos/scripts/upgrade-storage.sh
@@ -1,8 +1,8 @@
 set -ex
 
-if [ ! -e /var/tezos/node/data/context ]
+if [ ! -e /var/tezos/node/data/context/store.dict ]
 then
-  printf "No context in data dir found, probably initial start, doing nothing."
+  printf "No store in data dir found, probably initial start, doing nothing."
   exit 0
 fi
 octez-node upgrade storage --config /etc/tezos/config.json
diff --git a/charts/tezos/templates/_containers.tpl b/charts/tezos/templates/_containers.tpl
@@ -150,6 +150,10 @@
       name: config-volume
     - mountPath: /var/tezos
       name: var-volume
+    {{- if .local_storage }}
+    - mountPath: /var/persistent
+      name: persistent-volume
+    {{- end }}
     {{- if .with_secret }}
     - mountPath: /etc/secret-volume
       name: tezos-accounts
@@ -264,6 +268,7 @@
                                                 "type"        "octez-node"
                                                 "image"       "octez"
                                                 "with_config" 0
+                                                "local_storage" $.node_vals.local_storage
                                                 "resources"   $.node_vals.resources
     ) | nindent 0 }}
 {{- end }}

diff --git a/charts/tezos/templates/nodes.yaml b/charts/tezos/templates/nodes.yaml
@@ -66,10 +66,16 @@ spec:
 {{- if $.node_vals.local_storage | default false }}
         - emptyDir: {}
           name: var-volume
-{{- else }}
+{{- end }}
   volumeClaimTemplates:
     - metadata:
+{{- if $.node_vals.local_storage | default false }}
+        # since we are using local storage, deploy a small persistent volume
+        # to store identity.json and peers.json
+        name: persistent-volume
+{{- else }}
         name: var-volume
+{{- end }}
         namespace: {{ $.Release.Namespace }}
       spec:
         accessModes:
@@ -79,6 +85,9 @@ spec:
         {{- end }}
         resources:
           requests:
+{{- if $.node_vals.local_storage | default false }}
+            storage: "1Gi"
+{{- else }}
             storage: {{ default "15Gi" $.node_vals.storage_size }}
 {{- end }}
 ---

diff --git a/test/charts/mainnet.expect.yaml b/test/charts/mainnet.expect.yaml
@@ -341,7 +341,7 @@ spec:
                   exit 0
               fi
               
-              if [ -d ${node_data_dir}/context ]; then
+              if [ -e ${node_data_dir}/context/store.dict ]; then
                   echo "Blockchain has already been imported. If a tarball"
                   echo "instead of a regular tezos snapshot was used, it was"
                   echo "imported in the snapshot-downloader container."
@@ -392,9 +392,9 @@ spec:
             - |
               set -ex
               
-              if [ ! -e /var/tezos/node/data/context ]
+              if [ ! -e /var/tezos/node/data/context/store.dict ]
               then
-                printf "No context in data dir found, probably initial start, doing nothing."
+                printf "No store in data dir found, probably initial start, doing nothing."
                 exit 0
               fi
               octez-node upgrade storage --config /etc/tezos/config.json

diff --git a/test/charts/mainnet2.expect.yaml b/test/charts/mainnet2.expect.yaml
@@ -450,7 +450,7 @@ spec:
                   exit 0
               fi
               
-              if [ -d ${node_data_dir}/context ]; then
+              if [ -e ${node_data_dir}/context/store.dict ]; then
                   echo "Blockchain has already been imported. If a tarball"
                   echo "instead of a regular tezos snapshot was used, it was"
                   echo "imported in the snapshot-downloader container."
@@ -503,9 +503,9 @@ spec:
             - |
               set -ex
               
-              if [ ! -e /var/tezos/node/data/context ]
+              if [ ! -e /var/tezos/node/data/context/store.dict ]
               then
-                printf "No context in data dir found, probably initial start, doing nothing."
+                printf "No store in data dir found, probably initial start, doing nothing."
                 exit 0
               fi
               octez-node upgrade storage --config /etc/tezos/config.json
@@ -828,7 +828,7 @@ spec:
                   exit 0
               fi
               
-              if [ -d ${node_data_dir}/context ]; then
+              if [ -e ${node_data_dir}/context/store.dict ]; then
                   echo "Blockchain has already been imported. If a tarball"
                   echo "instead of a regular tezos snapshot was used, it was"
                   echo "imported in the snapshot-downloader container."
@@ -881,9 +881,9 @@ spec:
             - |
               set -ex
               
-              if [ ! -e /var/tezos/node/data/context ]
+              if [ ! -e /var/tezos/node/data/context/store.dict ]
               then
-                printf "No context in data dir found, probably initial start, doing nothing."
+                printf "No store in data dir found, probably initial start, doing nothing."
                 exit 0
               fi
               octez-node upgrade storage --config /etc/tezos/config.json

diff --git a/test/charts/private-chain.expect.yaml b/test/charts/private-chain.expect.yaml
@@ -440,9 +440,9 @@ spec:
             - |
               set -ex
               
-              if [ ! -e /var/tezos/node/data/context ]
+              if [ ! -e /var/tezos/node/data/context/store.dict ]
               then
-                printf "No context in data dir found, probably initial start, doing nothing."
+                printf "No store in data dir found, probably initial start, doing nothing."
                 exit 0
               fi
               octez-node upgrade storage --config /etc/tezos/config.json
@@ -1013,9 +1013,9 @@ spec:
             - |
               set -ex
               
-              if [ ! -e /var/tezos/node/data/context ]
+              if [ ! -e /var/tezos/node/data/context/store.dict ]
               then
-                printf "No context in data dir found, probably initial start, doing nothing."
+                printf "No store in data dir found, probably initial start, doing nothing."
                 exit 0
               fi
               octez-node upgrade storage --config /etc/tezos/config.json
@@ -1217,9 +1217,9 @@ spec:
             - |
               set -ex
               
-              if [ ! -e /var/tezos/node/data/context ]
+              if [ ! -e /var/tezos/node/data/context/store.dict ]
               then
-                printf "No context in data dir found, probably initial start, doing nothing."
+                printf "No store in data dir found, probably initial start, doing nothing."
                 exit 0
               fi
               octez-node upgrade storage --config /etc/tezos/config.json

diff --git a/utils/config-generator.sh b/utils/config-generator.sh
@@ -6,7 +6,7 @@ cat /etc/tezos/data/config.json
 echo ------------------------------------------------------------
 
 mkdir -p /var/tezos/client
-chmod -R 777 /var/tezos
+mkdir -p /var/tezos/node/data
 set -e
 python3 /config-generator.py "$@"
 set +e
@@ -17,6 +17,15 @@ set +e
 # tezos docker image.
 
 MY_CLASS=$(echo $NODES | jq -r ".\"${MY_NODE_CLASS}\"")
+
+# Set up symlinks for local storage
+local_storage=$(echo $MY_CLASS | jq -r ".local_storage")
+if [ "${local_storage}" == "true" ]; then
+  echo '{ "version": "2.0" }' > /var/tezos/node/data/version.json
+  ln -s /var/persistent/peers.json /var/tezos/node/data/peers.json
+  ln -s /var/persistent/identity.json /var/tezos/node/data/identity.json
+fi
+
 AM_I_BAKER=0
 if [ "$MY_CLASS" != null ]; then
     AM_I_BAKER=$(echo $MY_CLASS | \
@@ -39,3 +48,6 @@ if [ "$AM_I_BAKER" -eq 1 ]; then
 
     echo "$my_baker_account" > /etc/tezos/baker-account
 fi
+
+# make sure tezos user owns everything in /var/tezos
+chown -R 1000:1000 /var/tezos
diff --git a/utils/snapshot-downloader.sh b/utils/snapshot-downloader.sh
@@ -12,7 +12,7 @@ if [ ! -d "$data_dir" ]; then
   exit 1
 fi
 
-if [ -d "$node_data_dir/context" ]; then
+if [ -e "$node_data_dir/context/store.dict" ]; then
   echo "Blockchain has already been imported. Exiting."
   exit 0
 fi