-
-
Notifications
You must be signed in to change notification settings - Fork 32
/
run.sh
executable file
·2393 lines (2126 loc) · 94.9 KB
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
#####################################################################################################
# Steem node manager
# Released under GNU AGPL by Someguy123
#
# Github: https://github.com/Someguy123/steem-docker
#
# **Steem-in-a-box** is a toolkit for using the Steem Docker images[1] published by @someguy123.
# It's purpose is to simplify the deployment of `steemd` nodes.
#
# For more information, see README.md - or run `./run.sh help`
#
# [1] https://hub.docker.com/r/someguy123/steem/tags/
#
#####################################################################################################
BOLD="" RED="" GREEN="" YELLOW="" BLUE="" MAGENTA="" CYAN="" WHITE="" RESET=""
if [ -t 1 ]; then
BOLD="$(tput bold)" RED="$(tput setaf 1)" GREEN="$(tput setaf 2)" YELLOW="$(tput setaf 3)" BLUE="$(tput setaf 4)"
MAGENTA="$(tput setaf 5)" CYAN="$(tput setaf 6)" WHITE="$(tput setaf 7)" RESET="$(tput sgr0)"
fi
SIAB_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Array of Privex ShellCore modules to be loaded during ShellCore initialisation.
SG_LOAD_LIBS=(gnusafe helpers trap_helper traplib)
# Minimum required version of Privex ShellCore
# If a version older than this is installed, an update will be forced immediately
SIAB_MIN_SC_VER="0.4.3"
# Global variable used by 000_shellcore_setup:_sc_force_update to inform run.sh if it needs to restart the script
_SIAB_RELOAD=0
source "${SIAB_DIR}/scripts/siab_libs.sh"
source "${SIAB_DIR}/scripts/000_shellcore_setup.sh"
siab_load_lib shellcoresetup
_setup_shellcore
# _siab_sc_init "$@"
_sc_version_check "$@"
if (( _SIAB_RELOAD == 1 )); then
_SIAB_RELOAD=0
echo -e "${GREEN} >>> Attempting to restart run.sh using command + args: ${RESET}\n"
echo " $0 $*"
echo
echo -e "${GREEN} [...] Re-executing run.sh ...${RESET}\n\n"
set +u # Ignore undefined variables until the script is finished reloading
cleanup_env # Remove old SG_ and SRCED_ env vars before reloading to avoid conflicts
exec "$0" "$@" # Reload run.sh by replacing the currently running script with run.sh, using the same arguments as before.
exit
fi
autoupdate_shellcore
# Privex ShellCore Error Handler
# 0 = Abort immediately upon a non-zero return code
# 1 = Ignore the next non-zero return code, then re-enable strict mode (0)
# 2 = Fully disable non-zero error handling, until manually re-enabled via 'error_control 0' or 'error_control 1'
error_control 2
# If set to 1, the run.sh function siab_exit() will ALWAYS print a full traceback at the end of each run.sh execution
# even if no error was detected.
: ${SIAB_TRACE_EXIT=0}
_ERROR_TRIGGERED=0
_SIAB_HANDLE_EXIT=1
print_traceback() {
local trace_depth=1
(( $# > 0 )) && trace_depth=$(($1))
msgerr nots bold blue "\nTraceback:\n\n${RESET}${BOLD}$(trap_traceback $trace_depth)\n"
}
siab_error() {
local error_code="$?"
msg
msgerr bold red "A fatal error has occurred and SIAB run.sh must exit."
(( $# >= 1 )) && msgerr bold red "Line number which triggered this: $1"
(( $# >= 2 )) && msgerr bold red "Bash command / function which triggered this: $2"
msg
_ERROR_TRIGGERED=$error_code
(( error_code == 0 )) && _ERROR_TRIGGERED=1
exit $error_code
}
siab_exit() {
local error_code="$?"
(( _ERROR_TRIGGERED > 0 )) && error_code=$_ERROR_TRIGGERED
if (( _SIAB_HANDLE_EXIT == 1 )); then
msg
# (( error_code == 0 )) && msgerr green "run.sh has finished - exiting SIAB run.sh cleanly."
(( error_code != 0 )) && msgerr bold red "[ERROR] SIAB not exiting cleanly. Detected non-zero error code while exiting: $error_code"
fi
if (( SIAB_TRACE_EXIT == 1 )); then
msgerr bold red "[DEBUGGING] Detected SIAB_TRACE_EXIT == 1 - always running traceback on exit. Exit code: $error_code"
print_traceback -1
fi
exit $error_code
}
CLEANUP_FILES=()
CLEANUP_FOLDERS=()
siab_cleanup() {
local path_len f
for f in "${CLEANUP_FILES[@]}"; do
path_len=$(len "$f")
if (( path_len < 5 )); then
msgerr bold yellow "WARNING: Not cleaning up leftover file '$f' as path is shorter than 5 chars."
msgerr bold yellow " This is to prevent accidental deletion of important system files."
else
_debug "Removing leftover file: $f"
rm "$f"
fi
done
for f in "${CLEANUP_FOLDERS[@]}"; do
path_len=$(len "$f")
if (( path_len < 5 )); then
msgerr bold yellow "WARNING: Not cleaning up leftover folder '$f' as path is shorter than 5 chars."
msgerr bold yellow " This is to prevent accidental deletion of important system folders."
else
_debug "Removing leftover folder: $f"
rm -r "$f"
fi
done
}
siab_abort() {
local error_code="$?" s_line="$1" s_cmd="$2" s_signal="$3"
msg "\n"
msgerr bold red "[ERROR] Detected signal '$s_signal' while executing line number $s_line - last command: $s_cmd"
print_traceback
[[ "$s_signal" == "SIGINT" ]] && msgerr bold red "[ERROR] SIGINT (CTRL-C) detected. User requested SIAB to exit immediately. Stopping run.sh ..."
msg "\n"
_SIAB_HANDLE_EXIT=0 && error_control 2
exit 5
}
declare -f -t siab_abort siab_error siab_exit print_traceback
trap_add 'siab_exit' EXIT # ! ! ! TRAP EXIT ! ! !
trap_add 'siab_error ${LINENO} "$BASH_COMMAND"' ERR # ! ! ! TRAP ERR ! ! !
trap_add 'siab_abort ${LINENO} "$BASH_COMMAND" SIGINT' SIGINT
trap_add 'siab_abort ${LINENO} "$BASH_COMMAND" SIGTERM' SIGTERM
trap_add 'siab_abort ${LINENO} "$BASH_COMMAND" SIGHUP' SIGHUP
# trap_add 'siab_error ${LINENO} "$BASH_COMMAND"' SIGINT # ! ! ! TRAP ERR ! ! !
# trap_add 'siab_error ${LINENO} "$BASH_COMMAND"' SIGTERM # ! ! ! TRAP ERR ! ! !
# trap_add 'siab_error ${LINENO} "$BASH_COMMAND"' SIGHUP # ! ! ! TRAP ERR ! ! !
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
: ${DOCKER_DIR="$DIR/dkr"}
: ${FULL_DOCKER_DIR="$DIR/dkr_fullnode"}
: ${DATADIR="$DIR/data"}
: ${DOCKER_NAME="seed"}
STEEM_RUN_ARGS=()
STEEM_RUN_ARGS_EXTRA=()
DKR_RUN_ARGS=()
: "${DKR_DRY_RUN=0}"
DKR_VOLUMES=() EXTRA_VOLUMES=()
STEEM_REPLAY_ARGS=()
if [[ -f .env ]]; then
source .env
fi
: ${CONFIG_FILE="${DATADIR}/witness_node_data_dir/config.ini"}
: ${NETWORK="steem"}
if [[ "$NETWORK" == "hive" ]]; then
: ${DOCKER_IMAGE="hive"}
: ${STEEM_SOURCE="https://github.com/openhive-network/hive.git"}
: ${NETWORK_NAME="Hive"}
: ${SELF_NAME="Hive-in-a-box"}
: ${BC_HTTP="http://files.privex.io/hive/block_log.lz4"} # HTTP or HTTPS url to grab the blockchain from. Set compression in BC_HTTP_CMP
: ${BC_HTTP_RAW="http://files.privex.io/hive/block_log"} # Uncompressed block_log over HTTP
: ${BC_HTTP_CMP="lz4"} # Compression type, can be "xz", "lz4", or "no" (for no compression)
: ${BC_RSYNC="rsync://files.privex.io/hive/block_log"} # Anonymous rsync daemon URL to the raw block_log
: ${ROCKSDB_RSYNC="rsync://files.privex.io/hive/rocksdb/"} # Rsync URL for MIRA RocksDB files
: ${DK_TAG_BASE="someguy123/hive"}
: ${REMOTE_WS="wss://hived.privex.io"}
: ${REMOTE_RPC="https://hived.privex.io"}
: ${STOP_TIME=600} # Amount of time in seconds to allow the docker container to stop before killing it.
: ${STEEM_RPC_PORT="8091"} # Local steemd RPC port, used by commands such as 'monitor' which need to query your steemd's HTTP RPC
: ${DKR_DATA_MOUNT="/steem"} # Mount $DATADIR onto this folder within the container
: ${DKR_SHM_MOUNT="/shm"} # Mount $SHM_DIR onto this folder within the container
: ${DKR_RUN_BIN="steemd"} # Run this executable within the container
fi
: ${DKR_DATA_MOUNT="/steem"} # Mount $DATADIR onto this folder within the container
: ${DKR_SHM_MOUNT="/shm"} # Mount $SHM_DIR onto this folder within the container
: ${DKR_RUN_BIN="steemd"} # Run this executable within the container
# if (( ${#STEEM_RUN_ARGS[@]} == 0 )); then
# STEEM_RUN_ARGS+=("--data-dir=${DKR_DATA_MOUNT}/witness_node_data_dir")
# fi
(( ${#STEEM_RUN_ARGS_EXTRA[@]} > 0 )) && STEEM_RUN_ARGS+=("${STEEM_RUN_ARGS_EXTRA[@]}")
# the tag to use when running/replaying steemd
: ${DOCKER_IMAGE="steem"}
: ${NETWORK_NAME="Steem"}
: ${SELF_NAME="Steem-in-a-box"}
# HTTP or HTTPS url to grab the blockchain from. Set compression in BC_HTTP_CMP
: ${BC_HTTP="http://files.privex.io/steem/block_log.lz4"}
# Uncompressed block_log over HTTP, used for getting size for truncation, and
# potentially resuming downloads
: ${BC_HTTP_RAW="http://files.privex.io/steem/block_log"}
# Compression type, can be "xz", "lz4", or "no" (for no compression)
# Uses on-the-fly de-compression while downloading, to conserve disk space
# and save time by not having to decompress after the download is finished
: ${BC_HTTP_CMP="lz4"}
# Anonymous rsync daemon URL to the raw block_log, for repairing/resuming
# a damaged/incomplete block_log. Set to "no" to disable rsync when resuming.
: ${BC_RSYNC="rsync://files.privex.io/steem/block_log"}
# Rsync URL for MIRA RocksDB files
: ${ROCKSDB_RSYNC="rsync://files.privex.io/steem/rocksdb/"}
: ${DK_TAG_BASE="someguy123/steem"}
: ${DK_TAG="${DK_TAG_BASE}:latest"}
: ${DK_TAG_FULL="${DK_TAG_BASE}:latest-full"}
: ${SHM_DIR="/dev/shm"}
: ${REMOTE_WS="wss://steemd.privex.io"}
: ${REMOTE_RPC="https://steemd.privex.io"}
# Amount of time in seconds to allow the docker container to stop before killing it.
# Default: 600 seconds (10 minutes)
: ${STOP_TIME=600}
# Git repository to use when building Steem - containing steemd code
: ${STEEM_SOURCE="https://github.com/steemit/steem.git"}
# Local steemd RPC port, used by commands such as 'monitor' which need to query your steemd's HTTP RPC
: ${STEEM_RPC_PORT="8091"}
# Comma separated list of ports to expose to the internet.
# By default, only port 2001 will be exposed (the P2P seed port)
: ${PORTS="2001"}
# blockchain folder, used by dlblocks
: ${BC_FOLDER="$DATADIR/witness_node_data_dir/blockchain"}
: ${EXAMPLE_MIRA="$DATADIR/witness_node_data_dir/database.cfg.example"}
: ${MIRA_FILE="$DATADIR/witness_node_data_dir/database.cfg"}
: ${EXAMPLE_CONF="$DATADIR/witness_node_data_dir/config.ini.example"}
: ${CONF_FILE="$DATADIR/witness_node_data_dir/config.ini"}
# Set these environment vars to skip the yes/no prompts during fix-blocks
# 0 = default (prompt user for action), 1 = automatically answer "yes", 2 = automatically answer "no"
: ${AUTO_FIX_BLOCKLOG=0}
: ${AUTO_FIX_BLOCKINDEX=0}
: ${AUTO_FIX_ROCKSDB=0}
# If AUTO_FIX_BLOCKLOG is set to 1, this controls whether we verify block_log via checksummed rsync, in the
# event that the local block_log is the same size as the remote block_log
# 1 = (default) Do not attempt to verify/repair block_log if the size is equal to the remote server
# 0 = Attempt to verify/repair block_log even if the size is equal to the remote server
: ${AUTO_IGNORE_EQUAL=1}
BUILD_FULL=0 # Internal variable. Set to 1 by build_full to inform child functions
CUST_TAG="steem" # Placeholder for custom tag var CUST_TAG (shared between functions)
BUILD_VER="" # Placeholder for BUILD_VER shared between functions
# Array of additional arguments to be passed to Docker during builds
# Generally populated using arguments passed to build/build_full
# But you can specify custom additional build parameters by setting BUILD_ARGS
# as an array in .env
# e.g.
#
# BUILD_ARGS=('--rm' '-q' '--compress')
#
BUILD_ARGS=()
# MSG_TS_DEFAULT controls whether timestamps are automatically added to any
# message that doesn't opt-out using 'nots', or whether messages need to
# specify 'ts' to enable timestamps.
# This allows timestamps to be temporarily disabled per function, preventing the need to
# constantly add "nots".
#
# 1 = msg timestamps are opt-out (to disable timestamps: msg nots green "some message")
# 0 = msg timestamps are opt-in (to enable timestamps: msg ts green "some message")
MSG_TS_DEFAULT=1
# easy coloured messages function
# written by @someguy123
function msg () {
if [[ "$#" -eq 0 ]]; then echo ""; return; fi;
if [[ "$#" -eq 1 ]]; then
echo -e "$1"
return
fi
_msg=""
if (( MSG_TS_DEFAULT == 1 )); then
[[ "$1" == "ts" ]] && shift
{ [[ "$1" == "nots" ]] && shift; } || _msg="[$(date +'%Y-%m-%d %H:%M:%S %Z')] "
else
[[ "$1" == "nots" ]] && shift
[[ "$1" == "ts" ]] && shift && _msg="[$(date +'%Y-%m-%d %H:%M:%S %Z')] "
fi
if [[ "$#" -gt 2 ]] && [[ "$1" == "bold" ]]; then
echo -n "${BOLD}"
shift
fi
(($#==1)) && _msg+="$@" || _msg+="${@:2}"
case "$1" in
bold) echo -e "${BOLD}${_msg}${RESET}";;
BLUE|blue) echo -e "${BLUE}${_msg}${RESET}";;
YELLOW|yellow) echo -e "${YELLOW}${_msg}${RESET}";;
RED|red) echo -e "${RED}${_msg}${RESET}";;
GREEN|green) echo -e "${GREEN}${_msg}${RESET}";;
CYAN|cyan) echo -e "${CYAN}${_msg}${RESET}";;
MAGENTA|magenta|PURPLE|purple) echo -e "${MAGENTA}${_msg}${RESET}";;
* ) echo -e "${_msg}";;
esac
}
export -f msg
export RED GREEN YELLOW BLUE BOLD NORMAL RESET
# load helpers
# source "${SIAB_DIR}/scripts/010_helpers.sh"
siab_load_lib helpers docker stateshot
# if the config file doesn't exist, try copying the example config
if [[ ! -f "$CONF_FILE" ]]; then
if [[ -f "$EXAMPLE_CONF" ]]; then
echo "${YELLOW}File config.ini not found. copying example (seed)${RESET}"
cp -vi "$EXAMPLE_CONF" "$CONF_FILE"
echo "${GREEN} > Successfully installed example config for seed node.${RESET}"
echo " > You may want to adjust this if you're running a witness, e.g. disable p2p-endpoint"
else
echo "${YELLOW}WARNING: You don't seem to have a config file and the example config couldn't be found...${RESET}"
echo "${YELLOW}${BOLD}You may want to check these files exist, or you won't be able to launch Steem${RESET}"
echo "Example Config: $EXAMPLE_CONF"
echo "Main Config: $CONF_FILE"
fi
fi
if [[ ! -f "$MIRA_FILE" ]]; then
if [[ -f "$EXAMPLE_MIRA" ]]; then
echo "${YELLOW}File database.cfg not found. copying example ${RESET}"
cp -vi "$EXAMPLE_MIRA" "$MIRA_FILE"
echo "${GREEN} > Successfully installed example MIRA config.${RESET}"
echo " > You may want to adjust this depending on your resources and type of node:"
echo " - - > https://github.com/steemit/steem/blob/master/doc/mira.md"
else
echo "${YELLOW}WARNING: You don't seem to have a MIRA config file (data/database.cfg) and the example config couldn't be found...${RESET}"
echo "${YELLOW}${BOLD}You may want to check these files exist, or you won't be able to use Steem with MIRA${RESET}"
echo "Example Config: $EXAMPLE_MIRA"
echo "Main Config: $MIRA_FILE"
fi
fi
IFS=","
DPORTS=()
for i in $PORTS; do
if [[ $i != "" ]]; then
if grep -q ":" <<< "$i"; then
DPORTS+=("-p$i")
else
DPORTS+=("-p0.0.0.0:$i:$i")
fi
fi
done
if ! (( ${#DKR_VOLUMES[@]} )); then
DKR_VOLUMES=(
"${SHM_DIR}:${DKR_SHM_MOUNT}"
"${DATADIR}:${DKR_DATA_MOUNT}"
)
fi
if (( ${#EXTRA_VOLUMES[@]} )); then
IFS="," read -r -a _EXTRA_VOLS <<< "$EXTRA_VOLUMES"
DKR_VOLUMES+=("${_EXTRA_VOLS[@]}")
fi
# load docker hub API
# source "${SIAB_DIR}/scripts/030_docker.sh"
# source "${SIAB_DIR}/scripts/040_stateshot.sh"
help() {
echo "Usage: $0 COMMAND [DATA]"
echo
echo "Commands:
start - starts ${NETWORK_NAME} container
stop - stops ${NETWORK_NAME} container
kill - force stop ${NETWORK_NAME} container (in event of steemd hanging indefinitely)
restart - restarts ${NETWORK_NAME} container
replay - starts ${NETWORK_NAME} container (in replay mode)
memory_replay - starts ${NETWORK_NAME} container (in replay mode, with --memory-replay - for use with MIRA-enabled images only)
status - show status of ${NETWORK_NAME} container
ver - check version of ${SELF_NAME}, your ${NETWORK_NAME} docker image, and detect if any updates are available
fix-blocks - downloads / repairs your blockchain, block index, and/or rocksdb.
check '$0 fix-blocks help' for more info
clean - Remove blockchain, p2p, and/or shared mem folder contents (warns beforehand)
dlblocks - download and decompress the blockchain and block_log.index to speed up your first start
dlblockindex - download/repair just the block index (block_log.index)
dlrocksdb - download / replace RocksDB files - for use with MIRA-enabled ${NETWORK_NAME} images
shm_size - resizes /dev/shm to size given, e.g. ./run.sh shm_size 10G
install_docker - install docker
install - pulls latest docker image from server (no compiling)
install_full - pulls latest (FULL NODE FOR RPC) docker image from server (no compiling)
rebuild - builds ${NETWORK_NAME} container (from docker file), and then restarts it
build - only builds ${NETWORK_NAME} container (from docker file)
logs - show all logs inc. docker logs, and ${NETWORK_NAME} logs
wallet - open cli_wallet in the container
remote_wallet - open cli_wallet in the container connecting to a remote seed
enter - enter a bash session in the currently running container
shell - launch the ${NETWORK_NAME} container with appropriate mounts, then open bash for inspection
"
echo
exit
}
APT_UPDATED="n"
pkg_not_found() {
# check if a command is available
# if not, install it from the package specified
# Usage: pkg_not_found [cmd] [apt-package]
# e.g. pkg_not_found git git
if [[ $# -lt 2 ]]; then
echo "${RED}ERR: pkg_not_found requires 2 arguments (cmd) (package)${NORMAL}"
exit
fi
local cmd=$1
local pkg=$2
if ! [ -x "$(command -v $cmd)" ]; then
echo "${YELLOW}WARNING: Command $cmd was not found. installing now...${NORMAL}"
if [[ "$APT_UPDATED" == "n" ]]; then
sudo apt update -y
APT_UPDATED="y"
fi
sudo apt install -y "$pkg"
fi
}
optimize() {
echo 75 | sudo tee /proc/sys/vm/dirty_background_ratio
echo 1000 | sudo tee /proc/sys/vm/dirty_expire_centisecs
echo 80 | sudo tee /proc/sys/vm/dirty_ratio
echo 30000 | sudo tee /proc/sys/vm/dirty_writeback_centisecs
}
parse_build_args() {
BUILD_VER=$1
CUST_TAG="steem:$BUILD_VER"
if (( $BUILD_FULL == 1 )); then
CUST_TAG+="-full"
fi
BUILD_ARGS+=('--build-arg' "steemd_version=${BUILD_VER}")
shift
if (( $# >= 2 )); then
if [[ "$1" == "tag" ]]; then
CUST_TAG="$2"
msg yellow " >> Custom re-tag specified. Will tag new image with '${CUST_TAG}'"
shift; shift; # Get rid of the two tag arguments. Everything after is now build args
fi
fi
local has_steem_src='n'
if (( $# >= 1 )); then
msg yellow " >> Additional build arguments specified."
for a in "$@"; do
msg yellow " ++ Build argument: ${BOLD}${a}"
BUILD_ARGS+=('--build-arg' "$a")
if grep -q 'STEEM_SOURCE' <<< "$a"; then
has_steem_src='y'
fi
done
fi
if [[ "$has_steem_src" == "y" ]]; then
msg bold yellow " [!!] STEEM_SOURCE has been specified in the build arguments. Using source from build args instead of global"
else
msg bold yellow " [!!] Did not find STEEM_SOURCE in build args. Using STEEM_SOURCE from environment:"
msg bold yellow " [!!] STEEM_SOURCE = ${STEEM_SOURCE}"
BUILD_ARGS+=('--build-arg' "STEEM_SOURCE=${STEEM_SOURCE}")
fi
msg blue " ++ CUSTOM BUILD SPECIFIED. Building from branch/tag ${BOLD}${BUILD_VER}"
msg blue " ++ Tagging final image as: ${BOLD}${CUST_TAG}"
msg yellow " -> Docker build arguments: ${BOLD}${BUILD_ARGS[*]}"
}
build_local() {
STEEM_SOURCE="local_src_folder"
DOCKER_DIR="${DIR}/dkr_local"
if [[ ! -d "${DOCKER_DIR}/src" ]]; then
msg bold red "ERROR: You must place the source code inside of ${DOCKER_DIR}/src"
return 1
fi
msg green " >>> Local build requested."
msg green " >>> Will build Steem using code stored in '${DOCKER_DIR}/src' instead of remote git repo"
build "$@"
}
# Build standard low memory node as a docker image
# Usage: ./run.sh build [version] [tag tag_name] [build_args]
# Version is prefixed with v, matching steem releases
# e.g. build v0.20.6
#
# Override destination tag:
# ./run.sh build v0.21.0 tag 'steem:latest'
#
# Additional build args:
# ./run.sh build v0.21.0 ENABLE_MIRA=OFF
#
# Or combine both:
# ./run.sh build v0.21.0 tag 'steem:mira' ENABLE_MIRA=ON
#
build() {
fmm="Low Memory Mode (For Seed / Witness nodes)"
(( $BUILD_FULL == 1 )) && fmm="Full Memory Mode (For RPC nodes)" && DOCKER_DIR="$FULL_DOCKER_DIR"
BUILD_MSG=" >> Building docker container [[ ${fmm} ]]"
if (( $# >= 1 )); then
parse_build_args "$@"
sleep 2
cd "$DOCKER_DIR"
msg bold green "$BUILD_MSG"
docker build "${BUILD_ARGS[@]}" -t "$CUST_TAG" .
ret=$?
if (( $ret == 0 )); then
echo "${RED}
!!! !!! !!! !!! !!! !!! READ THIS !!! !!! !!! !!! !!! !!!
!!! !!! !!! !!! !!! !!! READ THIS !!! !!! !!! !!! !!! !!!
For your safety, we've tagged this image as $CUST_TAG
To use it in this ${SELF_NAME}, run:
${GREEN}${BOLD}
docker tag $CUST_TAG ${DOCKER_IMAGE}:latest
${RESET}${RED}
!!! !!! !!! !!! !!! !!! READ THIS !!! !!! !!! !!! !!! !!!
!!! !!! !!! !!! !!! !!! READ THIS !!! !!! !!! !!! !!! !!!
${RESET}
"
msg bold green " +++ Successfully built steemd"
msg green " +++ Steem node type: ${BOLD}${fmm}"
msg green " +++ Version/Branch: ${BOLD}${BUILD_VER}"
msg green " +++ Build args: ${BOLD}${BUILD_ARGS[@]}"
msg green " +++ Docker tag: ${CUST_TAG}"
else
msg bold red " !!! ERROR: Something went wrong during the build process."
msg red " !!! Please scroll up and check for any error output during the build."
fi
return
fi
msg bold green "$BUILD_MSG"
cd "$DOCKER_DIR"
docker build -t "$DOCKER_IMAGE" .
ret=$?
if (( $ret == 0 )); then
msg bold green " +++ Successfully built current stable steemd"
msg green " +++ Steem node type: ${BOLD}${fmm}"
msg green " +++ Docker tag: ${DOCKER_IMAGE}"
else
msg bold red " !!! ERROR: Something went wrong during the build process."
msg red " !!! Please scroll up and check for any error output during the build."
fi
}
# Build full memory node (for RPC nodes) as a docker image
# Usage: ./run.sh build_full [version]
# Version is prefixed with v, matching steem releases
# e.g. build_full v0.20.6
build_full() {
BUILD_FULL=1
build "$@"
}
# Usage: ./run.sh dlblocks [override_dlmethod] [url] [compress]
# Download the block_log from a remote server and de-compress it on-the-fly to save space,
# then places it correctly into $BC_FOLDER
# Automatically attempts to resume partially downloaded block_log's using rsync, or http if
# rsync is disabled in .env
#
# override_dlmethod - use this to force downloading a certain way (OPTIONAL)
# choices:
# - rsync - download via rsync, resume if exists, using append-verify and ignore times
# - rsync-replace - download whole file via rsync, delete block_log before download
# - http - download via http. if uncompressed, try to resume when possible
# - http-replace - do not attempt to resume. delete block_log before download
#
# url - Download/install block log using the supplied dlmethod from this url. (OPTIONAL)
#
# compress - Only valid for http/http-replace. Decompress the file on the fly. (OPTIONAL)
# options: xz, lz4, no (no compression)
# if a custom url is supplied, but no compression method, it is assumed it is raw and not compressed.
#
# Example: The default compressed lz4 download failed, but left it's block_log in place.
# You don't want to use rsync to resume, because your network is very fast
# Instead, you can continue your download using the uncompressed version over HTTP:
#
# ./run.sh dlblocks http "http://files.privex.io/steem/block_log"
#
# Or just re-download the whole uncompressed file instead of resuming:
#
# ./run.sh dlblocks http-replace "http://files.privex.io/steem/block_log"
#
dlblocks() {
pkg_not_found rsync rsync
pkg_not_found lz4 liblz4-tool
pkg_not_found xz xz-utils
[[ ! -d "$BC_FOLDER" ]] && mkdir -p "$BC_FOLDER"
[[ -f "$BC_FOLDER/block_log.index" ]] && msg "Removing old block index" && sudo rm -vf "$BC_FOLDER/block_log.index" 2> /dev/null
if (( $# > 0 )); then
custom-dlblocks "$@"
return $?
fi
if [[ -f "$BC_FOLDER/block_log" ]]; then
msg yellow "It looks like block_log already exists"
if [[ "$BC_RSYNC" == "no" ]]; then
msg red "As BC_RSYNC is set to 'no', we're just going to try to retry the http download"
msg "If your HTTP source is uncompressed, we'll try to resume it"
dl-blocks-http "$BC_HTTP" "$BC_HTTP_CMP"
return
else
msg green "We'll now use rsync to attempt to repair any corruption, or missing pieces from your block_log."
dl-blocks-rsync "$BC_RSYNC"
return
fi
fi
msg "No existing block_log found. Will use standard http to download, and will\n also decompress lz4 while downloading, to save time."
msg "If you encounter an error while downloading the block_log, just run dlblocks again,\n and it will use rsync to resume and repair it"
dl-blocks-http "$BC_HTTP" "$BC_HTTP_CMP"
msg "FINISHED. Blockchain installed to ${BC_FOLDER}/block_log (make sure to check for any errors above)"
msg red "If you encountered an error while downloading the block_log, just run dlblocks again\n and it will use rsync to resume and repair it"
echo "Remember to resize your /dev/shm, and run with replay!"
echo "$ ./run.sh shm_size SIZE (e.g. 8G)"
echo "$ ./run.sh replay"
}
# internal helper function for handling AUTO_FIX_ prompts
# if first arg is 1, will return 0 (true) immediately
# if first arg is 2, will return 1 (false) immediately
# if first arg is any other number, e.g. 0, then will display a prompt using 'yesno' with arg 2+
#
# usage:
# if _fixbl_prompt ${AUTO_FIX_BLOCKLOG} "Trim blocklog? (y/N) > " defno; then
# echo "trimming blocklog"
# else
# echo "not trimming blocklog"
# fi
#
_fixbl_prompt() {
(( $1 == 1 )) || { (( $1 != 2 )) && yesno "${@:2}"; };
}
local-file-size() {
local curr_os=$(uname -s)
if grep -qi "linux" <<< "$curr_os"; then
stat --printf="%s" "$1"
elif egrep -qi "darwin|freebsd|openbsd" <<< "$curr_os"; then
stat -f "%z" "$1"
else
du --apparent-size --block-size=1 "$1"
fi
}
# usage: fix-blocks-blocklog (path to local block_log)
#
# example:
# # With no arguments, it will default to ${BC_FOLDER}/block_log
# fix-blocks-blocklog
# # Otherwise, you can specify the path to the local block_log to be repaired
# fix-blocks-blocklog "/steem/data/witness_node_data_dir/blockchain/block_log"
#
fix-blocks-blocklog() {
msg
msg bold green " ========================================================================"
msg bold green " = ="
msg bold green " = Updating, trimming, and validating your block_log ="
msg bold green " = ="
msg bold green " ========================================================================"
msg
local local_bsz=0 local_bl="${BC_FOLDER}/block_log"
local remote_bsz=$(remote-file-size "$BC_HTTP_RAW" | tr -d '\r')
(( $# > 0 )) && local_bl="$1"
_debug "before local-file-size"
if [[ -f "$local_bl" ]]; then
local_bsz=$(local-file-size "$local_bl")
fi
_debug "before casting local_bsz / remote_bsz"
_debug "local_bsz is: $local_bsz"
_debug "remote_bsz is: $remote_bsz"
_debug "casting remote_bsz"
remote_bsz=$((remote_bsz))
_debug "casting local_bsz"
local_bsz=$((local_bsz))
msg
msg nots cyan " Local block_log path: ${BOLD}$local_bl"
msg nots cyan " Local block_log size: ${BOLD}$local_bsz bytes"
msg
msg nots cyan " Remote block_log URL: ${BOLD}$BC_HTTP_RAW"
msg nots cyan " Remote block_log size: ${BOLD}$remote_bsz bytes"
msg
_debug "before if (( $local_bsz > $remote_bsz ))"
if (( $local_bsz > $remote_bsz )); then
msg nots yellow " >> Your block_log file is larger than the remote block_log at $BC_HTTP_RAW"
msg
msg nots yellow " >> To repair your block_log - we'll need to trim it"
msg nots yellow " >> You won't be able to use the remote server's copy of block_log.index, and/or RocksDB unless your block_log"
msg nots yellow " >> is the same size as the server's copy."
msg
if _fixbl_prompt "$AUTO_FIX_BLOCKLOG" " ${MAGENTA}Do you want to trim your block_log down to $remote_bsz bytes?${RESET} (y/n) > "; then
msg green " [...] Trimming local block_log down to $remote_bsz bytes..."
truncate -s "$remote_bsz" "$local_bl"
msg green " [+++] Truncated block_log down to $remote_bsz bytes."
else
msg red " [!!!] Not trimming block_log"
fi
elif (( $local_bsz < $remote_bsz )); then
msg nots yellow " >> Your block_log file is smaller than the remote block_log at $BC_HTTP_RAW"
msg
msg nots yellow " >> To repair your block_log - we'll need to download the rest of the block_log using rsync, "
msg nots yellow " >> which will append to your block_log, instead of downloading it from scratch."
msg nots yellow " >> You won't be able to use the remote server's copy of block_log.index, and/or RocksDB unless your block_log"
msg nots yellow " >> is the same size as the server's copy."
msg
if _fixbl_prompt "$AUTO_FIX_BLOCKLOG" " ${MAGENTA}Do you want to download the rest of the block_log?${RESET} (y/n) > "; then
msg green " [...] Downloading block_log via rsync using --append ..."
rsync -Ivh --progress --append "$BC_RSYNC" "$local_bl"
msg green " [+++] Finished downloading block_log"
else
msg red " [!!!] Not downloading block_log"
fi
else
msg nots yellow " >> It appears your local block_log is the same size as the remote block_log at $BC_HTTP_RAW"
msg
msg nots yellow " >> If you believe your block_log is corrupted, we can repair it using rsync."
msg nots yellow " >> WARNING: This may take several hours, depending on your disk speed, network, and CPU performance."
msg nots yellow " >> You can say no, and we'll continue with the next fix-blocks step."
msg
# If AUTO_FIX_BLOCK_LOG is set to 1 (auto yes), but AUTO_IGNORE_EQUAL is also set to 1 (ignore equal size),
# then we need to change _auto_fix to 2 (auto no), since the block_log is the same size as the remote server.
local _auto_fix=$(($AUTO_FIX_BLOCKLOG))
(( AUTO_FIX_BLOCKLOG == 1 )) && (( AUTO_IGNORE_EQUAL == 1 )) && _auto_fix=2
if _fixbl_prompt "$_auto_fix" " ${MAGENTA}Do you want to check your block_log for corruption via rsync?${RESET} (y/N) > " defno; then
msg green " [...] Verifying / repairing block_log via rsync using --inplace ..."
msg green " [...] It may take 30-60 minutes before you see any progress here ..."
rsync -Ivhc --progress --inplace "$BC_RSYNC" "$local_bl"
msg green " [+++] Finished validating block_log"
else
msg nots red "\n [!!!] Not validating block_log"
fi
fi
return 0
}
fix-blocks-index() {
error_control 0
msg
msg bold green " ========================================================================"
msg bold green " = ="
msg bold green " = Downloading / replacing your block_log.index ="
msg bold green " = ="
msg bold green " ========================================================================"
msg
local local_idx="${BC_FOLDER}/block_log.index"
(( $# > 0 )) && local_idx="$1"
if _fixbl_prompt "$AUTO_FIX_BLOCKINDEX" "${MAGENTA}Do you want to replace your block_log.index to match the server?${RESET} (Y/n) > " defyes; then
msg green " [...] Updating block_log.index to match the remote server's copy ..."
# raise_error "something went wrong"
rsync -Ivhc --partial-dir="${DIR}/.rsync-partial" --progress "${BC_RSYNC}.index" "${local_idx}"
msg green " [+++] Finished downloading/validating block_log.index"
else
msg nots red "\n [!!!] Not replacing block_log.index"
fi
}
fix-blocks-rocksdb() {
msg
msg bold green " ========================================================================"
msg bold green " = ="
msg bold green " = Synchronising RocksDB Files for MIRA nodes ="
msg bold green " = ="
msg bold green " ========================================================================"
msg
msg nots yellow " >> If you use a MIRA-enabled image, i.e. using RocksDB instead of shared_memory.bin, then your"
msg nots yellow " >> RocksDB files must have been generated using the same block_log and block_log.index"
msg
msg nots cyan " Local RocksDB/SHM_DIR folder: ${BOLD}$SHM_DIR"
msg nots cyan " Remote RocksDB source: ${BOLD}$ROCKSDB_RSYNC"
msg
msg nots yellow " >> If you've repaired your block_log or block_log.index, and you use MIRA, then it's important that"
msg nots yellow " >> your RocksDB files match the remote server's copy exactly."
msg
if _fixbl_prompt "$AUTO_FIX_ROCKSDB" " ${MAGENTA}Do you want to synchronise your MIRA RocksDB files with the server?${RESET} (y/N) > " defno; then
msg
msg nots green "\n [...] Updating RocksDB files to match the remote server's copy ..."
_SILENCE_RDB_INTRO=1 dlrocksdb
msg
msg green " [+++] Finished downloading/validating RocksDB files into $SHM_DIR \n"
else
msg nots red "\n [!!!] Not synchronising RocksDB files with remote server \n"
fi
}
_fix_blocks_help() {
MSG_TS_DEFAULT=0
msg
msg green "Usage:"
msg green " $0 fix-blocks (blocklog|index|rocksdb|all) (auto)"
msg
msg yellow "Examples:"
msg bold "\t # Download/replace/repair the block_log, block_log.index, and rocksdb files - prompting user before starting each action"
msg bold "\t $0 fix-blocks"
msg
msg bold "\t # Compare the local block_log size against the remote server block_log, will show a yes/no prompt allowing user to decide"
msg bold "\t # whether or not to update / verify / truncate their local block_log."
msg bold "\t $0 fix-blocks blocklog"
msg
msg bold "\t # Synchronise RocksDB files with server, and skip the yes/no prompt"
msg bold "\t $0 fix-blocks rocksdb auto"
msg
msg bold "\t # Automatically attempt to repair block_log and block_log.index non-interactively, while skipping the RocksDB download/repair step."
msg bold "\t AUTO_FIX_BLOCKLOG=1 AUTO_FIX_BLOCKINDEX=1 AUTO_FIX_ROCKSDB=2 $0 fix-blocks"
msg
msg yellow "If no blockchain component is specified (e.g. blocklog or index), fix-blocks will try to repair (with prompts) in order:"
msg
msg yellow " - block_log"
msg yellow " - block_log.index"
msg yellow " - rocksdb"
msg
msg bold green "Environment Variables"
msg
msg green " Several AUTO_FIX_ env vars are available, allowing the fix-blocks yes/no prompts to be automatically answered."
msg green " They're most useful when running fix-blocks from within a non-interactive script, e.g. via crontab. "
msg
msg green " All AUTO_FIX_ env vars can be one of three values:"
msg green " 0 = prompt before taking action (default)"
msg green " 1 = automatically answer yes to prompts"
msg green " 2 = automatically answer no to prompts"
msg
msg green " - AUTO_FIX_BLOCKLOG (def: 0) - Controls whether block_log update/verify/truncate prompt is automatically responded to"
msg green " - AUTO_FIX_BLOCKINDEX (def: 0) - Controls whether block_log.index synchronization prompt is automatically responded to"
msg green " - AUTO_FIX_ROCKSDB (def: 0) - Controls whether rocksdb synchronization prompt is automatically responded to"
msg
msg green " - AUTO_IGNORE_EQUAL (def: 1) - Additional control when AUTO_FIX_BLOCKLOG is set to 1 (auto yes) - controls whether block_log"
msg green " should be rsync verified against the remote server in the event that the local block_log is the same size"
msg green " as the remote block_log."
msg
msg green " When set to 1: Do not attempt to verify/repair the block_log if it's the same size as the remote server"
msg green " When set to 0: Always verify/repair the block_log, even when it's the same size as the remote server"
msg
msg
msg bold green "Available fix-blocks actions and descriptions"
msg
msg green " blocklog / block_log / blocks - Fast and easy repair of local block_log, using size comparisons against the remote"
msg green " blockchain mirror. Uses rsync to quickly append to block_log if it's too small, while"
msg green " using inplace rsync with checksumming to test block_log for corruption if block_log is the same size. "
msg green " If local block_log is larger than the remote server's, the command can automatically trim the "
msg green " block_log to the correct size."
msg
msg green " index / blockindex / block_index - Download / replace / repair local block_log.index from remote blockchain mirror, using rsync with checksumming. "
msg
msg green " rocksdb / rocks / mira - Download / replace / repair local RocksDB files (for MIRA images) from remote blockchain mirror, "
msg green " using rsync with checksumming, and partial-dir allowing for resuming of download if it fails. "
msg
MSG_TS_DEFAULT=1
}
fix-blocks() {
error_control 0
local local_bl="${BC_FOLDER}/block_log"
if (( $# > 0 )); then
case "$1" in
help|HELP|--help|-h|-?)
_fix_blocks_help
return $?;;
blocklog|blocks|block_log|blockchain)
fix-blocks-blocklog
return $?;;
index|blockindex|block_index|block_log.index)
fix-blocks-index
return $?;;
rocks*|mira|MIRA)
fix-blocks-rocksdb
return $?;;
all)
echo
;;
*)
msg bold red "\n[!!!] Invalid option '$1' ...\n"
msg red " > Displaying fix-blocks help."
sleep 0.5
_fix_blocks_help
return 0
;;
esac
fi
msg
fix-blocks-blocklog "$local_bl"
msg "\n"
fix-blocks-index "${local_bl}.index"
msg "\n"
fix-blocks-rocksdb
msg "\n"
return 0
}
# usage: insert_env [env_line] (env_file)
# example:
#
# insert_env "SHM_DIR=${DATADIR}/rocksdb"
#
# insert_env "DATADIR=/steem/data" "/steem/.env"
#
insert_env() {
local env_line="$1" env_file="${DIR}/.env"
(( $# >= 2 )) && env_file="$2"
local env_dir="$(dirname "$env_file")"
# If the .env file doesn't exist, then we need to attempt to create it
if [[ ! -f "$env_file" ]]; then
msg yellow " [...] File '$env_file' does not exist. Creating it."
# Check if we
if ! ( [[ -w "$env_dir" ]] && [[ -x "$env_dir" ]] ) && ! can_write "$env_dir"; then
msg bold red " [!!!] ERROR: Your user does not have permission to write to '$env_dir'"
if ! sudo -n ls >/dev/null; then
msg bold red " [!!!] Attempted to test whether sudo works, but failed. Giving up."
msg bold red " Please manually add the line '$env_line' to '$env_file'"
return 3
fi
msg yellow " >> The 'sudo' command appears to work. Attempting to create .env with correct permissions using sudo..."
sudo -n touch "$env_file"
sudo -n chown "$(whoami):$(whoami)" "$env_dir" "$env_file"
sudo -n chmod 700 "$env_file"
# echo "SHM_DIR=${out_dir}" | sudo -n tee -a "$env_file" > /dev/null
else
touch "$env_file"
fi