From e4c7b83f69ae23b8035cafea1613df09ef850dec Mon Sep 17 00:00:00 2001 From: Artur Troian Date: Tue, 24 Sep 2024 08:55:55 -0500 Subject: [PATCH] fix(lint): shell lint Signed-off-by: Artur Troian --- .envrc | 15 ++ .github/workflows/release.yaml | 10 +- .github/workflows/tests.yaml | 64 +++++++- make/test-upgrade.mk | 27 ---- script/tools.sh | 269 ++++++++++++++++++++------------- script/upgrades.sh | 15 +- tests/upgrade/upgrade_test.go | 7 - 7 files changed, 253 insertions(+), 154 deletions(-) diff --git a/.envrc b/.envrc index f5ce87435..47c5fefb2 100644 --- a/.envrc +++ b/.envrc @@ -48,6 +48,21 @@ if ! has readlink; then exit 1 fi +if ! has pv; then + echo "pv is not installed" + exit 1 +fi + +if ! has lz4; then + echo "lz4 is not installed" + exit 1 +fi + +if ! has git-cliff; then + echo "git-cliff is not installed" + exit 1 +fi + if [ -z "$GOPATH" ]; then GOPATH=$(go env GOPATH) export GOPATH diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e4c41638d..9ecbf8884 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,6 +14,8 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: ./script/tools.sh gha-deps - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -47,7 +49,7 @@ jobs: # using PAT as homebrew is located in different repo GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test-network-upgrade-on-release: - runs-on: upgrade-tester + runs-on: gh-runner-test needs: - publish steps: @@ -57,6 +59,12 @@ jobs: run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') echo "GOVERSION=${toolchain}" >> $GITHUB_ENV + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - uses: actions/setup-go@v5 with: go-version: "${{ env.GOVERSION }}" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 3f1b4ae40..f7b1d2ba6 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -20,8 +20,12 @@ jobs: steps: - name: Install dependencies run: | - brew install bash direnv + brew install bash direnv pv lz4 git-cliff sudo chsh -s /usr/local/bin/bash + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Hook direnv to bash run: echo 'eval "$(direnv hook bash)"' >> $HOME/.bashrc - uses: actions/checkout@v4 @@ -44,6 +48,12 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -64,6 +74,12 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -82,6 +98,12 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -101,6 +123,12 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -119,6 +147,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - run: git fetch --prune --unshallow - name: Detect required Go version run: | @@ -140,6 +174,12 @@ jobs: env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -165,6 +205,12 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -189,7 +235,7 @@ jobs: run: make release network-upgrade-names: - runs-on: upgrade-tester + runs-on: gh-runner-test steps: - name: Cleanup build folder run: | @@ -197,6 +243,12 @@ jobs: sudo rm -rf ./.??* || true - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -222,7 +274,7 @@ jobs: ./script/semver.sh validate "$upgrade" done <<< $(find "$dir" ! -path "$dir" -maxdepth 1 -type d -exec basename {} \;) network-upgrade: - runs-on: upgrade-tester + runs-on: gh-runner-test steps: - name: Cleanup build folder run: | @@ -234,6 +286,12 @@ jobs: run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') echo "GOVERSION=${toolchain}" >> $GITHUB_ENV + - name: Install dependencies + run: sudo apt install -y make build-essentials direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - uses: actions/setup-go@v5 with: go-version: "${{ env.GOVERSION }}" diff --git a/make/test-upgrade.mk b/make/test-upgrade.mk index 4b74a7ebd..e0c48b310 100644 --- a/make/test-upgrade.mk +++ b/make/test-upgrade.mk @@ -42,33 +42,6 @@ test: $(COSMOVISOR) init -upgrade-version="$(UPGRADE_BINARY_VERSION)" \ -test-cases=test-cases.json -$(COSMOVISOR_DEBUG_VERSION_FILE): $(AKASH_DEVCACHE) - @echo "installing cosmovisor for remote debug $(COSMOVISOR_VERSION) ..." - rm -f $(COSMOVISOR_DEBUG) - wget -qO- "https://github.com/cosmos/cosmos-sdk/releases/download/cosmovisor/$(COSMOVISOR_VERSION)/cosmovisor-$(COSMOVISOR_VERSION)-$(GOOS)-$(GOARCH).tar.gz" | \ - tar xvz -C $(AKASH_RUN_BIN) cosmovisor - rm -rf "$(dir $@)" - mkdir -p "$(dir $@)" - touch $@ -$(COSMOVISOR_DEBUG): $(COSMOVISOR_DEBUG_VERSION_FILE) - -.PHONY: test-remote-prep -test-remote-prep: $(COSMOVISOR_DEBUG) bins - $(GO_TEST) -c -tags e2e.upgrade -timeout 60m \ - -o $(AKASH_DEVCACHE_BIN)/upgrade.test \ - -gcflags "all=-N -l" pkg.akt.dev/node/tests/upgrade - chmod +x $(AKASH_DEVCACHE_BIN)/upgrade.test - rsync -Pl $(AKASH_DEVCACHE_BIN)/upgrade.test $(REMOTE_TEST_HOST):$(REMOTE_TEST_WORKDIR)/ - rsync -Prl --delete $(AKASH_RUN) $(REMOTE_TEST_HOST):$(REMOTE_TEST_WORKDIR)/ - rsync -Pl $(AKASH_ROOT)/tests/upgrade/test-config.json $(REMOTE_TEST_HOST):$(REMOTE_TEST_WORKDIR) - rsync -Pl $(AKASH_ROOT)/tests/upgrade/test-cases.json $(REMOTE_TEST_HOST):$(REMOTE_TEST_WORKDIR) - -.PHONY: test-remote-start -test-remote-start: - ssh -t $(REMOTE_TEST_HOST) 'cd $(REMOTE_TEST_WORKDIR); bash -ic "\ - dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./upgrade.test \ - -- -test.v=test2json -test.paniconexit0 -test.run ^\QTestUpgrade\E$ -cosmovisor=$$(pwd)/bin/cosmovisor -workdir=$$(pwd)/run/upgrade/validators -config=test-config.json -upgrade-name=$(UPGRADE_TO) -upgrade-version=$(UPGRADE_BINARY_VERSION) -test-cases=test-cases.json"' - .PHONY: test-reset test-reset: $(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --config="$(PWD)/config.json" --uto=$(UPGRADE_TO) clean diff --git a/script/tools.sh b/script/tools.sh index 187775d38..8772e707a 100755 --- a/script/tools.sh +++ b/script/tools.sh @@ -6,117 +6,182 @@ SEMVER=$SCRIPT_DIR/semver.sh gomod="$SCRIPT_DIR/../go.mod" function get_gotoolchain() { - local gotoolchain - local goversion - local local_goversion - - gotoolchain=$(grep -E '^toolchain go[0-9]{1,}.[0-9]{1,}.[0-9]{1,}$' < "$gomod" | cut -d ' ' -f 2 | tr -d '\n') - goversion=$(grep -E '^go [0-9]{1,}.[0-9]{1,}(.[0-9]{1,})?$' < "$gomod" | cut -d ' ' -f 2 | tr -d '\n') - - if [[ ${gotoolchain} == "" ]]; then - # determine go toolchain from go version in go.mod - if which go > /dev/null 2>&1 ; then - local_goversion=$(GOTOOLCHAIN=local go version | cut -d ' ' -f 3 | sed 's/go*//' | tr -d '\n') - if [[ $($SEMVER compare "v$local_goversion" v"$goversion") -ge 0 ]]; then - goversion=$local_goversion - else - local_goversion= - fi - fi - - if [[ "$local_goversion" == "" ]]; then - goversion=$(curl -s "https://go.dev/dl/?mode=json&include=all" | jq -r --arg regexp "^go$goversion" '.[] | select(.stable == true) | select(.version | match($regexp)) | .version' | head -n 1 | sed -e s/^go//) - fi - - if [[ $goversion != "" ]] && [[ $($SEMVER compare "v$goversion" v1.21.0) -ge 0 ]]; then - gotoolchain=go${goversion} - else - gotoolchain=go$(grep -E '^go [0-9]{1,}.[0-9]{1,}$' < "$gomod" | cut -d ' ' -f 2 | tr -d '\n').0 - fi - fi - - echo -n "$gotoolchain" + local gotoolchain + local goversion + local local_goversion + + gotoolchain=$(grep -E '^toolchain go[0-9]{1,}.[0-9]{1,}.[0-9]{1,}$' <"$gomod" | cut -d ' ' -f 2 | tr -d '\n') + goversion=$(grep -E '^go [0-9]{1,}.[0-9]{1,}(.[0-9]{1,})?$' <"$gomod" | cut -d ' ' -f 2 | tr -d '\n') + + if [[ ${gotoolchain} == "" ]]; then + # determine go toolchain from go version in go.mod + if which go >/dev/null 2>&1; then + local_goversion=$(GOTOOLCHAIN=local go version | cut -d ' ' -f 3 | sed 's/go*//' | tr -d '\n') + if [[ $($SEMVER compare "v$local_goversion" v"$goversion") -ge 0 ]]; then + goversion=$local_goversion + else + local_goversion= + fi + fi + + if [[ "$local_goversion" == "" ]]; then + goversion=$(curl -s "https://go.dev/dl/?mode=json&include=all" | jq -r --arg regexp "^go$goversion" '.[] | select(.stable == true) | select(.version | match($regexp)) | .version' | head -n 1 | sed -e s/^go//) + fi + + if [[ $goversion != "" ]] && [[ $($SEMVER compare "v$goversion" v1.21.0) -ge 0 ]]; then + gotoolchain=go${goversion} + else + gotoolchain=go$(grep -E '^go [0-9]{1,}.[0-9]{1,}$' <"$gomod" | cut -d ' ' -f 2 | tr -d '\n').0 + fi + fi + + echo -n "$gotoolchain" } replace_paths() { - local file="${1}" - local cimport="${2}" - local nimport="${3}" - local sedcmd=sed + local file="${1}" + local cimport="${2}" + local nimport="${3}" + local sedcmd=sed - if [[ "$OSTYPE" == "darwin"* ]]; then - sedcmd=gsed - fi + if [[ "$OSTYPE" == "darwin"* ]]; then + sedcmd=gsed + fi - $sedcmd -ri "s~$cimport~$nimport~" "${file}" + $sedcmd -ri "s~$cimport~$nimport~" "${file}" } function replace_import_path() { - local next_major_version=$1 - local curr_module_name - local curr_version - local new_module_name - - curr_module_name=$(cd go || exit; go list -m) - curr_version=$(echo "$curr_module_name" | sed -n 's/.*v\([0-9]*\).*/\1/p') - new_module_name=${curr_module_name%/"v$curr_version"}/$next_major_version - - echo "current import paths are $curr_module_name, replacing with $new_module_name" - - declare -a modules_to_upgrade_manually - - modules_to_upgrade_manually+=("./go/go.mod") - - echo "preparing files to replace" - - declare -a files - - while IFS= read -r line; do - files+=("$line") - done < <(find . -type f -not \( \ - -path "./install.sh" \ - -or -path "./upgrades/*" \ - -or -path "./.cache/*" \ - -or -path "./dist/*" \ - -or -path "./.git*" \ - -or -name "*.md" \ - -or -path "./.idea/*" \)) - - echo "updating all files" - - for file in "${files[@]}"; do - if test -f "$file"; then - # skip files that need manual upgrading - for excluded_file in "${modules_to_upgrade_manually[@]}"; do - if [[ "$file" == *"$excluded_file"* ]]; then - continue 2 - fi - done - - replace_paths "$file" "\"$curr_module_name" "\"$new_module_name" - fi - done - - echo "updating go.mod" - for retract in $(cd go || exit; go mod edit --json | jq -cr '.Retract | if . != null then .[] else empty end'); do - local low - local high - - low=$(jq -r '.Low' <<<"$retract") - high=$(jq -r '.High' <<<"$retract") - echo " dropping retract: [$low, $high]" - go mod edit -dropretract=["$low","$high"] - done - - replace_paths "./go/go.mod" "$curr_module_name" "$new_module_name" + local next_major_version=$1 + local curr_module_name + local curr_version + local new_module_name + + curr_module_name=$( + cd go || exit + go list -m + ) + curr_version=$(echo "$curr_module_name" | sed -n 's/.*v\([0-9]*\).*/\1/p') + new_module_name=${curr_module_name%/"v$curr_version"}/$next_major_version + + echo "current import paths are $curr_module_name, replacing with $new_module_name" + + declare -a modules_to_upgrade_manually + + modules_to_upgrade_manually+=("./go/go.mod") + + echo "preparing files to replace" + + declare -a files + + while IFS= read -r line; do + files+=("$line") + done < <(find . -type f -not \( \ + -path "./install.sh" \ + -or -path "./upgrades/*" \ + -or -path "./.cache/*" \ + -or -path "./dist/*" \ + -or -path "./.git*" \ + -or -name "*.md" \ + -or -path "./.idea/*" \)) + + echo "updating all files" + + for file in "${files[@]}"; do + if test -f "$file"; then + # skip files that need manual upgrading + for excluded_file in "${modules_to_upgrade_manually[@]}"; do + if [[ "$file" == *"$excluded_file"* ]]; then + continue 2 + fi + done + + replace_paths "$file" "\"$curr_module_name" "\"$new_module_name" + fi + done + + echo "updating go.mod" + for retract in $( + cd go || exit + go mod edit --json | jq -cr '.Retract | if . != null then .[] else empty end' + ); do + local low + local high + + low=$(jq -r '.Low' <<<"$retract") + high=$(jq -r '.High' <<<"$retract") + echo " dropping retract: [$low, $high]" + go mod edit -dropretract=["$low","$high"] + done + + replace_paths "./go/go.mod" "$curr_module_name" "$new_module_name" +} + +function install_gha_dependencies() { + local debian_deps=( + "make" + "build-essentials" + "direnv" + "unzip" + "lz4" + "wget" + "curl" + "npm" + "jq" + "pv" + "coreutils" + ) + + tools= + + # shellcheck disable=SC2068 + for dep in ${debian_deps[@]}; do + echo -n "detecting $dep ..." + status="(installed)" + if ! dpkg -l "$dep" ; then + tools="$tools $dep" + status="(not installed)" + fi + echo " $status" + done + + cmd="apt-get" + + if is_command sudo; then + cmd="sudo $cmd" + fi + + if [[ "$tools" != "" ]]; then + $cmd update + # don't put quotes around $tools! + # shellcheck disable=SC2086 + (set -x; $cmd install -y $tools) + else + echo "All requirements already met. Nothing to install" + fi + +# if [[ "$OSTYPE" == "linux-gnu"* ]]; then +# sudo apt install pv lz4 npm jq curl unzip wget make +# # version="1.0.0" +# local version +# version=2.6.0 +# wget -qO- "https://github.com/orhun/git-cliff/releases/download/v${version}/git-cliff-${version}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xvz -C /usr/local/bin --strip-components=1 git-cliff-${version}/git-cliff +# elif [[ "$OSTYPE" == "darwin"* ]]; then +# brew install bash direnv pv lz4 git-cliff +# chsh -s /usr/local/bin/bash +# fi } case "$1" in -gotoolchain) - get_gotoolchain - ;; -replace-import-path) - shift - replace_import_path "$@" - ;; + gotoolchain) + get_gotoolchain + ;; + replace-import-path) + shift + replace_import_path "$@" + ;; + gha-deps) + shift + install_gha_dependencies + ;; esac diff --git a/script/upgrades.sh b/script/upgrades.sh index 0a5b257df..10e3c0a8f 100755 --- a/script/upgrades.sh +++ b/script/upgrades.sh @@ -258,13 +258,6 @@ function init() { cp "$validators_dir/.akash0/cosmovisor/upgrades/$UPGRADE_TO/bin/akash" "$upgrade_bin/akash" fi - pushd "$(pwd)" - cd "$cosmovisor_dir" - - ln -snf "current" "genesis" - - popd - AKASH=$genesis_bin/akash $AKASH init --home "$valdir" "$(jq -rc '.moniker' <<<"$val")" >/dev/null 2>&1 @@ -286,6 +279,7 @@ function init() { tar_cmd=$(content_type "$(content_name "$GENESIS_ORIG")") + # shellcheck disable=SC2086 wget -nv -O - "$GENESIS_ORIG" | pv $pv_args | eval "$tar_cmd" else echo "unable to download genesis" @@ -441,13 +435,6 @@ function bins() { cp "$validators_dir/.akash0/cosmovisor/upgrades/$UPGRADE_TO/bin/akash" "$upgrade_bin/akash" fi - pushd "$(pwd)" - cd "$cosmovisor_dir" - - ln -snf "genesis" "current" - - popd - ((cnt++)) || true done } diff --git a/tests/upgrade/upgrade_test.go b/tests/upgrade/upgrade_test.go index 4bbea9123..8c26a3dc8 100644 --- a/tests/upgrade/upgrade_test.go +++ b/tests/upgrade/upgrade_test.go @@ -546,13 +546,6 @@ func TestUpgrade(t *testing.T) { fail = true } } - - // select { - // case errs := <-vl.testErrsCh: - // - // - // case <-vl.ctx.Done(): - // } } if fail {