diff --git a/.envrc b/.envrc index f5ce87435..47c5fefb2 100644 --- a/.envrc +++ b/.envrc @@ -48,6 +48,21 @@ if ! has readlink; then exit 1 fi +if ! has pv; then + echo "pv is not installed" + exit 1 +fi + +if ! has lz4; then + echo "lz4 is not installed" + exit 1 +fi + +if ! has git-cliff; then + echo "git-cliff is not installed" + exit 1 +fi + if [ -z "$GOPATH" ]; then GOPATH=$(go env GOPATH) export GOPATH diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e4c41638d..c9a9ac234 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,6 +14,8 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: ./script/tools.sh gha-deps - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -47,7 +49,7 @@ jobs: # using PAT as homebrew is located in different repo GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test-network-upgrade-on-release: - runs-on: upgrade-tester + runs-on: self-hosted1 needs: - publish steps: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 3f1b4ae40..8e46ef496 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -20,7 +20,7 @@ jobs: steps: - name: Install dependencies run: | - brew install bash direnv + brew install bash direnv pv lz4 git-cliff sudo chsh -s /usr/local/bin/bash - name: Hook direnv to bash run: echo 'eval "$(direnv hook bash)"' >> $HOME/.bashrc @@ -44,6 +44,8 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: ./script/tools.sh gha-deps - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -64,6 +66,8 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: ./script/tools.sh gha-deps - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -82,6 +86,8 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: ./script/tools.sh gha-deps - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -101,6 +107,8 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: ./script/tools.sh gha-deps - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -119,6 +127,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install dependencies + run: ./script/tools.sh gha-deps - run: git fetch --prune --unshallow - name: Detect required Go version run: | @@ -140,6 +150,8 @@ jobs: env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - uses: actions/checkout@v4 + - name: Install dependencies + run: ./script/tools.sh gha-deps - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -165,6 +177,8 @@ jobs: steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow + - name: Install dependencies + run: ./script/tools.sh gha-deps - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') @@ -189,7 +203,7 @@ jobs: run: make release network-upgrade-names: - runs-on: upgrade-tester + runs-on: self-hosted1 steps: - name: Cleanup build folder run: | @@ -222,7 +236,7 @@ jobs: ./script/semver.sh validate "$upgrade" done <<< $(find "$dir" ! -path "$dir" -maxdepth 1 -type d -exec basename {} \;) network-upgrade: - runs-on: upgrade-tester + runs-on: self-hosted1 steps: - name: Cleanup build folder run: | diff --git a/make/test-upgrade.mk b/make/test-upgrade.mk index 4b74a7ebd..ef42d976d 100644 --- a/make/test-upgrade.mk +++ b/make/test-upgrade.mk @@ -22,6 +22,8 @@ UPGRADE_BINARY_VERSION ?= local REMOTE_TEST_WORKDIR ?= ~/go/src/github.com/akash-network/node REMOTE_TEST_HOST ?= +COSMOVISOR := /Users/amr/go/src/github.com/cosmos/cosmos-sdk/tools/cosmovisor/cosmovisor + $(AKASH_INIT): $(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --gbv=$(GENESIS_BINARY_VERSION) --ufrom=$(UPGRADE_FROM) --uto=$(UPGRADE_TO) --config="$(PWD)/config.json" init touch $@ diff --git a/script/tools.sh b/script/tools.sh index 187775d38..53da753ac 100755 --- a/script/tools.sh +++ b/script/tools.sh @@ -6,117 +6,140 @@ SEMVER=$SCRIPT_DIR/semver.sh gomod="$SCRIPT_DIR/../go.mod" function get_gotoolchain() { - local gotoolchain - local goversion - local local_goversion - - gotoolchain=$(grep -E '^toolchain go[0-9]{1,}.[0-9]{1,}.[0-9]{1,}$' < "$gomod" | cut -d ' ' -f 2 | tr -d '\n') - goversion=$(grep -E '^go [0-9]{1,}.[0-9]{1,}(.[0-9]{1,})?$' < "$gomod" | cut -d ' ' -f 2 | tr -d '\n') - - if [[ ${gotoolchain} == "" ]]; then - # determine go toolchain from go version in go.mod - if which go > /dev/null 2>&1 ; then - local_goversion=$(GOTOOLCHAIN=local go version | cut -d ' ' -f 3 | sed 's/go*//' | tr -d '\n') - if [[ $($SEMVER compare "v$local_goversion" v"$goversion") -ge 0 ]]; then - goversion=$local_goversion - else - local_goversion= - fi - fi - - if [[ "$local_goversion" == "" ]]; then - goversion=$(curl -s "https://go.dev/dl/?mode=json&include=all" | jq -r --arg regexp "^go$goversion" '.[] | select(.stable == true) | select(.version | match($regexp)) | .version' | head -n 1 | sed -e s/^go//) - fi - - if [[ $goversion != "" ]] && [[ $($SEMVER compare "v$goversion" v1.21.0) -ge 0 ]]; then - gotoolchain=go${goversion} - else - gotoolchain=go$(grep -E '^go [0-9]{1,}.[0-9]{1,}$' < "$gomod" | cut -d ' ' -f 2 | tr -d '\n').0 - fi - fi - - echo -n "$gotoolchain" + local gotoolchain + local goversion + local local_goversion + + gotoolchain=$(grep -E '^toolchain go[0-9]{1,}.[0-9]{1,}.[0-9]{1,}$' <"$gomod" | cut -d ' ' -f 2 | tr -d '\n') + goversion=$(grep -E '^go [0-9]{1,}.[0-9]{1,}(.[0-9]{1,})?$' <"$gomod" | cut -d ' ' -f 2 | tr -d '\n') + + if [[ ${gotoolchain} == "" ]]; then + # determine go toolchain from go version in go.mod + if which go >/dev/null 2>&1; then + local_goversion=$(GOTOOLCHAIN=local go version | cut -d ' ' -f 3 | sed 's/go*//' | tr -d '\n') + if [[ $($SEMVER compare "v$local_goversion" v"$goversion") -ge 0 ]]; then + goversion=$local_goversion + else + local_goversion= + fi + fi + + if [[ "$local_goversion" == "" ]]; then + goversion=$(curl -s "https://go.dev/dl/?mode=json&include=all" | jq -r --arg regexp "^go$goversion" '.[] | select(.stable == true) | select(.version | match($regexp)) | .version' | head -n 1 | sed -e s/^go//) + fi + + if [[ $goversion != "" ]] && [[ $($SEMVER compare "v$goversion" v1.21.0) -ge 0 ]]; then + gotoolchain=go${goversion} + else + gotoolchain=go$(grep -E '^go [0-9]{1,}.[0-9]{1,}$' <"$gomod" | cut -d ' ' -f 2 | tr -d '\n').0 + fi + fi + + echo -n "$gotoolchain" } replace_paths() { - local file="${1}" - local cimport="${2}" - local nimport="${3}" - local sedcmd=sed + local file="${1}" + local cimport="${2}" + local nimport="${3}" + local sedcmd=sed - if [[ "$OSTYPE" == "darwin"* ]]; then - sedcmd=gsed - fi + if [[ "$OSTYPE" == "darwin"* ]]; then + sedcmd=gsed + fi - $sedcmd -ri "s~$cimport~$nimport~" "${file}" + $sedcmd -ri "s~$cimport~$nimport~" "${file}" } function replace_import_path() { - local next_major_version=$1 - local curr_module_name - local curr_version - local new_module_name - - curr_module_name=$(cd go || exit; go list -m) - curr_version=$(echo "$curr_module_name" | sed -n 's/.*v\([0-9]*\).*/\1/p') - new_module_name=${curr_module_name%/"v$curr_version"}/$next_major_version - - echo "current import paths are $curr_module_name, replacing with $new_module_name" - - declare -a modules_to_upgrade_manually - - modules_to_upgrade_manually+=("./go/go.mod") - - echo "preparing files to replace" - - declare -a files - - while IFS= read -r line; do - files+=("$line") - done < <(find . -type f -not \( \ - -path "./install.sh" \ - -or -path "./upgrades/*" \ - -or -path "./.cache/*" \ - -or -path "./dist/*" \ - -or -path "./.git*" \ - -or -name "*.md" \ - -or -path "./.idea/*" \)) - - echo "updating all files" - - for file in "${files[@]}"; do - if test -f "$file"; then - # skip files that need manual upgrading - for excluded_file in "${modules_to_upgrade_manually[@]}"; do - if [[ "$file" == *"$excluded_file"* ]]; then - continue 2 - fi - done - - replace_paths "$file" "\"$curr_module_name" "\"$new_module_name" - fi - done - - echo "updating go.mod" - for retract in $(cd go || exit; go mod edit --json | jq -cr '.Retract | if . != null then .[] else empty end'); do - local low - local high - - low=$(jq -r '.Low' <<<"$retract") - high=$(jq -r '.High' <<<"$retract") - echo " dropping retract: [$low, $high]" - go mod edit -dropretract=["$low","$high"] - done - - replace_paths "./go/go.mod" "$curr_module_name" "$new_module_name" + local next_major_version=$1 + local curr_module_name + local curr_version + local new_module_name + + curr_module_name=$( + cd go || exit + go list -m + ) + curr_version=$(echo "$curr_module_name" | sed -n 's/.*v\([0-9]*\).*/\1/p') + new_module_name=${curr_module_name%/"v$curr_version"}/$next_major_version + + echo "current import paths are $curr_module_name, replacing with $new_module_name" + + declare -a modules_to_upgrade_manually + + modules_to_upgrade_manually+=("./go/go.mod") + + echo "preparing files to replace" + + declare -a files + + while IFS= read -r line; do + files+=("$line") + done < <(find . -type f -not \( \ + -path "./install.sh" \ + -or -path "./upgrades/*" \ + -or -path "./.cache/*" \ + -or -path "./dist/*" \ + -or -path "./.git*" \ + -or -name "*.md" \ + -or -path "./.idea/*" \)) + + echo "updating all files" + + for file in "${files[@]}"; do + if test -f "$file"; then + # skip files that need manual upgrading + for excluded_file in "${modules_to_upgrade_manually[@]}"; do + if [[ "$file" == *"$excluded_file"* ]]; then + continue 2 + fi + done + + replace_paths "$file" "\"$curr_module_name" "\"$new_module_name" + fi + done + + echo "updating go.mod" + for retract in $( + cd go || exit + go mod edit --json | jq -cr '.Retract | if . != null then .[] else empty end' + ); do + local low + local high + + low=$(jq -r '.Low' <<<"$retract") + high=$(jq -r '.High' <<<"$retract") + echo " dropping retract: [$low, $high]" + go mod edit -dropretract=["$low","$high"] + done + + replace_paths "./go/go.mod" "$curr_module_name" "$new_module_name" +} + +function install_gha_dependencies() { + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + sudo apt install pv lz4 npm jq curl unzip wget make + # version="1.0.0" + local version + version=2.6.0 + wget -qO- "https://github.com/orhun/git-cliff/releases/download/v${version}/git-cliff-${version}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xvz -C /usr/local/bin --strip-components=1 git-cliff-${version}/git-cliff + elif [[ "$OSTYPE" == "darwin"* ]]; then + brew install bash direnv pv lz4 git-cliff + chsh -s /usr/local/bin/bash + fi } case "$1" in -gotoolchain) - get_gotoolchain - ;; -replace-import-path) - shift - replace_import_path "$@" - ;; + gotoolchain) + get_gotoolchain + ;; + replace-import-path) + shift + replace_import_path "$@" + ;; + gha-deps) + shift + install_gha_dependencies + ;; esac diff --git a/script/upgrades.sh b/script/upgrades.sh index 0a5b257df..10e3c0a8f 100755 --- a/script/upgrades.sh +++ b/script/upgrades.sh @@ -258,13 +258,6 @@ function init() { cp "$validators_dir/.akash0/cosmovisor/upgrades/$UPGRADE_TO/bin/akash" "$upgrade_bin/akash" fi - pushd "$(pwd)" - cd "$cosmovisor_dir" - - ln -snf "current" "genesis" - - popd - AKASH=$genesis_bin/akash $AKASH init --home "$valdir" "$(jq -rc '.moniker' <<<"$val")" >/dev/null 2>&1 @@ -286,6 +279,7 @@ function init() { tar_cmd=$(content_type "$(content_name "$GENESIS_ORIG")") + # shellcheck disable=SC2086 wget -nv -O - "$GENESIS_ORIG" | pv $pv_args | eval "$tar_cmd" else echo "unable to download genesis" @@ -441,13 +435,6 @@ function bins() { cp "$validators_dir/.akash0/cosmovisor/upgrades/$UPGRADE_TO/bin/akash" "$upgrade_bin/akash" fi - pushd "$(pwd)" - cd "$cosmovisor_dir" - - ln -snf "genesis" "current" - - popd - ((cnt++)) || true done } diff --git a/tests/upgrade/upgrade_test.go b/tests/upgrade/upgrade_test.go index 4bbea9123..8c26a3dc8 100644 --- a/tests/upgrade/upgrade_test.go +++ b/tests/upgrade/upgrade_test.go @@ -546,13 +546,6 @@ func TestUpgrade(t *testing.T) { fail = true } } - - // select { - // case errs := <-vl.testErrsCh: - // - // - // case <-vl.ctx.Done(): - // } } if fail {