From 418abdda69686822fde05303346dbeb748efa6b8 Mon Sep 17 00:00:00 2001 From: Arjun Kathuria Date: Sat, 16 Dec 2023 10:05:54 +0530 Subject: [PATCH] [Rel-Eng]: Adds script to automatically generate GHCup metadata for Cabal releases (#9411) * [Rel-Eng]: Add initial version of the release metadata script Adds the initial version of the release script that generates the metadata for a given release automatically in the correct format which can be used by GHCup as-is which makes it way easier to add that release to be distributed via GHCup. This would evolve based on the feedback from the maintainers but the basic core functionality still works. To Resolve: #9298 * [Rel-Eng]: Change top shebang from '/bin/bash' to '/usr/bin/env bash' Modifies the top shebang from being #!/bin/bash to #!/usr/bin/env bash as requested in the PR review * [Rel-Eng]: Add script to download cabal-install binaries for a release The script to generate GHCup metadata needs the artifacts already downloaded. This script was made to be used in conjunction to the ghcup metadata generating script. This script:- * Makes the directories that ./create-release-metadata-for-ghcup.sh CDs into * Downloads the binary release files given a particular release number into correct directories. * Verifies that the downloaded checksums match the expected values upstream. * [Rel-Eng]: Integrate the release binary downloading script into GHCup metadata generating script Checks if binary-release folders for the release passed exist. "create-release-metadata-for-ghcup.sh" needs the release files pre-downloaded. Echos user to run the download script first if correct binaries folder not found. * [Rel-Eng]: Add release-scripts binary-downloads to gitignore We do not want to commit the downloaded binary files for a cabal release. This prevents us from doing that, even on accident. * [Rel-Eng]: Change from '/bin/bash' to '/usr/bin/env bash' in the download script Modifies the top shebang from being #!/bin/bash to #!/usr/bin/env bash in the 'download-cabal-install-release-binaries.sh' The later seems more portable as learned from the PR review comments. --- .gitignore | 8 ++ .../create-release-metadata-for-ghcup.sh | 125 ++++++++++++++++++ ...download-cabal-install-release-binaries.sh | 42 ++++++ 3 files changed, 175 insertions(+) create mode 100755 scripts/release/create-release-metadata-for-ghcup.sh create mode 100755 scripts/release/download-cabal-install-release-binaries.sh diff --git a/.gitignore b/.gitignore index e9ec3b6322f..db4444f6ce6 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,11 @@ bench.html # Emacs .projectile + +# I'm unsure how to ignore these generated golden files +cabal-testsuite/PackageTests/NewUpdate/RejectFutureIndexStates/cabal.out + +## Release Scripts + +# ignore the downloaded binary files +scripts/release/binary-downloads/ diff --git a/scripts/release/create-release-metadata-for-ghcup.sh b/scripts/release/create-release-metadata-for-ghcup.sh new file mode 100755 index 00000000000..c8683c2b85c --- /dev/null +++ b/scripts/release/create-release-metadata-for-ghcup.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +# This script, when passed the cabal release number as the first and only argument +# generates the metadata in the correct format to be useable as is by GHCup +# for eg:- +# $ create-release-metadata-for-ghcup.sh 3.10.2.0 or "3.10.2.0" + +# Note:- Please run ./download-cabal-install-release-binaries.sh before running this script. +set -eu +set -o pipefail + +RELEASE=$1 +## FixMe:// What dir to use here? + +if [ -d "binary-downloads/cabal-install-${RELEASE}-binaries" ]; then + echo "binary downloads folder for release ${RELEASE} found, starting generating GHCup metadata..." +else + echo "The binary downloads for release ${RELEASE} not found." + echo "Please run the script to download them first." +fi + +cd "binary-downloads/cabal-install-${RELEASE}-binaries" + +cat < /dev/stdout + $RELEASE: + viTags: + - Latest + viChangeLog: https://github.com/haskell/cabal/blob/master/release-notes/cabal-install-$RELEASE.md + viPostInstall: *cabal-${RELEASE//./}-post-install + viArch: + A_64: + Linux_UnknownLinux: + unknown_versioning: &cabal-${RELEASE//./}-64 + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-linux-alpine3_12.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-linux-alpine3_12.tar.xz" | awk '{ print $1 }') + Linux_Alpine: + unknown_versioning: &cabal-${RELEASE//./}-64 + Linux_CentOS: + unknown_versioning: &cabal-${RELEASE//./}-64-centos7 + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-linux-centos7.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-linux-centos7.tar.xz" | awk '{ print $1 }') + Linux_Debian: + ' ( >= 9 && < 10)': &cabal-${RELEASE//./}-64-debian + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-linux-deb9.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-linux-deb9.tar.xz" | awk '{ print $1 }') + ' ( == 10 && < 11)': + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-linux-deb10.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-linux-deb10.tar.xz" | awk '{ print $1 }') + ' ( >= 11)': + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-linux-deb11.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-linux-deb11.tar.xz" | awk '{ print $1 }') + unknown_versioning: &cabal-${RELEASE//./}-64-debian + Linux_Fedora: + '>= 33': + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-linux-fedora33.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-linux-fedora33.tar.xz" | awk '{ print $1 }') + unknown_versioning: &cabal-${RELEASE//./}-64-centos7 + Linux_Ubuntu: + '< 20': &cabal-${RELEASE//./}-64-ubuntu18 + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-linux-ubuntu18_04.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-linux-ubuntu18_04.tar.xz" | awk '{ print $1 }') + '>= 20': &cabal-${RELEASE//./}-64-ubuntu20 + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-linux-ubuntu20_04.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-linux-ubuntu20_04.tar.xz" | awk '{ print $1 }') + unknown_versioning: *cabal-${RELEASE//./}-64-ubuntu18 + Linux_Mint: + '< 20': *cabal-${RELEASE//./}-64-ubuntu18 + '>= 20': *cabal-${RELEASE//./}-64-ubuntu20 + unknown_versioning: *cabal-${RELEASE//./}-64-ubuntu18 + Darwin: + unknown_versioning: + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-darwin.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-darwin.tar.xz" | awk '{ print $1 }') + Windows: + unknown_versioning: + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-windows.zip + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-windows.zip" | awk '{ print $1 }') + FreeBSD: + unknown_versioning: + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-x86_64-freebsd.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-x86_64-freebsd.tar.xz" | awk '{ print $1 }') + A_32: + Linux_UnknownLinux: + unknown_versioning: &cabal-${RELEASE//./}-32 + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-i386-linux-alpine3_12.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-i386-linux-alpine3_12.tar.xz" | awk '{ print $1 }') + Linux_Alpine: + unknown_versioning: *cabal-${RELEASE//./}-32 + Linux_Debian: + '( >= 9 )': + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-i386-linux-deb9.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-i386-linux-deb9.tar.xz" | awk '{ print $1 }') + unknown_versioning: *cabal-${RELEASE//./}-32 + A_ARM64: + Darwin: + unknown_versioning: + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-aarch64-darwin.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-aarch64-darwin.tar.xz" | awk '{ print $1 }') + Linux_Debian: + '( >= 10 && < 11)': &cabal-31020-arm64 + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-aarch64-linux-deb10.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-aarch64-linux-deb10.tar.xz" | awk '{ print $1 }') + '( >= 11)': + dlUri: https://downloads.haskell.org/~cabal/cabal-install-$RELEASE/cabal-install-$RELEASE-aarch64-linux-deb11.tar.xz + dlSubdir: cabal-install-$RELEASE + dlHash: $(sha256sum "cabal-install-$RELEASE-aarch64-linux-deb11.tar.xz" | awk '{ print $1 }') + unknown_versioning: *cabal-${RELEASE//./}-arm64 + Linux_UnknownLinux: + unknown_versioning: *cabal-${RELEASE//./}-arm64 +EOF diff --git a/scripts/release/download-cabal-install-release-binaries.sh b/scripts/release/download-cabal-install-release-binaries.sh new file mode 100755 index 00000000000..4547fd910e8 --- /dev/null +++ b/scripts/release/download-cabal-install-release-binaries.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# A script to download the release binary files for a given cabal release +# from upstream "downlods.haskell.org". +# It accepts the first and only argument as the release number. +# +# useage:- +# $ download-cabal-install-release-binaries.sh "3.10.1.0" +# +# This was initally made to be used with ./create-release-metadata-for-ghcup.sh + +set -eu +set -o pipefail + +RELEASE=$1 + +echo "RELEASE: $RELEASE" + +for com in wget sha256sum ; do + command -V ${com} >/dev/null 2>&1 +done + +[ ! -d "binary-downloads/cabal-install-${RELEASE}-binaries" ] + +mkdir -p "binary-downloads/cabal-install-${RELEASE}-binaries" + +cd "binary-downloads/cabal-install-${RELEASE}-binaries" + +## Download release files +echo "Downloading form: \"https://downloads.haskell.org/~cabal/cabal-install-${RELEASE}/\"" +wget --no-parent -r --reject "index.html*" --no-directories "https://downloads.haskell.org/~cabal/cabal-install-${RELEASE}/" + +## Verify that sha256 sums of downloaded files match the ones mentioned in the upstream SHA256SUMS file +echo "verifying checksums for downloaded files..." + +if sha256sum --check ./SHA256SUMS; then + echo "All checksums match!" + echo "Successfully downloaded binaries for release: ${RELEASE}" +else + echo "checksums of downloaded files do no match the ones listed in upstream SHA256SUMS file." + echo "please try deleting \"binary-downloads/cabal-install-${RELEASE}-binaries\" folder and downloading again." +fi