From 0cc4d6d9e1b1a7a3b60531404aac9b9be1e42e92 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:32:52 +0100 Subject: [PATCH] Fix JVM native lib wrappers CI (#23) fix coordinator tests CI --- .github/actions/setup-nodejs/action.yml | 2 +- ...nsys.zkevm.linea-native-libs-helper.gradle | 95 ++++++------------- coordinator/Dockerfile | 2 +- .../contract/l1/Web3JFunctionBuilders.kt | 2 +- .../l2/L2MessageServiceGasLimitEstimate.kt | 4 +- .../persistence/PostgresBatchesRepository.kt | 2 +- docker/compose.yml | 6 +- .../finalized-tag-updater-jar-downloader.sh | 32 ------- finalized-tag-updater/jreleaser.yml | 4 +- jvm-libs/blob-compressor/build.gradle | 29 +----- jvm-libs/blob-shnarf-calculator/build.gradle | 29 +----- .../consensys/linea/traces/TracingModule.kt | 1 - package.json | 2 +- traces-api-facade/Dockerfile | 2 +- 14 files changed, 45 insertions(+), 167 deletions(-) delete mode 100644 docker/scripts/finalized-tag-updater-jar-downloader.sh diff --git a/.github/actions/setup-nodejs/action.yml b/.github/actions/setup-nodejs/action.yml index d9c2825ef..600643660 100644 --- a/.github/actions/setup-nodejs/action.yml +++ b/.github/actions/setup-nodejs/action.yml @@ -5,7 +5,7 @@ inputs: node-version: description: 'The version of node to use' required: true - default: '18.15.0' + default: '20.17.0' pnpm-version: description: 'The version of pnpm to use' required: true diff --git a/buildSrc/src/main/groovy/net.consensys.zkevm.linea-native-libs-helper.gradle b/buildSrc/src/main/groovy/net.consensys.zkevm.linea-native-libs-helper.gradle index a4dccc53c..be2fc0333 100644 --- a/buildSrc/src/main/groovy/net.consensys.zkevm.linea-native-libs-helper.gradle +++ b/buildSrc/src/main/groovy/net.consensys.zkevm.linea-native-libs-helper.gradle @@ -1,59 +1,17 @@ -import groovy.json.JsonSlurper - import java.nio.file.FileAlreadyExistsException import java.nio.file.Files import java.nio.file.Path import java.time.Duration import java.time.Instant -static def downloadAndParseJson( - String url, - Map headers = [:] -) { - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection() - connection.requestMethod = 'GET' - connection.setRequestProperty('Accept', 'application/json') - headers.each { key, value -> - connection.setRequestProperty(key, value) - } - connection.connect() - if (connection.responseCode == HttpURLConnection.HTTP_OK) { - def jsonResponse = connection.inputStream.text - def jsonSlurper = new JsonSlurper() - return jsonSlurper.parseText(jsonResponse) - } else { - throw new GradleException("Failed load JSON from $url HTTP error code: ${connection.responseCode}") - } -} - - -static def getReleaseAsset( - String releaseTagName, - String githubAccessToken -) { - def urlStr = 'https://api.github.com/repos/ConsenSys/zkevm-monorepo/releases' - def json = downloadAndParseJson(urlStr, ['Authorization': "token ${githubAccessToken}"]) - def release = json.find { it.tag_name == releaseTagName } - if (release == null) { - def releases = json.collect { it.tag_name } - throw new GradleException("Release ${releaseTagName} not found! releases: ${releases}") - } - if (release.assets.size() == 0) { - throw new GradleException("Release ${releaseTagName} has no assets!") - } - def asset = release.assets.find { it.name.contains(releaseTagName) } - //println(JsonOutput.prettyPrint(JsonOutput.toJson(asset))) - asset -} - void downloadFileUsingWget( String url, - String githubAccessToken, String outputFilePath ) { - println("Downloading ${url.replace(githubAccessToken, "*****")} into ${outputFilePath}") - def command = "curl -L -H 'Accept:application/octet-stream' -u ${githubAccessToken}: -o ${outputFilePath} ${url}" -// println("# " + command) + println("Downloading ${url} into ${outputFilePath}") + + String command = "curl -L -H 'Accept:application/octet-stream' -o ${outputFilePath} ${url}" + // println("# " + command) def execResult = exec { commandLine 'bash', '-c', command @@ -72,12 +30,11 @@ ext.architectureResourceDirMapping = [ "linux_x86_64" : "linux-x86-64" ] -private String downloadReleaseAsset( +private String downloadAssetIfNotPresent( + String libsZipUrl, String releaseTag, - String outputDir, - String githubAccessToken + String outputDir ) { - def releaseAssetJsonInfo = getReleaseAsset(releaseTag.toString(), githubAccessToken) def fileName = releaseTag + ".zip" def outputFilePath = Path.of(outputDir).resolve(fileName) @@ -92,7 +49,7 @@ private String downloadReleaseAsset( } } - downloadFileUsingWget(releaseAssetJsonInfo.url.toString(), githubAccessToken, outputFilePath.toString()) + downloadFileUsingWget(libsZipUrl, outputFilePath.toString()) return outputFilePath.toString() } @@ -156,16 +113,12 @@ def lazyUnzipWithRetry( } } -def downloadReleaseAndExtractToResources( - String releaseTag, - String libName, - String outputDir, - String githubAccessToken +def extractLibToResources( + Path zipFile, + Path outputUnzipDir, + String libName ) { - def outputUnzipDir = Path.of(outputDir).resolve(releaseTag) - def outputFile = downloadReleaseAsset(releaseTag, outputDir, githubAccessToken) - lazyUnzipWithRetry(Path.of(outputFile), outputUnzipDir, Duration.ofSeconds(60)) - + lazyUnzipWithRetry(zipFile, outputUnzipDir, Duration.ofSeconds(60)) fileTree(outputUnzipDir.toFile()) .filter { it.name.contains(libName) && (it.name.endsWith(".so") || it.name.endsWith(".dylib")) } .each { File file -> @@ -190,14 +143,22 @@ def downloadReleaseAndExtractToResources( } } -ext.fetchLib = { +def downloadZipReleaseAndExtractToResources( + String libsZipUrl, String releaseTag, String libName, - String outputDir, - String githubAccessToken = System.getenv("GITHUB_TOKEN") + String outputDir +) { + def zipFile = downloadAssetIfNotPresent(libsZipUrl, releaseTag, outputDir) + def outputUnzipDir = Path.of(outputDir).resolve(releaseTag) + extractLibToResources(Path.of(zipFile), outputUnzipDir, libName) +} + +ext.fetchLibFromZip = { + String libsZipUrl, + String libName, + String outputDir -> - if (githubAccessToken == null) { - throw new GradleException("GITHUB_TOKEN is required") - } - downloadReleaseAndExtractToResources(releaseTag, libName, outputDir, githubAccessToken) + def releaseTag = libsZipUrl.split("/").last().replace(".zip", "") + downloadZipReleaseAndExtractToResources(libsZipUrl, releaseTag, libName, outputDir) } diff --git a/coordinator/Dockerfile b/coordinator/Dockerfile index fb5e688d5..10ff0b233 100644 --- a/coordinator/Dockerfile +++ b/coordinator/Dockerfile @@ -25,7 +25,7 @@ LABEL org.label-schema.build-date=$BUILD_DATE \ org.label-schema.description="Coordinator for Linea" \ org.label-schema.url="https://consensys.io/" \ org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url="https://github.com/ConsenSys/zkevm-monorepo" \ + org.label-schema.vcs-url="https://github.com/ConsenSys/linea-monorepo" \ org.label-schema.vendor="ConsenSys" \ org.label-schema.version=$VERSION \ org.label-schema.schema-version="1.0" diff --git a/coordinator/clients/smart-contract-client/src/main/kotlin/net/consensys/linea/contract/l1/Web3JFunctionBuilders.kt b/coordinator/clients/smart-contract-client/src/main/kotlin/net/consensys/linea/contract/l1/Web3JFunctionBuilders.kt index acc33b817..9a7389403 100644 --- a/coordinator/clients/smart-contract-client/src/main/kotlin/net/consensys/linea/contract/l1/Web3JFunctionBuilders.kt +++ b/coordinator/clients/smart-contract-client/src/main/kotlin/net/consensys/linea/contract/l1/Web3JFunctionBuilders.kt @@ -27,7 +27,7 @@ internal fun buildSubmitBlobsFunction( /*snarkHash*/ blobCompressionProof.snarkHash ) - LineaRollup.BlobSubmissionData( + BlobSubmissionData( /*submissionData*/ supportingSubmissionData, /*dataEvaluationClaim*/ BigInteger(blobCompressionProof.expectedY), /*kzgCommitment*/ blobCompressionProof.commitment, diff --git a/coordinator/clients/smart-contract-client/src/main/kotlin/net/consensys/linea/contract/l2/L2MessageServiceGasLimitEstimate.kt b/coordinator/clients/smart-contract-client/src/main/kotlin/net/consensys/linea/contract/l2/L2MessageServiceGasLimitEstimate.kt index cf10df451..70436c106 100644 --- a/coordinator/clients/smart-contract-client/src/main/kotlin/net/consensys/linea/contract/l2/L2MessageServiceGasLimitEstimate.kt +++ b/coordinator/clients/smart-contract-client/src/main/kotlin/net/consensys/linea/contract/l2/L2MessageServiceGasLimitEstimate.kt @@ -115,8 +115,8 @@ class L2MessageServiceGasLimitEstimate( FUNC_ANCHORL1L2MESSAGEHASHES, listOf>( DynamicArray( - org.web3j.abi.datatypes.generated.Bytes32::class.java, - messageHashes.map { org.web3j.abi.datatypes.generated.Bytes32(it) } + Bytes32::class.java, + messageHashes.map { Bytes32(it) } ), Uint256(startingMessageNumber), Uint256(finalMessageNumber), diff --git a/coordinator/persistence/batch/src/main/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/PostgresBatchesRepository.kt b/coordinator/persistence/batch/src/main/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/PostgresBatchesRepository.kt index 7ea7bd09a..6f2a74a24 100644 --- a/coordinator/persistence/batch/src/main/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/PostgresBatchesRepository.kt +++ b/coordinator/persistence/batch/src/main/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/PostgresBatchesRepository.kt @@ -9,7 +9,7 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture * WARNING: Existing mappings should not chane. Otherwise, can break production New One can be added * though. */ -public fun batchStatusToDbValue(status: Batch.Status): Int { +fun batchStatusToDbValue(status: Batch.Status): Int { // using manual mapping to catch errors at compile time instead of runtime return when (status) { Batch.Status.Finalized -> 1 diff --git a/docker/compose.yml b/docker/compose.yml index 5d544644d..0197c5670 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -77,10 +77,10 @@ services: - ../tmp/linea-besu-sequencer/plugins:/linea-besu-sequencer/ linea-besu-sequencer-finalized-tag-updater-plugin-downloader: - image: badouralix/curl-jq - command: [ "sh", "/finalized-tag-updater-jar-downloader.sh", "${GITHUB_TOKEN}", "0.0.1", "/linea-besu-sequencer" ] + image: busybox:1.36.1 + command: [ "sh", "/file-downloader.sh", "https://github.com/Consensys/linea-monorepo/releases/download/finalized-tag-updater-v0.0.2/finalized-tag-updater-v0.0.2.jar", "/linea-besu-sequencer" ] volumes: - - ./scripts/finalized-tag-updater-jar-downloader.sh:/finalized-tag-updater-jar-downloader.sh:ro + - ./scripts/file-downloader.sh:/file-downloader.sh:ro - ../tmp/linea-besu-sequencer/plugins:/linea-besu-sequencer/ l2-node: diff --git a/docker/scripts/finalized-tag-updater-jar-downloader.sh b/docker/scripts/finalized-tag-updater-jar-downloader.sh deleted file mode 100644 index f11fb1820..000000000 --- a/docker/scripts/finalized-tag-updater-jar-downloader.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -if [ $# -lt 2 ]; then - echo "Usage: $0 github_access_token plugin_version destination_folder" - echo "Example: $0 ghp_xxx 0.0.1 /path/to/destination/" - exit -fi - -check_and_download_file() { - local github_access_token="$1" - local plugin_version="$2" - local destination_folder="$3" - local asset_name="finalized-tag-updater-v$plugin_version" - local file_name="$asset_name".jar - - # Check if the file exists in the directory - if [[ ! -f "$destination_folder/$file_name" ]]; then - # File does not exist, download it - download_url=$(\ - curl -sL -H 'Accept:application/json' \ - -u $1: https://api.github.com/repos/Consensys/zkevm-monorepo/releases | jq -rc \ - 'map(select(.tag_name | contains('"\"$asset_name\""'))) | .[] .assets[] | select(.name | contains('"\"$file_name\""')) .url'\ - ) - echo "Downloading $file_name from url=$download_url ..." - curl -L -H 'Accept:application/octet-stream' -u "$github_access_token": -o "$destination_folder/$file_name" "$download_url" - echo "Download complete!" - else - echo "File $file_name already exists in $destination_folder" - fi -} -echo "$0 github_access_token $2 $3" -check_and_download_file "$1" "$2" "$3" diff --git a/finalized-tag-updater/jreleaser.yml b/finalized-tag-updater/jreleaser.yml index a229daba5..07fbe759a 100644 --- a/finalized-tag-updater/jreleaser.yml +++ b/finalized-tag-updater/jreleaser.yml @@ -3,7 +3,7 @@ project: description: Finalized Tag Updater Fat Jar longDescription: Finalized Tag Updater (linea-besu plugin) links: - homepage: https://github.com/Consensys/zkevm-monorepo + homepage: https://github.com/Consensys/linea-monorepo authors: - Linea automations license: APACHE-2.0 @@ -28,4 +28,4 @@ distributions: finalized-tag-updater: type: SINGLE_JAR artifacts: - - path: 'build/libs/{{tagName}}.jar' \ No newline at end of file + - path: 'build/libs/{{tagName}}.jar' diff --git a/jvm-libs/blob-compressor/build.gradle b/jvm-libs/blob-compressor/build.gradle index 3ba407889..7e590c213 100644 --- a/jvm-libs/blob-compressor/build.gradle +++ b/jvm-libs/blob-compressor/build.gradle @@ -29,33 +29,8 @@ def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get() task downloadNativeLibs { doLast { - fetchLib("blob-libs-v0.1.0", "blob_compressor", libsZipDownloadOutputDir) - fetchLib("blob-libs-v1.0.1", "blob_compressor", libsZipDownloadOutputDir) - } -} - -// GITHUB_TOKEN=ghp_XXX ./gradlew jvm-libs:blob-compressor:downloadNativeLib -PreleaseTag=blob-libs-v0.1.0 -PlibName=blob_compressor -tasks.register('downloadNativeLib') { - def releaseTag - def libName - - doLast { - if (project.hasProperty("releaseTag")) { - releaseTag = project.releaseTag - } else { - throw new GradleException("releaseTag is required") - } - if (project.hasProperty("libName")) { - libName = project.libName - } else { - throw new GradleException("libName is required") - } - - fetchLib( - releaseTag, - libName, - libsZipDownloadOutputDir - ) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v0.1.0/linea-blob-libs-v0.1.0.zip", "blob_compressor", libsZipDownloadOutputDir) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.0.1/linea-blob-libs-v1.0.1.zip", "blob_compressor", libsZipDownloadOutputDir) } } diff --git a/jvm-libs/blob-shnarf-calculator/build.gradle b/jvm-libs/blob-shnarf-calculator/build.gradle index 0abcab5a8..f8e572a75 100644 --- a/jvm-libs/blob-shnarf-calculator/build.gradle +++ b/jvm-libs/blob-shnarf-calculator/build.gradle @@ -28,33 +28,8 @@ def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get() task downloadNativeLibs { doLast { - fetchLib("blob-libs-v0.1.0", "shnarf_calculator", libsZipDownloadOutputDir) - fetchLib("blob-libs-v1.0.1", "shnarf_calculator", libsZipDownloadOutputDir) - } -} - -// GITHUB_TOKEN=ghp_XXX ./gradlew jvm-libs:blob-compressor:downloadNativeLib -PreleaseTag=blob-libs-v0.1.0 -PlibName=blob_compressor -tasks.register('downloadNativeLib') { - def releaseTag - def libName - - doLast { - if (project.hasProperty("releaseTag")) { - releaseTag = project.releaseTag - } else { - throw new GradleException("releaseTag is required") - } - if (project.hasProperty("libName")) { - libName = project.libName - } else { - throw new GradleException("libName is required") - } - - fetchLib( - releaseTag, - libName, - libsZipDownloadOutputDir - ) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v0.1.0/linea-blob-libs-v0.1.0.zip", "shnarf_calculator", libsZipDownloadOutputDir) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.0.1/linea-blob-libs-v1.0.1.zip", "shnarf_calculator", libsZipDownloadOutputDir) } } diff --git a/jvm-libs/traces/src/main/kotlin/net/consensys/linea/traces/TracingModule.kt b/jvm-libs/traces/src/main/kotlin/net/consensys/linea/traces/TracingModule.kt index b3abf5467..143077c8a 100644 --- a/jvm-libs/traces/src/main/kotlin/net/consensys/linea/traces/TracingModule.kt +++ b/jvm-libs/traces/src/main/kotlin/net/consensys/linea/traces/TracingModule.kt @@ -4,7 +4,6 @@ sealed interface TracingModule { val name: String } -/** More info: https://github.com/ConsenSys/zkevm-monorepo/issues/525 */ enum class TracingModuleV1 : TracingModule { // EMV Module limits ADD, diff --git a/package.json b/package.json index b7f31fd85..658ddb57d 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "zkevm-monorepo", + "name": "linea-monorepo", "author": "Consensys Software Inc.", "engines": { "node": ">=20", diff --git a/traces-api-facade/Dockerfile b/traces-api-facade/Dockerfile index 4f78e72d0..5717ad018 100644 --- a/traces-api-facade/Dockerfile +++ b/traces-api-facade/Dockerfile @@ -21,7 +21,7 @@ LABEL org.label-schema.build-date=$BUILD_DATE \ org.label-schema.description="Linea Traces API" \ org.label-schema.url="https://consensys.io/" \ org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url="https://github.com/ConsenSys/zkevm-monorepo" \ + org.label-schema.vcs-url="https://github.com/ConsenSys/linea-monorepo" \ org.label-schema.vendor="ConsenSys" \ org.label-schema.version=$VERSION \ org.label-schema.schema-version="1.0"