update

docker/scripts/file-downloader.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 file_url destination_folder"
+    echo "Example: $0 https://example.com/somefile.txt /path/to/destination/"
+    exit
+fi
+
+check_and_download_file() {
+    local file_url="$1"
+    local directory="$2"
+    local filename="${file_url##*/}"  # Extracts the filename from the URL
+
+    # Check if the file exists in the directory
+    if [[ ! -f "$directory/$filename" ]]; then
+        # File does not exist, download it
+        echo "Downloading $file_url ..."
+        wget "$file_url" -P "$directory"
+        echo "Download complete!"
+    else
+        echo "File $filename already exists in $directory."
+    fi
+}
+echo "$0 $1 $2"
+check_and_download_file "$1" "$2"

docker/scripts/functions.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -Eeu
+
+function prepareDatadir {
+  datadir=$1
+  genesisfile=$2
+  if [ ! -d $datadir/geth ]; then
+    echo -e "\n\n----------> A new data directory '$datadir' will be created!"
+    geth --datadir $datadir init $genesisfile
+    echo -e "----------> A new data directory '$datadir' created!\n\n"
+  else
+    echo -e "\n\n----------> Data directory '$datadir' already exists! Contents:"
+    find $datadir
+    echo -e "\n\n"
+  fi
+}

docker/scripts/node.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -Eeu
+
+if [ $# -lt 4 ]; then
+    echo "Usage: $0 networkid genesisfile gasprice gaslimit"
+    echo Example: $0 12345 /genesis.json
+    exit
+fi
+
+source ./scripts/functions.sh
+networkid=$1
+genesisfile=$2
+gasprice=$3
+gaslimit=$4
+
+: ${BOOTNODES:=""} # default to empty
+: ${ETHSTATS_URL:=""} #default to empty
+
+echo DATA_DIR=$DATA_DIR
+echo networkid=$networkid
+echo genesisfile=$genesisfile
+echo gasprice=$gasprice
+echo gaslimit=$gaslimit
+echo BOOTNODES=$BOOTNODES
+echo ETHSTATS_URL=$ETHSTATS_URL
+echo NETRESTRICT=$NETRESTRICT
+
+prepareDatadir $DATA_DIR $genesisfile
+
+if [ ${#BOOTNODES} -ge 1 ]; then
+#    bootnode=$(getent hosts $bootnodehost | awk '{print $1}')
+    echo "Starting geth connecting to $BOOTNODES"
+    geth --datadir $DATA_DIR \
+      --networkid $networkid \
+	    --miner.gasprice $gasprice \
+      --miner.gaslimit $gaslimit \
+      --http --http.addr '0.0.0.0' --http.port 8545 --http.corsdomain '*' --http.api 'admin,eth,miner,net,web3,personal,txpool,debug' --http.vhosts="*" \
+      --ws --ws.addr '0.0.0.0' --ws.port 8546 --ws.origins '*' --ws.api 'admin,eth,miner,net,web3,personal,txpool,debug' \
+      --bootnodes $BOOTNODES \
+      --ethstats "$ETHSTATS_URL" \
+      --ipcdisable \
+      --netrestrict "$NETRESTRICT" \
+      --verbosity 3 \
+      --syncmode "full"
+else
+    echo "Starting geth with BOOTNODE enabled on port '$BOOTNODE_PORT' with '/boot.key'"
+    geth --datadir $DATA_DIR \
+     --networkid $networkid \
+	 --miner.gasprice $gasprice \
+     --miner.gaslimit $gaslimit \
+     --http --http.addr '0.0.0.0' --http.port 8545 --http.corsdomain '*' --http.api 'admin,eth,miner,net,web3,personal,txpool,debug' --http.vhosts="*" \
+     --ws --ws.addr '0.0.0.0' --ws.port 8546 --ws.origins '*' --ws.api 'admin,eth,miner,net,web3,personal,txpool,debug' \
+     --nodekey "/boot.key" \
+     --port $BOOTNODE_PORT \
+     --ethstats "$ETHSTATS_URL" \
+     --ipcdisable \
+     --netrestrict "$NETRESTRICT" \
+     --verbosity 3 \
+     --txpool.nolocals  \
+     --syncmode "full"
+fi

docker/scripts/validator.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -Eeu
+
+if [ $# -ne 5 ]; then
+    echo "Usage: $0 etherbase networkid genesis_file gasprice gaslimit"
+    echo "Example: $0 0x6d976c9b8ceee705d4fe8699b44e5eb58242f484 24709958 genesis.json 1 0x1C9C380"
+    exit
+fi
+
+source ./scripts/functions.sh
+
+etherbase=$1
+networkid=$2
+genesisfile=$3
+gasprice=$4
+gaslimit=$5
+: ${ETHSTATS_URL:=""} #default to empty
+
+echo etherbase=$etherbase
+echo networkid=$networkid
+echo genesisfile=$genesisfile
+echo gasprice=$gasprice
+echo gaslimit=$gaslimit
+echo ETHSTATS_URL=$ETHSTATS_URL
+echo BOOTNODES=$BOOTNODES
+
+mkdir -p $DATA_DIR/keystore
+cp /keystore/$etherbase.json $DATA_DIR/keystore
+prepareDatadir $DATA_DIR $genesisfile
+
+geth --datadir $DATA_DIR \
+     --networkid $networkid \
+     --miner.etherbase $etherbase \
+     --miner.gasprice $gasprice \
+     --miner.gaslimit $gaslimit \
+     --bootnodes $BOOTNODES \
+     --allow-insecure-unlock \
+     --http --http.addr '0.0.0.0' --http.port 8545 --http.corsdomain '*' --http.api 'admin,eth,miner,net,web3,personal,txpool,debug' --http.vhosts="*" \
+     --ethstats=$ETHSTATS_URL \
+     --netrestrict $NETRESTRICT \
+     --unlock $etherbase \
+     --password "/dev/null" \
+     --ipcdisable \
+     --verbosity 3 \
+     --mine \
+     --syncmode "full"

docker/web3signer/conf/config.yaml

@@ -0,0 +1,8 @@
+# Chain
+#chain-id: 1337
+
+# Path
+data-path: "~/web3signerdata" # Path
+
+# Signing key locations
+key-store-path: "/keyFiles/"

docker/web3signer/keyFiles/anchoring-signer.yaml

@@ -0,0 +1,6 @@
+type: "file-raw"
+keyType: "SECP256K1"
+# WARNING: FOR LOCAL DEV ONLY - DO NOT REUSE THESE KEYS ELSEWHERE
+# Key for anchoring on L2
+# Public key: 4a788ad6fa008beed58de6418369717d7492f37d173d70e2c26d9737e2c6eeae929452ef8602a19410844db3e200a0e73f5208fd76259a8766b73953fc3e7023
+privateKey: "0x4d01ae6487860981699236a58b68f807ee5f17b12df5740b85cf4c4653be0f55"

docker/web3signer/keyFiles/data-submission-signer.yaml

@@ -0,0 +1,6 @@
+type: "file-raw"
+keyType: "SECP256K1"
+# WARNING: FOR LOCAL DEV ONLY - DO NOT REUSE THESE KEYS ELSEWHERE
+# Key for data submission
+# Public key: 9d9031e97dd78ff8c15aa86939de9b1e791066a0224e331bc962a2099a7b1f0464b8bbafe1535f2301c72c2cb3535b172da30b02686ab0393d348614f157fbdb
+privateKey: "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a"

docker/web3signer/keyFiles/finalization-signer.yaml

@@ -0,0 +1,6 @@
+type: "file-raw"
+keyType: "SECP256K1"
+# WARNING: FOR LOCAL DEV ONLY - DO NOT REUSE THESE KEYS ELSEWHERE
+# Key for aggregation finalization
+# Public key: ba5734d8f7091719471e7f7ed6b9df170dc70cc661ca05e688601ad984f068b0d67351e5f06073092499336ab0839ef8a521afd334e53807205fa2f08eec74f4
+privateKey: "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"

docs/DEVELOPMENT_GUIDELINES.md

@@ -0,0 +1,116 @@
+# Linea Developer Guidelines <!-- omit in toc -->
+
+<!--
+ToC can be automatically updated with:
+A) VC Code "Markdown All In One" plugin - https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one#table-of-contents
+B) IntelliJ - https://www.jetbrains.com/help/idea/markdown.html#table-of-contents
+-->
+
+- [Logging Guidelines](#logging-guidelines)
+  - [Log Levels](#log-levels)
+  - [Log Format](#log-format)
+    - [Considerations](#considerations)
+    - [Exceptions Stack Traces](#exceptions-stack-traces)
+  - [Java/Kotlin Guidelines](#javakotlin-guidelines)
+    - [favour argument placeholder instead of String templating](#favour-argument-placeholder-instead-of-string-templating)
+    - [Performance](#performance)
+  - [GO Guidelines](#go-guidelines)
+  - [JavaScript/Typescript Guidelines](#javascripttypescript-guidelines)
+
+## Logging Guidelines
+
+Effective logging is crucial for understanding the behavior of our applications, diagnosing issues, and ensuring that our system is running smoothly. We have established the following logging guidelines, please adhere to these practices to help us keep our logs informative, actionable, and manageable.
+
+### Log Levels
+- **error** - for errors that compromise application/linea functionality, may have impact on user experience and should trigger an investigation right away. Consider including stack trace for unknown errors. Examples:
+  - Transaction reverted with "Invalid Proof" error.
+  - Message anchoring TX reverted
+- **warn** - for error that application can recover/retry from but will compromise the system if the problem persists overtime. e.g. 3 consecutive connection errors to upstream service within 30s period.
+- **info** - Application key events and state transitions, e.g Start/Stop, new batches/blob/aggregations generated and proven. Debounce repetitive log lines with 15s, also "heartbeat" logs informing why something expected is not happening, otherwise may look like a crash. INFO should offer the right amount of logs to Production.
+- **debug** - For extra events/state transitions that to help troubleshoot debug the application. Performance overhead should not be over 20%.
+
+`error` or `warn` logs will trigger alerts in our system, please use these levels carefully.
+
+
+### Log Format
+Our logs aim to be both human and machine friendly. We use [Grafana and Loki](https://grafana.com/docs/loki/latest/query/log_queries/#logfmt) so our logs must be **single-lined** and follow [logfmt](https://brandur.org/logfmt) format: `varName1=value1 varName2=value2`. Note, values must be white space delimited without trailing punctuation or special characters.
+
+Log entry
+```
+time=<iso8601_utc_time> level=<log_level> message=<message> | <other_runtime_fields> | <inline_error_stack_trace>
+```
+
+Message format
+```
+<Event/action description>: contextField1=value1 contextField2=value2
+```
+
+Examples
+```syslog
+time=2024-04-10T13:59:38.328+00:00 level=INFO message=new blob: blob=[9726..9729]4 trigger=TIME_LIMIT startBlockNumber=9726 endBlockNumber=9729 blobSizeBytes=11808 blobBatchesCount=4 blobBatchesLimit=299 blobBatchesList=[[9726..9726]1, [9727..9727]1, [9728..9728]1, [9729..9729]1]4 | class=GlobalBlobAwareConflationCalculator thread=pool-2-thread-13
+time=2024-04-10T15:40:18.1880-00:00 level=INFO message=blobs to submit: lastFinalizedBlockNumber=9572 totalBlobs=8 maxBlobsToSubmitPerTick=5 newBlobsToSubmit=[[9887..9888]2,[9889..9908]20]2  startBlockNumber=9887 endBlockNumber=9888 alreadySubmittedBlobs=[[9573..9573]1, [9574..9721]148, [9722..9722]1, [9723..9725]3, [9726..9885]160, [9886..9886]1]6 | class=BlobSubmissionCoordinatorImpl thread=pool-2-thread-140
+time=2024-04-16T14:21:02.388Z level=ERROR message=failed to fetch batches: errorMessage=Index 4 out of bounds for length 3 | logger=Playground thread=test-error-timer | java.lang.ArrayIndexOutOfBoundsException: Index 4 out of bounds for length 3\n	at java.base/java.util.Arrays$ArrayList.get(Arrays.java:4165)\n	at net.consensys.zkevm.coordinator.app.PlaygroundKt$testLog$$inlined$timer$2.run(Timer.kt:151)\n	at java.base/java.util.TimerThread.mainLoop(Timer.java:566)\n	at java.base/java.util.TimerThread.run(Timer.java:516)
+```
+**Note**: time needs to be in [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) format with milliseconds and timezone, like in the one of 3 examples above (invalid `2024-04-10T15:40:18.188`) otherwise promtail won't parse it correctly.
+
+Query example to filter logs for blocks numbers between 9720 and 9888
+```json
+{cluster="linea-devnet", pod="coordinator"} | logfmt | startBlockNumber >= 9720 and endBlockNumber <= 9888
+```
+
+
+#### Considerations
+- all the info/context associated with an event shall be in a single log message instead of fragmented across multiple log messages;
+- Consistency: log the same event/entity/concept exactly the same way everywhere across the application. Examples:
+  - continuous block intervals: `[10..15]6`
+- sensitive information: when logging a field make sure it does not have sensitive information. Examples:
+  - application start up configs may contain APP Keys, passwords, etc - use masked fields instead;
+  - debug/trace HTTP requests with credentials in the headers/urls
+- do not log "non-happy path" valid cases as error/warnings, e.g HTTP server logging "404 not path not found" can open window for logging DoS/noise;
+
+#### Exceptions Stack Traces
+
+Error's Stack Traces shall be logged in-line with `\n` character representing line breaks so Loki can escape them and display them in multi-line.
+
+```syslog
+level=warn time=2024-04-11T11:56:39.147 msg=Anchoring attempt failed! Anchoring will be re-attempted shortly. | class=MessageAnchoringService thread=vert.x-eventloop-thread-8 | java.util.concurrent.CompletionException: java.lang.InterruptedException: sleep interrupted\n	at java.base/java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:368)\n	at java.base/java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:377)\n	at java.base/java.util.concurrent.CompletableFuture$UniRelay.tryFire(CompletableFuture.java:1097)\n	at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n	at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162)\n	at tech.pegasys.teku.infrastructure.async.SafeFuture.lambda$propagateResult$3(SafeFuture.java:146)\n	at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)\n	at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)\nat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n	at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162)\n	at io.vertx.core.Future.lambda$toCompletionStage$3(Future.java:583)\n	at io.vertx.core.impl.future.FutureImpl$4.onFailure(FutureImpl.java:188)\n	at io.vertx.core.impl.future.FutureBase.lambda$emitFailure$1(FutureBase.java:75)\n	at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)\n	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)\n	at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)\n	at io.netty.channel.kqueue.KQueueEventLoop.run(KQueueEventLoop.java:300)\n	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)\n	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n	at java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: java.lang.InterruptedException: sleep interrupted\n	at java.base/java.lang.Thread.sleep(Native Method)\n	at net.consensys.zkevm.ethereum.coordination.messageanchoring.L1EventQuerierImpl.collectEvents(L1EventQuerierImpl.kt:111)\n	at net.consensys.zkevm.ethereum.coordination.messageanchoring.L1EventQuerierImpl.getSendMessageEventsForAnchoredMessage$lambda-0(L1EventQuerierImpl.kt:74)\n	at io.vertx.core.impl.ContextImpl.lambda$executeBlocking$1(ContextImpl.java:190)\n	at io.vertx.core.impl.ContextInternal.dispatch(ContextInternal.java:276)\n	at io.vertx.core.impl.ContextImpl.lambda$internalExecuteBlocking$2(ContextImpl.java:209)\n	at io.vertx.core.impl.TaskQueue.run(TaskQueue.java:76)\n	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n	... 2 more
+```
+
+With **Escape newlines** enabled in Grafana, the stack trace will be displayed as multi-line,but still a single log entry in Loki.
+![](assets/grafana-loki-singleline-strack-trace-displayed-as-multiline.png)
+
+### Java/Kotlin Guidelines
+
+These guidelines reelect some specificities of our logging library [Log4J2](https://logging.apache.org/log4j/2.x/)
+
+#### favour argument placeholder instead of String templating
+```kotlin
+// Do
+log.info("already proven: batch={}, reusedResponse={}", batch, fileName)
+
+// Don't - new string created every time, even if INFO is disabled
+log.info("already proven: batch=${batch}, reusedResponse=${fileName}")
+```
+
+#### Performance
+Avoid heavy computation for logging arguments. Consider using argument suppliers in Debug/Trace
+
+
+```kotlin
+// Ok
+log.debug(
+  "state={}",
+  { getTreeStateAsString() }  // argument supplier fun, lazy evaluated
+)
+// OK
+if(log.isDebugEnabled) {
+  log.debug("state={}", getTreeStateAsString())
+}
+
+// Not OK - will always evaluate getTreeStateAsString() function even debug is not enabled
+log.debug("state={}”, getTreeStateAsString())
+```
+
+### GO Guidelines
+
+### JavaScript/Typescript Guidelines

e2e/.env.template

@@ -0,0 +1,12 @@
+UAT_L1_RPC_KEY=
+UAT_L2_RPC_KEY=
+UAT_ACCOUNT_0_PRIVATE_KEY=
+
+DEV_DEPLOYER_ACCOUNT_PRIVATE_KEY=
+DEV_ACCOUNT_0_PRIVATE_KEY=
+DEV_OPERATOR_0_PRIVATE_KEY=
+
+LOCAL_DEPLOYER_ACCOUNT_PRIVATE_KEY=
+LOCAL_ACCOUNT_0_PRIVATE_KEY=
+LOCAL_OPERATOR_0_PRIVATE_KEY=
+LOCAL_MESSAGE_SERVICE_PRIVATE_KEY=

e2e/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

e2e/.eslintrc.js

@@ -0,0 +1,10 @@
+module.exports = {
+  extends: ["../.eslintrc.js"],
+  parserOptions: {
+    sourceType: "module",
+  },
+  env: {
+    node: true,
+    jest: true,
+  },
+};

e2e/.prettierignore

@@ -0,0 +1,3 @@
+node_modules
+dist
+typechain

e2e/.prettierrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require("../.prettierrc.js"),
+};