Merge from upstream v0.6.7

Merge from upstream v0.6.7

ci/e2e-group-dac-1/cli_test.go

@@ -35,7 +35,7 @@ func TestSetDataAvailabilityProtocol(t *testing.T) {
 	clientL1, err := ethclient.Dial(operations.DefaultL1NetworkURL)
 	require.NoError(t, err)
 
-	auth, err := operations.GetAuth("0xde3ca643a52f5543e84ba984c4419ff40dbabd0e483c31c1d09fee8168d68e38", operations.DefaultL1ChainID)
+	auth, err := operations.GetAuth("0x815405dddb0e2a99b12af775fd2929e526704e1d1aea6a0b4e74dc33e2f7fcd2", operations.DefaultL1ChainID)
 	require.NoError(t, err)
 
 	// New DAC Setup

config/config_test.go

@@ -101,6 +101,10 @@ func Test_Defaults(t *testing.T) {
 			path:          "Sequencer.Finalizer.ResourceExhaustedMarginPct",
 			expectedValue: uint32(10),
 		},
+		{
+			path:          "Sequencer.Finalizer.StateRootSyncInterval",
+			expectedValue: types.NewDuration(3600 * time.Second),
+		},
 		{
 			path:          "Sequencer.Finalizer.ForcedBatchesL1BlockConfirmations",
 			expectedValue: uint64(64),
@@ -127,7 +131,7 @@ func Test_Defaults(t *testing.T) {
 		},
 		{
 			path:          "Sequencer.Finalizer.BatchMaxDeltaTimestamp",
-			expectedValue: types.NewDuration(10 * time.Second),
+			expectedValue: types.NewDuration(1800 * time.Second),
 		},
 		{
 			path:          "Sequencer.Finalizer.Metrics.Interval",

config/default.go

@@ -143,7 +143,15 @@ TrustedSequencerURL = "" # If it is empty or not specified, then the value is re
 SyncBlockProtection = "safe" # latest, finalized, safe
 L1SynchronizationMode = "sequential"
 L1SyncCheckL2BlockHash = true
-L1SyncCheckL2BlockNumberhModulus = 30
+L1SyncCheckL2BlockNumberhModulus = 600
+	[Synchronizer.L1BlockCheck]
+		Enable = true
+		L1SafeBlockPoint = "finalized"
+		L1SafeBlockOffset = 0
+		ForceCheckBeforeStart = true
+		PreCheckEnable = true
+		L1PreSafeBlockPoint = "safe"
+		L1PreSafeBlockOffset = 0
 	[Synchronizer.L1ParallelSynchronization]
 		MaxClients = 10
 		MaxPendingNoProcessedBlocks = 25
@@ -176,12 +184,13 @@ StateConsistencyCheckInterval = "5s"
 		ForcedBatchesCheckInterval = "10s"
 		L1InfoTreeL1BlockConfirmations = 64
 		L1InfoTreeCheckInterval = "10s"
-		BatchMaxDeltaTimestamp = "10s"
+		BatchMaxDeltaTimestamp = "1800s"
 		L2BlockMaxDeltaTimestamp = "3s"
 		ResourceExhaustedMarginPct = 10
+		StateRootSyncInterval = "3600s"
 		HaltOnBatchNumber = 0
 		SequentialBatchSanityCheck = false
-		SequentialProcessL2Block = true
+		SequentialProcessL2Block = false
 		FullBatchSleepDuration = "0s"
 	[Sequencer.Finalizer.Metrics]
 		Interval = "60m"

config/environments/local/local.node.config.toml

@@ -103,9 +103,10 @@ StateConsistencyCheckInterval = "5s"
 		BatchMaxDeltaTimestamp = "120s"
 		L2BlockMaxDeltaTimestamp = "3s"
 		ResourceExhaustedMarginPct = 10
+		StateRootSyncInterval = "360s"
 		HaltOnBatchNumber = 0
 		SequentialBatchSanityCheck = false
-		SequentialProcessL2Block = true
+		SequentialProcessL2Block = false
 	[Sequencer.Finalizer.Metrics]
 		Interval = "60m"
 		EnableLog = true		

db/migrations/pool/1002.sql

@@ -0,0 +1,12 @@
+-- +migrate Up
+CREATE TABLE IF NOT EXISTS pool.readytx(
+                             id SERIAL PRIMARY KEY NOT NULL,
+                             count INT,
+                             updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+INSERT INTO pool.readytx(id, count) VALUES(1, 0)
+    ON CONFLICT(id) do UPDATE
+    SET count = 0;
+
+-- +migrate Down
+DROP TABLE IF EXISTS pool.readytx;

db/migrations/pool/1003.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+CREATE TABLE IF NOT EXISTS pool.innertx (
+                              hash VARCHAR(128) PRIMARY KEY NOT NULL,
+                              innertx text,
+                              created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- +migrate Down
+DROP TABLE IF EXISTS pool.innertx;

db/migrations/state/0019.sql

@@ -0,0 +1,25 @@
+-- +migrate Up
+
+-- the update below fix the wrong receipt TX indexes
+WITH map_fix_tx_index AS (
+    SELECT t.l2_block_num AS block_num
+         , t.hash AS tx_hash
+         , r.tx_index AS current_index
+         , (ROW_NUMBER() OVER (PARTITION BY t.l2_block_num ORDER BY r.tx_index))-1 AS correct_index
+      FROM state.receipt r
+     INNER JOIN state."transaction" t
+        ON t.hash = r.tx_hash
+)
+UPDATE state.receipt AS r
+   SET tx_index = m.correct_index
+  FROM map_fix_tx_index m
+ WHERE m.block_num = r.block_num
+   AND m.tx_hash = r.tx_hash
+   AND m.current_index = r.tx_index
+   AND m.current_index != m.correct_index;
+
+
+-- +migrate Down
+
+-- no action is needed, the data fixed by the 
+-- migrate up must remain fixed

db/migrations/state/0019_test.go

@@ -0,0 +1,145 @@
+package migrations_test
+
+import (
+	"database/sql"
+	"testing"
+
+	"github.com/0xPolygonHermez/zkevm-node/hex"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/require"
+)
+
+type migrationTest0019TestCase struct {
+	Name  string
+	Block migrationTest0019TestCaseBlock
+}
+
+type migrationTest0019TestCaseBlock struct {
+	Transactions []migrationTest0019TestCaseTransaction
+}
+
+type migrationTest0019TestCaseTransaction struct {
+	CurrentIndex uint
+}
+
+type migrationTest0019 struct {
+	TestCases []migrationTest0019TestCase
+}
+
+func (m migrationTest0019) InsertData(db *sql.DB) error {
+	const addBlock0 = "INSERT INTO state.block (block_num, received_at, block_hash) VALUES (0, now(), '0x0')"
+	if _, err := db.Exec(addBlock0); err != nil {
+		return err
+	}
+
+	const addBatch0 = `
+		INSERT INTO state.batch (batch_num, global_exit_root, local_exit_root, acc_input_hash, state_root, timestamp, coinbase, raw_txs_data, forced_batch_num, wip) 
+		VALUES (0,'0x0000', '0x0000', '0x0000', '0x0000', now(), '0x0000', null, null, true)`
+	if _, err := db.Exec(addBatch0); err != nil {
+		return err
+	}
+
+	const addL2Block = "INSERT INTO state.l2block (block_num, block_hash, header, uncles, parent_hash, state_root, received_at, batch_num, created_at) VALUES ($1, $2, '{}', '{}', '0x0', '0x0', now(), 0, now())"
+	const addTransaction = "INSERT INTO state.transaction (hash, encoded, decoded, l2_block_num, effective_percentage, l2_hash) VALUES ($1, 'ABCDEF', '{}', $2, 255, $1)"
+	const addReceipt = "INSERT INTO state.receipt (tx_hash, type, post_state, status, cumulative_gas_used, gas_used, effective_gas_price, block_num, tx_index, contract_address) VALUES ($1, 1, null, 1, 1234, 1234, 1, $2, $3, '')"
+
+	txUnique := 0
+	for tci, testCase := range m.TestCases {
+		blockNum := uint64(tci + 1)
+		blockHash := common.HexToHash(hex.EncodeUint64(blockNum)).String()
+		if _, err := db.Exec(addL2Block, blockNum, blockHash); err != nil {
+			return err
+		}
+		for _, tx := range testCase.Block.Transactions {
+			txUnique++
+			txHash := common.HexToHash(hex.EncodeUint64(uint64(txUnique))).String()
+			if _, err := db.Exec(addTransaction, txHash, blockNum); err != nil {
+				return err
+			}
+			if _, err := db.Exec(addReceipt, txHash, blockNum, tx.CurrentIndex); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (m migrationTest0019) RunAssertsAfterMigrationUp(t *testing.T, db *sql.DB) {
+	const getReceiptsByBlock = "SELECT r.tx_index FROM state.receipt r WHERE r.block_num = $1 ORDER BY r.tx_index"
+
+	for tci := range m.TestCases {
+		blockNum := uint64(tci + 1)
+
+		rows, err := db.Query(getReceiptsByBlock, blockNum)
+		require.NoError(t, err)
+
+		var expectedIndex = uint(0)
+		var txIndex uint
+		for rows.Next() {
+			err := rows.Scan(&txIndex)
+			require.NoError(t, err)
+			require.Equal(t, expectedIndex, txIndex)
+			expectedIndex++
+		}
+	}
+}
+
+func (m migrationTest0019) RunAssertsAfterMigrationDown(t *testing.T, db *sql.DB) {
+	m.RunAssertsAfterMigrationUp(t, db)
+}
+
+func TestMigration0019(t *testing.T) {
+	runMigrationTest(t, 19, migrationTest0019{
+		TestCases: []migrationTest0019TestCase{
+			{
+				Name: "single tx with correct index",
+				Block: migrationTest0019TestCaseBlock{
+					Transactions: []migrationTest0019TestCaseTransaction{
+						{CurrentIndex: 0},
+					},
+				},
+			},
+			{
+				Name: "multiple txs indexes are correct",
+				Block: migrationTest0019TestCaseBlock{
+					Transactions: []migrationTest0019TestCaseTransaction{
+						{CurrentIndex: 0},
+						{CurrentIndex: 1},
+						{CurrentIndex: 2},
+					},
+				},
+			},
+			{
+				Name: "single tx with wrong tx index",
+				Block: migrationTest0019TestCaseBlock{
+					Transactions: []migrationTest0019TestCaseTransaction{
+						{CurrentIndex: 3},
+					},
+				},
+			},
+			{
+				Name: "multiple txs missing 0 index",
+				Block: migrationTest0019TestCaseBlock{
+					Transactions: []migrationTest0019TestCaseTransaction{
+						{CurrentIndex: 1},
+						{CurrentIndex: 2},
+						{CurrentIndex: 3},
+						{CurrentIndex: 4},
+					},
+				},
+			},
+			{
+				Name: "multiple has index 0 but also txs index gap",
+				Block: migrationTest0019TestCaseBlock{
+					Transactions: []migrationTest0019TestCaseTransaction{
+						{CurrentIndex: 0},
+						{CurrentIndex: 2},
+						{CurrentIndex: 4},
+						{CurrentIndex: 6},
+					},
+				},
+			},
+		},
+	})
+}

db/migrations/state/0020.sql

@@ -0,0 +1,28 @@
+-- +migrate Up
+
+-- This migration will delete all empty blocks
+DELETE FROM state.block
+WHERE NOT EXISTS (SELECT *
+              FROM state.virtual_batch 
+              WHERE state.virtual_batch.block_num = state.block.block_num) 
+      AND NOT EXISTS (SELECT *
+              FROM state.verified_batch  
+              WHERE state.verified_batch.block_num = state.block.block_num)
+      AND NOT EXISTS (SELECT *
+              FROM state.forced_batch 
+              WHERE state.forced_batch.block_num = state.block.block_num)
+      AND NOT EXISTS (SELECT *
+              FROM state.exit_root 
+              WHERE state.exit_root.block_num = state.block.block_num)
+      AND NOT EXISTS (SELECT *
+              FROM state.monitored_txs 
+              WHERE state.monitored_txs.block_num = state.block.block_num)
+      AND NOT EXISTS (SELECT *
+              FROM state.fork_id 
+              WHERE state.fork_id.block_num = state.block.block_num);
+
+
+
+-- +migrate Down
+
+-- no action is needed, the data must remain deleted as it is useless