Update tests with should_succeed_complete (#18)

* Update tests with should_succeed_complete * Handle attestations too * Update semantics in README
eth-clients · Dec 4, 2023 · d966e64 · d966e64
1 parent 658f719
commit d966e64
Show file tree

Hide file tree

Showing 29 changed files with 313 additions and 160 deletions.
diff --git a/README.md b/README.md
@@ -84,16 +84,16 @@ Each block in `blocks` is structured as:
   "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
   "slot": "1",
   "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-  "should_succeed": false
+  "should_succeed": false,
+  "should_succeed_complete": true
 }
 ```
 
-Your test-runner should attempt to sign a block with `signing_root` at the
-given slot from the given `pubkey`. The `should_succeed` field describes
-whether this signing should be accepted (true) or rejected (false). Clients
-using a slashing protection mechanism that admits false positives may consider
-a rejection as success even if `should_succeed` is true. If the block is signed
-successfully it should be incorporated into the slashing protection database.
+Your test-runner should attempt to sign a block with `signing_root` at the given slot from the given
+`pubkey`. The `should_succeed` field describes whether this signing should be accepted (true) or
+rejected (false) _by a client using a minimal strategy_. Clients using a complete strategy should
+instead use the `should_succeed_complete` field which allows signing to succeed in more cases. If
+the block is signed successfully it should be incorporated into the slashing protection database.
 
 Each attestation in `attestations` is structured as:
 
@@ -103,13 +103,16 @@ Each attestation in `attestations` is structured as:
   "source_epoch": "11",
   "target_epoch": "12",
   "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-  "should_succeed": true
+  "should_succeed": true,
+  "should_succeed_complete": true
 }
 ```
 
 Similarly to above, your test-runner should attempt to sign an attestation with these parameters
-using the given `pubkey`, and succeed based on the value of `should_succeed`. Again, false positives
-may be treated as successes, and signed attestations should be incorporated into the database.
+using the given `pubkey`, and succeed based on the value of
+`should_succeed`/`should_succeed_complete`.  Again, implementations that use the _complete_ strategy
+should use `should_succeed_complete`. All implementations should incorporate signed attestations
+into the database.
 
 Note that the top-level `genesis_validators_root` is not necessarily the same
 as the GVR contained in the interchange, to allow us to test the case where

diff --git a/tests/generated/duplicate_pubkey_not_slashable.json b/tests/generated/duplicate_pubkey_not_slashable.json
@@ -52,19 +52,22 @@
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "10",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "13",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "14",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         }
       ],
       "attestations": [
@@ -73,14 +76,16 @@
           "source_epoch": "0",
           "target_epoch": "2",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "source_epoch": "1",
           "target_epoch": "3",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         }
       ]
     }

diff --git a/tests/generated/duplicate_pubkey_slashable_attestation.json b/tests/generated/duplicate_pubkey_slashable_attestation.json
@@ -41,28 +41,32 @@
           "source_epoch": "0",
           "target_epoch": "1",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "source_epoch": "0",
           "target_epoch": "2",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "source_epoch": "0",
           "target_epoch": "4",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "source_epoch": "1",
           "target_epoch": "4",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         }
       ]
     }

diff --git a/tests/generated/duplicate_pubkey_slashable_block.json b/tests/generated/duplicate_pubkey_slashable_block.json
@@ -46,13 +46,15 @@
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "10",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "11",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         }
       ],
       "attestations": []

diff --git a/tests/generated/multiple_interchanges_multiple_validators_repeat_idem.json b/tests/generated/multiple_interchanges_multiple_validators_repeat_idem.json
@@ -128,31 +128,36 @@
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "0",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "3",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": false,
+          "should_succeed_complete": true
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "7",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "3",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": false,
+          "should_succeed_complete": true
         },
         {
           "pubkey": "0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b",
           "slot": "0",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         }
       ],
       "attestations": [
@@ -161,14 +166,16 @@
           "source_epoch": "0",
           "target_epoch": "4",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b",
           "source_epoch": "0",
           "target_epoch": "4",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         }
       ]
     }

diff --git a/tests/generated/multiple_interchanges_overlapping_validators_merge_stale.json b/tests/generated/multiple_interchanges_overlapping_validators_merge_stale.json
@@ -116,37 +116,43 @@
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "100",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b",
           "slot": "101",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4de3d755733bf7e4a9b",
           "slot": "102",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "103",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         },
         {
           "pubkey": "0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b",
           "slot": "104",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         },
         {
           "pubkey": "0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4de3d755733bf7e4a9b",
           "slot": "105",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         }
       ],
       "attestations": [
@@ -155,63 +161,72 @@
           "source_epoch": "12",
           "target_epoch": "13",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "source_epoch": "11",
           "target_epoch": "14",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b",
           "source_epoch": "12",
           "target_epoch": "13",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b",
           "source_epoch": "11",
           "target_epoch": "14",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4de3d755733bf7e4a9b",
           "source_epoch": "12",
           "target_epoch": "13",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4de3d755733bf7e4a9b",
           "source_epoch": "11",
           "target_epoch": "14",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "source_epoch": "12",
           "target_epoch": "14",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         },
         {
           "pubkey": "0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b",
           "source_epoch": "13",
           "target_epoch": "14",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         },
         {
           "pubkey": "0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4de3d755733bf7e4a9b",
           "source_epoch": "13",
           "target_epoch": "14",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": true
+          "should_succeed": true,
+          "should_succeed_complete": true
         }
       ]
     }

diff --git a/tests/generated/multiple_interchanges_overlapping_validators_repeat_idem.json b/tests/generated/multiple_interchanges_overlapping_validators_repeat_idem.json
@@ -192,21 +192,24 @@
           "source_epoch": "0",
           "target_epoch": "4",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b",
           "source_epoch": "1",
           "target_epoch": "2",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4de3d755733bf7e4a9b",
           "source_epoch": "1",
           "target_epoch": "2",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         }
       ]
     }

diff --git a/tests/generated/multiple_interchanges_single_validator_fail_iff_imported.json b/tests/generated/multiple_interchanges_single_validator_fail_iff_imported.json
@@ -53,13 +53,15 @@
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "20",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         },
         {
           "pubkey": "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c",
           "slot": "50",
           "signing_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
-          "should_succeed": false
+          "should_succeed": false,
+          "should_succeed_complete": false
         }
       ],
       "attestations": []