Repository: ethereum/consensus-specs

Issue:
Attestation bitfield length checks 
See #1171:

> When there are two bitfields (Aggregation, and custody1), and we derive the indices for custody0 (in convert_to_indexed), should we check len(participants) < len(custody1)? E.g. the bitfields could be verified to be committee size, yet the custody bits could be more full than the actual participants, resulting in extra custody-1 indices. Or is this prevented elsewhere?

@JustinDrake:

> Oh, I think you're right that something is fishy in convert_to_indexed. Specifically, we want to check that every bit in custody_bitfield is zero whenever the corresponding bit in aggregation_bitfield is zero. It's possible this invariant was lost with the refactor to IndexedAttestation. cc @djrtwo


Separating this from the #1171 PR, as it's unnecessarily blocking, somewhat open, and may need more work (or not).


Discussion:
> It's possible this invariant was lost with the refactor to IndexedAttestation.

This invariant used to be in `verify_attestation` but was removed because during phase 0, the code-path could never be reached. The phase 0 enforcement that all custody bits were zero prevented the second check from ever being triggered
meaning, it was removed before the translation to IndexedAttestation!
> The phase 0 enforcement that all custody bits were zero prevented the second check from ever being triggered

Oh right. The check that all custody bits were zero prevents anyone from exploiting this in phase 0. Still good to add because both `assert`s can now be triggered.

Resolving patch:
diff --git a/scripts/build_spec.py b/scripts/build_spec.py
--- a/scripts/build_spec.py
+++ b/scripts/build_spec.py
@@ -20,6 +20,11 @@
     Tuple,
 )
 
+from dataclasses import (
+    dataclass,
+    field,
+)
+
 from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
     signing_root,
@@ -48,6 +53,11 @@
     Tuple,
 )
 
+from dataclasses import (
+    dataclass,
+    field,
+)
+
 from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
     signing_root,
@@ -223,9 +233,11 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     return functions, custom_types, constants, ssz_objects, inserts
 
 
-def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]:
-    functions, custom_types, constants, ssz_objects, inserts = get_spec(sourcefile)
-    spec = objects_to_spec(functions, custom_types, constants, ssz_objects, inserts, PHASE0_IMPORTS)
+def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]:
+    phase0_spec = get_spec(phase0_sourcefile)
+    fork_choice_spec = get_spec(fork_choice_sourcefile)
+    spec_objects = combine_spec_objects(phase0_spec, fork_choice_spec)
+    spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:
             out.write(spec)
@@ -235,12 +247,14 @@ def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]:
 def build_phase1_spec(phase0_sourcefile: str,
                       phase1_custody_sourcefile: str,
                       phase1_shard_sourcefile: str,
+                      fork_choice_sourcefile: str,
                       outfile: str=None) -> Optional[str]:
     phase0_spec = get_spec(phase0_sourcefile)
     phase1_custody = get_spec(phase1_custody_sourcefile)
     phase1_shard_data = get_spec(phase1_shard_sourcefile)
+    fork_choice_spec = get_spec(fork_choice_sourcefile)
     spec_objects = phase0_spec
-    for value in [phase1_custody, phase1_shard_data]:
+    for value in [phase1_custody, phase1_shard_data, fork_choice_spec]:
         spec_objects = combine_spec_objects(spec_objects, value)
     spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS)
     if outfile is not None:
@@ -254,13 +268,15 @@ def build_phase1_spec(phase0_sourcefile: str,
 Build the specs from the md docs.
 If building phase 0:
     1st argument is input spec.md
-    2nd argument is output spec.py
+    2nd argument is input fork_choice.md
+    3rd argument is output spec.py
 
 If building phase 1:
     1st argument is input spec_phase0.md
     2nd argument is input spec_phase1_custody.md
     3rd argument is input spec_phase1_shard_data.md
-    4th argument is output spec.py
+    4th argument is input fork_choice.md
+    5th argument is output spec.py
 '''
     parser = ArgumentParser(description=description)
     parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #")
@@ -268,14 +284,14 @@ def build_phase1_spec(phase0_sourcefile: str,
 
     args = parser.parse_args()
     if args.phase == 0:
-        if len(args.files) == 2:
+        if len(args.files) == 3:
             build_phase0_spec(*args.files)
         else:
-            print(" Phase 0 requires an output as well as an input file.")
+            print(" Phase 0 requires an output as well as spec and forkchoice files.")
     elif args.phase == 1:
-        if len(args.files) == 4:
+        if len(args.files) == 5:
             build_phase1_spec(*args.files)
         else:
-            print(" Phase 1 requires an output as well as 3 input files (phase0.md and phase1.md, phase1.md)")
+            print(" Phase 1 requires an output as well as 4 input files (phase0.md and phase1.md, phase1.md, fork_choice.md)")
     else:
         print("Invalid phase: {0}".format(args.phase))
diff --git a/scripts/function_puller.py b/scripts/function_puller.py
--- a/scripts/function_puller.py
+++ b/scripts/function_puller.py
@@ -29,6 +29,7 @@ def get_spec(file_name: str) -> SpecObject:
     inserts = {}
     function_matcher = re.compile(FUNCTION_REGEX)
     inserts_matcher = re.compile(BEGIN_INSERT_REGEX)
+    is_ssz = False
     custom_types = {}
     for linenum, line in enumerate(open(file_name).readlines()):
         line = line.rstrip()


Test patch:
diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py
new file mode 100644
--- /dev/null
+++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py
@@ -0,0 +1,62 @@
+from typing import List
+
+from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root
+
+from eth2spec.test.context import with_all_phases, with_state, bls_switch
+
+from eth2spec.test.helpers.block import build_empty_block_for_next_slot
+from eth2spec.test.helpers.attestations import get_valid_attestation
+from eth2spec.test.helpers.state import next_slot
+
+
+@with_all_phases
+@with_state
+@bls_switch
+def test_basic(spec, state):
+    state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody()))
+
+    # Initialization
+    store = spec.get_genesis_store(state)
+    blocks = []
+    time = 100
+    spec.on_tick(store, time)
+    assert store.time == time
+
+    # On receiving a block of `GENESIS_SLOT + 1` slot
+    block = build_empty_block_for_next_slot(spec, state)
+    blocks.append(block)
+    spec.on_block(store, block)
+    assert store.blocks[signing_root(block)] == block
+
+    # On receiving a block of next epoch
+    store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH
+    block = build_empty_block_for_next_slot(spec, state)
+    block.slot += spec.SLOTS_PER_EPOCH
+    blocks.append(block)
+
+    spec.on_block(store, block)
+    assert store.blocks[signing_root(block)] == block
+
+    # TODO: add tests for justified_root and finalized_root
+
+
+@with_all_phases
+@with_state
+@bls_switch
+def test_on_attestation(spec, state):
+    store = spec.get_genesis_store(state)
+    time = 100
+    spec.on_tick(store, time)
+
+    next_slot(spec, state)
+
+    attestation = get_valid_attestation(spec, state, slot=1)
+    indexed_attestation = spec.convert_to_indexed(state, attestation)
+    spec.on_attestation(store, attestation)
+    assert (
+        store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] ==
+        spec.Target(
+            epoch=attestation.data.target_epoch,
+            root=attestation.data.target_root,
+        )
+    )
diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py
--- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py
+++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py
@@ -1,6 +1,13 @@
+import copy
 from types import GeneratorType
-from typing import List, Iterable, TypeVar, Type, NewType
-from typing import Union
+from typing import (
+    List,
+    Iterable,
+    TypeVar,
+    Type,
+    NewType,
+    Union,
+)
 from typing_inspect import get_origin
 
 # SSZ integers
@@ -145,6 +152,9 @@ def __eq__(self, other):
     def __hash__(self):
         return hash(self.hash_tree_root())
 
+    def copy(self):
+        return copy.deepcopy(self)
+
     @classmethod
     def get_fields_dict(cls):
         return dict(cls.__annotations__)
@@ -343,6 +353,8 @@ def __instancecheck__(self, other):
         return _is_bytes_n_instance_of(self, other.__class__)
 
     def __eq__(self, other):
+        if other == ():
+            return False
         return _is_equal_bytes_n_type(self, other)
 
     def __ne__(self, other):
diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt
--- a/test_libs/pyspec/requirements.txt
+++ b/test_libs/pyspec/requirements.txt
@@ -3,4 +3,5 @@ eth-typing>=2.1.0,<3.0.0
 pycryptodome==3.7.3
 py_ecc>=1.6.0
 typing_inspect==0.4.0
+dataclasses==0.6
 ssz==0.1.0a10
diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py
--- a/test_libs/pyspec/setup.py
+++ b/test_libs/pyspec/setup.py
@@ -10,6 +10,7 @@
         "pycryptodome==3.7.3",
         "py_ecc>=1.6.0",
         "typing_inspect==0.4.0",
-        "ssz==0.1.0a10"
+        "ssz==0.1.0a10",
+        "dataclasses==0.6",
     ]
 )