Repository: ethereum/consensus-specs

Issue:
is_genesis_trigger fails to verify deposit merkle branch
## Issue
Deposits are processed against empty state with undefined `eth1_data` while `genesis_eth1_data` should be in place.



Discussion:

Resolving patch:
diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py
--- a/deposit_contract/contracts/validator_registration.v.py
+++ b/deposit_contract/contracts/validator_registration.v.py
@@ -6,7 +6,7 @@
 AMOUNT_LENGTH: constant(uint256) = 8  # bytes
 SIGNATURE_LENGTH: constant(uint256) = 96  # bytes
 
-Deposit: event({
+DepositEvent: event({
     pubkey: bytes[48],
     withdrawal_credentials: bytes[32],
     amount: bytes[8],
@@ -42,8 +42,9 @@ def to_little_endian_64(value: uint256) -> bytes[8]:
 
 @public
 @constant
-def get_deposit_root() -> bytes32:
-    node: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000
+def get_hash_tree_root() -> bytes32:
+    zero_bytes32: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000
+    node: bytes32 = zero_bytes32
     size: uint256 = self.deposit_count
     for height in range(DEPOSIT_CONTRACT_TREE_DEPTH):
         if bitwise_and(size, 1) == 1:  # More gas efficient than `size % 2 == 1`
@@ -51,7 +52,7 @@ def get_deposit_root() -> bytes32:
         else:
             node = sha256(concat(node, self.zero_hashes[height]))
         size /= 2
-    return node
+    return sha256(concat(node, self.to_little_endian_64(self.deposit_count), slice(zero_bytes32, start=0, len=24)))
 
 
 @public
@@ -75,11 +76,11 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH],
     assert len(withdrawal_credentials) == WITHDRAWAL_CREDENTIALS_LENGTH
     assert len(signature) == SIGNATURE_LENGTH
 
-    # Emit `Deposit` log
+    # Emit `DepositEvent` log
     amount: bytes[8] = self.to_little_endian_64(deposit_amount)
-    log.Deposit(pubkey, withdrawal_credentials, amount, signature, self.to_little_endian_64(self.deposit_count))
+    log.DepositEvent(pubkey, withdrawal_credentials, amount, signature, self.to_little_endian_64(self.deposit_count))
 
-    # Compute `DepositData` root
+    # Compute `DepositData` hash tree root
     zero_bytes32: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000
     pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes32, start=0, len=64 - PUBKEY_LENGTH)))
     signature_root: bytes32 = sha256(concat(
@@ -91,7 +92,7 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH],
         sha256(concat(amount, slice(zero_bytes32, start=0, len=32 - AMOUNT_LENGTH), signature_root)),
     ))
 
-    # Add `DepositData` root to Merkle tree (update a single `branch` node)
+    # Add `DepositData` hash tree root to Merkle tree (update a single `branch` node)
     self.deposit_count += 1
     size: uint256 = self.deposit_count
     for height in range(DEPOSIT_CONTRACT_TREE_DEPTH):


Test patch:
diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py
--- a/deposit_contract/tests/contracts/test_deposit.py
+++ b/deposit_contract/tests/contracts/test_deposit.py
@@ -15,26 +15,12 @@
     DepositData,
 )
 from eth2spec.utils.hash_function import hash
+from eth2spec.utils.ssz.ssz_typing import List
 from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
 )
 
 
-def compute_merkle_root(leaf_nodes):
-    assert len(leaf_nodes) >= 1
-    empty_node = b'\x00' * 32
-    child_nodes = leaf_nodes[:]
-    for _ in range(DEPOSIT_CONTRACT_TREE_DEPTH):
-        parent_nodes = []
-        if len(child_nodes) % 2 == 1:
-            child_nodes.append(empty_node)
-        for j in range(0, len(child_nodes), 2):
-            parent_nodes.append(hash(child_nodes[j] + child_nodes[j + 1]))
-        child_nodes = parent_nodes
-        empty_node = hash(empty_node + empty_node)
-    return child_nodes[0]
-
-
 @pytest.fixture
 def deposit_input():
     """
@@ -110,8 +96,8 @@ def test_deposit_inputs(registration_contract,
         )
 
 
-def test_deposit_log(registration_contract, a0, w3, deposit_input):
-    log_filter = registration_contract.events.Deposit.createFilter(
+def test_deposit_event_log(registration_contract, a0, w3, deposit_input):
+    log_filter = registration_contract.events.DepositEvent.createFilter(
         fromBlock='latest',
     )
 
@@ -131,13 +117,14 @@ def test_deposit_log(registration_contract, a0, w3, deposit_input):
         assert log['signature'] == deposit_input[2]
         assert log['index'] == i.to_bytes(8, 'little')
 
+
 def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input):
-    log_filter = registration_contract.events.Deposit.createFilter(
+    log_filter = registration_contract.events.DepositEvent.createFilter(
         fromBlock='latest',
     )
 
     deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(10)]
-    leaf_nodes = []
+    deposit_data_list = []
     for i in range(0, 10):
         tx_hash = registration_contract.functions.deposit(
             *deposit_input,
@@ -151,13 +138,12 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input
 
         assert log["index"] == i.to_bytes(8, 'little')
 
-        deposit_data = DepositData(
+        deposit_data_list.append(DepositData(
             pubkey=deposit_input[0],
             withdrawal_credentials=deposit_input[1],
             amount=deposit_amount_list[i],
             signature=deposit_input[2],
-        )
-        hash_tree_root_result = hash_tree_root(deposit_data)
-        leaf_nodes.append(hash_tree_root_result)
-        root = compute_merkle_root(leaf_nodes)
-        assert root == registration_contract.functions.get_deposit_root().call()
+        ))
+
+        root = hash_tree_root(List[DepositData, 2**32](*deposit_data_list))
+        assert root == registration_contract.functions.get_hash_tree_root().call()
diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py
--- a/test_libs/pyspec/eth2spec/test/context.py
+++ b/test_libs/pyspec/eth2spec/test/context.py
@@ -27,9 +27,13 @@ def entry(*args, **kw):
 DEFAULT_BLS_ACTIVE = False
 
 
+def spectest_with_bls_switch(fn):
+    return bls_switch(spectest()(fn))
+
+
 # shorthand for decorating @with_state @spectest()
 def spec_state_test(fn):
-    return with_state(bls_switch(spectest()(fn)))
+    return with_state(spectest_with_bls_switch(fn))
 
 
 def expect_assertion_error(fn):
diff --git a/test_libs/pyspec/eth2spec/test/genesis/__init__.py b/test_libs/pyspec/eth2spec/test/genesis/__init__.py
new file mode 100644
diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py b/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py
new file mode 100644
--- /dev/null
+++ b/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py
@@ -0,0 +1,30 @@
+from eth2spec.test.context import spectest_with_bls_switch, with_phases
+from eth2spec.test.helpers.deposits import (
+    prepare_genesis_deposits,
+)
+
+
+@with_phases(['phase0'])
+@spectest_with_bls_switch
+def test_initialize_beacon_state_from_eth1(spec):
+    deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
+    deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
+
+    eth1_block_hash = b'\x12' * 32
+    eth1_timestamp = spec.MIN_GENESIS_TIME
+
+    yield 'eth1_block_hash', eth1_block_hash
+    yield 'eth1_timestamp', eth1_timestamp
+    yield 'deposits', deposits
+
+    # initialize beacon_state
+    state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)
+
+    assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.SECONDS_PER_DAY + 2 * spec.SECONDS_PER_DAY
+    assert len(state.validators) == deposit_count
+    assert state.eth1_data.deposit_root == deposit_root
+    assert state.eth1_data.deposit_count == deposit_count
+    assert state.eth1_data.block_hash == eth1_block_hash
+
+    # yield state
+    yield 'state', state
diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py
new file mode 100644
--- /dev/null
+++ b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py
@@ -0,0 +1,86 @@
+from eth2spec.test.context import spectest_with_bls_switch, with_phases
+from eth2spec.test.helpers.deposits import (
+    prepare_genesis_deposits,
+)
+
+
+def create_valid_beacon_state(spec):
+    deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
+    deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
+
+    eth1_block_hash = b'\x12' * 32
+    eth1_timestamp = spec.MIN_GENESIS_TIME
+    return spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)
+
+
+def run_is_valid_genesis_state(spec, state, valid=True):
+    """
+    Run ``is_valid_genesis_state``, yielding:
+      - state ('state')
+      - is_valid ('is_valid')
+    If ``valid == False``, run expecting ``AssertionError``
+    """
+    yield state
+    is_valid = spec.is_valid_genesis_state(state)
+    yield 'is_valid', is_valid
+
+
+@with_phases(['phase0'])
+@spectest_with_bls_switch
+def test_is_valid_genesis_state_true(spec):
+    state = create_valid_beacon_state(spec)
+
+    yield from run_is_valid_genesis_state(spec, state, valid=True)
+
+
+@with_phases(['phase0'])
+@spectest_with_bls_switch
+def test_is_valid_genesis_state_false_invalid_timestamp(spec):
+    state = create_valid_beacon_state(spec)
+    state.genesis_time = spec.MIN_GENESIS_TIME - 1
+
+    yield from run_is_valid_genesis_state(spec, state, valid=True)
+
+
+@with_phases(['phase0'])
+@spectest_with_bls_switch
+def test_is_valid_genesis_state_true_more_balance(spec):
+    state = create_valid_beacon_state(spec)
+    state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE + 1
+
+    yield from run_is_valid_genesis_state(spec, state, valid=True)
+
+
+@with_phases(['phase0'])
+@spectest_with_bls_switch
+def test_is_valid_genesis_state_false_not_enough_balance(spec):
+    state = create_valid_beacon_state(spec)
+    state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE - 1
+
+    yield from run_is_valid_genesis_state(spec, state, valid=False)
+
+
+@with_phases(['phase0'])
+@spectest_with_bls_switch
+def test_is_valid_genesis_state_true_one_more_validator(spec):
+    deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + 1
+    deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
+
+    eth1_block_hash = b'\x12' * 32
+    eth1_timestamp = spec.MIN_GENESIS_TIME
+    state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)
+
+    yield from run_is_valid_genesis_state(spec, state, valid=True)
+
+
+@with_phases(['phase0'])
+@spectest_with_bls_switch
+def test_is_valid_genesis_state_false_not_enough_validator(spec):
+    deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1
+    deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
+
+    eth1_block_hash = b'\x12' * 32
+    eth1_timestamp = spec.MIN_GENESIS_TIME
+    state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)
+
+    yield from run_is_valid_genesis_state(spec, state, valid=False)
diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py
--- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py
+++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py
@@ -1,66 +1,88 @@
 from eth2spec.test.helpers.keys import pubkeys, privkeys
 from eth2spec.utils.bls import bls_sign
-from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_root, get_merkle_proof
-from eth2spec.utils.ssz.ssz_impl import signing_root
+from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_proof
+from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root
+from eth2spec.utils.ssz.ssz_typing import List
 
 
-def build_deposit_data(spec, state, pubkey, privkey, amount, withdrawal_credentials, signed=False):
+def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=None, signed=False):
     deposit_data = spec.DepositData(
         pubkey=pubkey,
         withdrawal_credentials=withdrawal_credentials,
         amount=amount,
     )
     if signed:
-        sign_deposit_data(spec, state, deposit_data, privkey)
+        sign_deposit_data(spec, deposit_data, privkey, state)
     return deposit_data
 
 
-def sign_deposit_data(spec, state, deposit_data, privkey):
-    signature = bls_sign(
-        message_hash=signing_root(deposit_data),
-        privkey=privkey,
-        domain=spec.get_domain(
+def sign_deposit_data(spec, deposit_data, privkey, state=None):
+    if state is None:
+        # Genesis
+        domain = spec.bls_domain(spec.DOMAIN_DEPOSIT)
+    else:
+        domain = spec.get_domain(
             state,
             spec.DOMAIN_DEPOSIT,
         )
+
+    signature = bls_sign(
+        message_hash=signing_root(deposit_data),
+        privkey=privkey,
+        domain=domain,
     )
     deposit_data.signature = signature
 
 
 def build_deposit(spec,
                   state,
-                  deposit_data_leaves,
+                  deposit_data_list,
                   pubkey,
                   privkey,
                   amount,
                   withdrawal_credentials,
                   signed):
-    deposit_data = build_deposit_data(spec, state, pubkey, privkey, amount, withdrawal_credentials, signed)
-
-    item = deposit_data.hash_tree_root()
-    index = len(deposit_data_leaves)
-    deposit_data_leaves.append(item)
-    tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves))
-    root = get_merkle_root((tuple(deposit_data_leaves)))
-    proof = list(get_merkle_proof(tree, item_index=index))
-    assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root)
-
-    deposit = spec.Deposit(
-        proof=list(proof),
-        index=index,
-        data=deposit_data,
-    )
+    deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed)
+    index = len(deposit_data_list)
+    deposit_data_list.append(deposit_data)
+    root = hash_tree_root(List[spec.DepositData, 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH](*deposit_data_list))
+    tree = calc_merkle_tree_from_leaves(tuple([d.hash_tree_root() for d in deposit_data_list]))
+    proof = list(get_merkle_proof(tree, item_index=index)) + [(index + 1).to_bytes(32, 'little')]
+    leaf = deposit_data.hash_tree_root()
+    assert spec.verify_merkle_branch(leaf, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH + 1, index, root)
+    deposit = spec.Deposit(proof=proof, data=deposit_data)
+
+    return deposit, root, deposit_data_list
+
+
+def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False):
+    deposit_data_list = []
+    genesis_deposits = []
+    for validator_index in range(genesis_validator_count):
+        pubkey = pubkeys[validator_index]
+        privkey = privkeys[validator_index]
+        # insecurely use pubkey as withdrawal key if no credentials provided
+        withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:]
+        deposit, root, deposit_data_list = build_deposit(
+            spec,
+            None,
+            deposit_data_list,
+            pubkey,
+            privkey,
+            amount,
+            withdrawal_credentials,
+            signed,
+        )
+        genesis_deposits.append(deposit)
 
-    return deposit, root, deposit_data_leaves
+    return genesis_deposits, root
 
 
 def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False):
     """
     Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount.
     """
-    pre_validator_count = len(state.validators)
-    # fill previous deposits with zero-hash
-    deposit_data_leaves = [spec.ZERO_HASH] * pre_validator_count
+    deposit_data_list = []
 
     pubkey = pubkeys[validator_index]
     privkey = privkeys[validator_index]
@@ -69,10 +91,10 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c
     if withdrawal_credentials is None:
         withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:]
 
-    deposit, root, deposit_data_leaves = build_deposit(
+    deposit, root, deposit_data_list = build_deposit(
         spec,
         state,
-        deposit_data_leaves,
+        deposit_data_list,
         pubkey,
         privkey,
         amount,
@@ -80,6 +102,7 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c
         signed,
     )
 
+    state.eth1_deposit_index = 0
     state.eth1_data.deposit_root = root
-    state.eth1_data.deposit_count = len(deposit_data_leaves)
+    state.eth1_data.deposit_count = len(deposit_data_list)
     return deposit
diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py
--- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py
+++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py
@@ -117,7 +117,7 @@ def test_invalid_withdrawal_credentials_top_up(spec, state):
 @with_all_phases
 @spec_state_test
 def test_wrong_deposit_for_deposit_count(spec, state):
-    deposit_data_leaves = [spec.ZERO_HASH] * len(state.validators)
+    deposit_data_leaves = [spec.DepositData() for _ in range(len(state.validators))]
 
     # build root for deposit_1
     index_1 = len(deposit_data_leaves)
@@ -167,6 +167,6 @@ def test_bad_merkle_proof(spec, state):
     # mess up merkle branch
     deposit.proof[5] = spec.ZERO_HASH
 
-    sign_deposit_data(spec, state, deposit.data, privkeys[validator_index])
+    sign_deposit_data(spec, deposit.data, privkeys[validator_index], state=state)
 
     yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False)
diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py
--- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py
+++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py
@@ -1,4 +1,5 @@
 from .hash_function import hash
+from math import log2
 
 
 ZERO_BYTES32 = b'\x00' * 32
@@ -8,11 +9,10 @@
     zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1]))
 
 
-# Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree
-def calc_merkle_tree_from_leaves(values):
+def calc_merkle_tree_from_leaves(values, layer_count=32):
     values = list(values)
     tree = [values[::]]
-    for h in range(32):
+    for h in range(layer_count):
         if len(values) % 2 == 1:
             values.append(zerohashes[h])
         values = [hash(values[i] + values[i + 1]) for i in range(0, len(values), 2)]
@@ -20,8 +20,11 @@ def calc_merkle_tree_from_leaves(values):
     return tree
 
 
-def get_merkle_root(values):
-    return calc_merkle_tree_from_leaves(values)[-1][0]
+def get_merkle_root(values, pad_to=1):
+    layer_count = int(log2(pad_to))
+    if len(values) == 0:
+        return zerohashes[layer_count]
+    return calc_merkle_tree_from_leaves(values, layer_count)[-1][0]
 
 
 def get_merkle_proof(tree, item_index):
@@ -32,19 +35,7 @@ def get_merkle_proof(tree, item_index):
     return proof
 
 
-def next_power_of_two(v: int) -> int:
-    """
-    Get the next power of 2. (for 64 bit range ints).
-    0 is a special case, to have non-empty defaults.
-    Examples:
-    0 -> 1, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64
-    """
-    if v == 0:
-        return 1
-    return 1 << (v - 1).bit_length()
-
-
-def merkleize_chunks(chunks, pad_to: int = 1):
+def merkleize_chunks(chunks, pad_to: int=1):
     count = len(chunks)
     depth = max(count - 1, 0).bit_length()
     max_depth = max(depth, (pad_to - 1).bit_length())
diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py
--- a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py
+++ b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py
@@ -1,5 +1,5 @@
 import pytest
-from .merkle_minimal import zerohashes, merkleize_chunks
+from .merkle_minimal import zerohashes, merkleize_chunks, get_merkle_root
 from .hash_function import hash
 
 
@@ -53,6 +53,7 @@ def z(i: int) -> bytes:
     'depth,count,pow2,value',
     cases,
 )
-def test_merkleize_chunks(depth, count, pow2, value):
+def test_merkleize_chunks_and_get_merkle_root(depth, count, pow2, value):
     chunks = [e(i) for i in range(count)]
     assert merkleize_chunks(chunks, pad_to=pow2) == value
+    assert get_merkle_root(chunks, pad_to=pow2) == value