Repository: ethereum/web3.py

Issue:
bug: Unable to get pending block when connecting to Hardhat node (HTTPProvider)
* Version: 5.26.0
* Python: 3.9.7
* OS: macOS
* `pip freeze` output

```
aiohttp==3.8.1
aiosignal==1.2.0
alabaster==0.7.12
ape-alchemy==0.1.0b2
ape-solidity==0.1.0b2
appnope==0.1.2
argcomplete==1.12.3
asttokens==2.0.5
async-timeout==4.0.2
attrs==21.4.0
Babel==2.9.1
backcall==0.2.0
base58==2.1.1
bitarray==1.2.2
black==21.12b0
bleach==4.1.0
cached-property==1.5.2
certifi==2021.10.8
cffi==1.15.0
cfgv==3.3.1
charset-normalizer==2.0.10
click==8.0.3
colorama==0.4.4
commitizen==2.19.0
commonmark==0.9.1
coverage==6.2
cytoolz==0.11.2
dataclassy==0.10.4
decli==0.5.2
decorator==5.1.1
Deprecated==1.2.13
distlib==0.3.4
docopt==0.6.2
docutils==0.16
e==1.4.5
eth-abi==2.1.1
eth-account==0.5.6
eth-bloom==1.0.4
eth-hash==0.3.2
eth-keyfile==0.5.1
eth-keys==0.3.3
eth-rlp==0.2.1
eth-tester==0.6.0b4
eth-typing==2.2.2
eth-utils==1.10.0
ethpm-types==0.1.0b2
execnet==1.9.0
executing==0.8.2
filelock==3.4.2
flake8==3.9.2
flake8-breakpoint==1.1.0
flake8-plugin-utils==1.3.2
flake8-print==4.0.0
Flask==2.0.2
frozenlist==1.3.0
hexbytes==0.2.2
htmlmin==0.1.12
hypothesis==6.36.0
hypothesis-jsonschema==0.19.0
identify==2.4.4
idna==3.3
imagesize==1.3.0
importlib-metadata==4.10.1
iniconfig==1.1.1
ipdb==0.13.9
ipfshttpclient==0.8.0a2
ipython==8.0.1
isort==5.10.1
itsdangerous==2.0.1
jedi==0.18.1
Jinja2==3.0.3
jsonschema==3.2.0
keyring==21.8.0
lru-dict==1.1.7
markdown-it-py==2.0.0
MarkupSafe==2.0.1
matplotlib-inline==0.1.3
mccabe==0.6.1
mdit-py-plugins==0.3.0
mdurl==0.1.0
multiaddr==0.0.9
multidict==5.2.0
mypy==0.931
mypy-extensions==0.4.3
myst-parser==0.16.0
netaddr==0.8.0
nodeenv==1.6.0
packaging==20.9
parsimonious==0.8.1
parso==0.8.3
pathspec==0.9.0
pexpect==4.8.0
pickleshare==0.7.5
pkginfo==1.8.2
platformdirs==2.4.1
pluggy==0.13.1
pockets==0.9.1
pre-commit==2.17.0
prompt-toolkit==3.0.24
protobuf==3.19.3
ptyprocess==0.7.0
pure-eval==0.2.1
py==1.11.0
py-ecc==4.1.0
py-evm==0.5.0a1
py-geth==3.7.0
py-solc-x==1.1.1
pycodestyle==2.7.0
pycparser==2.21
pycryptodome==3.12.0
pydantic==1.9.0
pyethash==0.1.27
pyflakes==2.3.1
PyGithub==1.55
Pygments==2.11.2
PyJWT==2.3.0
PyNaCl==1.5.0
pyparsing==3.0.6
pyrsistent==0.18.1
pysha3==1.0.2
pytest==6.2.5
pytest-cov==3.0.0
pytest-forked==1.4.0
pytest-mock==3.6.1
pytest-watch==4.2.0
pytest-xdist==2.5.0
pytz==2021.3
PyYAML==5.4.1
questionary==1.10.0
readme-renderer==32.0
requests==2.27.1
requests-toolbelt==0.9.1
rfc3986==2.0.0
rich==10.16.2
rlp==2.0.1
semantic-version==2.8.5
setuptools-scm==6.4.2
six==1.16.0
snowballstemmer==2.2.0
sortedcontainers==2.4.0
Sphinx==3.5.4
sphinx-click==3.0.2
sphinx-rtd-theme==0.5.2
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-napoleon==0.7
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
stack-data==0.1.4
termcolor==1.1.0
tokenlists==0.1.1
toml==0.10.2
tomli==1.2.3
tomlkit==0.8.0
toolz==0.11.2
tqdm==4.62.3
traitlets==5.1.1
trie==2.0.0a5
twine==3.7.1
types-PyYAML==6.0.3
types-requests==2.27.7
types-urllib3==1.26.7
typing-extensions==3.10.0.2
urllib3==1.26.8
varint==1.0.2
virtualenv==20.13.0
watchdog==2.1.6
wcwidth==0.2.5
web3==5.26.0
webencodings==0.5.1
websockets==9.1
Werkzeug==2.0.2
wrapt==1.13.3
yarl==1.7.2
zipp==3.7.0
```


### What was wrong?

I am unable to get the pending block when connected to a hardhat node.  Here are reproduction steps:

1. Start hardhat node in a project via `npx hardhat node` (It says starting up at `http://127.0.0.1:8545`).
2. Execute the follow python code

```python
from web3 import Web3, HTTPProvider

web3 = Web3(HTTPProvider("http://127.0.0.1:8545/"))
pending_block = web3.eth.get_block("pending")              # <--  Fails here
print(pending_block)
```
3. Notice that it fails with the following error data:

```
...
 33, in middleware
    response = make_request(method, params)
  File "cytoolz/functoolz.pyx", line 254, in cytoolz.functoolz.curry.__call__
  File "cytoolz/functoolz.pyx", line 250, in cytoolz.functoolz.curry.__call__
  File "/Users/jules/virtualenvs/three-ten/lib/python3.10/site-packages/web3/middleware/formatting.py", line 83, in apply_formatters
    formatter(response["result"]),
  File "cytoolz/functoolz.pyx", line 254, in cytoolz.functoolz.curry.__call__
  File "cytoolz/functoolz.pyx", line 250, in cytoolz.functoolz.curry.__call__
  File "/Users/jules/virtualenvs/three-ten/lib/python3.10/site-packages/eth_utils/applicators.py", line 72, in apply_formatter_if
    return formatter(value)
  File "cytoolz/functoolz.pyx", line 254, in cytoolz.functoolz.curry.__call__
  File "cytoolz/functoolz.pyx", line 250, in cytoolz.functoolz.curry.__call__
  File "/Users/jules/virtualenvs/three-ten/lib/python3.10/site-packages/eth_utils/functional.py", line 45, in inner
    return callback(fn(*args, **kwargs))
  File "/Users/jules/virtualenvs/three-ten/lib/python3.10/site-packages/eth_utils/applicators.py", line 95, in apply_formatters_to_dict
    raise TypeError(new_error_message) from exc
TypeError: Could not format invalid type of None for field 'mixHash'
```

When analyzing this more in Postman, I notice that the `mixHash` of the pending block from a hardhat node is `null` whereas the `mixHash` of a pending block from a `geth --dev` node 0x0000000000000000000000000000000000000000000000000000000000000000, which does not cause this bug.

### How can it be fixed?

Adjust middleware to allow `null` mixHash? I don't know.


Discussion:
Thanks @unparalleled-js... getting a fix in asap!

Resolving patch:
diff --git a/web3/_utils/method_formatters.py b/web3/_utils/method_formatters.py
--- a/web3/_utils/method_formatters.py
+++ b/web3/_utils/method_formatters.py
@@ -210,7 +210,7 @@ def apply_list_to_array_formatter(formatter: Any) -> Callable[..., Any]:
 
 BLOCK_FORMATTERS = {
     'baseFeePerGas': to_integer_if_hex,
-    'extraData': to_hexbytes(32, variable_length=True),
+    'extraData': apply_formatter_if(is_not_null, to_hexbytes(32, variable_length=True)),
     'gasLimit': to_integer_if_hex,
     'gasUsed': to_integer_if_hex,
     'size': to_integer_if_hex,
@@ -218,21 +218,21 @@ def apply_list_to_array_formatter(formatter: Any) -> Callable[..., Any]:
     'hash': apply_formatter_if(is_not_null, to_hexbytes(32)),
     'logsBloom': apply_formatter_if(is_not_null, to_hexbytes(256)),
     'miner': apply_formatter_if(is_not_null, to_checksum_address),
-    'mixHash': to_hexbytes(32),
+    'mixHash': apply_formatter_if(is_not_null, to_hexbytes(32)),
     'nonce': apply_formatter_if(is_not_null, to_hexbytes(8, variable_length=True)),
     'number': apply_formatter_if(is_not_null, to_integer_if_hex),
     'parentHash': apply_formatter_if(is_not_null, to_hexbytes(32)),
     'sha3Uncles': apply_formatter_if(is_not_null, to_hexbytes(32)),
     'uncles': apply_list_to_array_formatter(to_hexbytes(32)),
     'difficulty': to_integer_if_hex,
-    'receiptsRoot': to_hexbytes(32),
-    'stateRoot': to_hexbytes(32),
+    'receiptsRoot': apply_formatter_if(is_not_null, to_hexbytes(32)),
+    'stateRoot': apply_formatter_if(is_not_null, to_hexbytes(32)),
     'totalDifficulty': to_integer_if_hex,
     'transactions': apply_one_of_formatters((
         (is_array_of_dicts, apply_list_to_array_formatter(transaction_result_formatter)),
         (is_array_of_strings, apply_list_to_array_formatter(to_hexbytes(32))),
     )),
-    'transactionsRoot': to_hexbytes(32),
+    'transactionsRoot': apply_formatter_if(is_not_null, to_hexbytes(32)),
 }
 
 


Test patch:
diff --git a/tests/core/eth-module/test_block_api.py b/tests/core/eth-module/test_block_api.py
new file mode 100644
--- /dev/null
+++ b/tests/core/eth-module/test_block_api.py
@@ -0,0 +1,64 @@
+import pytest
+
+from web3._utils.rpc_abi import (
+    RPC,
+)
+from web3.middleware import (
+    construct_result_generator_middleware,
+)
+
+
+@pytest.fixture(autouse=True)
+def wait_for_first_block(web3, wait_for_block):
+    wait_for_block(web3)
+
+
+def test_uses_default_block(web3, extra_accounts,
+                            wait_for_transaction):
+    assert(web3.eth.default_block == 'latest')
+    web3.eth.default_block = web3.eth.block_number
+    assert(web3.eth.default_block == web3.eth.block_number)
+
+
+def test_uses_defaultBlock_with_warning(web3, extra_accounts,
+                                        wait_for_transaction):
+    with pytest.warns(DeprecationWarning):
+        assert web3.eth.defaultBlock == 'latest'
+
+    with pytest.warns(DeprecationWarning):
+        web3.eth.defaultBlock = web3.eth.block_number
+
+    with pytest.warns(DeprecationWarning):
+        assert(web3.eth.defaultBlock == web3.eth.block_number)
+
+
+def test_get_block_formatters_with_null_values(web3):
+    null_values_block = {
+        'baseFeePerGas': None,
+        'extraData': None,
+        'gasLimit': None,
+        'gasUsed': None,
+        'size': None,
+        'timestamp': None,
+        'hash': None,
+        'logsBloom': None,
+        'miner': None,
+        'mixHash': None,
+        'nonce': None,
+        'number': None,
+        'parentHash': None,
+        'sha3Uncles': None,
+        'difficulty': None,
+        'receiptsRoot': None,
+        'statesRoot': None,
+        'totalDifficulty': None,
+        'transactionsRoot': None,
+    }
+    result_middleware = construct_result_generator_middleware({
+        RPC.eth_getBlockByNumber: lambda *_: null_values_block,
+    })
+
+    web3.middleware_onion.inject(result_middleware, 'result_middleware', layer=0)
+
+    received_block = web3.eth.get_block('pending')
+    assert received_block == null_values_block
diff --git a/tests/core/eth-module/test_default_block_api.py b/tests/core/eth-module/test_default_block_api.py
deleted file mode 100644
--- a/tests/core/eth-module/test_default_block_api.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import pytest
-
-
-@pytest.fixture(autouse=True)
-def wait_for_first_block(web3, wait_for_block):
-    wait_for_block(web3)
-
-
-def test_uses_default_block(web3, extra_accounts,
-                            wait_for_transaction):
-    assert(web3.eth.default_block == 'latest')
-    web3.eth.default_block = web3.eth.block_number
-    assert(web3.eth.default_block == web3.eth.block_number)
-
-
-def test_uses_defaultBlock_with_warning(web3, extra_accounts,
-                                        wait_for_transaction):
-    with pytest.warns(DeprecationWarning):
-        assert web3.eth.defaultBlock == 'latest'
-
-    with pytest.warns(DeprecationWarning):
-        web3.eth.defaultBlock = web3.eth.block_number
-
-    with pytest.warns(DeprecationWarning):
-        assert(web3.eth.defaultBlock == web3.eth.block_number)