Repository: rotki/rotki

Issue:
Going to the airdrops for some users fails with not enough values to unpack
## Problem Definition

After v1.22.0 some users reported that going to the airdrops page fails with `The backend query task died unexpectedly: not enough values to unpack (expected at least 2, got 0)`

## Logs

Relevant part from the logs of a user

```
[14/11/2021 03:44:18 Mountain Standard Time] ERROR rotkehlchen.api.rest Greenlet with id 2242469704808: Greenlet for task 15 dies with exception: not enough values to unpack (expected at least 2, got 0).
Exception Name: <class 'ValueError'>
Exception Info: not enough values to unpack (expected at least 2, got 0)
Traceback:
   File "src\\gevent\\greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
  File "rotkehlchen\api\rest.py", line 281, in _do_query_async
  File "rotkehlchen\api\rest.py", line 2232, in _get_ethereum_airdrops
  File "rotkehlchen\chain\ethereum\airdrops.py", line 250, in check_airdrops

[14/11/2021 03:45:02 Mountain Standard Time] ERROR rotkehlchen.api.rest Greenlet with id 2242472750952: Greenlet for task 16 dies with exception: not enough values to unpack (expected at least 2, got 0).
Exception Name: <class 'ValueError'>
Exception Info: not enough values to unpack (expected at least 2, got 0)
Traceback:
   File "src\\gevent\\greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
  File "rotkehlchen\api\rest.py", line 281, in _do_query_async
  File "rotkehlchen\api\rest.py", line 2232, in _get_ethereum_airdrops
  File "rotkehlchen\chain\ethereum\airdrops.py", line 250, in check_airdrops
```

### System Description


Operating system: Mac
Rotki version: v1.22.0



Discussion:

Resolving patch:
diff --git a/rotkehlchen/api/rest.py b/rotkehlchen/api/rest.py
--- a/rotkehlchen/api/rest.py
+++ b/rotkehlchen/api/rest.py
@@ -72,6 +72,7 @@
     RotkehlchenPermissionError,
     SystemPermissionError,
     TagConstraintError,
+    UnableToDecryptRemoteData,
     UnknownAsset,
     UnsupportedAsset,
 )
@@ -2231,8 +2232,10 @@ def _get_ethereum_airdrops(self) -> Dict[str, Any]:
                 addresses=self.rotkehlchen.chain_manager.accounts.eth,
                 data_dir=self.rotkehlchen.data_dir,
             )
-        except RemoteError as e:
+        except (RemoteError, UnableToDecryptRemoteData) as e:
             return wrap_in_fail_result(str(e), status_code=HTTPStatus.BAD_GATEWAY)
+        except OSError as e:
+            return wrap_in_fail_result(str(e), status_code=HTTPStatus.INSUFFICIENT_STORAGE)
 
         return _wrap_in_ok_result(process_result(data))
 
diff --git a/rotkehlchen/chain/ethereum/airdrops.py b/rotkehlchen/chain/ethereum/airdrops.py
--- a/rotkehlchen/chain/ethereum/airdrops.py
+++ b/rotkehlchen/chain/ethereum/airdrops.py
@@ -1,6 +1,7 @@
 import csv
 from collections import defaultdict
 from json.decoder import JSONDecodeError
+import logging
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, TextIO, Tuple
 
@@ -21,10 +22,15 @@
     A_UNI,
 )
 from rotkehlchen.constants.timing import DEFAULT_TIMEOUT_TUPLE
-from rotkehlchen.errors import RemoteError
+from rotkehlchen.errors import RemoteError, UnableToDecryptRemoteData
+from rotkehlchen.logging import RotkehlchenLogsAdapter
 from rotkehlchen.typing import ChecksumEthAddress
 from rotkehlchen.utils.serialization import jsonloads_dict, rlk_jsondumps
 
+logger = logging.getLogger(__name__)
+log = RotkehlchenLogsAdapter(logger)
+
+SMALLEST_AIRDROP_SIZE = 20900
 AIRDROPS = {
     'uniswap': (
         # is checksummed
@@ -40,7 +46,7 @@
     ),
     'tornado': (
         # is checksummed
-        'https://raw.githubusercontent.com/tornadocash/airdrop/master/airdrop.csv',
+        'https://raw.githubusercontent.com/rotki/data/main/airdrops/tornado.csv',
         A_TORN,
         'https://tornado.cash/',
     ),
@@ -198,13 +204,24 @@ def get_airdrop_data(name: str, data_dir: Path) -> Tuple[Iterator, TextIO]:
     if not filename.is_file():
         # if not cached, get it from the gist
         try:
-            request = requests.get(url=AIRDROPS[name][0], timeout=DEFAULT_TIMEOUT_TUPLE)
+            response = requests.get(url=AIRDROPS[name][0], timeout=DEFAULT_TIMEOUT_TUPLE)
         except requests.exceptions.RequestException as e:
             raise RemoteError(f'Airdrops Gist request failed due to {str(e)}') from e
-
-        with open(filename, 'w') as f:
-            f.write(request.content.decode('utf-8'))
-
+        try:
+            content = response.text
+            if (
+                not csv.Sniffer().has_header(content) or
+                len(response.content) < SMALLEST_AIRDROP_SIZE
+            ):
+                raise csv.Error
+            with open(filename, 'w') as f:
+                f.write(content)
+        except csv.Error as e:
+            log.debug(f'airdrop file {filename} contains invalid data {content}')
+            raise UnableToDecryptRemoteData(
+                f'File {filename} contains invalid data. Check logs.',
+            ) from e
+    # Verify the CSV file
     csvfile = open(filename, 'r')
     iterator = csv.reader(csvfile)
     next(iterator)  # skip header
@@ -247,7 +264,12 @@ def check_airdrops(
     found_data: Dict[ChecksumEthAddress, Dict] = defaultdict(lambda: defaultdict(dict))
     for protocol_name, airdrop_data in AIRDROPS.items():
         data, csvfile = get_airdrop_data(protocol_name, data_dir)
-        for addr, amount, *_ in data:
+        for row in data:
+            if len(row) < 2:
+                raise UnableToDecryptRemoteData(
+                    f'Airdrop CSV for {protocol_name} contains an invalid row: {row}',
+                )
+            addr, amount, *_ = row
             # not doing to_checksum_address() here since the file addresses are checksummed
             # and doing to_checksum_address() so many times hits performance
             if protocol_name in ('cornichon', 'tornado', 'grain', 'lido'):


Test patch:
diff --git a/rotkehlchen/tests/unit/test_ethereum_airdrops.py b/rotkehlchen/tests/unit/test_ethereum_airdrops.py
--- a/rotkehlchen/tests/unit/test_ethereum_airdrops.py
+++ b/rotkehlchen/tests/unit/test_ethereum_airdrops.py
@@ -1,9 +1,29 @@
+from unittest.mock import patch
+
 import pytest
 from rotkehlchen.chain.ethereum.airdrops import check_airdrops, AIRDROPS
 from rotkehlchen.constants.assets import A_UNI, A_1INCH, A_GRAIN
+from rotkehlchen.errors import UnableToDecryptRemoteData
 
 TEST_ADDR1 = '0x2B888954421b424C5D3D9Ce9bB67c9bD47537d12'
 TEST_ADDR2 = '0x51985CE8BB9AB1708746b24e22e37CD7A980Ec24'
+NOT_CSV_WEBPAGE = {
+    'test': (
+        'https://github.com/rotki/yabirgb',
+        A_UNI,
+        'https://github.com',
+    ),
+}
+
+
+@pytest.fixture(name='mock_airdrop_list')
+def fixture_airdrop_list(airdrop_list):
+    print(airdrop_list)
+    with patch(
+        'rotkehlchen.chain.ethereum.airdrops.AIRDROPS',
+        new=airdrop_list,
+    ):
+        yield
 
 
 @pytest.mark.parametrize('number_of_eth_accounts', [2])
@@ -43,3 +63,12 @@ def test_check_airdrops(ethereum_accounts, data_dir):
     # Test cache files are created
     for protocol_name in AIRDROPS:
         assert (data_dir / 'airdrops' / f'{protocol_name}.csv').is_file()
+
+
+@pytest.mark.parametrize('airdrop_list', [NOT_CSV_WEBPAGE])
+def test_airdrop_fail(data_dir, mock_airdrop_list):  # pylint: disable=unused-argument
+    with pytest.raises(UnableToDecryptRemoteData):
+        check_airdrops(
+            addresses=[TEST_ADDR1],
+            data_dir=data_dir,
+        )