Repository: rotki/rotki

Issue:
Figure out if the nonce lock is needed in the 3 exchanges
## Problem Definition

Kraken, binance, poloniex have a nonce lock in the api query. The reason stated in the comments is that:

The exchange will reject non-increasing API query nonces. So if two greenlets come in here at the same time one of them will fail

## Task

Is that really true? Check the documentation of these three exchanges. Also think about it a bit. The nonce is:  `req['nonce'] = int(time.time() * 1000)`.

Can there be a context switch fast enough so that the nonce is at the same second entering another call and thus two identical nonces end up being sent to the exchange?

- If yes write perhaps (if possible) a test for it.
- If no remove these locks.


Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/binance.py b/rotkehlchen/exchanges/binance.py
--- a/rotkehlchen/exchanges/binance.py
+++ b/rotkehlchen/exchanges/binance.py
@@ -20,7 +20,6 @@
 
 import gevent
 import requests
-from gevent.lock import Semaphore
 from typing_extensions import Literal
 
 from rotkehlchen.accounting.structures import Balance
@@ -241,7 +240,6 @@ def __init__(
             'X-MBX-APIKEY': self.api_key,
         })
         self.msg_aggregator = msg_aggregator
-        self.nonce_lock = Semaphore()
         self.offset_ms = 0
 
     def first_connection(self) -> None:
@@ -311,61 +309,57 @@ def api_query(
         call_options = options.copy() if options else {}
 
         while True:
-            with self.nonce_lock:
-                # Protect this region with a lock since binance will reject
-                # non-increasing nonces. So if two greenlets come in here at
-                # the same time one of them will fail
-                if 'signature' in call_options:
-                    del call_options['signature']
-
-                is_v3_api_method = api_type == 'api' and method in V3_METHODS
-                is_new_futures_api = api_type in ('fapi', 'dapi')
-                call_needs_signature = (
-                    (api_type == 'fapi' and method in FAPI_METHODS) or
-                    (api_type == 'dapi' and method in FAPI_METHODS) or  # same as fapi
-                    (api_type == 'sapi' and method in SAPI_METHODS) or
-                    (api_type == 'wapi' and method in WAPI_METHODS) or
-                    is_v3_api_method
-                )
-                if call_needs_signature:
-                    if api_type in ('sapi', 'dapi'):
-                        api_version = 1
-                    elif api_type == 'fapi':
-                        api_version = 2
-                    elif api_type == 'wapi' or is_v3_api_method:
-                        api_version = 3
-                    else:
-                        raise AssertionError(
-                            f'Should never get to signed binance api call for '
-                            f'api_type: {api_type} and method {method}',
-                        )
-
-                    # Recommended recvWindows is 5000 but we get timeouts with it
-                    call_options['recvWindow'] = 10000
-                    call_options['timestamp'] = str(ts_now_in_ms() + self.offset_ms)
-                    signature = hmac.new(
-                        self.secret,
-                        urlencode(call_options).encode('utf-8'),
-                        hashlib.sha256,
-                    ).hexdigest()
-                    call_options['signature'] = signature
-                elif api_type == 'api' and method in V1_METHODS:
+            if 'signature' in call_options:
+                del call_options['signature']
+
+            is_v3_api_method = api_type == 'api' and method in V3_METHODS
+            is_new_futures_api = api_type in ('fapi', 'dapi')
+            call_needs_signature = (
+                (api_type == 'fapi' and method in FAPI_METHODS) or
+                (api_type == 'dapi' and method in FAPI_METHODS) or  # same as fapi
+                (api_type == 'sapi' and method in SAPI_METHODS) or
+                (api_type == 'wapi' and method in WAPI_METHODS) or
+                is_v3_api_method
+            )
+            if call_needs_signature:
+                if api_type in ('sapi', 'dapi'):
                     api_version = 1
+                elif api_type == 'fapi':
+                    api_version = 2
+                elif api_type == 'wapi' or is_v3_api_method:
+                    api_version = 3
                 else:
-                    raise AssertionError(f'Unexpected {self.name} API method {method}')
+                    raise AssertionError(
+                        f'Should never get to signed binance api call for '
+                        f'api_type: {api_type} and method {method}',
+                    )
 
-                api_subdomain = api_type if is_new_futures_api else 'api'
-                request_url = (
-                    f'https://{api_subdomain}.{self.uri}{api_type}/v{str(api_version)}/{method}?'
-                )
-                request_url += urlencode(call_options)
-                log.debug(f'{self.name} API request', request_url=request_url)
-                try:
-                    response = self.session.get(request_url, timeout=DEFAULT_TIMEOUT_TUPLE)
-                except requests.exceptions.RequestException as e:
-                    raise RemoteError(
-                        f'{self.name} API request failed due to {str(e)}',
-                    ) from e
+                # Recommended recvWindows is 5000 but we get timeouts with it
+                call_options['recvWindow'] = 10000
+                call_options['timestamp'] = str(ts_now_in_ms() + self.offset_ms)
+                signature = hmac.new(
+                    self.secret,
+                    urlencode(call_options).encode('utf-8'),
+                    hashlib.sha256,
+                ).hexdigest()
+                call_options['signature'] = signature
+            elif api_type == 'api' and method in V1_METHODS:
+                api_version = 1
+            else:
+                raise AssertionError(f'Unexpected {self.name} API method {method}')
+
+            api_subdomain = api_type if is_new_futures_api else 'api'
+            request_url = (
+                f'https://{api_subdomain}.{self.uri}{api_type}/v{str(api_version)}/{method}?'
+            )
+            request_url += urlencode(call_options)
+            log.debug(f'{self.name} API request', request_url=request_url)
+            try:
+                response = self.session.get(request_url, timeout=DEFAULT_TIMEOUT_TUPLE)
+            except requests.exceptions.RequestException as e:
+                raise RemoteError(
+                    f'{self.name} API request failed due to {str(e)}',
+                ) from e
 
             if response.status_code not in (200, 418, 429):
                 code = 'no code found'
diff --git a/rotkehlchen/exchanges/kraken.py b/rotkehlchen/exchanges/kraken.py
--- a/rotkehlchen/exchanges/kraken.py
+++ b/rotkehlchen/exchanges/kraken.py
@@ -13,7 +13,6 @@
 
 import gevent
 import requests
-from gevent.lock import Semaphore
 from requests import Response
 
 from rotkehlchen.accounting.structures import Balance
@@ -271,7 +270,6 @@ def __init__(
         )
         self.msg_aggregator = msg_aggregator
         self.session.headers.update({'API-Key': self.api_key})
-        self.nonce_lock = Semaphore()
         self.set_account_type(kraken_account_type)
         self.call_counter = 0
         self.last_query_ts = 0
@@ -471,31 +469,28 @@ def _query_private(self, method: str, req: Optional[dict] = None) -> Union[Dict,
             req = {}
 
         urlpath = '/' + KRAKEN_API_VERSION + '/private/' + method
-
-        with self.nonce_lock:
-            # Protect this section, or else, non increasing nonces will be rejected
-            req['nonce'] = int(1000 * time.time())
-            post_data = urlencode(req)
-            # any unicode strings must be turned to bytes
-            hashable = (str(req['nonce']) + post_data).encode()
-            message = urlpath.encode() + hashlib.sha256(hashable).digest()
-            signature = hmac.new(
-                base64.b64decode(self.secret),
-                message,
-                hashlib.sha512,
+        req['nonce'] = int(1000 * time.time())
+        post_data = urlencode(req)
+        # any unicode strings must be turned to bytes
+        hashable = (str(req['nonce']) + post_data).encode()
+        message = urlpath.encode() + hashlib.sha256(hashable).digest()
+        signature = hmac.new(
+            base64.b64decode(self.secret),
+            message,
+            hashlib.sha512,
+        )
+        self.session.headers.update({
+            'API-Sign': base64.b64encode(signature.digest()),  # type: ignore
+        })
+        try:
+            response = self.session.post(
+                KRAKEN_BASE_URL + urlpath,
+                data=post_data.encode(),
+                timeout=DEFAULT_TIMEOUT_TUPLE,
             )
-            self.session.headers.update({
-                'API-Sign': base64.b64encode(signature.digest()),  # type: ignore
-            })
-            try:
-                response = self.session.post(
-                    KRAKEN_BASE_URL + urlpath,
-                    data=post_data.encode(),
-                    timeout=DEFAULT_TIMEOUT_TUPLE,
-                )
-            except requests.exceptions.RequestException as e:
-                raise RemoteError(f'Kraken API request failed due to {str(e)}') from e
-            self._manage_call_counter(method)
+        except requests.exceptions.RequestException as e:
+            raise RemoteError(f'Kraken API request failed due to {str(e)}') from e
+        self._manage_call_counter(method)
 
         return _check_and_get_response(response, method)
 
diff --git a/rotkehlchen/exchanges/poloniex.py b/rotkehlchen/exchanges/poloniex.py
--- a/rotkehlchen/exchanges/poloniex.py
+++ b/rotkehlchen/exchanges/poloniex.py
@@ -9,7 +9,6 @@
 
 import gevent
 import requests
-from gevent.lock import Semaphore
 from typing_extensions import Literal
 
 from rotkehlchen.accounting.structures import Balance
@@ -17,7 +16,7 @@
 from rotkehlchen.assets.converters import asset_from_poloniex
 from rotkehlchen.constants.assets import A_LEND
 from rotkehlchen.constants.misc import ZERO
-from rotkehlchen.constants.timing import QUERY_RETRY_TIMES, DEFAULT_TIMEOUT_TUPLE
+from rotkehlchen.constants.timing import DEFAULT_TIMEOUT_TUPLE, QUERY_RETRY_TIMES
 from rotkehlchen.errors import (
     DeserializationError,
     RemoteError,
@@ -236,7 +235,6 @@ def __init__(
         self.public_uri = self.uri + 'public?command='
         self.session.headers.update({'Key': self.api_key})
         self.msg_aggregator = msg_aggregator
-        self.nonce_lock = Semaphore()
 
     def first_connection(self) -> None:
         if self.first_connection_made:
@@ -292,20 +290,16 @@ def _single_query(self, command: str, req: Dict[str, Any]) -> Optional[requests.
             response = self.session.get(self.public_uri + command, timeout=DEFAULT_TIMEOUT_TUPLE)
         else:
             req['command'] = command
-            with self.nonce_lock:
-                # Protect this region with a lock since poloniex will reject
-                # non-increasing nonces. So if two greenlets come in here at
-                # the same time one of them will fail
-                req['nonce'] = ts_now_in_ms()
-                post_data = str.encode(urlencode(req))
-
-                sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
-                self.session.headers.update({'Sign': sign})
-                response = self.session.post(
-                    'https://poloniex.com/tradingApi',
-                    req,
-                    timeout=DEFAULT_TIMEOUT_TUPLE,
-                )
+            req['nonce'] = ts_now_in_ms()
+            post_data = str.encode(urlencode(req))
+
+            sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
+            self.session.headers.update({'Sign': sign})
+            response = self.session.post(
+                'https://poloniex.com/tradingApi',
+                req,
+                timeout=DEFAULT_TIMEOUT_TUPLE,
+            )
 
         if response.status_code == 504:
             # backoff and repeat


Test patch: