Repository: freqtrade/freqtrade

Issue:
Kraken Read timed out
## Step 1: Have you search for this issue before posting it?

After running the bot in dry-run mode for an hour there was a fatal exception due to a read timeout with using the Kraken api.

## Step 2: Describe your environment

  * Python Version: 3.7.3 (`python -V`)
  * CCXT version: 1.18.481 (`pip freeze | grep ccxt`)
  * Docker image tag: develop
  * Docker image ID: bba83f73ccd2
  
## Step 3: Describe the problem:

The bot crashed after running for an hour and getting a timeout while querying Kraken for buy signals.
  
### Observed Results:

  * What happened?
     The bot crashed.
  * What did you expect to happen?
     Getting a buy signal.

### Relevant code exceptions or logs:

  ```
2019-04-21 21:22:59,484 - freqtrade - ERROR - Fatal exception!
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 384, in _make_request
    six.raise_from(e, None)
  File "<string>", line 2, in raise_from
  File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 380, in _make_request
    httplib_response = conn.getresponse()
  File "/usr/local/lib/python3.7/http/client.py", line 1321, in getresponse
    response.begin()
  File "/usr/local/lib/python3.7/http/client.py", line 296, in begin
    version, status, reason = self._read_status()
  File "/usr/local/lib/python3.7/http/client.py", line 257, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/usr/local/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
  File "/usr/local/lib/python3.7/ssl.py", line 1052, in recv_into
    return self.read(nbytes, buffer)
  File "/usr/local/lib/python3.7/ssl.py", line 911, in read
    return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
    _stacktrace=sys.exc_info()[2])
  File "/usr/local/lib/python3.7/site-packages/urllib3/util/retry.py", line 368, in increment
    raise six.reraise(type(error), error, _stacktrace)
  File "/usr/local/lib/python3.7/site-packages/urllib3/packages/six.py", line 686, in reraise
    raise value
  File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
    chunked=chunked)
  File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 386, in _make_request
    self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
  File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 306, in _raise_timeout
    raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='api.kraken.com', port=443): Read timed out. (read timeout=10)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/ccxt/base/exchange.py", line 494, in fetch
    proxies=self.proxies
  File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 529, in send
    raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.kraken.com', port=443): Read timed out. (read timeout=10)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/freqtrade/freqtrade/main.py", line 42, in main
    worker.run()
  File "/freqtrade/freqtrade/worker.py", line 72, in run
    state = self._worker(old_state=state)
  File "/freqtrade/freqtrade/worker.py", line 110, in _worker
    self._throttle(func=self._process, min_secs=throttle_secs)
  File "/freqtrade/freqtrade/worker.py", line 123, in _throttle
    result = func(*args, **kwargs)
  File "/freqtrade/freqtrade/worker.py", line 133, in _process
    state_changed = self.freqtrade.process()
  File "/freqtrade/freqtrade/freqtradebot.py", line 101, in process
    self.exchange._reload_markets()
  File "/freqtrade/freqtrade/exchange/exchange.py", line 225, in _reload_markets
    self._api.load_markets(reload=True)
  File "/usr/local/lib/python3.7/site-packages/ccxt/base/exchange.py", line 1097, in load_markets
    currencies = self.fetch_currencies()
  File "/usr/local/lib/python3.7/site-packages/ccxt/kraken.py", line 365, in fetch_currencies
    response = self.publicGetAssets(params)
  File "/usr/local/lib/python3.7/site-packages/ccxt/base/exchange.py", line 404, in inner
    return entry(_self, **inner_kwargs)
  File "/usr/local/lib/python3.7/site-packages/ccxt/base/exchange.py", line 433, in request
    return self.fetch2(path, api, method, params, headers, body)
  File "/usr/local/lib/python3.7/site-packages/ccxt/base/exchange.py", line 429, in fetch2
    return self.fetch(request['url'], request['method'], request['headers'], request['body'])
  File "/usr/local/lib/python3.7/site-packages/ccxt/base/exchange.py", line 512, in fetch
    self.raise_error(RequestTimeout, method, url, e)
  File "/usr/local/lib/python3.7/site-packages/ccxt/base/exchange.py", line 414, in raise_error
    raise exception_type(output)
ccxt.base.errors.RequestTimeout: kraken GET https://api.kraken.com/0/public/Assets HTTPSConnectionPool(host='api.kraken.com', port=443): Read timed out. (read timeout=10)
  ```



Discussion:
Can be just a networking issue you experienced at that moment.
Is it repeatable/reproducible?

Though, anyway, the bot probably should not crash in case of temporary bad networking...

What are your `rateLimit` setings in your config?
I don't think that it's rate-limting - exchanges normally tell you if you're beeing blocked, this looks like a downtime of the exchange - or at least broken connectivity between the bot and the exchange.

I do agree however, should not crash the bot
Btw, my internet provider reloads theirs routers somewhere in the route to the Internet at 4 AM every night. So when I run the bot from home at nights I can experience similar issue (have not checked yet). Will try to catch the crash...

My rate limit config is at 200. The process crashes every 2-3 hours because of this problem and gets restarted again because of the docker restart policy.
The server is hosted on a public cloud platform, so I don't expect it to be a network problem but more like kraken being unavailable.

Resolving patch:
diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -222,8 +222,11 @@ def _reload_markets(self) -> None:
                 > arrow.utcnow().timestamp):
             return None
         logger.debug("Performing scheduled market reload..")
-        self._api.load_markets(reload=True)
-        self._last_markets_refresh = arrow.utcnow().timestamp
+        try:
+            self._api.load_markets(reload=True)
+            self._last_markets_refresh = arrow.utcnow().timestamp
+        except ccxt.BaseError:
+            logger.exception("Could not reload markets.")
 
     def validate_pairs(self, pairs: List[str]) -> None:
         """


Test patch:
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -301,6 +301,20 @@ def load_markets(*args, **kwargs):
     assert log_has('Performing scheduled market reload..', caplog.record_tuples)
 
 
+def test__reload_markets_exception(default_conf, mocker, caplog):
+    caplog.set_level(logging.DEBUG)
+
+    api_mock = MagicMock()
+    api_mock.load_markets = MagicMock(side_effect=ccxt.NetworkError)
+    default_conf['exchange']['markets_refresh_interval'] = 10
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance")
+
+    # less than 10 minutes have passed, no reload
+    exchange._reload_markets()
+    assert exchange._last_markets_refresh == 0
+    assert log_has_re(r"Could not reload markets.*", caplog.record_tuples)
+
+
 def test_validate_pairs(default_conf, mocker):  # test exchange.validate_pairs directly
     api_mock = MagicMock()
     type(api_mock).markets = PropertyMock(return_value={