Repository: freqtrade/freqtrade
Pull request: Added support for regex in whitelist
Introduce regex whitelist support

closes #4177

Problem statement:
UP/DOWN wildcards for whitelist
<!-- 
Note: this section will not show up in the issue.
Have you search for this feature before requesting it? It's highly likely that a similar request was already filed.
-->

## Describe your environment
(if applicable)

  * Operating system: ____
  * Python Version: _____ (`python -V`)
  * CCXT version: _____ (`pip freeze | grep ccxt`)
  * Freqtrade Version: ____ (`freqtrade -V` or `docker-compose run --rm freqtrade -V` for Freqtrade running in docker)


## Describe the enhancement

After having the "UP/DOWN" token wildcard feat. for the blacklist, it would be very helpful having the same for the whitelist to filter / unfilter leveraged markets

Hints from issue discussion:
I'm labling this as "good first issue" - which means i'll leave it open for an undefined period of time without working on it myself.
This can be a good issue for new contributors to freqtrade.

The logic for the pairs is available (used by blacklist) - and would simply need to be implemented to StaticPairlist - which makes this a pretty simple, quick issue.

Commit message: Added suppoort for regex in whitelist

Patch to review:
diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py
index 25c7d0436c8..1ce02eee500 100644
--- a/freqtrade/commands/data_commands.py
+++ b/freqtrade/commands/data_commands.py
@@ -10,6 +10,7 @@
                                     refresh_backtest_trades_data)
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
 
@@ -42,15 +43,17 @@ def start_download_data(args: Dict[str, Any]) -> None:
             "Downloading data requires a list of pairs. "
             "Please check the documentation on how to configure this.")
 
-    logger.info(f"About to download pairs: {config['pairs']}, "
-                f"intervals: {config['timeframes']} to {config['datadir']}")
-
     pairs_not_available: List[str] = []
 
     # Init exchange
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
     # Manual validations of relevant settings
     exchange.validate_pairs(config['pairs'])
+    expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
+
+    logger.info(f"About to download pairs: {expanded_pairs}, "
+                f"intervals: {config['timeframes']} to {config['datadir']}")
+
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
 
@@ -58,20 +61,20 @@ def start_download_data(args: Dict[str, Any]) -> None:
 
         if config.get('download_trades'):
             pairs_not_available = refresh_backtest_trades_data(
-                exchange, pairs=config['pairs'], datadir=config['datadir'],
+                exchange, pairs=expanded_pairs, datadir=config['datadir'],
                 timerange=timerange, erase=bool(config.get('erase')),
                 data_format=config['dataformat_trades'])
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
-                pairs=config['pairs'], timeframes=config['timeframes'],
+                pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format_ohlcv=config['dataformat_ohlcv'],
                 data_format_trades=config['dataformat_trades'],
-                )
+            )
         else:
             pairs_not_available = refresh_backtest_ohlcv_data(
-                exchange, pairs=config['pairs'], timeframes=config['timeframes'],
+                exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format=config['dataformat_ohlcv'])
 
diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py
index 037717c6820..e549a3701df 100644
--- a/freqtrade/edge/edge_positioning.py
+++ b/freqtrade/edge/edge_positioning.py
@@ -12,6 +12,7 @@
 from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
 from freqtrade.data.history import get_timerange, load_data, refresh_data
 from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.strategy.interface import SellType
 
 
@@ -80,10 +81,12 @@ def __init__(self, config: Dict[str, Any], exchange, strategy) -> None:
         if config.get('fee'):
             self.fee = config['fee']
         else:
-            self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
+            self.fee = self.exchange.get_fee(symbol=expand_pairlist(
+                self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
 
     def calculate(self) -> bool:
-        pairs = self.config['exchange']['pair_whitelist']
+        pairs = expand_pairlist(self.config['exchange']['pair_whitelist'],
+                                list(self.exchange.markets))
         heartbeat = self.edge_config.get('process_throttle_secs')
 
         if (self._last_updated > 0) and (
diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index b610b28f413..489f705289c 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -25,6 +25,7 @@
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier,
                                        retrier_async)
 from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
 CcxtModuleType = Any
@@ -335,8 +336,9 @@ def validate_pairs(self, pairs: List[str]) -> None:
         if not self.markets:
             logger.warning('Unable to validate pairs (assuming they are correct).')
             return
+        extended_pairs = expand_pairlist(pairs, list(self.markets))
         invalid_pairs = []
-        for pair in pairs:
+        for pair in extended_pairs:
             # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
             # TODO: add a support for having coins in BTC/USDT format
             if self.markets and pair not in self.markets:
diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py
index 40e3da9c926..996c5276c4f 100644
--- a/freqtrade/plot/plotting.py
+++ b/freqtrade/plot/plotting.py
@@ -13,6 +13,7 @@
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds
 from freqtrade.misc import pair_to_filename
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.strategy import IStrategy
 
@@ -29,16 +30,16 @@
     exit(1)
 
 
-def init_plotscript(config, startup_candles: int = 0):
+def init_plotscript(config, markets: List, startup_candles: int = 0):
     """
     Initialize objects needed for plotting
     :return: Dict with candle (OHLCV) data, trades and pairs
     """
 
     if "pairs" in config:
-        pairs = config['pairs']
+        pairs = expand_pairlist(config['pairs'], markets)
     else:
-        pairs = config['exchange']['pair_whitelist']
+        pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets)
 
     # Set timerange to use
     timerange = TimeRange.parse_timerange(config.get('timerange'))
@@ -177,7 +178,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
         trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, "
                                                   f"{row['sell_reason']}, "
                                                   f"{row['trade_duration']} min",
-                                                  axis=1)
+                                      axis=1)
         trade_buys = go.Scatter(
             x=trades["open_date"],
             y=trades["open_rate"],
@@ -527,7 +528,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
 
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
     IStrategy.dp = DataProvider(config, exchange)
-    plot_elements = init_plotscript(config, strategy.startup_candle_count)
+    plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count)
     timerange = plot_elements['timerange']
     trades = plot_elements['trades']
     pair_counter = 0
@@ -562,7 +563,8 @@ def plot_profit(config: Dict[str, Any]) -> None:
     But should be somewhat proportional, and therefor useful
     in helping out to find a good algorithm.
     """
-    plot_elements = init_plotscript(config)
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
+    plot_elements = init_plotscript(config, list(exchange.markets))
     trades = plot_elements['trades']
     # Filter trades to relevant pairs
     # Remove open pairs - we don't know the profit yet so can't calculate profit for these.
diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py
index 865aa90d6ac..f9809aeb16e 100644
--- a/freqtrade/plugins/pairlist/IPairList.py
+++ b/freqtrade/plugins/pairlist/IPairList.py
@@ -124,6 +124,15 @@ def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
         """
         return self._pairlistmanager.verify_blacklist(pairlist, logmethod)
 
+    def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]:
+        """
+        Proxy method to verify_whitelist for easy access for child classes.
+        :param pairlist: Pairlist to validate
+        :param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
+        :return: pairlist - whitelisted pairs
+        """
+        return self._pairlistmanager.verify_whitelist(pairlist, logmethod)
+
     def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]:
         """
         Check available markets and remove pair from whitelist if necessary
diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py
index dd592e0cafe..c216a322dc1 100644
--- a/freqtrade/plugins/pairlist/StaticPairList.py
+++ b/freqtrade/plugins/pairlist/StaticPairList.py
@@ -50,9 +50,10 @@ def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
         :return: List of pairs
         """
         if self._allow_inactive:
-            return self._config['exchange']['pair_whitelist']
+            return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)
         else:
-            return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
+            return self._whitelist_for_active_markets(
+                self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info))
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py
index ad7b46cb8de..c471d7c51f0 100644
--- a/freqtrade/plugins/pairlistmanager.py
+++ b/freqtrade/plugins/pairlistmanager.py
@@ -59,6 +59,11 @@ def expanded_blacklist(self) -> List[str]:
         """The expanded blacklist (including wildcard expansion)"""
         return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
 
+    @property
+    def expanded_whitelist(self) -> List[str]:
+        """The expanded whitelist (including wildcard expansion)"""
+        return expand_pairlist(self._whitelist, self._exchange.get_markets().keys())
+
     @property
     def name_list(self) -> List[str]:
         """Get list of loaded Pairlist Handler names"""
@@ -129,6 +134,21 @@ def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
                 pairlist.remove(pair)
         return pairlist
 
+    def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]:
+        """
+        Verify and remove items from pairlist - returning a filtered pairlist.
+        Logs a warning or info depending on `aswarning`.
+        Pairlist Handlers explicitly using this method shall use
+        `logmethod=logger.info` to avoid spamming with warning messages
+        :return: pairlist - blacklisted pairs
+        """
+        try:
+            whitelist = self.expanded_whitelist
+        except ValueError as err:
+            logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+            return []
+        return whitelist
+
     def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes:
         """
         Create list of pair tuples with (pair, timeframe)
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index 7d9954cb913..8cd2a9bca16 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -505,37 +505,38 @@ def test_validate_pairs(default_conf, mocker):  # test exchange.validate_pairs d
     Exchange(default_conf)
 
 
-def test_validate_pairs_not_available(default_conf, mocker):
-    api_mock = MagicMock()
-    type(api_mock).markets = PropertyMock(return_value={
-        'XRP/BTC': {'inactive': True}
-    })
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
-    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
-
-    with pytest.raises(OperationalException, match=r'not available'):
-        Exchange(default_conf)
-
-
-def test_validate_pairs_exception(default_conf, mocker, caplog):
-    caplog.set_level(logging.INFO)
-    api_mock = MagicMock()
-    mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
-
-    type(api_mock).markets = PropertyMock(return_value={})
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
-    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
-
-    with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
-        Exchange(default_conf)
-
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
-    Exchange(default_conf)
-    assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
+# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs
+# def test_validate_pairs_not_available(default_conf, mocker):
+#     api_mock = MagicMock()
+#     type(api_mock).markets = PropertyMock(return_value={
+#         'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'}
+#     })
+#     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
+#     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
+#     mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
+#     mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
+#
+#     with pytest.raises(OperationalException, match=r'not available'):
+#         Exchange(default_conf)
+#
+#
+# def test_validate_pairs_exception(default_conf, mocker, caplog):
+#     caplog.set_level(logging.INFO)
+#     api_mock = MagicMock()
+#     mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
+#
+#     type(api_mock).markets = PropertyMock(return_value={})
+#     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
+#     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
+#     mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
+#     mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
+#
+#     with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
+#         Exchange(default_conf)
+#
+#     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
+#     Exchange(default_conf)
+#     assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
 
 
 def test_validate_pairs_restricted(default_conf, mocker, caplog):
diff --git a/tests/test_plotting.py b/tests/test_plotting.py
index 8e7b0ef7c2f..8f3ac464e46 100644
--- a/tests/test_plotting.py
+++ b/tests/test_plotting.py
@@ -47,14 +47,15 @@ def test_init_plotscript(default_conf, mocker, testdatadir):
     default_conf['timeframe'] = "5m"
     default_conf["datadir"] = testdatadir
     default_conf['exportfilename'] = testdatadir / "backtest-result_test.json"
-    ret = init_plotscript(default_conf)
+    supported_markets = ["TRX/BTC", "ADA/BTC"]
+    ret = init_plotscript(default_conf, supported_markets)
     assert "ohlcv" in ret
     assert "trades" in ret
     assert "pairs" in ret
     assert 'timerange' in ret
 
     default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"]
-    ret = init_plotscript(default_conf, 20)
+    ret = init_plotscript(default_conf, supported_markets, 20)
     assert "ohlcv" in ret
     assert "TRX/BTC" in ret["ohlcv"]
     assert "ADA/BTC" in ret["ohlcv"]


Code review comments:
diff_hunk: @@ -50,9 +50,10 @@ def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
         :return: List of pairs
         """
         if self._allow_inactive:
-            return self._config['exchange']['pair_whitelist']
+            return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)
path: freqtrade/plugins/pairlist/StaticPairList.py
text: @user1:
I think this is a special case (when `allow_inactive = True`), in which we'd not wan't to expand the whitelist.

Problem here is, the whitelist expansion implicitly checks if the pair exists.
This flag was however explicitly added in #3939 to allow backtesting no longer existing pairs, which this will basically prevent.

Obviously, we'd need to add a note (`pairlist.md`) that this will prevent whitelist expansion to the docs ... 

@author:
From there, I called expand_pairlist with parameter keep invalid = True, so that invalid pairs are maintained in the whitelist.

@user1:
Great way to solve this! :+1:

diff_hunk: @@ -505,37 +505,38 @@ def test_validate_pairs(default_conf, mocker):  # test exchange.validate_pairs d
     Exchange(default_conf)
 
 
-def test_validate_pairs_not_available(default_conf, mocker):
-    api_mock = MagicMock()
-    type(api_mock).markets = PropertyMock(return_value={
-        'XRP/BTC': {'inactive': True}
-    })
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
-    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
-
-    with pytest.raises(OperationalException, match=r'not available'):
-        Exchange(default_conf)
-
-
-def test_validate_pairs_exception(default_conf, mocker, caplog):
-    caplog.set_level(logging.INFO)
-    api_mock = MagicMock()
-    mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
-
-    type(api_mock).markets = PropertyMock(return_value={})
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
-    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
-
-    with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
-        Exchange(default_conf)
-
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
-    Exchange(default_conf)
-    assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
+# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs
+# def test_validate_pairs_not_available(default_conf, mocker):
path: tests/exchange/test_exchange.py
text: @user1:
See above note why i think this might not be a good idea ... 

@author:
I reverted this, now this test passes, as expand_pairlist keeps the invalid pairs and so the exceptions are raised, if needs to be.