Repository: freqtrade/freqtrade

Issue:
Derived Strategy break Hyperopt
<!-- 
Have you searched for similar issues before posting it?

If you have discovered a bug in the bot, please [search the issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue). 
If it hasn't been reported, please create a new issue.

Please do not use bug reports to request new features.
-->

## Describe your environment

  * Operating system: Docker
  * Python Version: 3.10
  * Freqtrade Version: 2022.6
  

## Describe the problem:

If deriving you strategy from another one that has been defined outside of the current strategy file it causes hyperopt to fail with a Loky object failed to unserialize error.

### Steps to reproduce:

  1. Create a new strategy with freqtrade new-strategy *args
  2. Make a copy of that strategy in another file
  3. Inherit your first strategy from the 2. one
  
### Observed Results:

  * What happened?
  Hyperopt failed with unserialize error.
  * What did you expect to happen?
  Hyperopt to run normally.

### Relevant code exceptions or logs

```
2022-07-13 13:48:31,965 - freqtrade - ERROR - Fatal exception!
joblib.externals.loky.process_executor._RemoteTraceback:
"""
Traceback (most recent call last):
  File "/home/ftuser/.local/lib/python3.10/site-packages/joblib/externals/loky/process_executor.py", line 407, in _process_worker
    call_item = call_queue.get(block=True, timeout=timeout)
  File "/usr/local/lib/python3.10/multiprocessing/queues.py", line 122, in get
    return _ForkingPickler.loads(res)
  File "/home/ftuser/.local/lib/python3.10/site-packages/joblib/externals/loky/cloudpickle_wrapper.py", line 48, in _reconstruct_wrapper
    obj = loads(_pickled_object)
ModuleNotFoundError: No module named 'test'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/freqtrade/freqtrade/main.py", line 37, in main
    return_code = args['func'](args)
  File "/freqtrade/freqtrade/commands/optimize_commands.py", line 107, in start_hyperopt
    hyperopt.start()
  File "/freqtrade/freqtrade/optimize/hyperopt.py", line 523, in start
    f_val = self.run_optimizer_parallel(parallel, asked, i)
  File "/freqtrade/freqtrade/optimize/hyperopt.py", line 389, in run_optimizer_parallel
    return parallel(delayed(
  File "/home/ftuser/.local/lib/python3.10/site-packages/joblib/parallel.py", line 1056, in __call__
    self.retrieve()
  File "/home/ftuser/.local/lib/python3.10/site-packages/joblib/parallel.py", line 935, in retrieve
    self._output.extend(job.get(timeout=self.timeout))
  File "/home/ftuser/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py", line 542, in wrap_future_result
    return future.result(timeout=timeout)
  File "/usr/local/lib/python3.10/concurrent/futures/_base.py", line 446, in result
    return self.__get_result()
  File "/usr/local/lib/python3.10/concurrent/futures/_base.py", line 391, in __get_result
    raise self._exception
joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.

```



Discussion:
I've been aware of this in the back of my mind for quite some time - but there's no real solution i've come up with - as it's a dependency of unpickling to be able to "find" all elements (it won't find the parent class).

it will also depend a bit on how you're importing the strategy if i remember correctly.

Relative imports will not work (once the strategy is moved into the worker, it "forgets" where it came from.
absolute imports MAY work - assuming they're relative to the local dir (`from user_data.strategies.xxx import yyy)` - but require each dir to be a package (contain a `__init__.py` file).



Resolving patch:
diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -175,23 +175,11 @@ def __init__(self, config: Dict[str, Any], validate: bool = True) -> None:
         logger.info(f'Using Exchange "{self.name}"')
 
         if validate:
-            # Check if timeframe is available
-            self.validate_timeframes(config.get('timeframe'))
-
             # Initial markets load
             self._load_markets()
-
-            # Check if all pairs are available
-            self.validate_stakecurrency(config['stake_currency'])
-            if not exchange_config.get('skip_pair_validation'):
-                self.validate_pairs(config['exchange']['pair_whitelist'])
-            self.validate_ordertypes(config.get('order_types', {}))
-            self.validate_order_time_in_force(config.get('order_time_in_force', {}))
+            self.validate_config(config)
             self.required_candle_call_count = self.validate_required_startup_candles(
                 config.get('startup_candle_count', 0), config.get('timeframe', ''))
-            self.validate_trading_mode_and_margin_mode(self.trading_mode, self.margin_mode)
-            self.validate_pricing(config['exit_pricing'])
-            self.validate_pricing(config['entry_pricing'])
 
         # Converts the interval provided in minutes in config to seconds
         self.markets_refresh_interval: int = exchange_config.get(
@@ -214,6 +202,20 @@ def close(self):
             logger.info("Closing async ccxt session.")
             self.loop.run_until_complete(self._api_async.close())
 
+    def validate_config(self, config):
+        # Check if timeframe is available
+        self.validate_timeframes(config.get('timeframe'))
+
+        # Check if all pairs are available
+        self.validate_stakecurrency(config['stake_currency'])
+        if not config['exchange'].get('skip_pair_validation'):
+            self.validate_pairs(config['exchange']['pair_whitelist'])
+        self.validate_ordertypes(config.get('order_types', {}))
+        self.validate_order_time_in_force(config.get('order_time_in_force', {}))
+        self.validate_trading_mode_and_margin_mode(self.trading_mode, self.margin_mode)
+        self.validate_pricing(config['exit_pricing'])
+        self.validate_pricing(config['entry_pricing'])
+
     def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt,
                    ccxt_kwargs: Dict = {}) -> ccxt.Exchange:
         """
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -6,6 +6,7 @@
 
 import logging
 import random
+import sys
 import warnings
 from datetime import datetime, timezone
 from math import ceil
@@ -17,6 +18,7 @@
 from colorama import Fore, Style
 from colorama import init as colorama_init
 from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
+from joblib.externals import cloudpickle
 from pandas import DataFrame
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN
@@ -87,6 +89,7 @@ def __init__(self, config: Dict[str, Any]) -> None:
         self.backtesting._set_strategy(self.backtesting.strategylist[0])
         self.custom_hyperopt.strategy = self.backtesting.strategy
 
+        self.hyperopt_pickle_magic(self.backtesting.strategy.__class__.__bases__)
         self.custom_hyperoptloss: IHyperOptLoss = HyperOptLossResolver.load_hyperoptloss(
             self.config)
         self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function
@@ -137,6 +140,17 @@ def clean_hyperopt(self) -> None:
                 logger.info(f"Removing `{p}`.")
                 p.unlink()
 
+    def hyperopt_pickle_magic(self, bases) -> None:
+        """
+        Hyperopt magic to allow strategy inheritance across files.
+        For this to properly work, we need to register the module of the imported class
+        to pickle as value.
+        """
+        for modules in bases:
+            if modules.__name__ != 'IStrategy':
+                cloudpickle.register_pickle_by_value(sys.modules[modules.__module__])
+                self.hyperopt_pickle_magic(modules.__bases__)
+
     def _get_params_dict(self, dimensions: List[Dimension], raw_params: List[Any]) -> Dict:
 
         # Ensure the number of dimensions match


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -112,11 +112,8 @@ def patch_exchange(
     mock_supported_modes=True
 ) -> None:
     mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock(return_value={}))
-    mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.validate_config', MagicMock())
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
-    mocker.patch('freqtrade.exchange.Exchange.validate_ordertypes', MagicMock())
-    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency', MagicMock())
-    mocker.patch('freqtrade.exchange.Exchange.validate_pricing')
     mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value=id))
     mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value=id.title()))
     mocker.patch('freqtrade.exchange.Exchange.precisionMode', PropertyMock(return_value=2))
diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -1,7 +1,7 @@
 # pragma pylint: disable=missing-docstring,W0212,C0103
 from datetime import datetime, timedelta
 from pathlib import Path
-from unittest.mock import ANY, MagicMock
+from unittest.mock import ANY, MagicMock, PropertyMock
 
 import pandas as pd
 import pytest
@@ -18,8 +18,8 @@
 from freqtrade.optimize.optimize_reports import generate_strategy_stats
 from freqtrade.optimize.space import SKDecimal
 from freqtrade.strategy import IntParameter
-from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has, log_has_re, patch_exchange,
-                            patched_configuration_load_config_file)
+from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, get_markets, log_has, log_has_re,
+                            patch_exchange, patched_configuration_load_config_file)
 
 
 def generate_result_metrics():
@@ -855,7 +855,7 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None:
         'strategy': 'HyperoptableStrategy',
         'user_data_dir': Path(tmpdir),
         'hyperopt_random_state': 42,
-        'spaces': ['all']
+        'spaces': ['all'],
     })
     hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.exchange.get_max_leverage = MagicMock(return_value=1.0)
@@ -883,6 +883,45 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None:
         hyperopt.get_optimizer([], 2)
 
 
+def test_in_strategy_auto_hyperopt_with_parallel(mocker, hyperopt_conf, tmpdir, fee) -> None:
+    mocker.patch('freqtrade.exchange.Exchange.validate_config', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
+    mocker.patch('freqtrade.exchange.Exchange._load_markets')
+    mocker.patch('freqtrade.exchange.Exchange.markets',
+                 PropertyMock(return_value=get_markets()))
+    (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True)
+    # No hyperopt needed
+    hyperopt_conf.update({
+        'strategy': 'HyperoptableStrategy',
+        'user_data_dir': Path(tmpdir),
+        'hyperopt_random_state': 42,
+        'spaces': ['all'],
+        # Enforce parallelity
+        'epochs': 2,
+        'hyperopt_jobs': 2,
+        'fee': fee.return_value,
+    })
+    hyperopt = Hyperopt(hyperopt_conf)
+    hyperopt.backtesting.exchange.get_max_leverage = lambda *x, **xx: 1.0
+    hyperopt.backtesting.exchange.get_min_pair_stake_amount = lambda *x, **xx: 1.0
+    hyperopt.backtesting.exchange.get_max_pair_stake_amount = lambda *x, **xx: 100.0
+
+    assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto)
+    assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter)
+    assert hyperopt.backtesting.strategy.bot_loop_started is True
+
+    assert hyperopt.backtesting.strategy.buy_rsi.in_space is True
+    assert hyperopt.backtesting.strategy.buy_rsi.value == 35
+    assert hyperopt.backtesting.strategy.sell_rsi.value == 74
+    assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value == 30
+    buy_rsi_range = hyperopt.backtesting.strategy.buy_rsi.range
+    assert isinstance(buy_rsi_range, range)
+    # Range from 0 - 50 (inclusive)
+    assert len(list(buy_rsi_range)) == 51
+
+    hyperopt.start()
+
+
 def test_SKDecimal():
     space = SKDecimal(1, 2, decimals=2)
     assert 1.5 in space