Repository: freqtrade/freqtrade

Issue:
Conflicting filename wit 1m and 1M for case insensitive filesystems

Hi,

When case insensitive filesystem is used, there is a conflict with filenames for 1 minute and 1 month historical data.

For example, filename "BTC_USDT-1m.json" and "BTC_USDT-1M.json" are the same when using Windows with NTFS.

Thanks


Discussion:
Please get used to fill out the issue template. Repeat offenders to this will have their issues closed without further comment.
We're asking some very basic questions - by deleting the issue template, you're explicitly choosing not to answer our questions - which makes me doubt why you think we should chose to answer your questions.

---

That's probably correct.
For the moment, the only thing you can do is to use a case sensitive filesystem, on windows most likely by using WSL2.

While we may fix this eventually at some point, it's unlikely it will be fixed quickly.

Resolving patch:
diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py
--- a/freqtrade/data/history/hdf5datahandler.py
+++ b/freqtrade/data/history/hdf5datahandler.py
@@ -40,7 +40,7 @@ def ohlcv_get_available_data(
         return [
             (
                 cls.rebuild_pair_from_filename(match[1]),
-                match[2],
+                cls.rebuild_timeframe_from_filename(match[2]),
                 CandleType.from_string(match[3])
             ) for match in _tmp if match and len(match.groups()) > 1]
 
@@ -109,7 +109,11 @@ def _ohlcv_load(self, pair: str, timeframe: str,
         )
 
         if not filename.exists():
-            return pd.DataFrame(columns=self._columns)
+            # Fallback mode for 1M files
+            filename = self._pair_data_filename(
+                self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
+            if not filename.exists():
+                return pd.DataFrame(columns=self._columns)
         where = []
         if timerange:
             if timerange.starttype == 'date':
diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py
--- a/freqtrade/data/history/idatahandler.py
+++ b/freqtrade/data/history/idatahandler.py
@@ -26,7 +26,7 @@
 
 class IDataHandler(ABC):
 
-    _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)'
+    _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)'
 
     def __init__(self, datadir: Path) -> None:
         self._datadir = datadir
@@ -193,10 +193,14 @@ def _pair_data_filename(
         datadir: Path,
         pair: str,
         timeframe: str,
-        candle_type: CandleType
+        candle_type: CandleType,
+        no_timeframe_modify: bool = False
     ) -> Path:
         pair_s = misc.pair_to_filename(pair)
         candle = ""
+        if not no_timeframe_modify:
+            timeframe = cls.timeframe_to_file(timeframe)
+
         if candle_type != CandleType.SPOT:
             datadir = datadir.joinpath('futures')
             candle = f"-{candle_type}"
@@ -210,6 +214,18 @@ def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
         filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
         return filename
 
+    @staticmethod
+    def timeframe_to_file(timeframe: str):
+        return timeframe.replace('M', 'Mo')
+
+    @staticmethod
+    def rebuild_timeframe_from_filename(timeframe: str) -> str:
+        """
+        converts timeframe from disk to file
+        Replaces mo with M (to avoid problems on case-insensitive filesystems)
+        """
+        return re.sub('1mo', '1M', timeframe, flags=re.IGNORECASE)
+
     @staticmethod
     def rebuild_pair_from_filename(pair: str) -> str:
         """
diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py
--- a/freqtrade/data/history/jsondatahandler.py
+++ b/freqtrade/data/history/jsondatahandler.py
@@ -41,7 +41,7 @@ def ohlcv_get_available_data(
         return [
             (
                 cls.rebuild_pair_from_filename(match[1]),
-                match[2],
+                cls.rebuild_timeframe_from_filename(match[2]),
                 CandleType.from_string(match[3])
             ) for match in _tmp if match and len(match.groups()) > 1]
 
@@ -103,9 +103,14 @@ def _ohlcv_load(self, pair: str, timeframe: str,
         :param candle_type: Any of the enum CandleType (must match trading mode!)
         :return: DataFrame with ohlcv data, or empty DataFrame
         """
-        filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type)
+        filename = self._pair_data_filename(
+            self._datadir, pair, timeframe, candle_type=candle_type)
         if not filename.exists():
-            return DataFrame(columns=self._columns)
+            # Fallback mode for 1M files
+            filename = self._pair_data_filename(
+                self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
+            if not filename.exists():
+                return DataFrame(columns=self._columns)
         try:
             pairdata = read_json(filename, orient='values')
             pairdata.columns = self._columns


Test patch:
diff --git a/tests/data/test_history.py b/tests/data/test_history.py
--- a/tests/data/test_history.py
+++ b/tests/data/test_history.py
@@ -158,21 +158,22 @@ def test_testdata_path(testdatadir) -> None:
     assert str(Path('tests') / 'testdata') in str(testdatadir)
 
 
-@pytest.mark.parametrize("pair,expected_result,candle_type", [
-    ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""),
-    ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""),
-    ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""),
-    (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""),
-    ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""),
-    ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""),
-    ("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"),
-    ("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"),
+@pytest.mark.parametrize("pair,timeframe,expected_result,candle_type", [
+    ("ETH/BTC", "5m", "freqtrade/hello/world/ETH_BTC-5m.json", ""),
+    ("ETH/USDT", "1M", "freqtrade/hello/world/ETH_USDT-1Mo.json", ""),
+    ("Fabric Token/ETH", "5m", "freqtrade/hello/world/Fabric_Token_ETH-5m.json", ""),
+    ("ETHH20", "5m", "freqtrade/hello/world/ETHH20-5m.json", ""),
+    (".XBTBON2H", "5m", "freqtrade/hello/world/_XBTBON2H-5m.json", ""),
+    ("ETHUSD.d", "5m", "freqtrade/hello/world/ETHUSD_d-5m.json", ""),
+    ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/ACC_OLD_BTC-5m.json", ""),
+    ("ETH/BTC", "5m", "freqtrade/hello/world/futures/ETH_BTC-5m-mark.json", "mark"),
+    ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json", "index"),
 ])
-def test_json_pair_data_filename(pair, expected_result, candle_type):
+def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type):
     fn = JsonDataHandler._pair_data_filename(
         Path('freqtrade/hello/world'),
         pair,
-        '5m',
+        timeframe,
         CandleType.from_string(candle_type)
     )
     assert isinstance(fn, Path)
@@ -180,7 +181,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type):
     fn = JsonGzDataHandler._pair_data_filename(
         Path('freqtrade/hello/world'),
         pair,
-        '5m',
+        timeframe,
         candle_type=CandleType.from_string(candle_type)
     )
     assert isinstance(fn, Path)