Repository: sdv-dev/RDT

Issue:
`_safe_parse_datetime` causes `OverflowError` for large numbers
### Error Description

The `_safe_parse_datetime` method calls `dateutil.parser.parse` on the input value. When the value is a large integer, it will try to interpret it as a Unix timestamp but will fail to convert this large number to a C long (which is used internally for timestamp calculations), causing the overflow.

### Steps to Reproduce

Running `_safe_parse_datetime('20220902110443000000')` causes `OverflowError: Python int too large to convert to C long`.

### Additional Context

This error was first observed on [this](https://github.com/sdv-dev/SDV/actions/runs/14865377207/job/41740757122#step:5:497) test case in SDV.

Hints from issue comments:

Solution patch:
diff --git a/rdt/transformers/datetime.py b/rdt/transformers/datetime.py
index 419084f..d729c2c 100644
--- a/rdt/transformers/datetime.py
+++ b/rdt/transformers/datetime.py
@@ -74,34 +74,51 @@ class UnixTimestampEncoder(BaseTransformer):
         self._has_multiple_timezones = False
         self._timezone_offset = None
 
+    def _warn_if_mixed_timezones(self):
+        if self._has_multiple_timezones:
+            warnings.warn(
+                'Mixed timezones are not supported in SDV Community. Data will be converted to UTC.'
+            )
+
+    def _raise_appropriate_conversion_error(self, error):
+        message = str(error)
+        if 'Unknown string' in message or 'Unknown datetime string' in message:
+            raise TypeError('Data must be of dtype datetime, or castable to datetime.') from None
+
+        raise ValueError('Data does not match specified datetime format.') from None
+
+    def _learn_timezone_offest(self, data):
+        """Extracts and stores the timezone offset from the first valid datetime in the data."""
+        has_timezone = self.datetime_format and '%z' in self.datetime_format.lower()
+        if has_timezone and not self._has_multiple_timezones:
+            for val in data:
+                if pd.notna(val):
+                    try:
+                        dt = _safe_parse_datetime(
+                            str(val), warn=True, datetime_format=self.datetime_format
+                        )
+                        self._timezone_offset = dt.tzinfo
+                        break
+
+                    except (ValueError, TypeError, AttributeError):
+                        self._timezone_offset = None
+
     def _learn_has_multiple_timezones(self, data):
         """Determines if the data contains multiple timezones and stores the result."""
+        if self.datetime_format and '%z' not in self.datetime_format.lower():
+            return
+
         if not isinstance(data, pd.Series):
             data = data.to_series()
 
-        self._has_multiple_timezones = data_has_multiple_timezones(data)
-
-    def _convert_to_datetime(self, data):
-        if self._needs_datetime_conversion(data):
-            try:
-                self._learn_has_multiple_timezones(data)
-                parsed_data = self._to_datetime(data)
-                return parsed_data
-
-            except ValueError as error:
-                self._raise_appropiate_conversion_error(error)
-
-        return data
+        self._has_multiple_timezones = data_has_multiple_timezones(
+            data, datetime_format=self.datetime_format
+        )
 
     def _needs_datetime_conversion(self, data):
         """Determines if the data requires datetime conversion."""
         return self.datetime_format is not None or not is_numeric_dtype(data)
 
-    def _to_datetime(self, data):
-        """Converts data to datetime using the instance's datetime format."""
-        datetime_format = self._get_pandas_datetime_format()
-        return pd.to_datetime(data, format=datetime_format, utc=self._has_multiple_timezones)
-
     def _get_pandas_datetime_format(self):
         """Converts the instance's datetime format to a pandas-compatible format."""
         if self.datetime_format:
@@ -109,18 +126,22 @@ class UnixTimestampEncoder(BaseTransformer):
 
         return None
 
-    def _warn_if_mixed_timezones(self):
-        if self._has_multiple_timezones:
-            warnings.warn(
-                'Mixed timezones are not supported in SDV Community. Data will be converted to UTC.'
-            )
+    def _to_datetime(self, data):
+        """Converts data to datetime using the instance's datetime format."""
+        return pd.to_datetime(
+            data,
+            format=self._get_pandas_datetime_format(),
+            utc=getattr(self, '_has_multiple_timezones', False),  # Backward compatibility
+        )
 
-    def _raise_appropiate_conversion_error(self, error):
-        message = str(error)
-        if 'Unknown string' in message or 'Unknown datetime string' in message:
-            raise TypeError('Data must be of dtype datetime, or castable to datetime.') from None
+    def _convert_to_datetime(self, data):
+        if self._needs_datetime_conversion(data):
+            try:
+                return self._to_datetime(data)
+            except ValueError as error:
+                self._raise_appropriate_conversion_error(error)
 
-        raise ValueError('Data does not match specified datetime format.') from None
+        return data
 
     def _transform_helper(self, datetimes):
         """Transform datetime values to integer."""
@@ -132,29 +153,6 @@ class UnixTimestampEncoder(BaseTransformer):
 
         return transformed
 
-    def _reverse_transform_helper(self, data):
-        """Transform integer values back into datetimes."""
-        if not isinstance(data, np.ndarray):
-            data = data.to_numpy()
-
-        data = self.null_transformer.reverse_transform(data)
-        data = np.round(data.astype(np.float64))
-        return data
-
-    def _learn_timezone_offest(self, data):
-        """Extracts and stores the timezone offset from the first valid datetime in the data."""
-        has_timezone = self.datetime_format and '%z' in self.datetime_format.lower()
-        if has_timezone and not self._has_multiple_timezones:
-            for val in data:
-                if pd.notna(val):
-                    try:
-                        dt = _safe_parse_datetime(str(val), warn=True)
-                        self._timezone_offset = dt.tzinfo
-                        break
-
-                    except (ValueError, TypeError, AttributeError):
-                        self._timezone_offset = None
-
     def _fit(self, data):
         """Fit the transformer to the data.
 
@@ -167,8 +165,8 @@ class UnixTimestampEncoder(BaseTransformer):
             datetime_array = data[data.notna()].astype(str).to_numpy()
             self.datetime_format = _guess_datetime_format_for_array(datetime_array)
 
+        self._learn_has_multiple_timezones(data)
         transformed = self._transform_helper(data)
-
         self._learn_timezone_offest(data)
         self._warn_if_mixed_timezones()
 
@@ -265,6 +263,15 @@ class UnixTimestampEncoder(BaseTransformer):
 
         return datetime_data.dt.strftime(self.datetime_format).astype(self._dtype)
 
+    def _reverse_transform_helper(self, data):
+        """Transform integer values back into datetimes."""
+        if not isinstance(data, np.ndarray):
+            data = data.to_numpy()
+
+        data = self.null_transformer.reverse_transform(data)
+        data = np.round(data.astype(np.float64))
+        return data
+
     def _reverse_transform(self, data):
         """Convert float values back to datetimes.
 
diff --git a/rdt/transformers/utils.py b/rdt/transformers/utils.py
index 59f84fe..75e902e 100644
--- a/rdt/transformers/utils.py
+++ b/rdt/transformers/utils.py
@@ -12,7 +12,7 @@ from decimal import Decimal
 import numpy as np
 import pandas as pd
 from dateutil import parser
-from dateutil.parser import UnknownTimezoneWarning
+from dateutil.parser import ParserError, UnknownTimezoneWarning
 from dateutil.tz import UTC
 
 import sre_parse  # isort:skip
@@ -401,7 +401,7 @@ def _extract_timezone_from_a_string(dt_str):
     return match.group(1) if match else None
 
 
-def _safe_parse_datetime(value, warn=False):
+def _safe_parse_datetime(value, warn=False, datetime_format=None):
     """Safely parse a value into a datetime object, handling invalid inputs.
 
     Converts the input `value` into a `datetime.datetime` object using `dateutil.parser.parse`.
@@ -416,6 +416,9 @@ def _safe_parse_datetime(value, warn=False):
         warn (bool):
             If True, warns when an unrecognized timezone is converted to UTC.
             Defaults to False.
+        datetime_format (str):
+            Format of the datetime string.
+            Defaults to None.
 
     Returns:
         datetime.datetime | pd.NaT:
@@ -427,7 +430,15 @@ def _safe_parse_datetime(value, warn=False):
 
     try:
         with warnings.catch_warnings(record=True) as captured_warnings:
-            dt = parser.parse(value)
+            try:
+                dt = parser.parse(value)
+
+            # Strings of large numbers cause parser.parse to overflow
+            except (OverflowError, ParserError) as error:
+                if not datetime_format:
+                    raise error
+                value = str(pd.to_datetime(value, format=datetime_format))
+                dt = parser.parse(value)
 
         if any(issubclass(warned.category, UnknownTimezoneWarning) for warned in captured_warnings):
             input_timezone = _extract_timezone_from_a_string(value)
@@ -441,7 +452,7 @@ def _safe_parse_datetime(value, warn=False):
 
         return dt
 
-    except (ValueError, TypeError, AttributeError):
+    except (ValueError, TypeError, AttributeError, OverflowError):
         return None
 
 
@@ -452,19 +463,24 @@ def _get_utc_offset(dt):
         return None
 
 
-def data_has_multiple_timezones(data):
+def data_has_multiple_timezones(data, datetime_format=None):
     """Check if a Series of datetime values contains multiple timezones.
 
     Args:
         data (pd.Series):
             Series of datetime strings or datetime objects.
+        datetime_format (str):
+            Format of the datetime string.
+            Defaults to None.
 
     Returns:
         bool:
             True if multiple timezones are detected, False otherwise.
     """
     try:
-        parsed_datetimes = data.apply(_safe_parse_datetime).dropna()
+        parsed_datetimes = data.apply(
+            _safe_parse_datetime, datetime_format=datetime_format
+        ).dropna()
         offsets = parsed_datetimes.apply(_get_utc_offset).dropna()
         return offsets.nunique() > 1
 


Test patch:
diff --git a/tests/integration/transformers/test_datetime.py b/tests/integration/transformers/test_datetime.py
index 4daf5ee..e857554 100644
--- a/tests/integration/transformers/test_datetime.py
+++ b/tests/integration/transformers/test_datetime.py
@@ -316,6 +316,68 @@ class TestUnixTimestampEncoder:
         pd.testing.assert_frame_equal(data, reverse_transformed)
         assert not is_datetime64tz_dtype(reverse_transformed['my_datetime_col'])
 
+    def test_datetime_strings_large_numbers(self):
+        """Ensure it runs for strings of large numbers."""
+        # Setup
+        data = pd.DataFrame({
+            'my_datetime_col': ['20220902110443000001 UTC', '20220902110443000000 UTC']
+        })
+        transformer = UnixTimestampEncoder(datetime_format='%Y%m%d%H%M%S%f %Z')
+
+        # Run
+        transformer.fit(data, column='my_datetime_col')
+        transformer.set_random_state(np.random.RandomState(42), 'reverse_transform')
+        transformed = transformer.transform(data)
+        reverse_transformed = transformer.reverse_transform(transformed)
+
+        # Assert
+        expected_data = data = pd.DataFrame({
+            'my_datetime_col': ['20220902110443000001 UTC', '20220902110443000000 UTC']
+        })
+        pd.testing.assert_frame_equal(expected_data, reverse_transformed)
+        assert not is_datetime64tz_dtype(reverse_transformed['my_datetime_col'])
+
+    def test_datetime_strings_large_numbers_no_timezone(self):
+        """Ensure it runs for strings of large numbers when no timezone is given."""
+        # Setup
+        data = pd.DataFrame({'my_datetime_col': ['20220902110443000001', '20220902110443000000']})
+        transformer = UnixTimestampEncoder(datetime_format='%Y%m%d%H%M%S%f')
+
+        # Run
+        transformer.fit(data, column='my_datetime_col')
+        transformer.set_random_state(np.random.RandomState(42), 'reverse_transform')
+        transformed = transformer.transform(data)
+        reverse_transformed = transformer.reverse_transform(transformed)
+
+        # Assert
+        expected_data = data = pd.DataFrame({
+            'my_datetime_col': ['20220902110443000001', '20220902110443000000']
+        })
+        pd.testing.assert_frame_equal(expected_data, reverse_transformed)
+        assert not is_datetime64tz_dtype(reverse_transformed['my_datetime_col'])
+
+    def test_datetime_strings_large_numbers_with_timezone_offset(self):
+        """Ensure it runs for strings of large numbers with timezone offset."""
+        # Setup
+        data = pd.DataFrame({
+            'my_datetime_col': ['20220902110443000001+0200', '20220902110443000000+0200']
+        })
+        transformer = UnixTimestampEncoder(datetime_format='%Y%m%d%H%M%S%f%z')
+
+        # Run
+        transformer.fit(data, column='my_datetime_col')
+        transformer.set_random_state(np.random.RandomState(42), 'reverse_transform')
+        transformed = transformer.transform(data)
+        reverse_transformed = transformer.reverse_transform(transformed)
+
+        # Assert
+        # The timezone offset is 2 hours, so the datetime should be 2 hours behind
+        expected_data = data = pd.DataFrame({
+            'my_datetime_col': ['20220902110443000001+0200', '20220902110443000000+0200']
+        })
+        pd.testing.assert_frame_equal(expected_data, reverse_transformed)
+        assert not is_datetime64tz_dtype(reverse_transformed['my_datetime_col'])
+
     def test_datetime_strings_with_timezone_offset(self):
         """Ensure datetime strings with timezone offsets (%z) are correctly parsed and normalized.
 
@@ -401,6 +463,34 @@ class TestUnixTimestampEncoder:
         assert not is_datetime64tz_dtype(reverse_transformed['my_datetime_col'])
         pd.testing.assert_frame_equal(data, reverse_transformed)
 
+    def test_datetime_objects_without_timezone_but_with_format(self):
+        """Ensure datetime objects without timezone are correctly handled when a format is given.
+
+        This test verifies that naive datetime64 values (i.e., without timezone info) are:
+            - Properly transformed and reverse-transformed without introducing any timezone.
+            - Preserved with a timezone-naive dtype (`datetime64[ns]`) throughout.
+            - Returned exactly as the original input after the reverse transformation.
+        """
+        # Setup
+        data = pd.DataFrame({
+            'my_datetime_col': pd.to_datetime([
+                '2025-02-04 08:32:21.123456',
+                '2025-01-13 08:32:21.123456',
+            ])
+        })
+        transformer = UnixTimestampEncoder(datetime_format='%Y-%m-%d %H:%M:%S.%f')
+
+        # Run
+        transformer.fit(data, column='my_datetime_col')
+        transformer.set_random_state(np.random.RandomState(42), 'reverse_transform')
+        transformed = transformer.transform(data)
+        reverse_transformed = transformer.reverse_transform(transformed)
+
+        # Assert
+        assert is_datetime64_ns_dtype(reverse_transformed['my_datetime_col'])
+        assert not is_datetime64tz_dtype(reverse_transformed['my_datetime_col'])
+        pd.testing.assert_frame_equal(data, reverse_transformed)
+
     def test_mixed_timezone_data(self):
         """Ensure that mixed timezone datetime values are converted to UTC during transformation.
 
diff --git a/tests/unit/transformers/test_datetime.py b/tests/unit/transformers/test_datetime.py
index b960c72..3f0753a 100644
--- a/tests/unit/transformers/test_datetime.py
+++ b/tests/unit/transformers/test_datetime.py
@@ -776,21 +776,46 @@ class TestUnixTimestampEncoder:
         # Assert
         assert instance._timezone_offset is None
 
-    @patch('rdt.transformers.datetime.data_has_multiple_timezones')
-    def test__learn_has_multiple_timezones_with_no_valid_timezones(self, mock_data_has_multiple_tz):
+    def test__learn_has_multiple_timezones_no_z_format(self):
+        """Test it when datetime format does not contain %z."""
+        # Setup
+        instance = UnixTimestampEncoder()
+        instance.datetime_format = '%Y-%m-%d %H:%M:%S'
+        data = pd.Series(['2025-01-01 00:00:00', '2025-01-01 00:00:00'])
+
+        # Run
+        result = UnixTimestampEncoder._learn_has_multiple_timezones(instance, data)
+
+        # Assert
+        assert instance._has_multiple_timezones is False
+        assert result is None
+
+    def test__learn_has_multiple_timezones_not_series(self):
+        """Test it when data is not a series."""
+        # Setup
+        instance = UnixTimestampEncoder()
+        data = pd.Index(['2025-01-01 00:00:00'], name='datetime')
+
+        # Run
+        result = UnixTimestampEncoder._learn_has_multiple_timezones(instance, data)
+
+        # Assert
+        assert instance._has_multiple_timezones is False
+        assert result is None
+
+    def test__learn_has_multiple_timezones_with_no_valid_timezones(self):
         """Test that `_learn_has_multiple_timezones` handles data with no valid timezones."""
         # Setup
         instance = UnixTimestampEncoder()
         instance.datetime_format = '%Y-%m-%d %H:%M:%S%z'
         data = pd.Series([None, 'not-a-date', '---'])
-        mock_data_has_multiple_tz.return_value = False
 
         # Run
-        UnixTimestampEncoder._learn_has_multiple_timezones(instance, data)
+        result = UnixTimestampEncoder._learn_has_multiple_timezones(instance, data)
 
         # Assert
         assert instance._has_multiple_timezones is False
-        mock_data_has_multiple_tz.assert_called_once_with(data)
+        assert result is None
 
     @patch('rdt.transformers.datetime.is_numeric_dtype')
     def test__needs_datetime_conversion_with_non_numeric_data(self, mock_pd_is_numeric_dtype):
diff --git a/tests/unit/transformers/test_utils.py b/tests/unit/transformers/test_utils.py
index f430828..b27f86a 100644
--- a/tests/unit/transformers/test_utils.py
+++ b/tests/unit/transformers/test_utils.py
@@ -548,20 +548,33 @@ def test__safe_parse_datetime():
     """
     # Setup
     str_input = '2023-01-01 12:00:00+0200'
+    str_input_format = '20220902110443000000'
+    str_empty = ''
     dt_input = datetime.datetime(2023, 1, 1, 12, 0)
     ts_input = pd.Timestamp(dt_input)
+    str_format_tz = '2023-10-15 14:30:00 XYZ'
 
     # Run
     res_str = _safe_parse_datetime(str_input)
+    res_str_format = _safe_parse_datetime(str_input_format, datetime_format='%Y%m%d%H%M%S%f')
+    res_str_format_wrong = _safe_parse_datetime(str_input_format, datetime_format='%Y%m%d%H%M%S')
     res_dt = _safe_parse_datetime(dt_input)
     res_ts = _safe_parse_datetime(ts_input)
     res_invalid = _safe_parse_datetime('not-a-date')
+    res_empty = _safe_parse_datetime(str_empty)
+    res_str_format_wrong_tz = _safe_parse_datetime(str_format_tz, datetime_format='%Y%m%d%H%M%S')
+    res_large_number_no_format = _safe_parse_datetime(str_input_format)
 
     # Assert
     assert res_str.isoformat() == '2023-01-01T12:00:00+02:00'
+    assert res_str_format.isoformat() == '2022-09-02T11:04:43'
+    assert res_str_format_wrong is None
     assert res_dt == dt_input
     assert res_ts == ts_input
     assert res_invalid is None
+    assert res_empty is None
+    assert res_str_format_wrong_tz.isoformat() == '2023-10-15T14:30:00+00:00'
+    assert res_large_number_no_format is None
 
 
 def test__safe_parse_datetime_with_unrecognized_timezone_and_warning():