Repository: optuna/optuna

Issue:
Incompatibility with PySpark
Hola! 

I discovered that optuna somehow is not compatible with PySpark. There is also a related question on [SO] and temporary fix.

[SO]: https://stackoverflow.com/q/58183158/4952975

If there is `import pyspark` before `import optuna` (just one line) then it fails with `n_trials > 1`. If reverse (optuna before spark) everything works fine. 

Here is the code: 
```python
# import pyspark  # uncomment to have an error
import optuna


def objective(trial: optuna.Trial):
    param = trial.suggest_uniform('param', 0, 100)
    return param


study = optuna.create_study()
study.optimize(objective, n_trials=3, n_jobs=1)
```

When I run this code with commented `import pyspark` I have three successful trials (even when `import pyspark` is after optuna import). If I run with uncommented import then I have failed trials except the first one: 

```
[I 2019-10-02 15:40:05,742] Finished trial#0 resulted in value: 49.07164288419234. Current best value is 49.07164288419234 with parameters: {'param': 49.07164288419234}.
[W 2019-10-02 15:40:05,743] Setting status of trial#1 as TrialState.FAIL because of the following error: AttributeError("'_BaseUniformDistribution' object has no attribute 'to_internal_repr'")
Traceback (most recent call last):
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/study.py", line 448, in _run_trial
    result = func(trial)
  File "/Users/roseaysina/code/test.py", line 6, in objective
    param = trial.suggest_uniform('param', 0, 100)
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/trial.py", line 180, in suggest_uniform
    return self._suggest(name, distribution)
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/trial.py", line 453, in _suggest
    self.study, trial, name, distribution)
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/samplers/tpe/sampler.py", line 127, in sample_independent
    values, scores = _get_observation_pairs(study, param_name)
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/samplers/tpe/sampler.py", line 558, in _get_observation_pairs
    param_value = distribution.to_internal_repr(trial.params[param_name])
AttributeError: '_BaseUniformDistribution' object has no attribute 'to_internal_repr'
[W 2019-10-02 15:40:05,771] Setting status of trial#2 as TrialState.FAIL because of the following error: AttributeError("'_BaseUniformDistribution' object has no attribute 'to_internal_repr'")
Traceback (most recent call last):
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/study.py", line 448, in _run_trial
    result = func(trial)
  File "/Users/roseaysina/code/test.py", line 6, in objective
    param = trial.suggest_uniform('param', 0, 100)
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/trial.py", line 180, in suggest_uniform
    return self._suggest(name, distribution)
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/trial.py", line 453, in _suggest
    self.study, trial, name, distribution)
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/samplers/tpe/sampler.py", line 127, in sample_independent
    values, scores = _get_observation_pairs(study, param_name)
  File "/Users/roseaysina/anaconda3/envs/sauvage/lib/python3.7/site-packages/optuna/samplers/tpe/sampler.py", line 558, in _get_observation_pairs
    param_value = distribution.to_internal_repr(trial.params[param_name])
AttributeError: '_BaseUniformDistribution' object has no attribute 'to_internal_repr'
```

My environment: 
- Optuna version: `0.16.0`
- PySpark version: `2.4.4`
- Python version: `3.7.4`, Anaconda 3
- OS: macOS

Currently I am going to use fix with `RandomSampler` but I hope this strange (and almost uncatchable) bug will be fixed :) 

Thanks!





Discussion:

Resolving patch:
diff --git a/optuna/distributions.py b/optuna/distributions.py
--- a/optuna/distributions.py
+++ b/optuna/distributions.py
@@ -1,15 +1,14 @@
 import abc
 import json
 import six
-from typing import NamedTuple
-from typing import Tuple
-from typing import Union
 
 from optuna import type_checking
 
 if type_checking.TYPE_CHECKING:
     from typing import Any  # NOQA
     from typing import Dict  # NOQA
+    from typing import Tuple  # NOQA
+    from typing import Union  # NOQA
 
 
 @six.add_metaclass(abc.ABCMeta)
@@ -79,16 +78,37 @@ def _contains(self, param_value_in_internal_repr):
 
         raise NotImplementedError
 
-    @abc.abstractmethod
     def _asdict(self):
         # type: () -> Dict
 
-        raise NotImplementedError
+        return self.__dict__
+
+    def __eq__(self, other):
+        # type: (Any) -> bool
+
+        if not isinstance(other, type(self)):
+            return False
+
+        return self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        # type: (Any) -> bool
+
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        # type: () -> int
+
+        return hash(tuple(sorted(self.__dict__.items())))
+
+    def __repr__(self):
+        # type: () -> str
+
+        kwargs = ', '.join('{}={}'.format(k, v) for k, v in sorted(self.__dict__.items()))
+        return '{}({})'.format(self.__class__.__name__, kwargs)
 
 
-class UniformDistribution(
-        NamedTuple('_BaseUniformDistribution', [('low', float), ('high', float)]),
-        BaseDistribution):
+class UniformDistribution(BaseDistribution):
     """A uniform distribution in the linear domain.
 
     This object is instantiated by :func:`~optuna.trial.Trial.suggest_uniform`, and passed to
@@ -101,14 +121,15 @@ class UniformDistribution(
             Upper endpoint of the range of the distribution. ``high`` is excluded from the range.
     """
 
-    def __new__(cls, low, high):
-        # type: (float, float) -> UniformDistribution
+    def __init__(self, low, high):
+        # type: (float, float) -> None
 
         if low > high:
             raise ValueError("The `low` value must be smaller than or equal to the `high` value "
                              "(low={}, high={}).".format(low, high))
 
-        return super(UniformDistribution, cls).__new__(cls, low, high)
+        self.low = low
+        self.high = high
 
     def single(self):
         # type: () -> bool
@@ -125,9 +146,7 @@ def _contains(self, param_value_in_internal_repr):
             return self.low <= value and value < self.high
 
 
-class LogUniformDistribution(
-        NamedTuple('_BaseLogUniformDistribution', [('low', float), ('high', float)]),
-        BaseDistribution):
+class LogUniformDistribution(BaseDistribution):
     """A uniform distribution in the log domain.
 
     This object is instantiated by :func:`~optuna.trial.Trial.suggest_loguniform`, and passed to
@@ -140,14 +159,15 @@ class LogUniformDistribution(
             Upper endpoint of the range of the distribution. ``high`` is excluded from the range.
     """
 
-    def __new__(cls, low, high):
-        # type: (float, float) -> LogUniformDistribution
+    def __init__(self, low, high):
+        # type: (float, float) -> None
 
         if low > high:
             raise ValueError("The `low` value must be smaller than or equal to the `high` value "
                              "(low={}, high={}).".format(low, high))
 
-        return super(LogUniformDistribution, cls).__new__(cls, low, high)
+        self.low = low
+        self.high = high
 
     def single(self):
         # type: () -> bool
@@ -164,9 +184,7 @@ def _contains(self, param_value_in_internal_repr):
             return self.low <= value and value < self.high
 
 
-class DiscreteUniformDistribution(
-        NamedTuple('_BaseDiscreteUniformDistribution', [('low', float), ('high', float),
-                                                        ('q', float)]), BaseDistribution):
+class DiscreteUniformDistribution(BaseDistribution):
     """A discretized uniform distribution in the linear domain.
 
     This object is instantiated by :func:`~optuna.trial.Trial.suggest_discrete_uniform`, and passed
@@ -181,14 +199,16 @@ class DiscreteUniformDistribution(
             A discretization step.
     """
 
-    def __new__(cls, low, high, q):
-        # type: (float, float, float) -> DiscreteUniformDistribution
+    def __init__(self, low, high, q):
+        # type: (float, float, float) -> None
 
         if low > high:
             raise ValueError("The `low` value must be smaller than or equal to the `high` value "
                              "(low={}, high={}, q={}).".format(low, high, q))
 
-        return super(DiscreteUniformDistribution, cls).__new__(cls, low, high, q)
+        self.low = low
+        self.high = high
+        self.q = q
 
     def single(self):
         # type: () -> bool
@@ -202,9 +222,7 @@ def _contains(self, param_value_in_internal_repr):
         return self.low <= value and value <= self.high
 
 
-class IntUniformDistribution(
-        NamedTuple('_BaseIntUniformDistribution', [('low', int), ('high', int)]),
-        BaseDistribution):
+class IntUniformDistribution(BaseDistribution):
     """A uniform distribution on integers.
 
     This object is instantiated by :func:`~optuna.trial.Trial.suggest_int`, and passed to
@@ -217,14 +235,15 @@ class IntUniformDistribution(
             Upper endpoint of the range of the distribution. ``high`` is included in the range.
     """
 
-    def __new__(cls, low, high):
-        # type: (int, int) -> IntUniformDistribution
+    def __init__(self, low, high):
+        # type: (int, int) -> None
 
         if low > high:
             raise ValueError("The `low` value must be smaller than or equal to the `high` value "
                              "(low={}, high={}).".format(low, high))
 
-        return super(IntUniformDistribution, cls).__new__(cls, low, high)
+        self.low = low
+        self.high = high
 
     def to_external_repr(self, param_value_in_internal_repr):
         # type: (float) -> int
@@ -248,9 +267,7 @@ def _contains(self, param_value_in_internal_repr):
         return self.low <= value and value <= self.high
 
 
-class CategoricalDistribution(
-        NamedTuple('_BaseCategoricalDistribution', [('choices', Tuple[Union[float, str], ...])]),
-        BaseDistribution):
+class CategoricalDistribution(BaseDistribution):
     """A categorical distribution.
 
     This object is instantiated by :func:`~optuna.trial.Trial.suggest_categorical`, and
@@ -261,13 +278,13 @@ class CategoricalDistribution(
             Candidates of parameter values.
     """
 
-    def __new__(cls, choices):
-        # type: (Tuple[Union[float, str], ...]) -> CategoricalDistribution
+    def __init__(self, choices):
+        # type: (Tuple[Union[float, str], ...]) -> None
 
         if len(choices) == 0:
             raise ValueError("The `choices` must contains one or more elements.")
 
-        return super(CategoricalDistribution, cls).__new__(cls, choices)
+        self.choices = choices
 
     def to_external_repr(self, param_value_in_internal_repr):
         # type: (float) -> Union[float, str]


Test patch:
diff --git a/tests/test_distributions.py b/tests/test_distributions.py
--- a/tests/test_distributions.py
+++ b/tests/test_distributions.py
@@ -1,3 +1,4 @@
+import copy
 import json
 import pytest
 
@@ -64,19 +65,19 @@ def test_check_distribution_compatibility():
     # test dynamic value range (CategoricalDistribution)
     pytest.raises(
         ValueError, lambda: distributions.check_distribution_compatibility(
-            EXAMPLE_DISTRIBUTIONS['c2'], EXAMPLE_DISTRIBUTIONS['c2']._replace(
-                choice=('Roppongi', 'Akasaka'))))
+            EXAMPLE_DISTRIBUTIONS['c2'],
+            distributions.CategoricalDistribution(choices=('Roppongi', 'Akasaka'))))
 
     # test dynamic value range (except CategoricalDistribution)
     distributions.check_distribution_compatibility(
-        EXAMPLE_DISTRIBUTIONS['u'], EXAMPLE_DISTRIBUTIONS['u']._replace(low=-1.0, high=-2.0))
+        EXAMPLE_DISTRIBUTIONS['u'], distributions.UniformDistribution(low=-3.0, high=-2.0))
     distributions.check_distribution_compatibility(
-        EXAMPLE_DISTRIBUTIONS['l'], EXAMPLE_DISTRIBUTIONS['l']._replace(low=-0.1, high=1.0))
+        EXAMPLE_DISTRIBUTIONS['l'], distributions.LogUniformDistribution(low=-0.1, high=1.0))
     distributions.check_distribution_compatibility(
-        EXAMPLE_DISTRIBUTIONS['du'], EXAMPLE_DISTRIBUTIONS['du']._replace(
-            low=-1.0, high=10.0, q=3.))
+        EXAMPLE_DISTRIBUTIONS['du'],
+        distributions.DiscreteUniformDistribution(low=-1.0, high=10.0, q=3.))
     distributions.check_distribution_compatibility(
-        EXAMPLE_DISTRIBUTIONS['iu'], EXAMPLE_DISTRIBUTIONS['iu']._replace(low=-1, high=1))
+        EXAMPLE_DISTRIBUTIONS['iu'], distributions.IntUniformDistribution(low=-1, high=1))
 
 
 def test_contains():
@@ -185,3 +186,36 @@ def test_empty_distribution():
 
     with pytest.raises(ValueError):
         distributions.CategoricalDistribution(choices=()),
+
+
+def test_eq_ne_hash():
+    # type: () -> None
+
+    # Two instances of a class are regarded as equivalent if the fields have the same values.
+    for d in EXAMPLE_DISTRIBUTIONS.values():
+        assert d == copy.deepcopy(d)
+        assert hash(d) == hash(copy.deepcopy(d))
+
+    # Different field values.
+    d0 = distributions.UniformDistribution(low=1, high=2)
+    d1 = distributions.UniformDistribution(low=1, high=3)
+    assert d0 != d1
+    assert hash(d0) != hash(d1)
+
+    # Different classes.
+    d2 = distributions.IntUniformDistribution(low=1, high=2)
+    assert d0 != d2
+
+    # In the implementation of `__hash__`, only attributes are considered.
+    assert hash(d0) == hash(d2)
+
+
+def test_repr():
+    # type: () -> None
+
+    # The following variable is needed to apply `eval` to distribution
+    # instances that contain `float('inf')` as a field value.
+    inf = float('inf')  # NOQA
+
+    for d in EXAMPLE_DISTRIBUTIONS.values():
+        assert d == eval('distributions.' + repr(d))