Repository: pandas-dev/pandas

Issue:
BUG: pd.IntervalIndex.difference raises when empty difference
#### Code Sample, a copy-pastable example if possible

```python
>>> ii1 = pd.IntervalIndex.from_breaks([0,1,2])
>>> ii2 = pd.IntervalIndex.from_breaks([0,1])
>>> ii2.difference(ii1)
TypeError: category, object, and string subtypes are not supported for IntervalIndex
```
#### Problem description

A test in my PR #19021 that passed a few days ago, is now failing. The issue is  AFAIKT that ``IntervalIndex.difference`` incorrectly casts an empy difference as having dtype ``object``, where it should have dtype ``Interval[int64]``.

xref #19022, that seems to have surfaced if not caused this bug.

#### Expected Output
Expected was ``IntervalIndex([] closed='right', dtype='interval[int64]')``.

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: 26e7419e7de6d91981bd905e1585850a19f0a3c2
python: 3.6.3.final.0
python-bits: 32
OS: Windows
OS-release: 10
machine: AMD64
processor: Intel64 Family 6 Model 78 Stepping 3, GenuineIntel
byteorder: little
LC_ALL: None
LANG: None
LOCALE: None.None

pandas: 0.22.0.dev0+511.g26e7419
pytest: 3.3.1
pip: 9.0.1
setuptools: 38.2.5
Cython: 0.26.1
numpy: 1.13.3
scipy: 1.0.0
pyarrow: None
xarray: None
IPython: 6.2.1
sphinx: 1.6.3
patsy: 0.4.1
dateutil: 2.6.1
pytz: 2017.3
blosc: None
bottleneck: None
tables: None
numexpr: None
feather: None
matplotlib: 2.1.0
openpyxl: 2.4.9
xlrd: 1.1.0
xlwt: 1.3.0
xlsxwriter: None
lxml: None
bs4: None
html5lib: 1.0b10
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: 2.9.6
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None

</details>

  
  


Discussion:
Thanks, this appears to happen with any type of set operation that returns empty:
```python
In [15]: ii1 = pd.interval_range(0, 4)

In [16]: ii2 = pd.interval_range(5, 10)

In [17]: ii1.intersection(ii2)
---------------------------------------------------------------------------
TypeError: category, object, and string subtypes are not supported for IntervalIndex
```

Under the hood, `IntervalIndex` uses `MultiIndex` set ops, and then takes the resulting array of tuples to build the result.  When the result is empty, an empty `object` dtype array is returned by `MultiIndex`, so the inferred dtype is `object`, which is then rejected upon `IntervalIndex` creation.

Will submit a fix either later today or tomorrow.
It looks like there was a related bug in previous versions where dtype wasn't maintained for empty results:

```python
In [2]: pd.__version__
Out[2]: '0.22.0'

In [3]: ii = pd.interval_range(0, 4)

In [4]: ii.dtype
Out[4]: interval[int64]

In [5]: ii.difference(ii)
Out[5]:
IntervalIndex([]
              closed='right',
              dtype='interval[float64]')
```
Seems like the returned dtype should match what a non-empty result would have.

Resolving patch:
diff --git a/pandas/core/indexes/interval.py b/pandas/core/indexes/interval.py
--- a/pandas/core/indexes/interval.py
+++ b/pandas/core/indexes/interval.py
@@ -5,7 +5,7 @@
 from pandas.core.dtypes.missing import notna, isna
 from pandas.core.dtypes.generic import ABCDatetimeIndex, ABCPeriodIndex
 from pandas.core.dtypes.dtypes import IntervalDtype
-from pandas.core.dtypes.cast import maybe_convert_platform
+from pandas.core.dtypes.cast import maybe_convert_platform, find_common_type
 from pandas.core.dtypes.common import (
     _ensure_platform_int,
     is_list_like,
@@ -16,6 +16,7 @@
     is_integer_dtype,
     is_float_dtype,
     is_interval_dtype,
+    is_object_dtype,
     is_scalar,
     is_float,
     is_number,
@@ -1289,9 +1290,25 @@ def func(self, other):
             msg = ('can only do set operations between two IntervalIndex '
                    'objects that are closed on the same side')
             other = self._as_like_interval_index(other, msg)
+
+            # GH 19016: ensure set op will not return a prohibited dtype
+            subtypes = [self.dtype.subtype, other.dtype.subtype]
+            common_subtype = find_common_type(subtypes)
+            if is_object_dtype(common_subtype):
+                msg = ('can only do {op} between two IntervalIndex '
+                       'objects that have compatible dtypes')
+                raise TypeError(msg.format(op=op_name))
+
             result = getattr(self._multiindex, op_name)(other._multiindex)
             result_name = self.name if self.name == other.name else None
-            return type(self).from_tuples(result.values, closed=self.closed,
+
+            # GH 19101: ensure empty results have correct dtype
+            if result.empty:
+                result = result.values.astype(self.dtype.subtype)
+            else:
+                result = result.values
+
+            return type(self).from_tuples(result, closed=self.closed,
                                           name=result_name)
         return func
 


Test patch:
diff --git a/pandas/tests/indexes/interval/test_interval.py b/pandas/tests/indexes/interval/test_interval.py
--- a/pandas/tests/indexes/interval/test_interval.py
+++ b/pandas/tests/indexes/interval/test_interval.py
@@ -880,6 +880,16 @@ def test_union(self, closed):
         tm.assert_index_equal(index.union(index), index)
         tm.assert_index_equal(index.union(index[:1]), index)
 
+        # GH 19101: empty result, same dtype
+        index = IntervalIndex(np.array([], dtype='int64'), closed=closed)
+        result = index.union(index)
+        tm.assert_index_equal(result, index)
+
+        # GH 19101: empty result, different dtypes
+        other = IntervalIndex(np.array([], dtype='float64'), closed=closed)
+        result = index.union(other)
+        tm.assert_index_equal(result, index)
+
     def test_intersection(self, closed):
         index = self.create_index(closed=closed)
         other = IntervalIndex.from_breaks(range(5, 13), closed=closed)
@@ -893,14 +903,48 @@ def test_intersection(self, closed):
 
         tm.assert_index_equal(index.intersection(index), index)
 
+        # GH 19101: empty result, same dtype
+        other = IntervalIndex.from_breaks(range(300, 314), closed=closed)
+        expected = IntervalIndex(np.array([], dtype='int64'), closed=closed)
+        result = index.intersection(other)
+        tm.assert_index_equal(result, expected)
+
+        # GH 19101: empty result, different dtypes
+        breaks = np.arange(300, 314, dtype='float64')
+        other = IntervalIndex.from_breaks(breaks, closed=closed)
+        result = index.intersection(other)
+        tm.assert_index_equal(result, expected)
+
     def test_difference(self, closed):
         index = self.create_index(closed=closed)
         tm.assert_index_equal(index.difference(index[:1]), index[1:])
 
+        # GH 19101: empty result, same dtype
+        result = index.difference(index)
+        expected = IntervalIndex(np.array([], dtype='int64'), closed=closed)
+        tm.assert_index_equal(result, expected)
+
+        # GH 19101: empty result, different dtypes
+        other = IntervalIndex.from_arrays(index.left.astype('float64'),
+                                          index.right, closed=closed)
+        result = index.difference(other)
+        tm.assert_index_equal(result, expected)
+
     def test_symmetric_difference(self, closed):
-        idx = self.create_index(closed=closed)
-        result = idx[1:].symmetric_difference(idx[:-1])
-        expected = IntervalIndex([idx[0], idx[-1]])
+        index = self.create_index(closed=closed)
+        result = index[1:].symmetric_difference(index[:-1])
+        expected = IntervalIndex([index[0], index[-1]])
+        tm.assert_index_equal(result, expected)
+
+        # GH 19101: empty result, same dtype
+        result = index.symmetric_difference(index)
+        expected = IntervalIndex(np.array([], dtype='int64'), closed=closed)
+        tm.assert_index_equal(result, expected)
+
+        # GH 19101: empty result, different dtypes
+        other = IntervalIndex.from_arrays(index.left.astype('float64'),
+                                          index.right, closed=closed)
+        result = index.symmetric_difference(other)
         tm.assert_index_equal(result, expected)
 
     @pytest.mark.parametrize('op_name', [
@@ -909,17 +953,25 @@ def test_set_operation_errors(self, closed, op_name):
         index = self.create_index(closed=closed)
         set_op = getattr(index, op_name)
 
-        # test errors
+        # non-IntervalIndex
         msg = ('can only do set operations between two IntervalIndex objects '
                'that are closed on the same side')
         with tm.assert_raises_regex(ValueError, msg):
             set_op(Index([1, 2, 3]))
 
+        # mixed closed
         for other_closed in {'right', 'left', 'both', 'neither'} - {closed}:
             other = self.create_index(closed=other_closed)
             with tm.assert_raises_regex(ValueError, msg):
                 set_op(other)
 
+        # GH 19016: incompatible dtypes
+        other = interval_range(Timestamp('20180101'), periods=9, closed=closed)
+        msg = ('can only do {op} between two IntervalIndex objects that have '
+               'compatible dtypes').format(op=op_name)
+        with tm.assert_raises_regex(TypeError, msg):
+            set_op(other)
+
     def test_isin(self, closed):
         index = self.create_index(closed=closed)