Repository: pydata/xarray

Issue:
pandas-0.23 breaks stack with duplicated indices 
In this script:
```
import pandas
import xarray

df = pandas.DataFrame(
    [[1, 2], [3, 4]],
    index=['foo', 'foo'],
    columns=['bar', 'baz'])
print(df.stack())

a = xarray.DataArray(df)
print(a.stack(s=a.dims))
```

The first part works both with pandas 0.22 and 0.23.
The second part works in xarray 0.10. 4 + pandas 0.22, and crashes with pandas 0.23:

```
  File "/mnt/resource/tmp/anaconda_guido/lib/python3.6/site-packages/xarray/core/dataarray.py", line 1115, in stack
    ds = self._to_temp_dataset().stack(**dimensions)
  File "/mnt/resource/tmp/anaconda_guido/lib/python3.6/site-packages/xarray/core/dataset.py", line 2123, in stack
    result = result._stack_once(dims, new_dim)
  File "/mnt/resource/tmp/anaconda_guido/lib/python3.6/site-packages/xarray/core/dataset.py", line 2092, in _stack_once
    idx = utils.multiindex_from_product_levels(levels, names=dims)
  File "/mnt/resource/tmp/anaconda_guido/lib/python3.6/site-packages/xarray/core/utils.py", line 96, in multiindex_from_product_levels
    return pd.MultiIndex(levels, labels, sortorder=0, names=names)
  File "/mnt/resource/tmp/anaconda_guido/lib/python3.6/site-packages/pandas/core/indexes/multi.py", line 240, in __new__
    result._verify_integrity()
  File "/mnt/resource/tmp/anaconda_guido/lib/python3.6/site-packages/pandas/core/indexes/multi.py", line 283, in _verify_integrity
    level=i))
ValueError: Level values must be unique: ['foo', 'foo'] on level 0
```

Discussion:
jhamman: LGTM. @crusaderky, mind giving look to see if it resolves the bug you reported?

Patch:
diff --git a/doc/computation.rst b/doc/computation.rst
index 0f22a2ed..6793e667 100644
--- a/doc/computation.rst
+++ b/doc/computation.rst
@@ -185,7 +185,7 @@ windowed rolling, convolution, short-time FFT etc.
 
 Because the ``DataArray`` given by ``r.construct('window_dim')`` is a view
 of the original array, it is memory efficient.
-You can also use ``construct`` to compute a weighted rolling mean:
+You can also use ``construct`` to compute a weighted rolling sum:
 
 .. ipython:: python
 
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
index d9f43fa1..055369f0 100644
--- a/doc/whats-new.rst
+++ b/doc/whats-new.rst
@@ -59,12 +59,20 @@ Bug fixes
   dimension were improperly skipped.
   By `Stephan Hoyer <https://github.com/shoyer>`_
 
+- Fix :meth:`~DataArray.stack` with non-unique coordinates on pandas 0.23
+  (:issue:`2160`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_
+
 - Selecting data indexed by a length-1 ``CFTimeIndex`` with a slice of strings
   now behaves as it does when using a length-1 ``DatetimeIndex`` (i.e. it no
   longer falsely returns an empty array when the slice includes the value in
   the index) (:issue:`2165`).
   By `Spencer Clark <https://github.com/spencerkclark>`_.
-  
+
+- Fix Dataset.to_netcdf() cannot create group with engine="h5netcdf"
+  (:issue:`2177`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_
+
 .. _whats-new.0.10.4:
 
 v0.10.4 (May 16, 2018)
diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py
index f9e2b3de..6b3cd9eb 100644
--- a/xarray/backends/h5netcdf_.py
+++ b/xarray/backends/h5netcdf_.py
@@ -12,7 +12,7 @@ from .common import (
     HDF5_LOCK, DataStorePickleMixin, WritableCFDataStore, find_root)
 from .netCDF4_ import (
     BaseNetCDF4Array, _encode_nc4_variable, _extract_nc4_variable_encoding,
-    _get_datatype, _nc4_group)
+    _get_datatype, _nc4_require_group)
 
 
 class H5NetCDFArrayWrapper(BaseNetCDF4Array):
@@ -57,11 +57,16 @@ _extract_h5nc_encoding = functools.partial(
     lsd_okay=False, h5py_okay=True, backend='h5netcdf')
 
 
+def _h5netcdf_create_group(dataset, name):
+    return dataset.create_group(name)
+
+
 def _open_h5netcdf_group(filename, mode, group):
     import h5netcdf
     ds = h5netcdf.File(filename, mode=mode)
     with close_on_error(ds):
-        return _nc4_group(ds, group, mode)
+        return _nc4_require_group(
+            ds, group, mode, create_group=_h5netcdf_create_group)
 
 
 class H5NetCDFStore(WritableCFDataStore, DataStorePickleMixin):
diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py
index 14061a0f..5391a890 100644
--- a/xarray/backends/netCDF4_.py
+++ b/xarray/backends/netCDF4_.py
@@ -108,7 +108,11 @@ def _nc4_dtype(var):
     return dtype
 
 
-def _nc4_group(ds, group, mode):
+def _netcdf4_create_group(dataset, name):
+    return dataset.createGroup(name)
+
+
+def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
     if group in set([None, '', '/']):
         # use the root group
         return ds
@@ -123,7 +127,7 @@ def _nc4_group(ds, group, mode):
                 ds = ds.groups[key]
             except KeyError as e:
                 if mode != 'r':
-                    ds = ds.createGroup(key)
+                    ds = create_group(ds, key)
                 else:
                     # wrap error to provide slightly more helpful message
                     raise IOError('group not found: %s' % key, e)
@@ -210,7 +214,7 @@ def _open_netcdf4_group(filename, mode, group=None, **kwargs):
     ds = nc4.Dataset(filename, mode=mode, **kwargs)
 
     with close_on_error(ds):
-        ds = _nc4_group(ds, group, mode)
+        ds = _nc4_require_group(ds, group, mode)
 
     _disable_auto_decode_group(ds)
 
diff --git a/xarray/core/utils.py b/xarray/core/utils.py
index 06bb3ede..f6c5830c 100644
--- a/xarray/core/utils.py
+++ b/xarray/core/utils.py
@@ -76,13 +76,12 @@ def safe_cast_to_index(array):
 def multiindex_from_product_levels(levels, names=None):
     """Creating a MultiIndex from a product without refactorizing levels.
 
-    Keeping levels the same is faster, and also gives back the original labels
-    when we unstack.
+    Keeping levels the same gives back the original labels when we unstack.
 
     Parameters
     ----------
-    levels : sequence of arrays
-        Unique labels for each level.
+    levels : sequence of pd.Index
+        Values for each MultiIndex level.
     names : optional sequence of objects
         Names for each level.
 
@@ -90,8 +89,11 @@ def multiindex_from_product_levels(levels, names=None):
     -------
     pandas.MultiIndex
     """
-    labels_mesh = np.meshgrid(*[np.arange(len(lev)) for lev in levels],
-                              indexing='ij')
+    if any(not isinstance(lev, pd.Index) for lev in levels):
+        raise TypeError('levels must be a list of pd.Index objects')
+
+    split_labels, levels = zip(*[lev.factorize() for lev in levels])
+    labels_mesh = np.meshgrid(*split_labels, indexing='ij')
     labels = [x.ravel() for x in labels_mesh]
     return pd.MultiIndex(levels, labels, sortorder=0, names=names)
 


Test patch:
diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index 513f5f08..0768a942 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -892,7 +892,7 @@ class BaseNetCDF4Test(CFEncodedDataTest):
 
             # check equivalent ways to specify group
             for group in 'foo', '/foo', 'foo/', '/foo/':
-                with open_dataset(tmp_file, group=group) as actual:
+                with self.open(tmp_file, group=group) as actual:
                     assert_equal(actual['x'], expected['x'])
 
             # check that missing group raises appropriate exception
@@ -920,18 +920,18 @@ class BaseNetCDF4Test(CFEncodedDataTest):
 
             # check equivalent ways to specify group
             for group in 'foo/bar', '/foo/bar', 'foo/bar/', '/foo/bar/':
-                with open_dataset(tmp_file, group=group) as actual:
+                with self.open(tmp_file, group=group) as actual:
                     assert_equal(actual['x'], expected['x'])
 
     def test_write_groups(self):
         data1 = create_test_data()
         data2 = data1 * 2
         with create_tmp_file() as tmp_file:
-            data1.to_netcdf(tmp_file, group='data/1')
-            data2.to_netcdf(tmp_file, group='data/2', mode='a')
-            with open_dataset(tmp_file, group='data/1') as actual1:
+            self.save(data1, tmp_file, group='data/1')
+            self.save(data2, tmp_file, group='data/2', mode='a')
+            with self.open(tmp_file, group='data/1') as actual1:
                 assert_identical(data1, actual1)
-            with open_dataset(tmp_file, group='data/2') as actual2:
+            with self.open(tmp_file, group='data/2') as actual2:
                 assert_identical(data2, actual2)
 
     def test_roundtrip_string_with_fill_value_vlen(self):
diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py
index 35e270f0..a03d265c 100644
--- a/xarray/tests/test_dataarray.py
+++ b/xarray/tests/test_dataarray.py
@@ -1673,6 +1673,13 @@ class TestDataArray(TestCase):
         actual = DataArray(s, dims='z').unstack('z')
         assert_identical(expected, actual)
 
+    def test_stack_nonunique_consistency(self):
+        orig = DataArray([[0, 1], [2, 3]], dims=['x', 'y'],
+                         coords={'x': [0, 1], 'y': [0, 0]})
+        actual = orig.stack(z=['x', 'y'])
+        expected = DataArray(orig.to_pandas().stack(), dims='z')
+        assert_identical(expected, actual)
+
     def test_transpose(self):
         assert_equal(self.dv.variable.transpose(),
                      self.dv.transpose().variable)
diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py
index 0b3b0ee7..1f73743d 100644
--- a/xarray/tests/test_utils.py
+++ b/xarray/tests/test_utils.py
@@ -72,7 +72,8 @@ def test_safe_cast_to_index_datetime_datetime(enable_cftimeindex):
 
 
 def test_multiindex_from_product_levels():
-    result = utils.multiindex_from_product_levels([['b', 'a'], [1, 3, 2]])
+    result = utils.multiindex_from_product_levels(
+        [pd.Index(['b', 'a']), pd.Index([1, 3, 2])])
     np.testing.assert_array_equal(
         result.labels, [[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]])
     np.testing.assert_array_equal(result.levels[0], ['b', 'a'])
@@ -82,6 +83,15 @@ def test_multiindex_from_product_levels():
     np.testing.assert_array_equal(result.values, other.values)
 
 
+def test_multiindex_from_product_levels_non_unique():
+    result = utils.multiindex_from_product_levels(
+        [pd.Index(['b', 'a']), pd.Index([1, 1, 2])])
+    np.testing.assert_array_equal(
+        result.labels, [[0, 0, 0, 1, 1, 1], [0, 0, 1, 0, 0, 1]])
+    np.testing.assert_array_equal(result.levels[0], ['b', 'a'])
+    np.testing.assert_array_equal(result.levels[1], [1, 2])
+
+
 class TestArrayEquiv(TestCase):
     def test_0d(self):
         # verify our work around for pd.isnull not working for 0-dimensional