Repository: alteryx/woodwork (python)

Issue:
Preserve the underlying pandas DataFrame's index in initialization of Woodwork data structure
- As a user, I would like Woodwork to preserve the underlying pandas DataFrame's index if I did not specify a specific `index` column to use. Right now, in WW 0.0.8 and 0.0.9, the index is replaced with `RangeIndex`. However, in EvalML, we want to preserve the user input's index. This is currently tricky to do without adding a new index column and then removing it.

Talked to @tamargrey about https://github.com/alteryx/woodwork/pull/464; we agreed a happy balance could be: 
- If user doesn't specify `index` parameter in initialization of WW DataTable, use underlying Index.
- If user later sets `index` and then removes it, `RangeIndex` is a good default.

#### Code Example

```python
X_df = pd.DataFrame([0, 1, 2, 3, 4], index=list(range(-5, 0)))
X_ww = ww.DataTable(X_df)
assert X_ww.to_dataframe().index == list(range(-5, 0))
```


Pull request:
Preserve underlying index
Closes #569 by preserving the underlying index when returning to a data frame.

Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index 403ea76..7ba0534 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -7,6 +7,7 @@ Release Notes
     * Fixes
     * Changes
         * Avoid calculating mutualinfo for non-unique columns (:pr:`563`)
+        * Preserve underlying DataFrame index if index column is not specified (:pr:`588`)
     * Documentation Changes
     * Testing Changes
         * Update branch reference in tests workflow (:pr:`552`, :pr:`601`)
@@ -14,7 +15,7 @@ Release Notes
         * Refactor test_datatable.py (:pr:`574`)
 
     Thanks to the following people for contributing to this release:
-    :user:`gsheni`, :user:`tamargrey`, :user:`johnbridstrup`
+    :user:`gsheni`, :user:`jeff-hernandez`, :user:`johnbridstrup`, :user:`tamargrey`
 
 **v0.0.9 February 5, 2021**
     * Enhancements
diff --git a/woodwork/datatable.py b/woodwork/datatable.py
index d9988c9..93b032d 100644
--- a/woodwork/datatable.py
+++ b/woodwork/datatable.py
@@ -321,6 +321,8 @@ class DataTable(object):
         if self.index is not None and index is None:
             updated_index_col = self.columns[self.index].remove_semantic_tags('index')
             self._update_columns({self.index: updated_index_col})
+            self._dataframe = self._dataframe.reset_index(drop=True)
+
         elif index is not None:
             _update_index(self, index, self.index)
         # Update the underlying index
@@ -374,21 +376,15 @@ class DataTable(object):
         '''Sets the index of a DataTable's underlying dataframe on pandas DataTables.
 
         If the DataTable has an index, will be set to that index.
-        If no index is specified and the DataFrame's index isn't a RangeIndex, will reset the DataFrame's index,
-        meaning that the index will be a pd.RangeIndex starting from zero.
+        If no index is specified, the DataFrame's index will remain the same.
         '''
         needs_update = False
         new_df = self._dataframe
-        if isinstance(self._dataframe, pd.DataFrame):
-            if self.index is not None:
-                needs_update = True
-                new_df = self._dataframe.set_index(self.index, drop=False)
-                # Drop index name to not overlap with the original column
-                new_df.index.name = None
-            # Only reset the index if the index isn't a RangeIndex
-            elif not isinstance(self._dataframe.index, pd.RangeIndex):
-                needs_update = True
-                new_df = self._dataframe.reset_index(drop=True)
+        if isinstance(self._dataframe, pd.DataFrame) and self.index is not None:
+            needs_update = True
+            new_df = self._dataframe.set_index(self.index, drop=False)
+            # Drop index name to not overlap with the original column
+            new_df.index.name = None
 
         self._dataframe = new_df
         return needs_update
@@ -403,6 +399,9 @@ class DataTable(object):
             woodwork.DataColumn: DataColumn including logical type and semantic tags.
         """
         col = self[column_name]
+        if 'index' in col.semantic_tags:
+            self._dataframe = self._dataframe.reset_index(drop=True)
+
         del self.columns[column_name]
         self._dataframe = self._dataframe.drop(column_name, axis=1)
         return col
@@ -423,7 +422,10 @@ class DataTable(object):
         if not_present:
             raise ValueError(f'{not_present} not found in DataTable')
 
-        return self._new_dt_from_cols([col for col in self._dataframe.columns if col not in columns])
+        dt = self._new_dt_from_cols([col for col in self._dataframe.columns if col not in columns])
+        if dt.index is None and self.index is not None:
+            dt._dataframe = dt._dataframe.reset_index(drop=True)
+        return dt
 
     def rename(self, columns):
         """Renames columns in a DataTable
@@ -534,6 +536,8 @@ class DataTable(object):
             cols_to_update[col_name] = col
 
         new_dt._update_columns(cols_to_update)
+        if new_dt.index is None and self.index is not None:
+            new_dt._dataframe = new_dt._dataframe.reset_index(drop=True)
         return new_dt
 
     def add_semantic_tags(self, semantic_tags):
@@ -563,7 +567,10 @@ class DataTable(object):
             woodwork.DataTable: DataTable with the specified semantic tags removed
         """
         _check_semantic_tags(self._dataframe, semantic_tags)
-        return self._update_cols_and_get_new_dt('remove_semantic_tags', semantic_tags)
+        dt = self._update_cols_and_get_new_dt('remove_semantic_tags', semantic_tags)
+        if dt.index is None and self.index is not None:
+            dt._dataframe = dt._dataframe.reset_index(drop=True)
+        return dt
 
     def reset_semantic_tags(self, columns=None, retain_index_tags=False):
         """Reset the semantic tags for the specified columns to the default values and
@@ -589,7 +596,10 @@ class DataTable(object):
                               f"dataframe: '{', '.join(cols_not_found)}'")
         if not columns:
             columns = self._dataframe.columns
-        return self._update_cols_and_get_new_dt('reset_semantic_tags', columns, retain_index_tags)
+        dt = self._update_cols_and_get_new_dt('reset_semantic_tags', columns, retain_index_tags)
+        if dt.index is None and self.index is not None:
+            dt._dataframe = dt._dataframe.reset_index(drop=True)
+        return dt
 
     def _update_cols_and_get_new_dt(self, method, new_values, *args):
         """Helper method that can be used for updating columns by calling the column method


Test patch:
diff --git a/woodwork/tests/datatable/test_datatable_get_set.py b/woodwork/tests/datatable/test_datatable_get_set.py
index 2ef068a..c261603 100644
--- a/woodwork/tests/datatable/test_datatable_get_set.py
+++ b/woodwork/tests/datatable/test_datatable_get_set.py
@@ -50,7 +50,7 @@ def test_datatable_getitem_list_input(sample_df):
     new_dt = dt[columns]
     assert new_dt is not dt
     assert new_dt.to_dataframe() is not df
-    pd.testing.assert_frame_equal(to_pandas(df[columns]).reset_index(drop=True), to_pandas(new_dt.to_dataframe()))
+    pd.testing.assert_frame_equal(to_pandas(df[columns]), to_pandas(new_dt.to_dataframe()))
     assert all(new_dt.to_dataframe().columns == ['age', 'full_name'])
     assert set(new_dt.columns.keys()) == {'age', 'full_name'}
     assert new_dt.index is None
diff --git a/woodwork/tests/datatable/test_datatable_indexing.py b/woodwork/tests/datatable/test_datatable_indexing.py
index b23a686..d2601fc 100644
--- a/woodwork/tests/datatable/test_datatable_indexing.py
+++ b/woodwork/tests/datatable/test_datatable_indexing.py
@@ -98,8 +98,8 @@ def test_underlying_index_no_index(sample_df):
     assert type(sample_df.index) == pd.Int64Index
     dt = DataTable(sample_df)
 
-    assert type(dt._dataframe.index) == pd.RangeIndex
-    assert type(dt.to_dataframe().index) == pd.RangeIndex
+    assert type(dt._dataframe.index) == pd.Int64Index
+    assert type(dt.to_dataframe().index) == pd.Int64Index
 
 
 def test_underlying_index(sample_df):
@@ -162,6 +162,34 @@ def test_underlying_index_on_update(sample_df):
     assert type(dt._dataframe.index) == pd.Int64Index
     assert type(dt.to_dataframe().index) == pd.Int64Index
 
+    actual = dt.iloc[[0, 1]]
+    assert type(actual._dataframe.index) == pd.Index
+    assert type(actual.to_dataframe().index) == pd.Index
+
+    actual = dt.select(dt.index)
+    assert type(actual._dataframe.index) == pd.Int64Index
+    assert type(actual.to_dataframe().index) == pd.Int64Index
+
+    actual = dt[['age']]
+    assert type(actual._dataframe.index) == pd.Int64Index
+    assert type(actual.to_dataframe().index) == pd.Int64Index
+
+    actual = dt.drop(dt.index)
+    assert type(actual._dataframe.index) == pd.RangeIndex
+    assert type(actual.to_dataframe().index) == pd.RangeIndex
+
+    actual = dt.reset_semantic_tags(retain_index_tags=False)
+    assert type(actual._dataframe.index) == pd.RangeIndex
+    assert type(actual.to_dataframe().index) == pd.RangeIndex
+
+    actual = dt.set_types(retain_index_tags=False, semantic_tags={'id': 'numeric'})
+    assert type(actual._dataframe.index) == pd.RangeIndex
+    assert type(actual.to_dataframe().index) == pd.RangeIndex
+
+    dt.pop(dt.index)
+    assert type(dt._dataframe.index) == pd.RangeIndex
+    assert type(dt.to_dataframe().index) == pd.RangeIndex
+
 
 def test_set_time_index(sample_df):
     # Test setting time index with set_time_index()