Repository: TileDB-Inc/TileDB-Py

Issue:
`open_dataframe` does not match original input to `from_pandas`
### Versions
TileDB: `0.6.5`
LibTileDB: `(2, 0, 6)`

### Description
When using `tiledb.from_pandas` and `tiledb.open_dataframe` back-to-back the dataframes do not match. This happens when `from_pandas` is passed a dataframe where `index.name` is unset. `from_pandas` creates the dimension "rows" (as no index name is present). However the metadata shows that `__pandas_index_dims` is `{"null": "int64"}`. 

### Possible solutions
1. `from_pandas` should set `__pandas_index_dims` to `{"rows": "int64"}`
2. `open_dataframe` should check if `"null"` is in the `index_dims` metadata and use "rows" dim as an unamed index and not a column

### Notes
IMO the use of `"rows"` as an index name should change to another _scoped_  name such as `__df_rows`

### Reproducible example
```python
import numpy as np
import pandas as pd
import pandas._testing as tm
import tiledb

df_orig = pd.DataFrame(np.random.randint(0, 100, size=(10, 4)), columns=['A', 'B', 'C', 'D'])

array_name = 'dataframe'
tiledb.from_pandas(array_name, df_orig)

df_read = tiledb.open_dataframe(array_name)

tm.assert_frame_equal(df_orig, df_read)
```

### Workaround
You can get around this by manually setting an index name

```python
df.index.name = "__my_index"
```


Discussion:

Resolving patch:
diff --git a/tiledb/dataframe_.py b/tiledb/dataframe_.py
--- a/tiledb/dataframe_.py
+++ b/tiledb/dataframe_.py
@@ -231,8 +231,11 @@ def dim_for_column(ctx, name, dim_info, col, tile=None, full_domain=False, ndim=
 def get_index_metadata(dataframe):
     md = dict()
     for index in dataframe.index.names:
+        index_md_name = index
+        if index == None:
+            index_md_name = '__tiledb_rows'
         # Note: this may be expensive.
-        md[index] = dtype_from_column(dataframe.index.get_level_values(index)).dtype
+        md[index_md_name] = dtype_from_column(dataframe.index.get_level_values(index)).dtype
 
     return md
 
@@ -263,7 +266,7 @@ def create_dims(ctx, dataframe, index_dims,
             name = index.name
         else:
             index_dtype = np.dtype('uint64')
-            name = 'rows'
+            name = '__tiledb_rows'
 
         index_dict[name] = index.values
 
@@ -343,7 +346,6 @@ def write_array_metadata(array, attr_metadata = None, index_metadata = None):
     if attr_metadata:
         attr_md_dict = {n: str(t) for n,t in attr_metadata.items()}
         array.meta['__pandas_attribute_repr'] = json.dumps(attr_md_dict)
-
     if index_metadata:
         index_md_dict = {n: str(t) for n,t in index_metadata.items()}
         array.meta['__pandas_index_dims'] = json.dumps(index_md_dict)
@@ -530,6 +532,7 @@ def _tiledb_result_as_dataframe(readable_array, result_dict):
         index_dims = json.loads(readable_array.meta['__pandas_index_dims'])
 
     indexes = list()
+    rename_cols = dict()
 
     for col_name, col_val in result_dict.items():
         if repr_meta and col_name in repr_meta:
@@ -538,7 +541,14 @@ def _tiledb_result_as_dataframe(readable_array, result_dict):
         elif index_dims and col_name in index_dims:
             new_col = pd.Series(col_val, dtype=index_dims[col_name])
             result_dict[col_name] = new_col
-            indexes.append(col_name)
+            if col_name == '__tiledb_rows':
+                rename_cols['__tiledb_rows'] = None
+                indexes.append(None)
+            else:
+                indexes.append(col_name)
+
+    for col_key,col_name in rename_cols.items():
+        result_dict[col_name] = result_dict.pop(col_key)
 
     df = pd.DataFrame.from_dict(result_dict)
     if len(indexes) > 0:


Test patch:
diff --git a/tiledb/tests/test_pandas_dataframe.py b/tiledb/tests/test_pandas_dataframe.py
--- a/tiledb/tests/test_pandas_dataframe.py
+++ b/tiledb/tests/test_pandas_dataframe.py
@@ -167,8 +167,7 @@ def test_dataframe_basic1(self):
         tiledb.from_dataframe(uri, df, sparse=False, ctx=ctx)
 
         # TODO tiledb.read_dataframe
-        with tiledb.open(uri) as B:
-            df_readback = pd.DataFrame.from_dict(B[:])
+        df_readback = tiledb.open_dataframe(uri)
 
         tm.assert_frame_equal(df, df_readback)
 
@@ -509,16 +508,11 @@ def test_csv_fillna(self):
         tiledb.from_csv(tmp_array, tmp_csv, fillna={'v': 0})
 
         def check_array(path, df):
-            with tiledb.open(path) as A:
-                res = A[:]
-                # check the value in the raw array
-                self.assertEqual(res['v'][4], 0)
+            # update the value in the original dataframe to match what we expect on read-back
+            df['v'][4] = 0
 
-                df_bk = pd.DataFrame(res)
-                df_bk.pop('rows')
-
-                # update the value in the original dataframe to match what we expect on read-back
-                df['v'][4] = 0
+            with tiledb.open(path) as A:
+                df_bk = A.df[:] #pd.DataFrame.from_dict(res)
                 tm.assert_frame_equal(df_bk, df)
 
         check_array(tmp_array, copy.deepcopy(df))