Repository: TileDB-Inc/TileDB-Py

Issue:
Error when storing floats with negative values
Trying to store a pandas dataframe as a tiledb object but running into issues here. Not sure how to exactly debug this. 

```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/var/folders/qq/2xnllcyd0yl3zwtcmyrnz3fr0000gn/T/ipykernel_53099/629258542.py in <module>
      1 import os
----> 2 tiledb.from_pandas(os.getcwd() + "/test.tdb", df)

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in from_pandas(uri, dataframe, **kwargs)
    410 
    411     with tiledb.scope_ctx(tiledb_args.get("ctx")):
--> 412         _from_pandas(uri, dataframe, tiledb_args)
    413 
    414 

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in _from_pandas(uri, dataframe, tiledb_args)
    463     with tiledb.scope_ctx(tiledb_args.get("ctx")):
    464         if create_array:
--> 465             _create_array(
    466                 uri,
    467                 dataframe,

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in _create_array(uri, df, sparse, full_domain, index_dims, column_infos, tiledb_args)
    493 
    494 def _create_array(uri, df, sparse, full_domain, index_dims, column_infos, tiledb_args):
--> 495     dims, dim_metadata = create_dims(
    496         df,
    497         index_dims,

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in create_dims(df, index_dims, tile, full_domain, filters)
    334         return dim_tile if dim_tile is not None else default_dim_tile
    335 
--> 336     dims = [
    337         dim_for_column(
    338             name,

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in <listcomp>(.0)
    335 
    336     dims = [
--> 337         dim_for_column(
    338             name,
    339             values,

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in dim_for_column(name, values, dtype, tile, full_domain, dim_filters)
    252     if np.issubdtype(dtype, np.integer) or dtype.kind == "M":
    253         # we can't make a tile larger than the dimension range or lower than 1
--> 254         tile = max(1, min(tile, np.uint64(dim_max - dim_min)))
    255     elif np.issubdtype(dtype, np.floating):
    256         # this difference can be inf

TypeError: unsupported operand type(s) for -: 'str' and 'str'
```
Error when storing floats with negative values
Trying to store a pandas dataframe as a tiledb object but running into issues here. Not sure how to exactly debug this. 

```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/var/folders/qq/2xnllcyd0yl3zwtcmyrnz3fr0000gn/T/ipykernel_53099/629258542.py in <module>
      1 import os
----> 2 tiledb.from_pandas(os.getcwd() + "/test.tdb", df)

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in from_pandas(uri, dataframe, **kwargs)
    410 
    411     with tiledb.scope_ctx(tiledb_args.get("ctx")):
--> 412         _from_pandas(uri, dataframe, tiledb_args)
    413 
    414 

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in _from_pandas(uri, dataframe, tiledb_args)
    463     with tiledb.scope_ctx(tiledb_args.get("ctx")):
    464         if create_array:
--> 465             _create_array(
    466                 uri,
    467                 dataframe,

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in _create_array(uri, df, sparse, full_domain, index_dims, column_infos, tiledb_args)
    493 
    494 def _create_array(uri, df, sparse, full_domain, index_dims, column_infos, tiledb_args):
--> 495     dims, dim_metadata = create_dims(
    496         df,
    497         index_dims,

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in create_dims(df, index_dims, tile, full_domain, filters)
    334         return dim_tile if dim_tile is not None else default_dim_tile
    335 
--> 336     dims = [
    337         dim_for_column(
    338             name,

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in <listcomp>(.0)
    335 
    336     dims = [
--> 337         dim_for_column(
    338             name,
    339             values,

~/opt/miniconda3/envs/sketch/lib/python3.9/site-packages/tiledb/dataframe_.py in dim_for_column(name, values, dtype, tile, full_domain, dim_filters)
    252     if np.issubdtype(dtype, np.integer) or dtype.kind == "M":
    253         # we can't make a tile larger than the dimension range or lower than 1
--> 254         tile = max(1, min(tile, np.uint64(dim_max - dim_min)))
    255     elif np.issubdtype(dtype, np.floating):
    256         # this difference can be inf

TypeError: unsupported operand type(s) for -: 'str' and 'str'
```


Discussion:
Hi @jkanche, thanks for opening the issue. Are you able to share a dataframe that reproduces this, or your `df.dtypes`?
the dataframe is generated from a 2-d numpy array that contains float/nan's all the columns are floats. Is there a way to add row and column names after generating a tiledb object from a numpy array ?

![image](https://user-images.githubusercontent.com/399324/138376570-239d4764-4bd4-4f4a-8b24-a24cdfb56c9a.png)

Hi @jkanche,

I put together a quick script to generate some data like that (collapsed below):

script:
<details>

```
#%%
import tiledb, numpy as np, pandas as pd
import uuid
import tempfile

def gen_array(sz):
    # generate random floats in [a,b)
    a,b = -1000, 1000
    data = (b - a) * np.random.default_rng().random(size=sz) + a
    # mask some to nan
    mask = np.random.choice([1,0], data.shape)
    data[mask] = np.nan
    return data

#%%
col_size = 1000
n_cols = 30

df = pd.DataFrame(
    {
        k: v for k,v in [(str(uuid.uuid4())[:12], gen_array(col_size)) for _ in range(n_cols)]
    })

uri = tempfile.mkdtemp()
tiledb.from_pandas(uri, df)

# %%
with tiledb.open(uri) as A:
    print(A.df[:])
# %%
```

</details>

However, I wasn't able to reproduce yet - I will keep trying. Just to double-check, what version of TileDB-Py are you using? I am testing with 0.10.3.

> Is there a way to add row and column names after generating a tiledb object from a numpy array

Sorry, not quite following -- is this referring to using `from_numpy`, or to slicing?
Hi @ihnorton if you slightly modify this snippet to add index names - 

```
df.index = [str(uuid.uuid4())[:12] for i in range(df.shape[0])]
df.head()
```

and try to create a tiledb object, you will run into the same error 

I am also on version 0.10.3
Hi @jkanche, reproduced - thanks. We'll get a fix for this in ASAP, but a workaround is to set the name of the index to a (possibly empty) string:

```
df.index = pd.Series([str(uuid.uuid4())[:12] for i in range(df.shape[0])], name="")
```

Full script working for me locally:

<details>

```
#%%
import tiledb, numpy as np, pandas as pd
import uuid
import tempfile

def gen_array(sz):
    # generate random floats in [a,b)
    a,b = -1000, 1000
    data = (b - a) * np.random.default_rng().random(size=sz) + a
    # mask some to nan
    mask = np.random.choice([1,0], data.shape)
    data[mask] = np.nan
    return data

#%%
col_size = 1000
n_cols = 30

df = pd.DataFrame(
    {
        k: v for k,v in [(str(uuid.uuid4())[:12], gen_array(col_size)) for _ in range(n_cols)]
    })

df.index = pd.Series([str(uuid.uuid4())[:12] for i in range(df.shape[0])], name="")
df.head()

#%%
uri = tempfile.mkdtemp()
tiledb.from_pandas(uri, df)

# %%
with tiledb.open(uri) as A:
    df2 = A.df[:]
    print(df2)
# %%

```

</details>
Hi @jkanche, thanks for opening the issue. Are you able to share a dataframe that reproduces this, or your `df.dtypes`?
the dataframe is generated from a 2-d numpy array that contains float/nan's all the columns are floats. Is there a way to add row and column names after generating a tiledb object from a numpy array ?

![image](https://user-images.githubusercontent.com/399324/138376570-239d4764-4bd4-4f4a-8b24-a24cdfb56c9a.png)

Hi @jkanche,

I put together a quick script to generate some data like that (collapsed below):

script:
<details>

```
#%%
import tiledb, numpy as np, pandas as pd
import uuid
import tempfile

def gen_array(sz):
    # generate random floats in [a,b)
    a,b = -1000, 1000
    data = (b - a) * np.random.default_rng().random(size=sz) + a
    # mask some to nan
    mask = np.random.choice([1,0], data.shape)
    data[mask] = np.nan
    return data

#%%
col_size = 1000
n_cols = 30

df = pd.DataFrame(
    {
        k: v for k,v in [(str(uuid.uuid4())[:12], gen_array(col_size)) for _ in range(n_cols)]
    })

uri = tempfile.mkdtemp()
tiledb.from_pandas(uri, df)

# %%
with tiledb.open(uri) as A:
    print(A.df[:])
# %%
```

</details>

However, I wasn't able to reproduce yet - I will keep trying. Just to double-check, what version of TileDB-Py are you using? I am testing with 0.10.3.

> Is there a way to add row and column names after generating a tiledb object from a numpy array

Sorry, not quite following -- is this referring to using `from_numpy`, or to slicing?
Hi @ihnorton if you slightly modify this snippet to add index names - 

```
df.index = [str(uuid.uuid4())[:12] for i in range(df.shape[0])]
df.head()
```

and try to create a tiledb object, you will run into the same error 

I am also on version 0.10.3
Hi @jkanche, reproduced - thanks. We'll get a fix for this in ASAP, but a workaround is to set the name of the index to a (possibly empty) string:

```
df.index = pd.Series([str(uuid.uuid4())[:12] for i in range(df.shape[0])], name="")
```

Full script working for me locally:

<details>

```
#%%
import tiledb, numpy as np, pandas as pd
import uuid
import tempfile

def gen_array(sz):
    # generate random floats in [a,b)
    a,b = -1000, 1000
    data = (b - a) * np.random.default_rng().random(size=sz) + a
    # mask some to nan
    mask = np.random.choice([1,0], data.shape)
    data[mask] = np.nan
    return data

#%%
col_size = 1000
n_cols = 30

df = pd.DataFrame(
    {
        k: v for k,v in [(str(uuid.uuid4())[:12], gen_array(col_size)) for _ in range(n_cols)]
    })

df.index = pd.Series([str(uuid.uuid4())[:12] for i in range(df.shape[0])], name="")
df.head()

#%%
uri = tempfile.mkdtemp()
tiledb.from_pandas(uri, df)

# %%
with tiledb.open(uri) as A:
    df2 = A.df[:]
    print(df2)
# %%

```

</details>

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -47,21 +47,11 @@
 
 # Set deployment target for mac
 #
-# Need to ensure thatextensions are built for macos 10.9 when compiling on a
-# 10.9 system or above, overriding distutils behaviour which is to target
-# the version used to build the current python binary.
-#
 # TO OVERRIDE:
 #   set MACOSX_DEPLOYMENT_TARGET before calling setup.py
-#
-# From https://github.com/pandas-dev/pandas/pull/24274
-# 3-Clause BSD License: https://github.com/pandas-dev/pandas/blob/master/LICENSE
 if sys.platform == "darwin":
     if "MACOSX_DEPLOYMENT_TARGET" not in os.environ:
-        current_system = LooseVersion(platform.mac_ver()[0])
-        python_target = LooseVersion(get_config_var("MACOSX_DEPLOYMENT_TARGET"))
-        if python_target < "10.9" and current_system >= "10.9":
-            os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.9"
+        os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.14"
 
 # Is this process building a wheel?
 WHEEL_BUILD = ("bdist_wheel" in sys.argv) or ("TILEDB_WHEEL_BUILD" in os.environ)
@@ -564,7 +554,7 @@ def install_requires():
 # Global variables
 CXXFLAGS = os.environ.get("CXXFLAGS", "").split()
 if not is_windows():
-    CXXFLAGS.append("-std=c++11")
+    CXXFLAGS.append("-std=c++17")
     if TILEDBPY_WERROR:
         CXXFLAGS.append("-Werror")
     if not TILEDB_DEBUG_BUILD:
diff --git a/tiledb/dataframe_.py b/tiledb/dataframe_.py
--- a/tiledb/dataframe_.py
+++ b/tiledb/dataframe_.py
@@ -291,6 +291,9 @@ def _sparse_from_dtypes(dtypes, sparse=None):
 
 
 def create_dims(df, index_dims, tile=None, full_domain=False, filters=None):
+    check_dataframe_deps()
+    import pandas as pd
+
     per_dim_tile = isinstance(tile, dict)
     if tile is not None:
         tile_values = tile.values() if per_dim_tile else (tile,)
@@ -313,13 +316,12 @@ def create_dims(df, index_dims, tile=None, full_domain=False, filters=None):
             raise ValueError(f"Unknown column or index named {name!r}")
 
         dtype = ColumnInfo.from_values(values).dtype
+        internal_dtype = dtype
+
+        if name == "__tiledb_rows" and isinstance(index, pd.RangeIndex):
+            internal_dtype = np.dtype("uint64")
         if name is None:
             name = "__tiledb_rows"
-            # force unnamed index to to uint64
-            # TODO: this looks iffy, check if we should we keep doing this
-            internal_dtype = np.dtype("uint64")
-        else:
-            internal_dtype = dtype
 
         dim_metadata[name] = dtype
         name_dtype_values.append((name, internal_dtype, values))
@@ -749,6 +751,9 @@ def from_csv(uri, csv_file, **kwargs):
             if df_list is None:
                 break
             df = pandas.concat(df_list)
+            if not "index_col" in tiledb_args and df.index.name is None:
+                df.index.name = "__tiledb_rows"
+
             tiledb_args["row_start_idx"] = rows_written
 
             from_pandas(uri, df, tiledb_args=tiledb_args, pandas_args=pandas_args)
@@ -765,6 +770,9 @@ def from_csv(uri, csv_file, **kwargs):
         df_iter = pandas.read_csv(input_csv, **pandas_args)
         df = next(df_iter, None)
         while df is not None:
+            if not "index_col" in tiledb_args and df.index.name is None:
+                df.index.name = "__tiledb_rows"
+
             # tell from_pandas what row to start the next write
             tiledb_args["row_start_idx"] = rows_written
 
@@ -777,6 +785,8 @@ def from_csv(uri, csv_file, **kwargs):
 
     else:
         df = pandas.read_csv(csv_file, **kwargs)
+        if not "index_col" in tiledb_args and df.index.name is None:
+            df.index.name = "__tiledb_rows"
 
         kwargs.update(tiledb_args)
         from_pandas(uri, df, **kwargs)


Test patch:
diff --git a/tiledb/tests/test_pandas_dataframe.py b/tiledb/tests/test_pandas_dataframe.py
--- a/tiledb/tests/test_pandas_dataframe.py
+++ b/tiledb/tests/test_pandas_dataframe.py
@@ -6,9 +6,11 @@
 import copy
 import glob
 import os
+from pathlib import Path
 import random
 import string
-from pathlib import Path
+import tempfile
+import uuid
 
 import numpy as np
 from numpy.testing import assert_array_equal
@@ -19,6 +21,7 @@
     DiskTestCase,
     checked_path,
     dtype_max,
+    dtype_min,
     rand_ascii,
     rand_ascii_bytes,
     rand_datetime64_array,
@@ -306,8 +309,8 @@ def test_dataframe_basic1(self):
 
         with tiledb.open(uri) as A:
             dim = A.schema.domain.dim(0)
-            assert dim.domain[0] == 0
-            assert dim.domain[1] == dtype_max(np.uint64) - dim.tile
+            assert dim.domain[0] == dtype_min(np.int64)
+            assert dim.domain[1] == dtype_max(np.int64) - dim.tile
 
         for use_arrow in None, False, True:
             df_readback = tiledb.open_dataframe(uri, use_arrow=use_arrow)
@@ -1291,3 +1294,40 @@ def assert_filters_eq(left, right):
                 tiledb.from_pandas(uri, df, **{opt: f})
                 with tiledb.open(uri) as A:
                     assert_filters_eq(getter(A), f)
+
+
+###############################################################################
+
+# Regression tests for specific bugs
+
+
+def test_write_unnamed_index_py755(checked_path):
+    """Test writing array with unnamed non-RangeIndex"""
+
+    def gen_array(sz):
+        # generate random floats in [a,b)
+        a, b = -1000, 1000
+        data = (b - a) * np.random.default_rng().random(size=sz) + a
+        # mask some to nan
+        mask = np.random.choice([1, 0], data.shape)
+        data[mask] = np.nan
+        return data
+
+    col_size = 10
+    n_cols = 9
+
+    df = pd.DataFrame(
+        {
+            k: v
+            for k, v in [
+                (str(uuid.uuid4())[:12], gen_array(col_size)) for _ in range(n_cols)
+            ]
+        }
+    )
+    df.index = pd.Series([str(uuid.uuid4())[:12] for i in range(df.shape[0])], name="")
+
+    uri = checked_path.path()
+    tiledb.from_pandas(uri, df)
+
+    with tiledb.open(uri) as A:
+        tm.assert_frame_equal(df.sort_index(), A.df[:])