Repository: TileDB-Inc/TileDB-Py

Issue:
Multidimensional variable length array write fails
I got an exception running the following code. I am using Python 3.6.6, TileDB version 1.5.1 and TileDB-Py version 0.4.2 on Linux.

```
import numpy as np
import tiledb

array_name = 'example_array'
ctx = tiledb.Ctx()

dom = tiledb.Domain(tiledb.Dim(name="rows", domain=(0, 1), tile=2, dtype=np.int64),
                    tiledb.Dim(name="cols", domain=(0, 1), tile=2, dtype=np.int64), ctx=ctx)

schema = tiledb.ArraySchema(domain=dom, sparse=False,
                            attrs=[tiledb.Attr(name="a", dtype='S', ctx=ctx)],
                            ctx=ctx)

tiledb.DenseArray.create(array_name, schema)

with tiledb.DenseArray(array_name, mode='w', ctx=ctx) as A:
    A[...] = np.array([['A', 'B'], ['A', 'B']], dtype='S')
```

```
Traceback (most recent call last):
  File "./tiledb_example.py", line 17, in <module>
    A[...] = np.array([['A', 'B'], ['A', 'B']], dtype='S')
  File "tiledb/libtiledb.pyx", line 4394, in tiledb.libtiledb.DenseArray.__setitem__
  File "tiledb/libtiledb.pyx", line 4465, in tiledb.libtiledb.DenseArray._write_dense_subarray
  File "tiledb/libtiledb.pyx", line 250, in tiledb.libtiledb._raise_ctx_err
  File "tiledb/libtiledb.pyx", line 235, in tiledb.libtiledb._raise_tiledb_error
tiledb.libtiledb.TileDBError: [TileDB::Writer] Error: Buffer sizes check failed; Invalid number of cells given for attribute 'a' (2 != 4)
```

It seems like 2D numpy string arrays don't work with multidimensional dense arrays. Looking at the call to `_pack_varlen_bytes(self, object val)`, the for loops that iterate through `val` (`np.ndarray arr`) don't loop through the individual elements in the multidimensional case. https://github.com/TileDB-Inc/TileDB-Py/blob/ac9ab44d63d54e3fcab22dad00a226dddb977573/tiledb/libtiledb.pyx#L3713-L3723 
https://github.com/TileDB-Inc/TileDB-Py/blob/ac9ab44d63d54e3fcab22dad00a226dddb977573/tiledb/libtiledb.pyx#L3750-L3758 Iterating through a multidimensional array returns the corresponding subarrays, not the individual elements. Therefore the buffer offsets don't correspond to individual elements, but subarrays of the original array. Do you think it should iterate through this array using `arr.flatiter`, which loops through the individual elements?


Multidimensional variable length array write fails
I got an exception running the following code. I am using Python 3.6.6, TileDB version 1.5.1 and TileDB-Py version 0.4.2 on Linux.

```
import numpy as np
import tiledb

array_name = 'example_array'
ctx = tiledb.Ctx()

dom = tiledb.Domain(tiledb.Dim(name="rows", domain=(0, 1), tile=2, dtype=np.int64),
                    tiledb.Dim(name="cols", domain=(0, 1), tile=2, dtype=np.int64), ctx=ctx)

schema = tiledb.ArraySchema(domain=dom, sparse=False,
                            attrs=[tiledb.Attr(name="a", dtype='S', ctx=ctx)],
                            ctx=ctx)

tiledb.DenseArray.create(array_name, schema)

with tiledb.DenseArray(array_name, mode='w', ctx=ctx) as A:
    A[...] = np.array([['A', 'B'], ['A', 'B']], dtype='S')
```

```
Traceback (most recent call last):
  File "./tiledb_example.py", line 17, in <module>
    A[...] = np.array([['A', 'B'], ['A', 'B']], dtype='S')
  File "tiledb/libtiledb.pyx", line 4394, in tiledb.libtiledb.DenseArray.__setitem__
  File "tiledb/libtiledb.pyx", line 4465, in tiledb.libtiledb.DenseArray._write_dense_subarray
  File "tiledb/libtiledb.pyx", line 250, in tiledb.libtiledb._raise_ctx_err
  File "tiledb/libtiledb.pyx", line 235, in tiledb.libtiledb._raise_tiledb_error
tiledb.libtiledb.TileDBError: [TileDB::Writer] Error: Buffer sizes check failed; Invalid number of cells given for attribute 'a' (2 != 4)
```

It seems like 2D numpy string arrays don't work with multidimensional dense arrays. Looking at the call to `_pack_varlen_bytes(self, object val)`, the for loops that iterate through `val` (`np.ndarray arr`) don't loop through the individual elements in the multidimensional case. https://github.com/TileDB-Inc/TileDB-Py/blob/ac9ab44d63d54e3fcab22dad00a226dddb977573/tiledb/libtiledb.pyx#L3713-L3723 
https://github.com/TileDB-Inc/TileDB-Py/blob/ac9ab44d63d54e3fcab22dad00a226dddb977573/tiledb/libtiledb.pyx#L3750-L3758 Iterating through a multidimensional array returns the corresponding subarrays, not the individual elements. Therefore the buffer offsets don't correspond to individual elements, but subarrays of the original array. Do you think it should iterate through this array using `arr.flatiter`, which loops through the individual elements?




Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,13 @@ def get_zipfile(url):
 # Use `setup.py [] --debug` for a debug build of libtiledb
 TILEDB_DEBUG_BUILD = False
 
-# ALlow to override TILEDB_FORCE_ALL_DEPS with environment variable
+# Use `setup.py [] --modular` for a modular build of libtiledb_py
+#   Each .pyx file will be built as a separate shared library for faster
+#   compilation. This is disabled by default to avoid distributing multiple
+#   shared libraries.
+TILEDBPY_MODULAR = False
+
+# Allow to override TILEDB_FORCE_ALL_DEPS with environment variable
 TILEDB_FORCE_ALL_DEPS = "TILEDB_FORCE_ALL_DEPS" in os.environ
 TILEDB_SERIALIZATION = "TILEDB_SERIALIZATION" in os.environ
 CMAKE_GENERATOR = os.environ.get("CMAKE_GENERATOR", None)
@@ -427,7 +433,6 @@ def setup_requires():
 LIB_DIRS = []
 LIBS = ["tiledb"]
 DEF_MACROS = []
-SOURCES = ["tiledb/libtiledb.pyx"]
 
 # Pass command line flags to setup.py script
 # handle --tiledb=[PATH] --lflags=[FLAGS] --cxxflags=[FLAGS]
@@ -445,7 +450,9 @@ def setup_requires():
     if arg.find('--debug') == 0:
         TILEDB_DEBUG_BUILD = True
         sys.argv.remove(arg)
-
+    if arg.find('--modular') == 0:
+        TILEDBPY_MODULAR = True
+        sys.argv.remove(arg)
 
 if TILEDB_PATH != '':
     LIB_DIRS += [os.path.join(TILEDB_PATH, 'lib')]
@@ -461,24 +468,48 @@ def setup_requires():
 with open('README.rst') as f:
     README_RST = f.read()
 
-cy_extension=Extension(
+
+__extensions = [
+  Extension(
     "tiledb.libtiledb",
     include_dirs=INC_DIRS,
     define_macros=DEF_MACROS,
-    sources=SOURCES,
+    sources=["tiledb/libtiledb.pyx"],
     library_dirs=LIB_DIRS,
     libraries=LIBS,
     extra_link_args=LFLAGS,
     extra_compile_args=CXXFLAGS,
     language="c++"
     )
+]
+
+MODULAR_SOURCES = [
+  'tiledb/np2buf.pyx'
+  ]
+
+if TILEDBPY_MODULAR:
+  for source in MODULAR_SOURCES:
+    module_name = os.path.splitext(os.path.split(source)[-1])[0]
+    ext = Extension(
+        "tiledb.{}".format(module_name),
+        include_dirs=INC_DIRS,
+        define_macros=DEF_MACROS,
+        sources=[source],
+        library_dirs=LIB_DIRS,
+        libraries=LIBS,
+        extra_link_args=LFLAGS,
+        extra_compile_args=CXXFLAGS,
+        language="c++"
+        )
+    __extensions.append(ext)
 
 # Helper to set Extension attributes correctly based on python version
 def ext_attr_update(attr, value):
-  if sys.version_info < (3,0):
-      cy_extension.__dict__[attr] = value
-  else:
-      cy_extension.__setattr__(attr, value)
+  for x in __extensions:
+    if sys.version_info < (3,0):
+        x.__dict__[attr] = value
+    else:
+        x.__setattr__(attr, value)
 
 # Monkey patches to be forwarded to cythonize
 # some of these will error out if passed directly
@@ -496,6 +527,9 @@ def ext_attr_update(attr, value):
 if not is_windows():
   ext_attr_update('runtime_library_dirs', LIB_DIRS)
 
+# This must always be set so the compile-time conditional has a value
+ext_attr_update('cython_compile_time_env', {'TILEDBPY_MODULAR': TILEDBPY_MODULAR})
+
 setup(
     name='tiledb',
     description="Pythonic interface to the TileDB array storage manager",
@@ -512,9 +546,7 @@ def ext_attr_update(attr, value):
         'local_scheme': 'dirty-tag',
         'write_to': 'tiledb/version.py'
     },
-    ext_modules=[
-      cy_extension
-    ],
+    ext_modules=__extensions,
     setup_requires=setup_requires(),
     install_requires=[
         numpy_required_version,
diff --git a/tiledb/__init__.py b/tiledb/__init__.py
--- a/tiledb/__init__.py
+++ b/tiledb/__init__.py
@@ -60,13 +60,20 @@
      walk,
      remove,
      move,
+     schema_like,
      stats_enable,
      stats_disable,
      stats_reset,
      stats_dump,
 )
 
-from .highlevel import *
+from .highlevel import (
+     open,
+     save,
+     from_numpy,
+     empty_like,
+     array_exists
+)
 
 # Note: we use a modified namespace packaging to allow continuity of existing TileDB-Py imports.
 #       Therefore, 'tiledb/__init__.py' must *only* exist in this package.


Test patch:
diff --git a/tiledb/tests/test_libtiledb.py b/tiledb/tests/test_libtiledb.py
--- a/tiledb/tests/test_libtiledb.py
+++ b/tiledb/tests/test_libtiledb.py
@@ -967,12 +967,15 @@ def test_complex_attributes(self):
         attr = tiledb.Attr(ctx=ctx, dtype=np.complex64)
         schema = tiledb.ArraySchema(ctx=ctx, domain=dom, attrs=(attr,))
         tiledb.DenseArray.create(self.path("foo"), schema)
+
         A = np.random.rand(20).astype(np.float32).view(dtype=np.complex64)
-        attr.dump()
+
+        self.assertEqual(schema, tiledb.schema_like(A, dim_dtype=int))
         self.assertEqual(A.dtype, attr.dtype)
 
         with tiledb.DenseArray(self.path("foo"), mode='w', ctx=ctx) as T:
             T[:] = A
+
         with tiledb.DenseArray(self.path("foo"), mode='r', ctx=ctx) as T:
             assert_array_equal(A, T[:])
             assert_array_equal(A[:5], T[:5])
@@ -1035,6 +1038,48 @@ def test_multiple_attributes(self):
             with self.assertRaises(tiledb.TileDBError):
                 T[:] = V
 
+    def test_array_2d_s1(self):
+        # This array is currently read back with dtype object
+        A = np.array([['A', 'B'], ['A', 'B']], dtype='S')
+
+        uri = self.path("varlen_2d_s1")
+        ctx = tiledb.Ctx()
+        dom = tiledb.Domain(tiledb.Dim(name="rows", domain=(0, 1), tile=2, dtype=np.int64),
+                            tiledb.Dim(name="cols", domain=(0, 1), tile=2, dtype=np.int64), ctx=ctx)
+
+        schema = tiledb.ArraySchema(domain=dom, sparse=False,
+                                    attrs=[tiledb.Attr(name="a", dtype='S', ctx=ctx)],
+                                    ctx=ctx)
+
+        tiledb.DenseArray.create(uri, schema)
+
+        with tiledb.DenseArray(uri, mode='w', ctx=ctx) as T:
+            T[...] = A
+
+        with tiledb.DenseArray(uri) as T:
+            assert_array_equal(A, T)
+
+    def test_array_2d_s3_mixed(self):
+        # This array is currently read back with dtype object
+        A = np.array([['AAA', 'B'], ['AB', 'C']], dtype='S3')
+
+        uri = self.path("varlen_2d_s1")
+        ctx = tiledb.Ctx()
+        dom = tiledb.Domain(tiledb.Dim(name="rows", domain=(0, 1), tile=2, dtype=np.int64),
+                            tiledb.Dim(name="cols", domain=(0, 1), tile=2, dtype=np.int64), ctx=ctx)
+
+        schema = tiledb.ArraySchema(domain=dom, sparse=False,
+                                    attrs=[tiledb.Attr(name="a", dtype='S3', ctx=ctx)],
+                                    ctx=ctx)
+
+        tiledb.DenseArray.create(uri, schema)
+
+        with tiledb.DenseArray(uri, mode='w', ctx=ctx) as T:
+            T[...] = A
+
+        with tiledb.DenseArray(uri) as T:
+            assert_array_equal(A, T)
+
 class DenseVarlen(DiskTestCase):
     def test_varlen_write_bytes(self):
         A = np.array(['aa','bbb','ccccc','ddddddddddddddddddddd','ee','ffffff','g','hhhhhhhhhh'], dtype=bytes)
@@ -1055,14 +1100,14 @@ def test_varlen_write_bytes(self):
 
 
     def test_varlen_write_unicode(self):
-        A = np.array(['aa','bbb','ccccc','ddddddddddddddddddddd',
-                      'ee','ffffff','g','hhhhhhhhhh'],
-                     dtype=np.unicode_)
+        A = np.array(['aa','bbb',
+                      'ccccc','ddddddddddddddddddddd',
+                      'ee','ffffff','g','hhhhhhhhhh'], dtype=np.unicode_)
 
         # basic write
         ctx = tiledb.Ctx()
         dom = tiledb.Domain(tiledb.Dim(domain=(1, len(A)), tile=len(A), ctx=ctx), ctx=ctx)
-        att = tiledb.Attr(dtype=np.unicode_, ctx=ctx)
+        att = tiledb.Attr(dtype=np.unicode_, var=True, ctx=ctx)
 
         schema = tiledb.ArraySchema(dom, (att,), ctx=ctx)
 
@@ -1073,8 +1118,8 @@ def test_varlen_write_unicode(self):
         with tiledb.DenseArray(self.path("foo"), mode='r', ctx=ctx) as T:
             assert_array_equal(A[:], T[:])
 
-
     def test_varlen_write_floats(self):
+        # Generates 8 variable-length float64 subarrays (subarray len and content are randomized)
         A = np.array([np.random.rand(x) for x in np.random.randint(1,12,8)], dtype=np.object)
 
         # basic write
@@ -1084,6 +1129,7 @@ def test_varlen_write_floats(self):
 
         schema = tiledb.ArraySchema(dom, (att,), ctx=ctx)
         tiledb.DenseArray.create(self.path("foo"), schema)
+
         with tiledb.DenseArray(self.path("foo"), mode='w', ctx=ctx) as T:
             T[:] = A
 
@@ -1093,8 +1139,30 @@ def test_varlen_write_floats(self):
             # can't use assert_array_equal w/ np.object array
             self.assertTrue(all(np.array_equal(x,A[i]) for i,x in enumerate(T_)))
 
+    def test_varlen_write_floats_2d(self):
+        A = np.array([np.random.rand(x) for x in np.arange(1,10)], dtype=np.object).reshape(3,3)
+
+        # basic write
+        ctx = tiledb.Ctx()
+        dom = tiledb.Domain(tiledb.Dim(domain=(1, 3), tile=len(A)),
+                            tiledb.Dim(domain=(1, 3), tile=len(A)),
+                            ctx=ctx)
+        att = tiledb.Attr(dtype=np.float64, var=True, ctx=ctx)
+
+        schema = tiledb.ArraySchema(dom, (att,), ctx=ctx)
+        tiledb.DenseArray.create(self.path("foo"), schema)
+
+        with tiledb.DenseArray(self.path("foo"), mode='w', ctx=ctx) as T:
+            T[:] = A
+
+        with tiledb.DenseArray(self.path("foo"), mode='r', ctx=ctx) as T:
+            T_ = T[:]
+            self.assertEqual(len(A), len(T_))
+            # can't use assert_array_equal w/ np.object array
+            self.assertTrue(np.all([np.array_equal(A.flat[i], T[:].flat[i]) for i in np.arange(0, 9)]))
 
     def test_varlen_write_fixedbytes(self):
+        # The actual dtype of this array is 'S21'
         A = np.array(['aa','bbb','ccccc','ddddddddddddddddddddd','ee',
                       'ffffff','g','hhhhhhhhhh'], dtype=np.dtype('S'))
 
@@ -1168,13 +1236,15 @@ def test_varlen_wrong_domain(self):
                 T[:] = A
 
     def test_array_varlen_mismatched(self):
-        A = np.array([b'aa', b'bbb', b'cccc',
-                     np.array([1,3,4], dtype=np.uint64),
-                ], dtype = np.object)
+        # Test that we raise a TypeError when passing a heterogeneous object array.
+        A = np.array(
+                [  b'aa', b'bbb', b'cccc',
+                   np.uint64([1,3,4]), ],
+                dtype = np.object)
 
         ctx = tiledb.Ctx()
         dom = tiledb.Domain(tiledb.Dim(domain=(0, 3), tile=4, ctx=ctx), ctx=ctx)
-        att = tiledb.Attr(dtype=np.bytes_, ctx=ctx)
+        att = tiledb.Attr(dtype=np.bytes_, var=True, ctx=ctx)
 
         schema = tiledb.ArraySchema(dom, (att,), ctx=ctx)
 
@@ -1183,6 +1253,26 @@ def test_array_varlen_mismatched(self):
             with self.assertRaises(TypeError):
                 T[:] = A
 
+    def test_array_varlen_2d_s_fixed(self):
+        A = np.array([['AAAAAAAAAa', 'BBB'], ['ACCC', 'BBBCBCBCBCCCBBCBCBCCBC']], dtype='S')
+
+        uri = self.path("varlen_2d_s_fixed")
+        ctx = tiledb.Ctx()
+        dom = tiledb.Domain(tiledb.Dim(name="rows", domain=(0, 1), tile=2, dtype=np.int64),
+                            tiledb.Dim(name="cols", domain=(0, 1), tile=2, dtype=np.int64), ctx=ctx)
+
+        schema = tiledb.ArraySchema(domain=dom, sparse=False,
+                                    attrs=[tiledb.Attr(name="a", dtype='S', var=True, ctx=ctx)],
+                                    ctx=ctx)
+
+        tiledb.DenseArray.create(uri, schema)
+
+        with tiledb.DenseArray(uri, mode='w', ctx=ctx) as T:
+            T[...] = A
+
+        with tiledb.DenseArray(uri) as T:
+            assert_array_equal(A, T)
+
 
 class SparseArray(DiskTestCase):
 
@@ -1333,7 +1423,7 @@ def test_sparse_bytes(self):
         ctx = tiledb.Ctx()
         dom = tiledb.Domain(
             tiledb.Dim("x", domain=(1, 10000), tile=100, dtype=int, ctx=ctx), ctx=ctx)
-        att = tiledb.Attr("", dtype=np.bytes_, ctx=ctx)
+        att = tiledb.Attr("", var=True, dtype=np.bytes_, ctx=ctx)
         schema = tiledb.ArraySchema(domain=dom, attrs=(att,), sparse=True, ctx=ctx)
         tiledb.SparseArray.create(self.path("foo"), schema)
 
@@ -1360,12 +1450,13 @@ def test_sparse_unicode(self):
         ctx = tiledb.Ctx()
         dom = tiledb.Domain(
             tiledb.Dim("x", domain=(1, 10000), tile=100, dtype=int, ctx=ctx), ctx=ctx)
-        att = tiledb.Attr("", dtype=np.unicode_, ctx=ctx)
+        att = tiledb.Attr("", var=True, dtype=np.unicode_, ctx=ctx)
         schema = tiledb.ArraySchema(domain=dom, attrs=(att,), sparse=True, ctx=ctx)
         tiledb.SparseArray.create(self.path("foo"), schema)
 
         with tiledb.SparseArray(self.path("foo"), mode='r', ctx=ctx) as T:
             self.assertIsNone(T.nonempty_domain())
+
         A = np_array = np.array([u'1234545lkjalsdfj', u'mnopqrs', u'ijkl', u'gh', u'abcdef',
                                  u'aαbββcγγγdδδδδ', u'aαbββc', u'γγγdδδδδ'], dtype=object)