Repository: pandas-dev/pandas

Issue:
.memory_usage(deep=True) errors on some objects 
#### Code Sample, a copy-pastable example if possible

```python
import pandas as pd
pd.Series([(1,2), (3,4)]).memory_usage(deep=True)

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/series.py", line 3913, in memory_usage
    v = super(Series, self).memory_usage(deep=deep)
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/base.py", line 1410, in memory_usage
    v += lib.memory_usage_of_objects(self.array)
  File "pandas/_libs/lib.pyx", line 101, in pandas._libs.lib.memory_usage_of_objects
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/arrays/numpy_.py", line 226, in __getitem__
    result = type(self)(result)
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/arrays/numpy_.py", line 131, in __init__
    raise ValueError("'values' must be a NumPy array.")
ValueError: 'values' must be a NumPy array.
```
It seems to be related to calling `self.array` on the series which might makes sense given PandasArray is supposed be 1-dimensional
```python
pd.array([(1,2), (3,4)])
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/arrays/array_.py", line 273, in array
    result = PandasArray._from_sequence(data, dtype=dtype, copy=copy)
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/arrays/numpy_.py", line 150, in _from_sequence
    return cls(result)
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/arrays/numpy_.py", line 134, in __init__
    raise ValueError("PandasArray must be 1-dimensional.")
ValueError: PandasArray must be 1-dimensional.
```
This also happens with Python objects
```python
class Dummy(object):
    def __init__(self):
        pass

pd.array([Dummy(), Dummy()])
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/arrays/base.py", line 848, in __repr__
    indent_for_name=False).rstrip(', \n')
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/io/formats/printing.py", line 348, in format_object_summary
    first = formatter(obj[0])
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/arrays/numpy_.py", line 226, in __getitem__
    result = type(self)(result)
  File "/home/roy/ftpyenv/clean/lib/python3.7/site-packages/pandas/core/arrays/numpy_.py", line 131, in __init__
    raise ValueError("'values' must be a NumPy array.")
ValueError: 'values' must be a NumPy array.
```
#### Problem description

The current behavior is a problem because if a dataframe has a column with non-standard data like tuples the `.memory_usage(deep=True)` call fails.

#### Expected Output

```python
pd.Series([(1,2)]).memory_usage(deep=True)
176
```

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.7.1.final.0
python-bits: 64
OS: Linux
OS-release: 4.15.0-43-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8
LOCALE: en_US.UTF-8

pandas: 0.24.0
pytest: 3.8.2
pip: 18.1
setuptools: 40.6.2
Cython: None
numpy: 1.16.0
scipy: 1.1.0
pyarrow: None
xarray: None
IPython: 7.1.1
sphinx: 1.8.1
patsy: None
dateutil: 2.7.5
pytz: 2018.7
blosc: None
bottleneck: None
tables: None
numexpr: None
feather: None
matplotlib: 3.0.1
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml.etree: None
bs4: None
html5lib: None
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: 2.10
s3fs: 0.2.0
fastparquet: 0.2.1
pandas_gbq: None
pandas_datareader: None
gcsfs: None

</details>



Discussion:
Taking a look at this now. The fix is quite simple

```diff
diff --git a/pandas/core/arrays/numpy_.py b/pandas/core/arrays/numpy_.py
index 47517782e..791ff4430 100644
--- a/pandas/core/arrays/numpy_.py
+++ b/pandas/core/arrays/numpy_.py
@@ -222,7 +222,7 @@ class PandasArray(ExtensionArray, ExtensionOpsMixin, NDArrayOperatorsMixin):
             item = item._ndarray

         result = self._ndarray[item]
-        if not lib.is_scalar(result):
+        if not lib.is_scalar(item):
             result = type(self)(result)
         return result
```

but I'm trying to figure out a way to robustly test this.

Resolving patch:
diff --git a/pandas/core/arrays/numpy_.py b/pandas/core/arrays/numpy_.py
--- a/pandas/core/arrays/numpy_.py
+++ b/pandas/core/arrays/numpy_.py
@@ -222,7 +222,7 @@ def __getitem__(self, item):
             item = item._ndarray
 
         result = self._ndarray[item]
-        if not lib.is_scalar(result):
+        if not lib.is_scalar(item):
             result = type(self)(result)
         return result
 


Test patch:
diff --git a/pandas/tests/extension/numpy_/__init__.py b/pandas/tests/extension/numpy_/__init__.py
new file mode 100644
diff --git a/pandas/tests/extension/numpy_/conftest.py b/pandas/tests/extension/numpy_/conftest.py
new file mode 100644
--- /dev/null
+++ b/pandas/tests/extension/numpy_/conftest.py
@@ -0,0 +1,38 @@
+import numpy as np
+import pytest
+
+from pandas.core.arrays.numpy_ import PandasArray
+
+
+@pytest.fixture
+def allow_in_pandas(monkeypatch):
+    """
+    A monkeypatch to tell pandas to let us in.
+
+    By default, passing a PandasArray to an index / series / frame
+    constructor will unbox that PandasArray to an ndarray, and treat
+    it as a non-EA column. We don't want people using EAs without
+    reason.
+
+    The mechanism for this is a check against ABCPandasArray
+    in each constructor.
+
+    But, for testing, we need to allow them in pandas. So we patch
+    the _typ of PandasArray, so that we evade the ABCPandasArray
+    check.
+    """
+    with monkeypatch.context() as m:
+        m.setattr(PandasArray, '_typ', 'extension')
+        yield
+
+
+@pytest.fixture
+def na_value():
+    return np.nan
+
+
+@pytest.fixture
+def na_cmp():
+    def cmp(a, b):
+        return np.isnan(a) and np.isnan(b)
+    return cmp
diff --git a/pandas/tests/extension/test_numpy.py b/pandas/tests/extension/numpy_/test_numpy.py
similarity index 84%
rename from pandas/tests/extension/test_numpy.py
rename to pandas/tests/extension/numpy_/test_numpy.py
--- a/pandas/tests/extension/test_numpy.py
+++ b/pandas/tests/extension/numpy_/test_numpy.py
@@ -6,7 +6,7 @@
 from pandas.core.arrays.numpy_ import PandasArray, PandasDtype
 import pandas.util.testing as tm
 
-from . import base
+from .. import base
 
 
 @pytest.fixture
@@ -14,28 +14,6 @@ def dtype():
     return PandasDtype(np.dtype('float'))
 
 
-@pytest.fixture
-def allow_in_pandas(monkeypatch):
-    """
-    A monkeypatch to tells pandas to let us in.
-
-    By default, passing a PandasArray to an index / series / frame
-    constructor will unbox that PandasArray to an ndarray, and treat
-    it as a non-EA column. We don't want people using EAs without
-    reason.
-
-    The mechanism for this is a check against ABCPandasArray
-    in each constructor.
-
-    But, for testing, we need to allow them in pandas. So we patch
-    the _typ of PandasArray, so that we evade the ABCPandasArray
-    check.
-    """
-    with monkeypatch.context() as m:
-        m.setattr(PandasArray, '_typ', 'extension')
-        yield
-
-
 @pytest.fixture
 def data(allow_in_pandas, dtype):
     return PandasArray(np.arange(1, 101, dtype=dtype._dtype))
@@ -46,18 +24,6 @@ def data_missing(allow_in_pandas):
     return PandasArray(np.array([np.nan, 1.0]))
 
 
-@pytest.fixture
-def na_value():
-    return np.nan
-
-
-@pytest.fixture
-def na_cmp():
-    def cmp(a, b):
-        return np.isnan(a) and np.isnan(b)
-    return cmp
-
-
 @pytest.fixture
 def data_for_sorting(allow_in_pandas):
     """Length-3 array with a known sort order.
diff --git a/pandas/tests/extension/numpy_/test_numpy_nested.py b/pandas/tests/extension/numpy_/test_numpy_nested.py
new file mode 100644
--- /dev/null
+++ b/pandas/tests/extension/numpy_/test_numpy_nested.py
@@ -0,0 +1,286 @@
+"""
+Tests for PandasArray with nested data. Users typically won't create
+these objects via `pd.array`, but they can show up through `.array`
+on a Series with nested data.
+
+We partition these tests into their own file, as many of the base
+tests fail, as they aren't appropriate for nested data. It is easier
+to have a seperate file with its own data generating fixtures, than
+trying to skip based upon the value of a fixture.
+"""
+import pytest
+
+import pandas as pd
+from pandas.core.arrays.numpy_ import PandasArray, PandasDtype
+
+from .. import base
+
+# For NumPy <1.16, np.array([np.nan, (1,)]) raises
+# ValueError: setting an array element with a sequence.
+np = pytest.importorskip('numpy', minversion='1.16.0')
+
+
+@pytest.fixture
+def dtype():
+    return PandasDtype(np.dtype('object'))
+
+
+@pytest.fixture
+def data(allow_in_pandas, dtype):
+    return pd.Series([(i,) for i in range(100)]).array
+
+
+@pytest.fixture
+def data_missing(allow_in_pandas):
+    return PandasArray(np.array([np.nan, (1,)]))
+
+
+@pytest.fixture
+def data_for_sorting(allow_in_pandas):
+    """Length-3 array with a known sort order.
+
+    This should be three items [B, C, A] with
+    A < B < C
+    """
+    # Use an empty tuple for first element, then remove,
+    # to disable np.array's shape inference.
+    return PandasArray(
+        np.array([(), (2,), (3,), (1,)])[1:]
+    )
+
+
+@pytest.fixture
+def data_missing_for_sorting(allow_in_pandas):
+    """Length-3 array with a known sort order.
+
+    This should be three items [B, NA, A] with
+    A < B and NA missing.
+    """
+    return PandasArray(
+        np.array([(1,), np.nan, (0,)])
+    )
+
+
+@pytest.fixture
+def data_for_grouping(allow_in_pandas):
+    """Data for factorization, grouping, and unique tests.
+
+    Expected to be like [B, B, NA, NA, A, A, B, C]
+
+    Where A < B < C and NA is missing
+    """
+    a, b, c = (1,), (2,), (3,)
+    return PandasArray(np.array(
+        [b, b, np.nan, np.nan, a, a, b, c]
+    ))
+
+
+skip_nested = pytest.mark.skip(reason="Skipping for nested PandasArray")
+
+
+class BaseNumPyTests(object):
+    pass
+
+
+class TestCasting(BaseNumPyTests, base.BaseCastingTests):
+
+    @skip_nested
+    def test_astype_str(self, data):
+        pass
+
+
+class TestConstructors(BaseNumPyTests, base.BaseConstructorsTests):
+    @pytest.mark.skip(reason="We don't register our dtype")
+    # We don't want to register. This test should probably be split in two.
+    def test_from_dtype(self, data):
+        pass
+
+    @skip_nested
+    def test_array_from_scalars(self, data):
+        pass
+
+
+class TestDtype(BaseNumPyTests, base.BaseDtypeTests):
+
+    @pytest.mark.skip(reason="Incorrect expected.")
+    # we unsurprisingly clash with a NumPy name.
+    def test_check_dtype(self, data):
+        pass
+
+
+class TestGetitem(BaseNumPyTests, base.BaseGetitemTests):
+
+    @skip_nested
+    def test_getitem_scalar(self, data):
+        pass
+
+    @skip_nested
+    def test_take_series(self, data):
+        pass
+
+
+class TestGroupby(BaseNumPyTests, base.BaseGroupbyTests):
+    @skip_nested
+    def test_groupby_extension_apply(self, data_for_grouping, op):
+        pass
+
+
+class TestInterface(BaseNumPyTests, base.BaseInterfaceTests):
+    @skip_nested
+    def test_array_interface(self, data):
+        # NumPy array shape inference
+        pass
+
+
+class TestMethods(BaseNumPyTests, base.BaseMethodsTests):
+
+    @pytest.mark.skip(reason="TODO: remove?")
+    def test_value_counts(self, all_data, dropna):
+        pass
+
+    @pytest.mark.skip(reason="Incorrect expected")
+    # We have a bool dtype, so the result is an ExtensionArray
+    # but expected is not
+    def test_combine_le(self, data_repeated):
+        super(TestMethods, self).test_combine_le(data_repeated)
+
+    @skip_nested
+    def test_combine_add(self, data_repeated):
+        # Not numeric
+        pass
+
+    @skip_nested
+    def test_shift_fill_value(self, data):
+        # np.array shape inference. Shift implementation fails.
+        super().test_shift_fill_value(data)
+
+    @skip_nested
+    def test_unique(self, data, box, method):
+        # Fails creating expected
+        pass
+
+    @skip_nested
+    def test_fillna_copy_frame(self, data_missing):
+        # The "scalar" for this array isn't a scalar.
+        pass
+
+    @skip_nested
+    def test_fillna_copy_series(self, data_missing):
+        # The "scalar" for this array isn't a scalar.
+        pass
+
+    @skip_nested
+    def test_hash_pandas_object_works(self, data, as_frame):
+        # ndarray of tuples not hashable
+        pass
+
+    @skip_nested
+    def test_searchsorted(self, data_for_sorting, as_series):
+        # Test setup fails.
+        pass
+
+    @skip_nested
+    def test_where_series(self, data, na_value, as_frame):
+        # Test setup fails.
+        pass
+
+    @skip_nested
+    def test_repeat(self, data, repeats, as_series, use_numpy):
+        # Fails creating expected
+        pass
+
+
+class TestPrinting(BaseNumPyTests, base.BasePrintingTests):
+    pass
+
+
+class TestMissing(BaseNumPyTests, base.BaseMissingTests):
+
+    @skip_nested
+    def test_fillna_scalar(self, data_missing):
+        # Non-scalar "scalar" values.
+        pass
+
+    @skip_nested
+    def test_fillna_series_method(self, data_missing, method):
+        # Non-scalar "scalar" values.
+        pass
+
+    @skip_nested
+    def test_fillna_series(self, data_missing):
+        # Non-scalar "scalar" values.
+        pass
+
+    @skip_nested
+    def test_fillna_frame(self, data_missing):
+        # Non-scalar "scalar" values.
+        pass
+
+
+class TestReshaping(BaseNumPyTests, base.BaseReshapingTests):
+
+    @pytest.mark.skip("Incorrect parent test")
+    # not actually a mixed concat, since we concat int and int.
+    def test_concat_mixed_dtypes(self, data):
+        super(TestReshaping, self).test_concat_mixed_dtypes(data)
+
+    @skip_nested
+    def test_merge(self, data, na_value):
+        # Fails creating expected
+        pass
+
+    @skip_nested
+    def test_merge_on_extension_array(self, data):
+        # Fails creating expected
+        pass
+
+    @skip_nested
+    def test_merge_on_extension_array_duplicates(self, data):
+        # Fails creating expected
+        pass
+
+
+class TestSetitem(BaseNumPyTests, base.BaseSetitemTests):
+
+    @skip_nested
+    def test_setitem_scalar_series(self, data, box_in_series):
+        pass
+
+    @skip_nested
+    def test_setitem_sequence(self, data, box_in_series):
+        pass
+
+    @skip_nested
+    def test_setitem_sequence_mismatched_length_raises(self, data, as_array):
+        pass
+
+    @skip_nested
+    def test_setitem_sequence_broadcasts(self, data, box_in_series):
+        pass
+
+    @skip_nested
+    def test_setitem_loc_scalar_mixed(self, data):
+        pass
+
+    @skip_nested
+    def test_setitem_loc_scalar_multiple_homogoneous(self, data):
+        pass
+
+    @skip_nested
+    def test_setitem_iloc_scalar_mixed(self, data):
+        pass
+
+    @skip_nested
+    def test_setitem_iloc_scalar_multiple_homogoneous(self, data):
+        pass
+
+    @skip_nested
+    def test_setitem_mask_broadcast(self, data, setter):
+        pass
+
+    @skip_nested
+    def test_setitem_scalar_key_sequence_raise(self, data):
+        pass
+
+
+# Skip Arithmetics, NumericReduce, BooleanReduce, Parsing