Repository: pandas-dev/pandas
Pull request: TST #22663: Operation between DataFrame with non-numeric types and incomplete series
- [ x ] closes #22663
- [ 1 ] tests added / passed
- [ x ] passes `black pandas`
- [ x ] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`


Problem statement:
Operation between DataFrame with non-numeric types and incomplete series
#### Code Sample, a copy-pastable example if possible

```python
In [2]: df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=list('abc'))

In [3]: df + pd.Series([-1, -2], index=list('ab'))
Out[3]: 
     a    b   c
0  0.0  0.0 NaN
1  3.0  3.0 NaN

In [4]: df['c'] = 'c'

In [5]: df + pd.Series([-1, -2], index=list('ab'))
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~/nobackup/repo/pandas/pandas/core/ops.py in na_op(x, y)
   1714         try:
-> 1715             result = expressions.evaluate(op, str_rep, x, y, **eval_kwargs)
   1716         except TypeError:

~/nobackup/repo/pandas/pandas/core/computation/expressions.py in evaluate(op, op_str, a, b, use_numexpr, **eval_kwargs)
    204     if use_numexpr:
--> 205         return _evaluate(op, op_str, a, b, **eval_kwargs)
    206     return _evaluate_standard(op, op_str, a, b)

~/nobackup/repo/pandas/pandas/core/computation/expressions.py in _evaluate_numexpr(op, op_str, a, b, truediv, reversed, **eval_kwargs)
    119     if result is None:
--> 120         result = _evaluate_standard(op, op_str, a, b)
    121 

~/nobackup/repo/pandas/pandas/core/computation/expressions.py in _evaluate_standard(op, op_str, a, b, **eval_kwargs)
     64     with np.errstate(all='ignore'):
---> 65         return op(a, b)
     66 

TypeError: Can't convert 'float' object to str implicitly

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-5-c60f1e54c070> in <module>()
----> 1 df + pd.Series([-1, -2], index=list('ab'))

~/nobackup/repo/pandas/pandas/core/ops.py in f(self, other, axis, level, fill_value)
   1737             return _combine_series_frame(self, other, na_op,
   1738                                          fill_value=fill_value, axis=axis,
-> 1739                                          level=level, try_cast=True)
   1740         else:
   1741             if fill_value is not None:

~/nobackup/repo/pandas/pandas/core/ops.py in _combine_series_frame(self, other, func, fill_value, axis, level, try_cast)
   1655         # default axis is columns
   1656         return self._combine_match_columns(other, func, level=level,
-> 1657                                            try_cast=try_cast)
   1658 
   1659 

~/nobackup/repo/pandas/pandas/core/frame.py in _combine_match_columns(self, other, func, level, try_cast)
   4832         new_data = left._data.eval(func=func, other=right,
   4833                                    axes=[left.columns, self.index],
-> 4834                                    try_cast=try_cast)
   4835         return self._constructor(new_data)
   4836 

~/nobackup/repo/pandas/pandas/core/internals/managers.py in eval(self, **kwargs)
    527 
    528     def eval(self, **kwargs):
--> 529         return self.apply('eval', **kwargs)
    530 
    531     def quantile(self, **kwargs):

~/nobackup/repo/pandas/pandas/core/internals/managers.py in apply(self, f, axes, filter, do_integrity_check, consolidate, **kwargs)
    421 
    422             kwargs['mgr'] = self
--> 423             applied = getattr(b, f)(**kwargs)
    424             result_blocks = _extend_blocks(applied, result_blocks)
    425 

~/nobackup/repo/pandas/pandas/core/internals/blocks.py in eval(self, func, other, errors, try_cast, mgr)
   1416         try:
   1417             with np.errstate(all='ignore'):
-> 1418                 result = get_result(other)
   1419 
   1420         # if we have an invalid shape/broadcast error

~/nobackup/repo/pandas/pandas/core/internals/blocks.py in get_result(other)
   1384                 result = func(values, other)
   1385             else:
-> 1386                 result = func(values, other)
   1387 
   1388             # mask if needed

~/nobackup/repo/pandas/pandas/core/ops.py in na_op(x, y)
   1715             result = expressions.evaluate(op, str_rep, x, y, **eval_kwargs)
   1716         except TypeError:
-> 1717             result = masked_arith_op(x, y, op)
   1718 
   1719         result = missing.fill_zeros(result, x, y, op_name, fill_zeros)

~/nobackup/repo/pandas/pandas/core/ops.py in masked_arith_op(x, y, op)
    819             # errors in Py3 (TypeError) vs Py2 (ValueError)
    820             # Note: Only = an issue in DataFrame case
--> 821             raise ValueError('Cannot broadcast operands together.')
    822 
    823         if mask.any():

ValueError: Cannot broadcast operands together.
```

#### Problem description

``df + df[['a', 'b']]`` works, and the above also worked until recently (it works for instance in 0.19, although it raises a ``VisibleDeprecationWarning: boolean index did not match indexed array along dimension 0; dimension is 1 but corresponding boolean dimension is 2`` - which however makes no reference to the mixed dtype).

#### Expected Output

In 0.19, ``In [3]`` and ``In [5]`` would yield the same result.

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.5.3.final.0
python-bits: 64
OS: Linux
OS-release: 4.9.0-7-amd64
machine: x86_64
processor: 
byteorder: little
LC_ALL: None
LANG: it_IT.UTF-8
LOCALE: it_IT.UTF-8

pandas: 0.24.0.dev0+465.ge775f9ab6
pytest: 3.5.0
pip: 9.0.1
setuptools: 39.2.0
Cython: 0.28.4
numpy: 1.14.3
scipy: 0.19.0
pyarrow: None
xarray: None
IPython: 6.2.1
sphinx: 1.5.6
patsy: 0.5.0
dateutil: 2.7.3
pytz: 2018.4
blosc: None
bottleneck: 1.2.0dev
tables: 3.3.0
numexpr: 2.6.1
feather: 0.3.1
matplotlib: 2.2.2.post1634.dev0+ge8120cf6d
openpyxl: 2.3.0
xlrd: 1.0.0
xlwt: 1.3.0
xlsxwriter: 0.9.6
lxml: 4.1.1
bs4: 4.5.3
html5lib: 0.999999999
sqlalchemy: 1.0.15
pymysql: None
psycopg2: None
jinja2: 2.10
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: 0.2.1
gcsfs: None

</details>

Hints from issue discussion:
This looks to work on master now. Could use a test

```
In [12]: In [2]: df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=list('abc'))
    ...:
    ...: In [3]: df + pd.Series([-1, -2], index=list('ab'))
Out[12]:
     a    b   c
0  0.0  0.0 NaN
1  3.0  3.0 NaN

In [13]: In [4]: df['c'] = 'c'
    ...:
    ...: In [5]: df + pd.Series([-1, -2], index=list('ab'))
Out[13]:
     a    b    c
0  0.0  0.0  NaN
1  3.0  3.0  NaN
```

Commit message: TST #22663

Patch to review:
diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py
index e7b7f3e524d44..84d42601fbc7b 100644
--- a/pandas/tests/frame/test_arithmetic.py
+++ b/pandas/tests/frame/test_arithmetic.py
@@ -1536,3 +1536,14 @@ def test_dataframe_blockwise_slicelike():
 
         expected = pd.DataFrame({i: left[i] + right[i] for i in left.columns})
         tm.assert_frame_equal(res, expected)
+
+
+def test_dataframe_operation_with_non_numeric_types():
+    # GH #22663
+    df = pd.DataFrame([[1, 2], [4, 5]], columns=list("ab"))
+    result = df + pd.Series([-1], index=list("a"))
+    expected = pd.DataFrame([[0.0, np.nan], [3.0, np.nan]], columns=list("ab"))
+    tm.assert_frame_equal(result, expected)
+    df["b"] = "b"
+    expected = df + pd.Series([-1], index=list("a"))
+    tm.assert_frame_equal(result, expected, check_dtype=False)


Code review comments:
diff_hunk: @@ -1536,3 +1536,14 @@ def test_dataframe_blockwise_slicelike():
 
         expected = pd.DataFrame({i: left[i] + right[i] for i in left.columns})
         tm.assert_frame_equal(res, expected)
+
+
+def test_dataframe_operation_with_non_numeric_types():
+    # GH #22663
+    df = pd.DataFrame([[1, 2], [4, 5]], columns=list("ab"))
+    result = df + pd.Series([-1], index=list("a"))
+    expected = pd.DataFrame([[0.0, np.nan], [3.0, np.nan]], columns=list("ab"))
+    tm.assert_frame_equal(result, expected)
+    df["b"] = "b"
+    expected = df + pd.Series([-1], index=list("a"))
+    tm.assert_frame_equal(result, expected, check_dtype=False)
path: pandas/tests/frame/test_arithmetic.py
text: @user1:
don't use check_dtype

@author:
I believe it is necessary, because of the column filled with `NaN` values, which is unclear about `dtype`. Here is the test exlained:
```
result = pd.DataFrame([[1, "b"], [4, "b"]], columns=list("ab")) + pd.Series([-1], index=list("a"))
expected = pd.DataFrame([[0.0, np.nan], [3.0, np.nan]], columns=list("ab"))
print(result)
print(result.dtypes)
print(expected)
print(expected.dtypes)
```
outputs are:
```
     a    b
0  0.0  NaN
1  3.0  NaN
a    float64
b     object
dtype: object
     a   b
0  0.0 NaN
1  3.0 NaN
a    float64
b    float64
dtype: object
```

The dtype of column `result['b']` is `object`, whereas `expected['b']` is `float64`. I believe the default `dtype` is `float64` which would raise an exception if I do check dtype.

@user2:
Is it possible to construct `expected` with the correct dtypes instead of using `check_dtype=False`?

e.g. something like
```python
expected = pd.DataFrame([[0.0, np.nan], [3.0, np.nan]], columns=list("ab")).astype({"b": "object"})
```

@author:
Yes it is possible, however, I need to add `astype` to both `result` and `expected` as the 2 tests use different dtypes (int / float / str + undefined np.nan).
Is it OK this way ?

@user2:
I'm not sure about changing `result` - can't you parametrize the test over the dtype to assign to `b`?

@author:
OK, I got it. A new PR is going on : to simplify the tests comprehension, I've only worked with `float64` data + I've parametrize a new input data for col `b`

diff_hunk: @@ -1536,3 +1536,14 @@ def test_dataframe_blockwise_slicelike():
 
         expected = pd.DataFrame({i: left[i] + right[i] for i in left.columns})
         tm.assert_frame_equal(res, expected)
+
+
+def test_dataframe_operation_with_non_numeric_types():
+    # GH #22663
+    df = pd.DataFrame([[1, 2], [4, 5]], columns=list("ab"))
+    result = df + pd.Series([-1], index=list("a"))
+    expected = pd.DataFrame([[0.0, np.nan], [3.0, np.nan]], columns=list("ab"))
+    tm.assert_frame_equal(result, expected)
+    df["b"] = "b"
+    expected = df + pd.Series([-1], index=list("a"))
path: pandas/tests/frame/test_arithmetic.py
text: @user1:
don't you mean result= here?

@author:
The first fix is also fixing this issue.