Repository: pandas-dev/pandas

Issue:
BUG: Categoricals cannot compare to ndim=0 array values
Reproduction steps:

``` python
In [1]: cat = pd.Categorical([1,2,3])

In [2]: cat > cat[0]
Out[2]: array([False,  True,  True], dtype=bool)

In [3]: cat[0] < cat
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-3-5c4bb0678680> in <module>()
----> 1 cat[0] < cat

/home/dshpektorov/sources/pandas/pandas/core/categorical.pyc in f(self, other)
     52             msg = "Cannot compare a Categorical for op {op} with type {typ}. If you want to \n" \
     53                   "compare values, use 'np.asarray(cat) <op> other'."
---> 54             raise TypeError(msg.format(op=op,typ=type(other)))
     55 
     56     f.__name__ = op

TypeError: Cannot compare a Categorical for op __gt__ with type <type 'numpy.ndarray'>. If you want to compare values, use 'np.asarray(cat) <op> other'.
```

The problem is that when `cat[0]` is the first operand (`type == np.int64`) it orchestrates the execution and the first step is casting it to a rank-0 array (it becomes `array(1)` rather than `np.int64(1)`) and at that point it ceases to pass `np.isscalar` check and corresponding branch of execution is skipped.

The fix is trivial, feel free to beat me to it.

On a broader scale, we might want to review all uses of `np.isscalar` and see if they can also benefit from accepting rank-0 arrays. If that's true, we might want to add `pd.isscalar` function and use it everywhere instead.



Discussion:
There's an inconsistency between comparison and arithmetical operations in numpy, I've reported it [here](https://github.com/numpy/numpy/issues/5246).

a numpy bug - say it's not so :)

Sure looks like it to me.

Regardless though, I think rank-0 arrays should be treated the same as their pure scalar counterparts. Is there a reason not to do that?

that sounds right - not sure we can fix his without the upstream fix though ?

Well, in numpy terms, scalar and zerodim array are different entities with their own properties.  There were discussions about removing one of them in favour of the other, but AFAIC none has come to a solid decision to do so ([here](http://mail.scipy.org/pipermail/numpy-discussion/2013-January/064995.html) is an interesting summary).

My fix thoughts were about the fact that there are de facto two ways of expressing a scalar value in numpy and there may be places in pandas where only one of those is handled properly. Pandas has a long history of fixing numpy's shortcomings, maybe this case should be another manifestation of "doing it right" principle.

I agree with you in principle about an pd.isscalar - go for it if u have time

would make indexing logic detection maybe a bit simpler


Resolving patch:
diff --git a/pandas/core/categorical.py b/pandas/core/categorical.py
--- a/pandas/core/categorical.py
+++ b/pandas/core/categorical.py
@@ -42,7 +42,16 @@ def f(self, other):
                 # In other series, the leads to False, so do that here too
                 ret[na_mask] = False
             return ret
-        elif lib.isscalar(other):
+
+        # Numpy-1.9 and earlier may convert a scalar to a zerodim array during
+        # comparison operation when second arg has higher priority, e.g.
+        #
+        #     cat[0] < cat
+        #
+        # With cat[0], for example, being ``np.int64(1)`` by the time it gets
+        # into this function would become ``np.array(1)``.
+        other = lib.item_from_zerodim(other)
+        if lib.isscalar(other):
             if other in self.categories:
                 i = self.categories.get_loc(other)
                 return getattr(self._codes, op)(i)
diff --git a/pandas/core/common.py b/pandas/core/common.py
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -84,7 +84,7 @@ def _check(cls, inst):
 ABCSparseArray = create_pandas_abc_type("ABCSparseArray", "_subtyp",
                                         ('sparse_array', 'sparse_series'))
 ABCCategorical = create_pandas_abc_type("ABCCategorical","_typ",("categorical"))
-
+ABCPeriod = create_pandas_abc_type("ABCPeriod", "_typ", ("period",))
 
 class _ABCGeneric(type):
 
diff --git a/pandas/tseries/period.py b/pandas/tseries/period.py
--- a/pandas/tseries/period.py
+++ b/pandas/tseries/period.py
@@ -63,6 +63,7 @@ class Period(PandasObject):
     """
     __slots__ = ['freq', 'ordinal']
     _comparables = ['name','freqstr']
+    _typ = 'period'
 
     @classmethod
     def _from_ordinal(cls, ordinal, freq):
@@ -498,7 +499,6 @@ def strftime(self, fmt):
         base, mult = _gfc(self.freq)
         return tslib.period_format(self.ordinal, base, fmt)
 
-
 def _get_ordinals(data, freq):
     f = lambda x: Period(x, freq=freq).ordinal
     if isinstance(data[0], Period):


Test patch:
diff --git a/pandas/tests/test_categorical.py b/pandas/tests/test_categorical.py
--- a/pandas/tests/test_categorical.py
+++ b/pandas/tests/test_categorical.py
@@ -917,6 +917,12 @@ def test_datetime_categorical_comparison(self):
         self.assert_numpy_array_equal(dt_cat > dt_cat[0], [False, True, True])
         self.assert_numpy_array_equal(dt_cat[0] < dt_cat, [False, True, True])
 
+    def test_reflected_comparison_with_scalars(self):
+        # GH8658
+        cat = pd.Categorical([1, 2, 3])
+        self.assert_numpy_array_equal(cat > cat[0], [False, True, True])
+        self.assert_numpy_array_equal(cat[0] < cat, [False, True, True])
+
 
 class TestCategoricalAsBlock(tm.TestCase):
     _multiprocess_can_split_ = True
diff --git a/pandas/tests/test_lib.py b/pandas/tests/test_lib.py
new file mode 100644
--- /dev/null
+++ b/pandas/tests/test_lib.py
@@ -0,0 +1,72 @@
+from datetime import datetime, timedelta, date, time
+
+import numpy as np
+
+import pandas as pd
+from pandas.lib import isscalar, item_from_zerodim
+import pandas.util.testing as tm
+
+
+class TestIsscalar(tm.TestCase):
+    def test_isscalar_builtin_scalars(self):
+        self.assertTrue(isscalar(None))
+        self.assertTrue(isscalar(True))
+        self.assertTrue(isscalar(False))
+        self.assertTrue(isscalar(0.))
+        self.assertTrue(isscalar(np.nan))
+        self.assertTrue(isscalar('foobar'))
+        self.assertTrue(isscalar(b'foobar'))
+        self.assertTrue(isscalar(u'foobar'))
+        self.assertTrue(isscalar(datetime(2014, 1, 1)))
+        self.assertTrue(isscalar(date(2014, 1, 1)))
+        self.assertTrue(isscalar(time(12, 0)))
+        self.assertTrue(isscalar(timedelta(hours=1)))
+        self.assertTrue(isscalar(pd.NaT))
+
+    def test_isscalar_builtin_nonscalars(self):
+        self.assertFalse(isscalar({}))
+        self.assertFalse(isscalar([]))
+        self.assertFalse(isscalar([1]))
+        self.assertFalse(isscalar(()))
+        self.assertFalse(isscalar((1,)))
+        self.assertFalse(isscalar(slice(None)))
+        self.assertFalse(isscalar(Ellipsis))
+
+    def test_isscalar_numpy_array_scalars(self):
+        self.assertTrue(isscalar(np.int64(1)))
+        self.assertTrue(isscalar(np.float64(1.)))
+        self.assertTrue(isscalar(np.int32(1)))
+        self.assertTrue(isscalar(np.object_('foobar')))
+        self.assertTrue(isscalar(np.str_('foobar')))
+        self.assertTrue(isscalar(np.unicode_(u'foobar')))
+        self.assertTrue(isscalar(np.bytes_(b'foobar')))
+        self.assertTrue(isscalar(np.datetime64('2014-01-01')))
+        self.assertTrue(isscalar(np.timedelta64(1, 'h')))
+
+    def test_isscalar_numpy_zerodim_arrays(self):
+        for zerodim in [np.array(1),
+                        np.array('foobar'),
+                        np.array(np.datetime64('2014-01-01')),
+                        np.array(np.timedelta64(1, 'h'))]:
+            self.assertFalse(isscalar(zerodim))
+            self.assertTrue(isscalar(item_from_zerodim(zerodim)))
+
+    def test_isscalar_numpy_arrays(self):
+        self.assertFalse(isscalar(np.array([])))
+        self.assertFalse(isscalar(np.array([[]])))
+        self.assertFalse(isscalar(np.matrix('1; 2')))
+
+    def test_isscalar_pandas_scalars(self):
+        self.assertTrue(isscalar(pd.Timestamp('2014-01-01')))
+        self.assertTrue(isscalar(pd.Timedelta(hours=1)))
+        self.assertTrue(isscalar(pd.Period('2014-01-01')))
+
+    def test_isscalar_pandas_containers(self):
+        self.assertFalse(isscalar(pd.Series()))
+        self.assertFalse(isscalar(pd.Series([1])))
+        self.assertFalse(isscalar(pd.DataFrame()))
+        self.assertFalse(isscalar(pd.DataFrame([[1]])))
+        self.assertFalse(isscalar(pd.Panel()))
+        self.assertFalse(isscalar(pd.Panel([[[1]]])))
+        self.assertFalse(isscalar(pd.Index([])))
+        self.assertFalse(isscalar(pd.Index([1])))