jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def testDataFrameNumpy(self):
        df = DataFrame([[1, 2, 3], [4, 5, 6]], index=[
                       'a', 'b'], columns=['x', 'y', 'z'])

        # column indexed
        outp = DataFrame(ujson.decode(ujson.encode(df), numpy=True))
        self.assertTrue((df == outp).values.all())
        tm.assert_index_equal(df.columns, outp.columns)
        tm.assert_index_equal(df.index, outp.index)

        dec = _clean_dict(ujson.decode(ujson.encode(df, orient="split"),
                                       numpy=True))
        outp = DataFrame(**dec)
        self.assertTrue((df == outp).values.all())
        tm.assert_index_equal(df.columns, outp.columns)
        tm.assert_index_equal(df.index, outp.index)

        outp = DataFrame(ujson.decode(ujson.encode(df, orient="index"),
                                      numpy=True))
        self.assertTrue((df.transpose() == outp).values.all())
        tm.assert_index_equal(df.transpose().columns, outp.columns)
        tm.assert_index_equal(df.transpose().index, outp.index)
//  [python]
def test_iteritems(self):
        pser = pd.Series(["A", "B", "C"])
        kser = ks.from_pandas(pser)

        for (p_name, p_items), (k_name, k_items) in zip(pser.iteritems(), kser.iteritems()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_iteritems(self):
        pser = pd.Series(["A", "B", "C"])
        kser = ks.from_pandas(pser)

        for (p_name, p_items), (k_name, k_items) in zip(pser.items(), kser.items()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_float_index_at_iat(self):
        s = pd.Series([1, 2, 3], index=[0.1, 0.2, 0.3])
        for el, item in s.iteritems():
            assert s.at[el] == item
        for i in range(len(s)):
            assert s.iat[i] == i + 1
//  [python]
def test_float_index_at_iat(self):
        s = pd.Series([1, 2, 3], index=[0.1, 0.2, 0.3])
        for el, item in s.items():
            assert s.at[el] == item
        for i in range(len(s)):
            assert s.iat[i] == i + 1
//  [python]
def test_items(self):
        pser = pd.Series(["A", "B", "C"])
        psser = ps.from_pandas(pser)

        for (p_name, p_items), (k_name, k_items) in zip(pser.iteritems(), psser.iteritems()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_items(self):
        pser = pd.Series(["A", "B", "C"])
        psser = ps.from_pandas(pser)

        for (p_name, p_items), (k_name, k_items) in zip(pser.items(), psser.items()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_series_from_pandas_a(self) -> None:
        import pandas as pd

        pds = pd.Series([3,4,5], index=list('abc'))
        sfs = Series.from_pandas(pds)
        self.assertEqual(list(pds.items()), list(sfs.items()))

        # mutate Pandas
        pds['c'] = 50
        self.assertNotEqual(pds['c'], sfs['c'])

        # owning data
        pds = pd.Series([3,4,5], index=list('abc'))
        sfs = Series.from_pandas(pds, own_data=True)
        self.assertEqual(list(pds.items()), list(sfs.items()))
//  [python]
def test_binary_ops_align(self):

        # test aligning binary ops

        # GH 6681
        index=MultiIndex.from_product([list('abc'),
                                       ['one','two','three'],
                                       [1,2,3]],
                                      names=['first','second','third'])

        df = DataFrame(np.arange(27*3).reshape(27,3),
                       index=index,
                       columns=['value1','value2','value3']).sortlevel()

        idx = pd.IndexSlice
        for op in ['add','sub','mul','div','truediv']:
            opa = getattr(operator,op,None)
            if opa is None:
                continue

            x = Series([ 1.0, 10.0, 100.0], [1,2,3])
            result = getattr(df,op)(x,level='third',axis=0)

            expected = pd.concat([ opa(df.loc[idx[:,:,i],:],v) for i, v in x.iteritems() ]).sortlevel()
            assert_frame_equal(result, expected)

            x = Series([ 1.0, 10.0], ['two','three'])
            result = getattr(df,op)(x,level='second',axis=0)

            expected = pd.concat([ opa(df.loc[idx[:,i],:],v) for i, v in x.iteritems() ]).reindex_like(df).sortlevel()
            assert_frame_equal(result, expected)
//  [python]
def test_binary_ops_align(self):

        # test aligning binary ops

        # GH 6681
        index=MultiIndex.from_product([list('abc'),
                                       ['one','two','three'],
                                       [1,2,3]],
                                      names=['first','second','third'])

        df = DataFrame(np.arange(27*3).reshape(27,3),
                       index=index,
                       columns=['value1','value2','value3']).sortlevel()

        idx = pd.IndexSlice
        for op in ['add','sub','mul','div','truediv']:
            opa = getattr(operator,op,None)
            if opa is None:
                continue

            x = Series([ 1.0, 10.0, 100.0], [1,2,3])
            result = getattr(df,op)(x,level='third',axis=0)

            expected = pd.concat([ opa(df.loc[idx[:,:,i],:],v) for i, v in x.items() ]).sortlevel()
            assert_frame_equal(result, expected)

            x = Series([ 1.0, 10.0], ['two','three'])
            result = getattr(df,op)(x,level='second',axis=0)

            expected = pd.concat([ opa(df.loc[idx[:,i],:],v) for i, v in x.items() ]).reindex_like(df).sortlevel()
            assert_frame_equal(result, expected)
//  [python]
def test_series_from_pandas_b(self) -> None:
        import pandas as pd

        pds = pd.Series([3,4,5], index=list('abc'))
        if hasattr(pds, 'convert_dtypes'):
            pds = pds.convert_dtypes()
        sfs = Series.from_pandas(pds)
        self.assertEqual(list(pds.items()), list(sfs.items()))

        # mutate Pandas
        pds['c'] = 50
        self.assertNotEqual(pds['c'], sfs['c'])

        # owning data
        pds = pd.Series([3,4,5], index=list('abc'))
        sfs = Series.from_pandas(pds, own_data=True)
        self.assertEqual(list(pds.items()), list(sfs.items()))
//  [python]
def test_dataframe_dtypes():
    dtypes = pd.Series(
        [np.int32, np.float32, np.float64], index=["c", "a", "b"]
    )
    df = cudf.DataFrame({k: np.ones(10, dtype=v) for k, v in dtypes.iteritems()})
    assert df.dtypes.equals(dtypes)
//  [python]
def test_dataframe_dtypes():
    dtypes = pd.Series(
        [np.int32, np.float32, np.float64], index=["c", "a", "b"]
    )
    df = cudf.DataFrame({k: np.ones(10, dtype=v) for k, v in dtypes.items()})
    assert df.dtypes.equals(dtypes)
//  [python]
def test_binary_ops_align(self, op):
        # test aligning binary ops

        # GH 6681
        index = MultiIndex.from_product(
            [list("abc"), ["one", "two", "three"], [1, 2, 3]],
            names=["first", "second", "third"],
        )

        df = DataFrame(
            np.arange(27 * 3).reshape(27, 3),
            index=index,
            columns=["value1", "value2", "value3"],
        ).sort_index()

        idx = pd.IndexSlice
        opa = getattr(operator, op, None)
        if opa is None:
            return

        x = Series([1.0, 10.0, 100.0], [1, 2, 3])
        result = getattr(df, op)(x, level="third", axis=0)

        expected = pd.concat(
            [opa(df.loc[idx[:, :, i], :], v) for i, v in zip(x.index, x.values)]
        ).sort_index()
        tm.assert_frame_equal(result, expected)

        x = Series([1.0, 10.0], ["two", "three"])
        result = getattr(df, op)(x, level="second", axis=0)

        expected = (
            pd.concat([opa(df.loc[idx[:, i], :], v) for i, v in zip(x.index, x.values)])
            .reindex_like(df)
            .sort_index()
        )
        tm.assert_frame_equal(result, expected)
//  [python]
def test_binary_ops_align(self, op):
        # test aligning binary ops

        # GH 6681
        index = MultiIndex.from_product(
            [list("abc"), ["one", "two", "three"], [1, 2, 3]],
            names=["first", "second", "third"],
        )

        df = DataFrame(
            np.arange(27 * 3).reshape(27, 3),
            index=index,
            columns=["value1", "value2", "value3"],
        ).sort_index()

        idx = pd.IndexSlice
        opa = getattr(operator, op, None)
        if opa is None:
            return

        x = Series([1.0, 10.0, 100.0], [1, 2, 3])
        result = getattr(df, op)(x, level="third", axis=0)

        expected = pd.concat(
            [opa(df.loc[idx[:, :, i], :], v) for i, v in x.items()]
        ).sort_index()
        tm.assert_frame_equal(result, expected)

        x = Series([1.0, 10.0], ["two", "three"])
        result = getattr(df, op)(x, level="second", axis=0)

        expected = (
            pd.concat([opa(df.loc[idx[:, i], :], v) for i, v in x.items()])
            .reindex_like(df)
            .sort_index()
        )
        tm.assert_frame_equal(result, expected)
//  [python]
def test_binary_ops_align(self):

        # test aligning binary ops

        # GH 6681
        index = MultiIndex.from_product([list('abc'),
                                         ['one', 'two', 'three'],
                                         [1, 2, 3]],
                                        names=['first', 'second', 'third'])

        df = DataFrame(np.arange(27 * 3).reshape(27, 3),
                       index=index,
                       columns=['value1', 'value2', 'value3']).sort_index()

        idx = pd.IndexSlice
        for op in ['add', 'sub', 'mul', 'div', 'truediv']:
            opa = getattr(operator, op, None)
            if opa is None:
                continue

            x = Series([1.0, 10.0, 100.0], [1, 2, 3])
            result = getattr(df, op)(x, level='third', axis=0)

            expected = pd.concat([opa(df.loc[idx[:, :, i], :], v)
                                  for i, v in x.iteritems()]).sort_index()
            assert_frame_equal(result, expected)

            x = Series([1.0, 10.0], ['two', 'three'])
            result = getattr(df, op)(x, level='second', axis=0)

            expected = (pd.concat([opa(df.loc[idx[:, i], :], v)
                                   for i, v in x.iteritems()])
                        .reindex_like(df).sort_index())
            assert_frame_equal(result, expected)

        # GH9463 (alignment level of dataframe with series)

        midx = MultiIndex.from_product([['A', 'B'], ['a', 'b']])
        df = DataFrame(np.ones((2, 4), dtype='int64'), columns=midx)
        s = pd.Series({'a': 1, 'b': 2})

        df2 = df.copy()
        df2.columns.names = ['lvl0', 'lvl1']
        s2 = s.copy()
        s2.index.name = 'lvl1'

        # different cases of integer/string level names:
        res1 = df.mul(s, axis=1, level=1)
        res2 = df.mul(s2, axis=1, level=1)
        res3 = df2.mul(s, axis=1, level=1)
        res4 = df2.mul(s2, axis=1, level=1)
        res5 = df2.mul(s, axis=1, level='lvl1')
        res6 = df2.mul(s2, axis=1, level='lvl1')

        exp = DataFrame(np.array([[1, 2, 1, 2], [1, 2, 1, 2]], dtype='int64'),
                        columns=midx)

        for res in [res1, res2]:
            assert_frame_equal(res, exp)

        exp.columns.names = ['lvl0', 'lvl1']
        for res in [res3, res4, res5, res6]:
            assert_frame_equal(res, exp)
//  [python]
def test_binary_ops_align(self):

        # test aligning binary ops

        # GH 6681
        index = MultiIndex.from_product([list('abc'),
                                         ['one', 'two', 'three'],
                                         [1, 2, 3]],
                                        names=['first', 'second', 'third'])

        df = DataFrame(np.arange(27 * 3).reshape(27, 3),
                       index=index,
                       columns=['value1', 'value2', 'value3']).sort_index()

        idx = pd.IndexSlice
        for op in ['add', 'sub', 'mul', 'div', 'truediv']:
            opa = getattr(operator, op, None)
            if opa is None:
                continue

            x = Series([1.0, 10.0, 100.0], [1, 2, 3])
            result = getattr(df, op)(x, level='third', axis=0)

            expected = pd.concat([opa(df.loc[idx[:, :, i], :], v)
                                  for i, v in x.items()]).sort_index()
            assert_frame_equal(result, expected)

            x = Series([1.0, 10.0], ['two', 'three'])
            result = getattr(df, op)(x, level='second', axis=0)

            expected = (pd.concat([opa(df.loc[idx[:, i], :], v)
                                   for i, v in x.items()])
                        .reindex_like(df).sort_index())
            assert_frame_equal(result, expected)

        # GH9463 (alignment level of dataframe with series)

        midx = MultiIndex.from_product([['A', 'B'], ['a', 'b']])
        df = DataFrame(np.ones((2, 4), dtype='int64'), columns=midx)
        s = pd.Series({'a': 1, 'b': 2})

        df2 = df.copy()
        df2.columns.names = ['lvl0', 'lvl1']
        s2 = s.copy()
        s2.index.name = 'lvl1'

        # different cases of integer/string level names:
        res1 = df.mul(s, axis=1, level=1)
        res2 = df.mul(s2, axis=1, level=1)
        res3 = df2.mul(s, axis=1, level=1)
        res4 = df2.mul(s2, axis=1, level=1)
        res5 = df2.mul(s, axis=1, level='lvl1')
        res6 = df2.mul(s2, axis=1, level='lvl1')

        exp = DataFrame(np.array([[1, 2, 1, 2], [1, 2, 1, 2]], dtype='int64'),
                        columns=midx)

        for res in [res1, res2]:
            assert_frame_equal(res, exp)

        exp.columns.names = ['lvl0', 'lvl1']
        for res in [res3, res4, res5, res6]:
            assert_frame_equal(res, exp)
//  [python]
def test_ffill(self):
        ts = Series([0., 1., 2., 3., 4.], index=tm.makeDateIndex(5))
        ts[2] = np.NaN
        assert_series_equal(ts.fillna(method='ffill'), ts.fillna(method='ffill'))
//  [python]
def test_ffill(self):
        ts = Series([0., 1., 2., 3., 4.], index=tm.makeDateIndex(5))
        ts[2] = np.NaN
        assert_series_equal(ts.ffill(), ts.fillna(method='ffill'))
//  [python]
def test_pad(self):
        pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
        psser = ps.from_pandas(pser)

        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
            self.assert_eq(pser.pad(), psser.pad())

            # Test `inplace=True`
            pser.pad(inplace=True)
            psser.pad(inplace=True)
            self.assert_eq(pser, psser)
        else:
            expected = ps.Series([np.nan, 2, 3, 4, 4, 6], name="x")
            self.assert_eq(expected, psser.pad())

            # Test `inplace=True`
            psser.pad(inplace=True)
            self.assert_eq(expected, psser)
//  [python]
def test_pad(self):
        pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
        psser = ps.from_pandas(pser)

        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
            self.assert_eq(pser.ffill(), psser.ffill())

            # Test `inplace=True`
            pser.ffill(inplace=True)
            psser.ffill(inplace=True)
            self.assert_eq(pser, psser)
        else:
            expected = ps.Series([np.nan, 2, 3, 4, 4, 6], name="x")
            self.assert_eq(expected, psser.ffill())

            # Test `inplace=True`
            psser.ffill(inplace=True)
            self.assert_eq(expected, psser)
//  [python]
def test_pad(self):
        pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
        kser = ks.from_pandas(pser)

        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
            self.assert_eq(pser.pad(), kser.pad())

            # Test `inplace=True`
            pser.pad(inplace=True)
            kser.pad(inplace=True)
            self.assert_eq(pser, kser)
        else:
            expected = ks.Series([np.nan, 2, 3, 4, 4, 6], name="x")
            self.assert_eq(expected, kser.pad())

            # Test `inplace=True`
            kser.pad(inplace=True)
            self.assert_eq(expected, kser)
//  [python]
def test_pad(self):
        pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
        kser = ks.from_pandas(pser)

        if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
            self.assert_eq(pser.ffill(), kser.ffill())

            # Test `inplace=True`
            pser.ffill(inplace=True)
            kser.ffill(inplace=True)
            self.assert_eq(pser, kser)
        else:
            expected = ks.Series([np.nan, 2, 3, 4, 4, 6], name="x")
            self.assert_eq(expected, kser.ffill())

            # Test `inplace=True`
            kser.ffill(inplace=True)
            self.assert_eq(expected, kser)
//  [python]
def test_iteritems(self):
        pdf = pd.DataFrame(
            {"species": ["bear", "bear", "marsupial"], "population": [1864, 22000, 80000]},
            index=["panda", "polar", "koala"],
            columns=["species", "population"],
        )
        psdf = ps.from_pandas(pdf)

        for (p_name, p_items), (k_name, k_items) in zip(pdf.iteritems(), psdf.iteritems()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_iteritems(self):
        pdf = pd.DataFrame(
            {"species": ["bear", "bear", "marsupial"], "population": [1864, 22000, 80000]},
            index=["panda", "polar", "koala"],
            columns=["species", "population"],
        )
        psdf = ps.from_pandas(pdf)

        for (p_name, p_items), (k_name, k_items) in zip(pdf.items(), psdf.items()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_iteritems(self):
        pdf = pd.DataFrame(
            {"species": ["bear", "bear", "marsupial"], "population": [1864, 22000, 80000]},
            index=["panda", "polar", "koala"],
            columns=["species", "population"],
        )
        kdf = ks.from_pandas(pdf)

        for (p_name, p_items), (k_name, k_items) in zip(pdf.iteritems(), kdf.iteritems()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_iteritems(self):
        pdf = pd.DataFrame(
            {"species": ["bear", "bear", "marsupial"], "population": [1864, 22000, 80000]},
            index=["panda", "polar", "koala"],
            columns=["species", "population"],
        )
        kdf = ks.from_pandas(pdf)

        for (p_name, p_items), (k_name, k_items) in zip(pdf.items(), kdf.items()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def _align_core(terms):
    term_index = [i for i, term in enumerate(terms) if hasattr(term.value, "axes")]
    term_dims = [terms[i].value.ndim for i in term_index]

    from pandas import Series

    ndims = Series(dict(zip(term_index, term_dims)))

    # initial axes are the axes of the largest-axis'd term
    biggest = terms[ndims.idxmax()].value
    typ = biggest._constructor
    axes = biggest.axes
    naxes = len(axes)
    gt_than_one_axis = naxes > 1

    for value in (terms[i].value for i in term_index):
        is_series = isinstance(value, ABCSeries)
        is_series_and_gt_one_axis = is_series and gt_than_one_axis

        for axis, items in enumerate(value.axes):
            if is_series_and_gt_one_axis:
                ax, itm = naxes - 1, value.index
            else:
                ax, itm = axis, items

            if not axes[ax].is_(itm):
                axes[ax] = axes[ax].union(itm)

    for i, ndim in ndims.items():
        for axis, items in zip(range(ndim), axes):
            ti = terms[i].value

            if hasattr(ti, "reindex"):
                transpose = isinstance(ti, ABCSeries) and naxes > 1
                reindexer = axes[naxes - 1] if transpose else items

                term_axis_size = len(ti.axes[axis])
                reindexer_size = len(reindexer)

                ordm = np.log10(max(1, abs(reindexer_size - term_axis_size)))
                if (
                    get_option("performance_warnings")
                    and ordm >= 1
                    and reindexer_size >= 10000
                ):
                    w = (
                        f"Alignment difference on axis {axis} is larger "
                        f"than an order of magnitude on term {terms[i].name!r}, "
                        f"by more than {ordm:.4g}; performance may suffer."
                    )
                    warnings.warn(
                        w, category=PerformanceWarning, stacklevel=find_stack_level()
                    )

                obj = ti.reindex(reindexer, axis=axis)
                terms[i].update(obj)

        terms[i].update(terms[i].value.values)

    return typ, _zip_axes_from_type(typ, axes)
//  [python]
def turnover(
    x: pd.Series, y: Union[pd.Series, float, int], smooth_y_days: int = 250
) -> float:
    """
    Gives the turnover of x, once normalised for y

    Returned in annualised terms

    """

    daily_x = x.resample("1B").last()
    if isinstance(y, float) or isinstance(y, int):
        daily_y = pd.Series(np.full(daily_x.shape[0], float(y)), daily_x.index)
    else:
        daily_y = y.reindex(daily_x.index, method="ffill")
        ## need to apply a drag to this or will give zero turnover for constant risk
        daily_y = daily_y.ewm(smooth_y_days, min_periods=2).mean()

    x_normalised_for_y = daily_x / daily_y.fillna(method='ffill')

    avg_daily = float(x_normalised_for_y.diff().abs().mean())

    return avg_daily * BUSINESS_DAYS_IN_YEAR
//  [python]
def turnover(
    x: pd.Series, y: Union[pd.Series, float, int], smooth_y_days: int = 250
) -> float:
    """
    Gives the turnover of x, once normalised for y

    Returned in annualised terms

    """

    daily_x = x.resample("1B").last()
    if isinstance(y, float) or isinstance(y, int):
        daily_y = pd.Series(np.full(daily_x.shape[0], float(y)), daily_x.index)
    else:
        daily_y = y.reindex(daily_x.index, method="ffill")
        ## need to apply a drag to this or will give zero turnover for constant risk
        daily_y = daily_y.ewm(smooth_y_days, min_periods=2).mean()

    x_normalised_for_y = daily_x / daily_y.ffill()

    avg_daily = float(x_normalised_for_y.diff().abs().mean())

    return avg_daily * BUSINESS_DAYS_IN_YEAR
//  [python]
def test_binary_ops_align(self):

        # test aligning binary ops

        # GH 6681
        index = MultiIndex.from_product(
            [list("abc"), ["one", "two", "three"], [1, 2, 3]],
            names=["first", "second", "third"],
        )

        df = DataFrame(
            np.arange(27 * 3).reshape(27, 3),
            index=index,
            columns=["value1", "value2", "value3"],
        ).sort_index()

        idx = pd.IndexSlice
        for op in ["add", "sub", "mul", "div", "truediv"]:
            opa = getattr(operator, op, None)
            if opa is None:
                continue

            x = Series([1.0, 10.0, 100.0], [1, 2, 3])
            result = getattr(df, op)(x, level="third", axis=0)

            expected = pd.concat(
                [opa(df.loc[idx[:, :, i], :], v) for i, v in zip(x.index, x.values)]
            ).sort_index()
            tm.assert_frame_equal(result, expected)

            x = Series([1.0, 10.0], ["two", "three"])
            result = getattr(df, op)(x, level="second", axis=0)

            expected = (
                pd.concat([opa(df.loc[idx[:, i], :], v) for i, v in zip(x.index, x.values)])
                .reindex_like(df)
                .sort_index()
            )
            tm.assert_frame_equal(result, expected)

        # GH9463 (alignment level of dataframe with series)

        midx = MultiIndex.from_product([["A", "B"], ["a", "b"]])
        df = DataFrame(np.ones((2, 4), dtype="int64"), columns=midx)
        s = Series({"a": 1, "b": 2})

        df2 = df.copy()
        df2.columns.names = ["lvl0", "lvl1"]
        s2 = s.copy()
        s2.index.name = "lvl1"

        # different cases of integer/string level names:
        res1 = df.mul(s, axis=1, level=1)
        res2 = df.mul(s2, axis=1, level=1)
        res3 = df2.mul(s, axis=1, level=1)
        res4 = df2.mul(s2, axis=1, level=1)
        res5 = df2.mul(s, axis=1, level="lvl1")
        res6 = df2.mul(s2, axis=1, level="lvl1")

        exp = DataFrame(
            np.array([[1, 2, 1, 2], [1, 2, 1, 2]], dtype="int64"), columns=midx
        )

        for res in [res1, res2]:
            tm.assert_frame_equal(res, exp)

        exp.columns.names = ["lvl0", "lvl1"]
        for res in [res3, res4, res5, res6]:
            tm.assert_frame_equal(res, exp)
//  [python]
def test_binary_ops_align(self):

        # test aligning binary ops

        # GH 6681
        index = MultiIndex.from_product(
            [list("abc"), ["one", "two", "three"], [1, 2, 3]],
            names=["first", "second", "third"],
        )

        df = DataFrame(
            np.arange(27 * 3).reshape(27, 3),
            index=index,
            columns=["value1", "value2", "value3"],
        ).sort_index()

        idx = pd.IndexSlice
        for op in ["add", "sub", "mul", "div", "truediv"]:
            opa = getattr(operator, op, None)
            if opa is None:
                continue

            x = Series([1.0, 10.0, 100.0], [1, 2, 3])
            result = getattr(df, op)(x, level="third", axis=0)

            expected = pd.concat(
                [opa(df.loc[idx[:, :, i], :], v) for i, v in x.items()]
            ).sort_index()
            tm.assert_frame_equal(result, expected)

            x = Series([1.0, 10.0], ["two", "three"])
            result = getattr(df, op)(x, level="second", axis=0)

            expected = (
                pd.concat([opa(df.loc[idx[:, i], :], v) for i, v in x.items()])
                .reindex_like(df)
                .sort_index()
            )
            tm.assert_frame_equal(result, expected)

        # GH9463 (alignment level of dataframe with series)

        midx = MultiIndex.from_product([["A", "B"], ["a", "b"]])
        df = DataFrame(np.ones((2, 4), dtype="int64"), columns=midx)
        s = Series({"a": 1, "b": 2})

        df2 = df.copy()
        df2.columns.names = ["lvl0", "lvl1"]
        s2 = s.copy()
        s2.index.name = "lvl1"

        # different cases of integer/string level names:
        res1 = df.mul(s, axis=1, level=1)
        res2 = df.mul(s2, axis=1, level=1)
        res3 = df2.mul(s, axis=1, level=1)
        res4 = df2.mul(s2, axis=1, level=1)
        res5 = df2.mul(s, axis=1, level="lvl1")
        res6 = df2.mul(s2, axis=1, level="lvl1")

        exp = DataFrame(
            np.array([[1, 2, 1, 2], [1, 2, 1, 2]], dtype="int64"), columns=midx
        )

        for res in [res1, res2]:
            tm.assert_frame_equal(res, exp)

        exp.columns.names = ["lvl0", "lvl1"]
        for res in [res3, res4, res5, res6]:
            tm.assert_frame_equal(res, exp)
//  [python]
def test_items(using_copy_on_write):
    df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
    df_orig = df.copy()

    # Test this twice, since the second time, the item cache will be
    # triggered, and we want to make sure it still works then.
    for i in range(2):
        for name, ser in df.iteritems():
            assert np.shares_memory(get_array(ser, name), get_array(df, name))

            # mutating df triggers a copy-on-write for that column / block
            ser.iloc[0] = 0

            if using_copy_on_write:
                assert not np.shares_memory(get_array(ser, name), get_array(df, name))
                tm.assert_frame_equal(df, df_orig)
            else:
                # Original frame will be modified
                assert df.loc[0, name] == 0
//  [python]
def test_items(using_copy_on_write):
    df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
    df_orig = df.copy()

    # Test this twice, since the second time, the item cache will be
    # triggered, and we want to make sure it still works then.
    for i in range(2):
        for name, ser in df.items():
            assert np.shares_memory(get_array(ser, name), get_array(df, name))

            # mutating df triggers a copy-on-write for that column / block
            ser.iloc[0] = 0

            if using_copy_on_write:
                assert not np.shares_memory(get_array(ser, name), get_array(df, name))
                tm.assert_frame_equal(df, df_orig)
            else:
                # Original frame will be modified
                assert df.loc[0, name] == 0
//  [python]
def test_datetime64_fillna(self):

        s = Series([Timestamp('20130101'), Timestamp('20130101'),
                    Timestamp('20130102'), Timestamp('20130103 9:01:01')])
        s[2] = np.nan

        # reg fillna
        result = s.fillna(Timestamp('20130104'))
        expected = Series([Timestamp('20130101'), Timestamp('20130101'),
                           Timestamp('20130104'), Timestamp('20130103 9:01:01')])
        assert_series_equal(result, expected)

        from pandas import tslib
        result = s.fillna(tslib.NaT)
        expected = s
        assert_series_equal(result, expected)

        # ffill
        result = s.fillna(method='ffill')
        expected = Series([Timestamp('20130101'), Timestamp('20130101'),
                           Timestamp('20130101'), Timestamp('20130103 9:01:01')])
        assert_series_equal(result, expected)

        # bfill
        result = s.fillna(method='bfill')
        expected = Series([Timestamp('20130101'), Timestamp('20130101'),
                           Timestamp('20130103 9:01:01'),
                           Timestamp('20130103 9:01:01')])
        assert_series_equal(result, expected)

        # GH 6587
        # make sure that we are treating as integer when filling
        # this also tests inference of a datetime-like with NaT's
        s = Series([pd.NaT, pd.NaT, '2013-08-05 15:30:00.000001'])
        expected = Series(['2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001'], dtype='M8[ns]')
        result = s.fillna(method='backfill')
        assert_series_equal(result, expected)
//  [python]
def test_datetime64_fillna(self):

        s = Series([Timestamp('20130101'), Timestamp('20130101'),
                    Timestamp('20130102'), Timestamp('20130103 9:01:01')])
        s[2] = np.nan

        # reg fillna
        result = s.fillna(Timestamp('20130104'))
        expected = Series([Timestamp('20130101'), Timestamp('20130101'),
                           Timestamp('20130104'), Timestamp('20130103 9:01:01')])
        assert_series_equal(result, expected)

        from pandas import tslib
        result = s.fillna(tslib.NaT)
        expected = s
        assert_series_equal(result, expected)

        # ffill
        result = s.ffill()
        expected = Series([Timestamp('20130101'), Timestamp('20130101'),
                           Timestamp('20130101'), Timestamp('20130103 9:01:01')])
        assert_series_equal(result, expected)

        # bfill
        result = s.bfill()
        expected = Series([Timestamp('20130101'), Timestamp('20130101'),
                           Timestamp('20130103 9:01:01'),
                           Timestamp('20130103 9:01:01')])
        assert_series_equal(result, expected)

        # GH 6587
        # make sure that we are treating as integer when filling
        # this also tests inference of a datetime-like with NaT's
        s = Series([pd.NaT, pd.NaT, '2013-08-05 15:30:00.000001'])
        expected = Series(['2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001'], dtype='M8[ns]')
        result = s.fillna(method='backfill')
        assert_series_equal(result, expected)
//  [python]
def test_items(self):
        pdf = pd.DataFrame(
            {"species": ["bear", "bear", "marsupial"], "population": [1864, 22000, 80000]},
            index=["panda", "polar", "koala"],
            columns=["species", "population"],
        )
        psdf = ps.from_pandas(pdf)

        for (p_name, p_items), (k_name, k_items) in zip(pdf.iteritems(), psdf.items()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_items(self):
        pdf = pd.DataFrame(
            {"species": ["bear", "bear", "marsupial"], "population": [1864, 22000, 80000]},
            index=["panda", "polar", "koala"],
            columns=["species", "population"],
        )
        psdf = ps.from_pandas(pdf)

        for (p_name, p_items), (k_name, k_items) in zip(pdf.items(), psdf.items()):
            self.assert_eq(p_name, k_name)
            self.assert_eq(p_items, k_items)
//  [python]
def test_apply_differently_indexed():
    df = DataFrame(np.random.default_rng(2).standard_normal((20, 10)))

    result = df.apply(Series.describe, axis=0)
    expected = DataFrame({i: v.describe() for i, v in df.iteritems()}, columns=df.columns)
    tm.assert_frame_equal(result, expected)

    result = df.apply(Series.describe, axis=1)
    expected = DataFrame({i: v.describe() for i, v in df.T.iteritems()}, columns=df.index).T
    tm.assert_frame_equal(result, expected)
//  [python]
def test_apply_differently_indexed():
    df = DataFrame(np.random.default_rng(2).standard_normal((20, 10)))

    result = df.apply(Series.describe, axis=0)
    expected = DataFrame({i: v.describe() for i, v in df.items()}, columns=df.columns)
    tm.assert_frame_equal(result, expected)

    result = df.apply(Series.describe, axis=1)
    expected = DataFrame({i: v.describe() for i, v in df.T.items()}, columns=df.index).T
    tm.assert_frame_equal(result, expected)
//  [python]
def test_from_records_dictlike(self):

        # test the dict methods
        df = DataFrame(
            {
                "A": np.array(np.random.randn(6), dtype=np.float64),
                "A1": np.array(np.random.randn(6), dtype=np.float64),
                "B": np.array(np.arange(6), dtype=np.int64),
                "C": ["foo"] * 6,
                "D": np.array([True, False] * 3, dtype=bool),
                "E": np.array(np.random.randn(6), dtype=np.float32),
                "E1": np.array(np.random.randn(6), dtype=np.float32),
                "F": np.array(np.arange(6), dtype=np.int32),
            }
        )

        # columns is in a different order here than the actual items iterated
        # from the dict
        blocks = df._to_dict_of_blocks()
        columns = []
        for dtype, b in blocks.items():
            columns.extend(b.columns)

        asdict = dict(df.iteritems())
        asdict2 = {x: y.values for x, y in df.iteritems()}

        # dict of series & dict of ndarrays (have dtype info)
        results = []
        results.append(DataFrame.from_records(asdict).reindex(columns=df.columns))
        results.append(
            DataFrame.from_records(asdict, columns=columns).reindex(columns=df.columns)
        )
        results.append(
            DataFrame.from_records(asdict2, columns=columns).reindex(columns=df.columns)
        )

        for r in results:
            tm.assert_frame_equal(r, df)
//  [python]
def test_from_records_dictlike(self):

        # test the dict methods
        df = DataFrame(
            {
                "A": np.array(np.random.randn(6), dtype=np.float64),
                "A1": np.array(np.random.randn(6), dtype=np.float64),
                "B": np.array(np.arange(6), dtype=np.int64),
                "C": ["foo"] * 6,
                "D": np.array([True, False] * 3, dtype=bool),
                "E": np.array(np.random.randn(6), dtype=np.float32),
                "E1": np.array(np.random.randn(6), dtype=np.float32),
                "F": np.array(np.arange(6), dtype=np.int32),
            }
        )

        # columns is in a different order here than the actual items iterated
        # from the dict
        blocks = df._to_dict_of_blocks()
        columns = []
        for dtype, b in blocks.items():
            columns.extend(b.columns)

        asdict = {x: y for x, y in df.items()}
        asdict2 = {x: y.values for x, y in df.items()}

        # dict of series & dict of ndarrays (have dtype info)
        results = []
        results.append(DataFrame.from_records(asdict).reindex(columns=df.columns))
        results.append(
            DataFrame.from_records(asdict, columns=columns).reindex(columns=df.columns)
        )
        results.append(
            DataFrame.from_records(asdict2, columns=columns).reindex(columns=df.columns)
        )

        for r in results:
            tm.assert_frame_equal(r, df)
//  [python]
def test_apply_differently_indexed():
    df = DataFrame(np.random.randn(20, 10))

    result = df.apply(Series.describe, axis=0)
    expected = DataFrame({i: v.describe() for i, v in df.iteritems()}, columns=df.columns)
    tm.assert_frame_equal(result, expected)

    result = df.apply(Series.describe, axis=1)
    expected = DataFrame({i: v.describe() for i, v in df.T.iteritems()}, columns=df.index).T
    tm.assert_frame_equal(result, expected)
//  [python]
def test_apply_differently_indexed():
    df = DataFrame(np.random.randn(20, 10))

    result = df.apply(Series.describe, axis=0)
    expected = DataFrame({i: v.describe() for i, v in df.items()}, columns=df.columns)
    tm.assert_frame_equal(result, expected)

    result = df.apply(Series.describe, axis=1)
    expected = DataFrame({i: v.describe() for i, v in df.T.items()}, columns=df.index).T
    tm.assert_frame_equal(result, expected)
//  [python]
def test_dataframe_items(columns):
    df = pd.DataFrame([[1, 10], [2, 20], [3, 30], [4, 40]], columns=columns)
    ddf = dd.from_pandas(df, npartitions=2)
    for a, b in zip(df.iteritems(), ddf.iteritems()):
        assert a[0] == b[0]  # column name
        assert_eq(a[1], b[1].compute())
//  [python]
def test_dataframe_items(columns):
    df = pd.DataFrame([[1, 10], [2, 20], [3, 30], [4, 40]], columns=columns)
    ddf = dd.from_pandas(df, npartitions=2)
    for a, b in zip(df.items(), ddf.items()):
        assert a[0] == b[0]  # column name
        assert_eq(a[1], b[1].compute())
//  [python]
def test_set_index_bug(self):
        # GH1590
        df = DataFrame({'val': [0, 1, 2], 'key': ['a', 'b', 'c']})
        df2 = df.select(lambda indx: indx >= 1)
        rs = df2.set_index('key')
        xp = DataFrame({'val': [1, 2]},
                       Index(['b', 'c'], name='key'))
        assert_frame_equal(rs, xp)
//  [python]
def test_set_index_bug(self):
        # GH1590
        df = DataFrame({'val': [0, 1, 2], 'key': ['a', 'b', 'c']})
        df2 = df.loc[df.index >= 1]
        rs = df2.set_index('key')
        xp = DataFrame({'val': [1, 2]},
                       Index(['b', 'c'], name='key'))
        assert_frame_equal(rs, xp)
//  [python]
def test_groupby_level_no_obs(self):
        # #1697
        midx = MultiIndex.from_tuples([('f1', 's1'), ('f1', 's2'), (
            'f2', 's1'), ('f2', 's2'), ('f3', 's1'), ('f3', 's2')])
        df = DataFrame(
            [[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]], columns=midx)
        df1 = df.select(lambda u: u[0] in ['f2', 'f3'], axis=1)

        grouped = df1.groupby(axis=1, level=0)
        result = grouped.sum()
        assert (result.columns == ['f2', 'f3']).all()
//  [python]
def test_groupby_level_no_obs(self):
        # #1697
        midx = MultiIndex.from_tuples([('f1', 's1'), ('f1', 's2'), (
            'f2', 's1'), ('f2', 's2'), ('f3', 's1'), ('f3', 's2')])
        df = DataFrame(
            [[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]], columns=midx)
        df1 = df.loc[:, df.columns.get_level_values(0).isin(['f2', 'f3'])]

        grouped = df1.groupby(axis=1, level=0)
        result = grouped.sum()
        assert (result.columns == ['f2', 'f3']).all()
//  [python]
def fit(
        self, raw_data: DFLike, discrete_columns: Optional[Sequence[str]] = None
    ) -> None:
        """Fit the ``DataTransformer``.

        Fits a ``ClusterBasedNormalizer`` for continuous columns and a
        ``OneHotEncoder`` or ``BinaryEncodingTransformer`` for discrete columns depending
        on the number of unique values in columns.

        This step also counts the #columns in matrix data and span information.
        """
        self.dataframe = True

        if discrete_columns is None:
            discrete_columns = []

        if not isinstance(raw_data, pd.DataFrame):
            self.dataframe = False
            # work around for RDT issue #328 Fitting with numerical column names fails
            discrete_columns = [str(column) for column in discrete_columns]
            column_names = [str(num) for num in range(raw_data.shape[1])]
            raw_data = pd.DataFrame(raw_data, columns=column_names)

        # Rather than calling infer_objects() on raw_data as a whole, call it on
        # each column individually and stitch together the result mimicking the
        # layout of raw_data.infer_objects().dtypes. This is done to reduce memory
        # pressure, as raw_data.infer_objects() might create a copy of the entire
        # data, whereas in the approach below, we only need duplicate the data from
        # a single column at a time.
        self._column_raw_dtypes = pd.Series(
            {c: s.infer_objects().dtype for c, s in raw_data.iteritems()}
        )
        self._column_transform_info_list = []
        if self._verbose:
            logger.info(
                f"Starting data transforms on {len(raw_data.columns)} columns..."
            )
        for column_name in raw_data.columns:
            if column_name in discrete_columns:
                column_transform_info = self._fit_discrete(raw_data[[column_name]])
            else:
                column_transform_info = self._fit_continuous(raw_data[[column_name]])

            self._column_transform_info_list.append(column_transform_info)
//  [python]
def fit(
        self, raw_data: DFLike, discrete_columns: Optional[Sequence[str]] = None
    ) -> None:
        """Fit the ``DataTransformer``.

        Fits a ``ClusterBasedNormalizer`` for continuous columns and a
        ``OneHotEncoder`` or ``BinaryEncodingTransformer`` for discrete columns depending
        on the number of unique values in columns.

        This step also counts the #columns in matrix data and span information.
        """
        self.dataframe = True

        if discrete_columns is None:
            discrete_columns = []

        if not isinstance(raw_data, pd.DataFrame):
            self.dataframe = False
            # work around for RDT issue #328 Fitting with numerical column names fails
            discrete_columns = [str(column) for column in discrete_columns]
            column_names = [str(num) for num in range(raw_data.shape[1])]
            raw_data = pd.DataFrame(raw_data, columns=column_names)

        # Rather than calling infer_objects() on raw_data as a whole, call it on
        # each column individually and stitch together the result mimicking the
        # layout of raw_data.infer_objects().dtypes. This is done to reduce memory
        # pressure, as raw_data.infer_objects() might create a copy of the entire
        # data, whereas in the approach below, we only need duplicate the data from
        # a single column at a time.
        self._column_raw_dtypes = pd.Series(
            {c: s.infer_objects().dtype for c, s in raw_data.items()}
        )
        self._column_transform_info_list = []
        if self._verbose:
            logger.info(
                f"Starting data transforms on {len(raw_data.columns)} columns..."
            )
        for column_name in raw_data.columns:
            if column_name in discrete_columns:
                column_transform_info = self._fit_discrete(raw_data[[column_name]])
            else:
                column_transform_info = self._fit_continuous(raw_data[[column_name]])

            self._column_transform_info_list.append(column_transform_info)
//  [python]
def test_equivalence_of_pools_from_pandas_dataframe_with_different_cat_features_column_types():
    df = DataFrame()
    df['num_feat_0'] = [0, 1, 0, 2, 3, 1, 2]
    df['num_feat_1'] = [0.12, 0.8, 0.33, 0.11, 0.0, 1.0, 0.0]
    df['cat_feat_2'] = ['A', 'B', 'A', 'C', 'A', 'A', 'A']
    df['cat_feat_3'] = ['x', 'x', 'y', 'y', 'y', 'x', 'x']
    df['cat_feat_4'] = ['large', 'small', 'medium', 'large', 'small', 'small', 'medium']
    df['cat_feat_5'] = [0, 1, 0, 2, 3, 1, 2]

    labels = [0, 1, 1, 0, 1, 0, 1]

    cat_features = ['cat_feat_%i' % i for i in range(2, 6)]

    pool_from_df = Pool(df, labels, cat_features=cat_features)

    for cat_features_dtype in ['object', 'category']:
        columns_for_new_df = OrderedDict()
        for column_name in df.columns:
            column_data = df[column_name]
            if column_name in cat_features:
                column_data = column_data.astype(cat_features_dtype)
            columns_for_new_df.setdefault(column_name, column_data)

        new_df = DataFrame(columns_for_new_df)

        pool_from_new_df = Pool(new_df, labels, cat_features=cat_features)

        assert _have_equal_features(pool_from_df, pool_from_new_df)
//  [python]
def test_equivalence_of_pools_from_pandas_dataframe_with_different_cat_features_column_types():
    df = DataFrame()
    df['num_feat_0'] = [0, 1, 0, 2, 3, 1, 2]
    df['num_feat_1'] = [0.12, 0.8, 0.33, 0.11, 0.0, 1.0, 0.0]
    df['cat_feat_2'] = ['A', 'B', 'A', 'C', 'A', 'A', 'A']
    df['cat_feat_3'] = ['x', 'x', 'y', 'y', 'y', 'x', 'x']
    df['cat_feat_4'] = ['large', 'small', 'medium', 'large', 'small', 'small', 'medium']
    df['cat_feat_5'] = [0, 1, 0, 2, 3, 1, 2]

    labels = [0, 1, 1, 0, 1, 0, 1]

    cat_features = ['cat_feat_%i' % i for i in range(2, 6)]

    pool_from_df = Pool(df, labels, cat_features=cat_features)

    for cat_features_dtype in ['object', 'category']:
        columns_for_new_df = OrderedDict()
        for column_name, column_data in df.items():
            if column_name in cat_features:
                column_data = column_data.astype(cat_features_dtype)
            columns_for_new_df.setdefault(column_name, column_data)

        new_df = DataFrame(columns_for_new_df)

        pool_from_new_df = Pool(new_df, labels, cat_features=cat_features)

        assert _have_equal_features(pool_from_df, pool_from_new_df)
//  [python]
def _convert_batch_type_to_pandas(
    data: DataBatchType,
    cast_tensor_columns: bool = False,
) -> "pd.DataFrame":
    """Convert the provided data to a Pandas DataFrame.

    Args:
        data: Data of type DataBatchType
        cast_tensor_columns: Whether tensor columns should be cast to NumPy ndarrays.

    Returns:
        A pandas Dataframe representation of the input data.

    """
    pd = _lazy_import_pandas()

    if isinstance(data, np.ndarray):
        data = pd.DataFrame({TENSOR_COLUMN_NAME: _ndarray_to_column(data)})
    elif isinstance(data, dict):
        tensor_dict = {}
        for col_name, col in data.items():
            if not isinstance(col, np.ndarray):
                raise ValueError(
                    "All values in the provided dict must be of type "
                    f"np.ndarray. Found type {type(col)} for key {col_name} "
                    f"instead."
                )
            tensor_dict[col_name] = _ndarray_to_column(col)
        data = pd.DataFrame(tensor_dict)
    elif pyarrow is not None and isinstance(data, pyarrow.Table):
        data = data.to_pandas()
    elif not isinstance(data, pd.DataFrame):
        raise ValueError(
            f"Received data of type: {type(data)}, but expected it to be one "
            f"of {DataBatchType}"
        )
    if cast_tensor_columns:
        data = _cast_tensor_columns_to_ndarrays(data)
    return data
//  [python]
def test_first():
    i = pd.date_range("2010-04-09", periods=400, freq="2D")
    modin_df = pd.DataFrame({"A": list(range(400)), "B": list(range(400))}, index=i)
    pandas_df = pandas.DataFrame(
        {"A": list(range(400)), "B": list(range(400))}, index=i
    )
    with pytest.warns(FutureWarning, match="first is deprecated and will be removed"):
        modin_result = modin_df.first("3D")
    df_equals(modin_result, pandas_df.first("3D"))
    df_equals(modin_df.first("20D"), pandas_df.first("20D"))