Issue 1:
Hi,

I have recently come across an unexpexted behaviour of the dataframe "shift" function.

When column titles are duplicated (or present more often), a column-specific shift using index-based positioning (by "iloc") does not only shift the specified columns, but all columns sharing the same name. From the behaviour I guess that internally, the column names and not the indices are used, causing all identically named columns to shift.

This behaviour did not occur (at least to my knowledge ) with v. 0.13, but is present in both 0.15.1 and 0.15.2.

Please find attached a small code snippet which demonstrates the behaviour, and the system information.

Thanks a lot for any help

Kay

Begin example code

from pandas import *
import pandas as pd

data = DataFrame(np.random.randn(20, 5)) # Generating a random data frame
data2 = data.copy()
data3 = data.copy()
shifting_matrix = [1,2,3,4,5] # shifting matrix - we want to shift the columns by these values

#Shifting the columns: Expected behaviour. Column 1 is shifter by 1, Column 2 by 2 and so on

for position, movement in enumerate(shifting_matrix):
print position
print movement
data.iloc[:, position] = data.iloc[:,position].shift(movement)

## Changing the titles of the dataframe
titles = [1,1,1,1,1]
data2.columns = titles

# Shifting the columns again. Since we are using index based positions, the shifting behaviour should be the same as above
for position, movement in enumerate(shifting_matrix):
print position
print movement
data2.iloc[:, position] = data2.iloc[:,position].shift(movement)

titles2 = [1,1,2,2,1] # Making mixed column titles, duplicating
data3.columns = titles2
# Shifting the columns again. Since we are using index based positions, the shifting behaviour should be the same as above
for position, movement in enumerate(shifting_matrix):
print position
print movement
data3.iloc[:, position] = data3.iloc[:,position].shift(movement)

print data
print data2
print data3

End example code

INSTALLED VERSIONS

commit: None

python: 2.7.6.final.0

python-bits: 32

OS: Windows

OS-release: 7

machine: AMD64

processor: Intel64 Family 6 Model 58 Stepping 9, GenuineIntel

byteorder: little

LC_ALL: None

LANG: nb_NO

pandas: 0.15.2

nose: 1.3.3

Cython: 0.20.1

numpy: 1.8.1

scipy: 0.14.0

statsmodels: 0.5.0

IPython: 2.1.0

sphinx: 1.2.2

patsy: 0.2.1

dateutil: 2.2

pytz: 2014.3

bottleneck: 0.8.0

tables: None

numexpr: 2.4

matplotlib: 1.3.1

openpyxl: 2.1.0

xlrd: None

xlwt: 0.7.5

xlsxwriter: 0.5.8

lxml: 3.3.5

bs4: 4.3.2

html5lib: 0.999

httplib2: None

apiclient: None

rpy2: None

sqlalchemy: 0.9.4

pymysql: None

psycopg2: None

Issue 2:
Code Sample, a copy-pastable example if possible

class NumPyBackedExtensionArrayMixin(ExtensionArray):
"""
Geo-Specific Extension Array Mixin
"""
@property
def dtype(self):
"""The dtype for this extension array, GeoType"""
return self._dtype

@classmethod
def _constructor_from_sequence(cls, scalars):
return cls(scalars)

@classmethod
def _from_factorized(cls, values, original):
return cls(values)

@property
def shape(self):
return (len(self.data),)

def __len__(self):
return len(self.data)

def __getitem__(self, *args):
result = operator.getitem(self.data, *args)
return result
#if isinstance(result, tuple):
#return self._box_scalar(result)
#elif result.ndim == 0:
#return self._box_scalar(result.item())
#else:
#return type(self)(result)

def setitem(self, indexer, value):
"""Set the 'value' inplace.
"""

self[indexer] = value
return self

@property
def nbytes(self):
return self._itemsize * len(self)

def _formatting_values(self):
return np.array(self._format_values(), dtype='object')

def copy(self, deep=False):
return type(self)(self.data.copy())

@classmethod
def _concat_same_type(cls, to_concat):
return cls(np.concatenate([array.data for array in to_concat]))

def tolist(self):
return self.data.tolist()

def argsort(self, axis=-1, kind='quicksort', order=None):
return self.data.argsort()

def unique(self):
_, indices = np.unique(self.data, return_index=True)
data = self.data.take(np.sort(indices))
return self._from_ndarray(data)

class GeoType(ExtensionDtype):
name = 'geometry'
type = Geometry
kind = 'O'
_record_type = np.dtype('O')
na_value = None

@classmethod
def construct_from_string(cls, string):
if string == cls.name:
return cls()
else:
raise TypeError("Cannot construct a '{}' from "
"'{}'".format(cls, string))

class GeoArray(NumPyBackedExtensionArrayMixin):
"""Array for Geometry data.
"""
# What type(s) do we support?
# MAC-48 or EUI-64?
_dtype = GeoType()
_itemsize = 8
ndim = 1
can_hold_na = True

def __init__(self, values, copy=True):
# TODO: parse hex / strings
self.data = np.array(values, dtype='O', copy=copy)

@classmethod
def _from_ndarray(cls, data, copy=False):
return cls(data, copy=copy)

@property
def na_value(self):
return self.dtype.na_value

def __repr__(self):
formatted = self._format_values()
return "GeoArray({!r})".format(formatted)

def _format_values(self):
return [_format(x) for x in self.data]

@classmethod
def from_geometry(cls, data, copy=False):
""""""
if copy:
data = data.copy()
new = GeoArray([])
new.data = np.array(data)
return new

def __setitem__(self, key, value):
value = Geometry(value)
self.data[key] = value

def __iter__(self):
return iter(self.data.tolist())

def __eq__(self, other):
return self.data == other

def equals(self, other):
if not isinstance(other, type(self)):
raise TypeError
return (self.data == other.data).all()

def _values_for_factorize(self):
# Should hit pandas' UInt64Hashtable
return self, 0

def isna(self):
return (self.data == self._dtype.na_value)

@property
def _parser(self):
return lambda x: x

def take(self, indexer, allow_fill=True, fill_value=None):
mask = indexer == -1
result = self.data.take(indexer)
result[mask] = self.dtype.na_value
return type(self)(result, copy=False)

def _formatting_values(self):
return np.array(self._format_values(), dtype='object')

@classmethod
def _concat_same_type(cls, to_concat):
return cls(np.concatenate([array.data for array in to_concat]))

def take_nd(self, indexer, allow_fill=True, fill_value=None):
return self.take(indexer, allow_fill=allow_fill, fill_value=fill_value)

def copy(self, deep=False):
return type(self)(self.data.copy())

from arcgis.geometry import BaseGeometry
@property
def is_valid(self):
"""Checks if the Geometry is Valid"""
return pd.Series([g.is_valid for g in self])

#### TO GET THE ERROR
>>> geoms = [{
"rings" : [[[-97.06138,32.837],[-97.06133,32.836],[-97.06124,32.834],[-97.06127,32.832],
[-97.06138,32.837]],[[-97.06326,32.759],[-97.06298,32.755],[-97.06153,32.749],
[-97.06326,32.759]]],
"spatialReference" : {"wkid" : 4326}
}
]
>>> v = GeoArray(geoms)
>>> df = pd.DataFrame({"SHAPE": v})
>>> small = df.head(1)
>>> print(small)

Problem description

Here is where the error occurs with I do a print(df.head(1))

builtins.AttributeError: 'numpy.ndarray' object has no attribute '_formatting_values'

def formatting_values(self):
return self.values._formatting_values()

I've implemented the override in the GeoArray, but it doesn't seem to ever get called instead it's calling the _formatting_values() on the ExtensionBlock class.

Any suggestions on how to resolve this issue?

Expected Output

A html version of the dataframe.

Output of pd.show_versions()

[paste the output of pd.show_versions() here below this line]

INSTALLED VERSIONS

commit: None

python: 3.6.5.final.0

python-bits: 64

OS: Windows

OS-release: 10

machine: AMD64

processor: Intel64 Family 6 Model 94 Stepping 3, GenuineIntel

byteorder: little

LC_ALL: None

LANG: None

LOCALE: English_United States.1252

pandas: 0.23.0

pytest: 3.5.0

pip: 9.0.3

setuptools: 39.0.1

Cython: None

numpy: 1.14.2

scipy: 1.0.1

pyarrow: None

xarray: None

IPython: 6.3.1

sphinx: None

patsy: None

dateutil: 2.7.2

pytz: 2018.3

blosc: None

bottleneck: None

tables: None

numexpr: 2.6.4

feather: None

matplotlib: 2.2.2

openpyxl: 2.5.2

xlrd: 1.1.0

xlwt: 1.3.0

xlsxwriter: None

lxml: None

bs4: None

html5lib: 1.0.1

sqlalchemy: None

pymysql: None

psycopg2: None

jinja2: 2.10

s3fs: None

fastparquet: None

pandas_gbq: None

pandas_datareader: None