Repository: pandas-dev/pandas

Issue:
.to_excel() cuts off columns
Today I discovered a strange behavior: When I am writing a DataFrame with `.to_excel()`, it cuts columns. Compared with the same DataFrame with `.to_csv()` or `.head()`, you can see the difference, that the last 8 columns are missing.

You can reproduce this by downloading `Features.pkl` from [here](https://dl.dropboxusercontent.com/u/16985007/Features.pkl) and then:

``` python
import pandas as pd
df = pd.read_pickle('Features.pkl')
df.head() # see the last 8 columns!
df.to_excel('Features.xlsx', index=False, header=False)
# see the Excel, you do not have these last 8 columns
# in a .to_csv() you have them
```

Funny part: If you `df.ix[:,-71:].to_excel('Features.xlsx', index=False, header=False)` you have one of the missing columns. If you do `df.ix[:,-70:].to_excel('Features.xlsx', index=False, header=False)` you have two and so on...

``` python
INSTALLED VERSIONS
------------------
commit: None
python: 2.7.10.final.0
python-bits: 64
OS: Darwin
OS-release: 14.5.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: de_DE.UTF-8

pandas: 0.16.2
nose: 1.3.7
Cython: 0.22.1
numpy: 1.9.2
scipy: 0.16.0
statsmodels: 0.6.1
IPython: 3.2.0
sphinx: 1.3.1
patsy: 0.3.0
dateutil: 2.4.2
pytz: 2015.4
bottleneck: 1.0.0
tables: 3.2.0
numexpr: 2.4.3
matplotlib: 1.4.3
openpyxl: 1.8.6
xlrd: 0.9.3
xlwt: 1.0.0
xlsxwriter: 0.7.3
lxml: 3.4.4
bs4: 4.3.2
html5lib: None
httplib2: 0.9
apiclient: None
sqlalchemy: 1.0.5
pymysql: None
psycopg2: None
```



Discussion:
In an odd coincidence, I think this is another Excel-dup-column issue:

```
>>> len(df.columns)
380
>>> len(set(df.columns))
372
```

and there are eight columns missing.  By changing the window into df, you change which columns (if any) are duplicated.

Thats it! There are columns with exactly the same name on this position `[-70:]`. So no export of duplicated column names `to_excel`?


Resolving patch:
diff --git a/pandas/core/format.py b/pandas/core/format.py
--- a/pandas/core/format.py
+++ b/pandas/core/format.py
@@ -1683,12 +1683,12 @@ class ExcelFormatter(object):
     def __init__(self, df, na_rep='', float_format=None, cols=None,
                  header=True, index=True, index_label=None, merge_cells=False,
                  inf_rep='inf'):
-        self.df = df
         self.rowcounter = 0
         self.na_rep = na_rep
-        self.columns = cols
-        if cols is None:
-            self.columns = df.columns
+        self.df = df
+        if cols is not None:
+            self.df = df.loc[:, cols]
+        self.columns = self.df.columns
         self.float_format = float_format
         self.index = index
         self.index_label = index_label
@@ -1843,12 +1843,9 @@ def _format_regular_rows(self):
             for idx, idxval in enumerate(index_values):
                 yield ExcelCell(self.rowcounter + idx, 0, idxval, header_style)
 
-        # Get a frame that will account for any duplicates in the column names.
-        col_mapped_frame = self.df.loc[:, self.columns]
-
         # Write the body of the frame data series by series.
         for colidx in range(len(self.columns)):
-            series = col_mapped_frame.iloc[:, colidx]
+            series = self.df.iloc[:, colidx]
             for i, val in enumerate(series):
                 yield ExcelCell(self.rowcounter + i, colidx + coloffset, val)
 
@@ -1917,12 +1914,9 @@ def _format_hierarchical_rows(self):
                                         header_style)
                     gcolidx += 1
 
-        # Get a frame that will account for any duplicates in the column names.
-        col_mapped_frame = self.df.loc[:, self.columns]
-
         # Write the body of the frame data series by series.
         for colidx in range(len(self.columns)):
-            series = col_mapped_frame.iloc[:, colidx]
+            series = self.df.iloc[:, colidx]
             for i, val in enumerate(series):
                 yield ExcelCell(self.rowcounter + i, gcolidx + colidx, val)
 


Test patch:
diff --git a/pandas/io/tests/test_excel.py b/pandas/io/tests/test_excel.py
--- a/pandas/io/tests/test_excel.py
+++ b/pandas/io/tests/test_excel.py
@@ -1346,7 +1346,7 @@ def roundtrip2(df, header=True, parser_hdr=0, index=True):
 
 
     def test_duplicated_columns(self):
-        # Test for issue #5235.
+        # Test for issue #5235
         _skip_if_no_xlrd()
 
         with ensure_clean(self.ext) as path:
@@ -1358,7 +1358,20 @@ def test_duplicated_columns(self):
 
             read_frame = read_excel(path, 'test1')
             read_frame.columns = colnames
+            tm.assert_frame_equal(write_frame, read_frame)
+
+            # 11007 / #10970
+            write_frame = DataFrame([[1,2,3,4],[5,6,7,8]],
+                                    columns=['A','B','A','B'])
+            write_frame.to_excel(path, 'test1')
+            read_frame = read_excel(path, 'test1')
+            read_frame.columns = ['A','B','A','B']
+            tm.assert_frame_equal(write_frame, read_frame)
 
+            # 10982
+            write_frame.to_excel(path, 'test1', index=False, header=False)
+            read_frame = read_excel(path, 'test1', header=None)
+            write_frame.columns = [0, 1, 2, 3]
             tm.assert_frame_equal(write_frame, read_frame)
 
     def test_swapped_columns(self):
@@ -1375,6 +1388,23 @@ def test_swapped_columns(self):
             tm.assert_series_equal(write_frame['A'], read_frame['A'])
             tm.assert_series_equal(write_frame['B'], read_frame['B'])
 
+    def test_invalid_columns(self):
+        # 10982
+        _skip_if_no_xlrd()
+
+        with ensure_clean(self.ext) as path:
+            write_frame = DataFrame({'A': [1, 1, 1],
+                                     'B': [2, 2, 2]})
+
+            write_frame.to_excel(path, 'test1', columns=['B', 'C'])
+            expected = write_frame.loc[:, ['B','C']]
+            read_frame = read_excel(path, 'test1')
+            tm.assert_frame_equal(expected, read_frame)
+
+            with tm.assertRaises(KeyError):
+                write_frame.to_excel(path, 'test1', columns=['C', 'D'])
+
+
     def test_datetimes(self):
 
         # Test writing and reading datetimes. For issue #9139. (xref #9185)