Repository: pandas-dev/pandas

Issue:
MultiIndexes and large CSV files
I've run into a data-dependent bug with MultiIndex. Attached are a test script and a data file generation script:
- sales-data.py loads CSV data from a CSV data file with read_csv and then displays a multiindexed subset.
- sales-gen.sh is an AWK script that generates CSV test files. By default, it creates a 600MB file, which causes the failure. To make a smaller test file, which succeeds, just use "head -1000".

My problem is that sales-data.py breaks with large CSV files but works with smaller CSV files. When it fails it displays the following error message:

```
'MultiIndex lexsort depth 0, key was length 4'
```

I know that MultiIndexes must be pre-sorted. So sales-gen.sh makes sure that the first four columns in the auto-generated CSV files are ordered. BTW, I'm not sure why lexsort is getting called.
# sales-data.py

```
#!/usr/bin/env python

import numpy as np
import pandas as pd
from memory_profiler import profile

pd.set_option('display.height',            400)
pd.set_option('display.width',             400)
pd.set_option('display.max_rows',         1000)
pd.set_option('display.max_columns',        30)
pd.set_option('display.line_width',        200)

# @profile
def load_data():
    try:
        df = pd.read_csv(
            './sales-large.csv',
          # './sales-small.csv',
            header = None,
            na_values = ['NULL'],
            names = [
                'salesperson',
                'customer',
                'invoice_date',
                'ship_date',
                'product',
                'quantity',
                'price',
            ],
            index_col = [
                'salesperson',
                'customer',
                'invoice_date',
                'ship_date',
            ],
            parse_dates = [
                'invoice_date',
                'ship_date',
            ],
        )

        print(df.loc[(
            'A00000',                        # salesperson
            'A',                             # customer
            pd.datetime(2011,3,1,0,0,0),     # invoice_date
            pd.datetime(2011,3,6,0,0,0),     # ship_date
        )])

    except Exception as e:
        print(e)

if __name__== '__main__':
    load_data()
```
# sales-gen.sh

```
#!/usr/bin/env bash

# 'salesperson',
# 'customer',
# 'invoice_date',
# 'ship_date',
# 'product',
# 'quantity',
# 'price',

awk "BEGIN {                                                                                              \
          first = 65;                                                                                     \
          last = 91;                                                                                      \
          s_char = first;                                                                                 \
          c_char = first;                                                                                 \
                                                                                                          \
          n_sa = 100;                                                                                     \
          n_cu = 100;                                                                                     \
          n_dt = 10;                                                                                      \
          n_pr = 100;                                                                                     \
                                                                                                          \
          for (i = 0; i < n_sa; i++ s_char++) {                                                           \
              if (s_char == last) {                                                                       \
                  s_char = first;                                                                         \
              }                                                                                           \
              c_char = first;                                                                             \
              for (j = 0; j < n_cu; j++ c_char++) {                                                       \
                  if (c_char == last) {                                                                   \
                      c_char = first;                                                                     \
                  }                                                                                       \
                  for (k = 1; k <= n_dt; k++) {                                                           \
                      for (l = 0; l < n_pr; l++) {                                                        \
                          printf(\"%c%05d,%c,2011-03-%02d 00:00:00,2011-03-%02d 00:00:00,%d,%d,%f\n\",    \
                              s_char, i, c_char, k, k+5, l, 1 + int(rand() * 10), rand())                 \
                      }                                                                                   \
                  }                                                                                       \
              }                                                                                           \
          }                                                                                               \
      }"
```



Discussion:
Here's how to figure out the problem

big file like this best to read it in chunks, and set the index at the end
(it SHOULD be a bit faster and will have lower peak memory too)

```
df_iter = pd.read_csv(
    './sales-large.csv',
    header = None,
    na_values = ['NULL'],
    names = [
        'salesperson',
        'customer',
        'invoice_date',
        'ship_date',
        'product',
        'quantity',
        'price',
        ],
    parse_dates = [
        'invoice_date',
        'ship_date',
        ],
    chunksize=100000,
    )
```

Read from the iterator; you have duplicate values, this will show you where they are

```
for df in df_iter:
    dups = df.duplicated(['salesperson','customer','invoice_date','ship_date'])
    dups[dups]

    dfs.append(df)
```

The above is for you to inspect, but here's to create a non-duplicated frame

```
df = pd.concat([ df for df in df_iter ]).drop_duplicates(['salesperson','customer','invoice_date','ship_date'])
```

Set the index at the very end

```
df = df.set_index([
    'salesperson',
    'customer',
    'invoice_date',
    'ship_date',
    ])
```

It actually is not sorted, but doesn't matter (though you prob DO want to sort it)

```
df.index.lexsort_depth
```

Running the above script 

```
[goat-jreback-~] python gen.py
0
product     0.00000
quantity    1.00000
price       0.49057
Name: (A00000, A, 2011-03-01 00:00:00, 2011-03-06 00:00:00), dtype: float64
```

Thanks, Jeff. I hate it when it's pilot error.

On Thu, Aug 8, 2013 at 3:39 PM, jreback notifications@github.com wrote:

> Here's how to figure out the problem
> 
> big file like this best to read it in chunks, and set the index at the end
> 
> df_iter = pd.read_csv(
>     './sales-large.csv',
>     header = None,
>     na_values = ['NULL'],
>     names = [
>         'salesperson',
>         'customer',
>         'invoice_date',
>         'ship_date',
>         'product',
>         'quantity',
>         'price',
>         ],
>     parse_dates = [
>         'invoice_date',
>         'ship_date',
>         ],
>     chunksize=100000,
>     )
> 
> Read from the iterator; you have duplicate values, this will show you
> where they are
> 
> for df in df_iter:
>     dups = df.duplicated(['salesperson','customer','invoice_date','ship_date'])
>     dups[dups]
> 
> ```
> dfs.append(df)
> ```
> 
> The above is for you to inspect, but here's to create a non-duplicated
> frame
> 
> df = pd.concat([ df for df in df_iter ]).drop_duplicates(['salesperson','customer','invoice_date','ship_date'])
> 
> Set the index at the very ends
> 
> df = df.set_index([
>     'salesperson',
>     'customer',
>     'invoice_date',
>     'ship_date',
>     ])
> 
> It actually is not sorted, but doesn't matter (though you prob DO want to
> sort it)
> 
> df.index.lexsort_depth
> 
> Running the above script
> 
> [goat-jreback-~] python gen.py
> 0
> product     0.00000
> quantity    1.00000
> price       0.49057
> Name: (A00000, A, 2011-03-01 00:00:00, 2011-03-06 00:00:00), dtype: float64
> 
> —
> Reply to this email directly or view it on GitHubhttps://github.com/pydata/pandas/issues/4516#issuecomment-22363578
> .

@DanSears this actually is a bug, has to do with how a frame with a multi-index is reindexed when there are multiple dtypes in the data (e.g. in the example below there are floats and ints). This is not implemented, but not too hard. thanks for the report.

In your case, depending no how you are going to access the data, I actually wouldn't set an index at all. 

What are you going to do with this data, eg. are you going to simple look up fields like you posted?

```
In [6]: df = DataFrame([['foo','bar',1.0,1],['foo','bar',2.0,2],['bah','bam',3.0,3],
   ...: ['bah','bam',4.0,4],['foo','bar',5.0,5],['bah','bam',6.0,6]],
   ...: columns=list('ABCD'))

In [8]: df
Out[8]: 
     A    B  C  D
0  foo  bar  1  1
1  foo  bar  2  2
2  bah  bam  3  3
3  bah  bam  4  4
4  foo  bar  5  5
5  bah  bam  6  6
In [9]: df = df.set_index(['A','B'])

In [10]: df
Out[10]: 
         C  D
A   B        
foo bar  1  1
    bar  2  2
bah bam  3  3
    bam  4  4
foo bar  5  5
bah bam  6  6

In [11]: df.sortlevel(0)
Exception: Reindexing only valid with uniquely valued Index objects

```


Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -3304,10 +3304,13 @@ def sortlevel(self, level=0, axis=0, ascending=True, inplace=False):
         new_axis, indexer = the_axis.sortlevel(level, ascending=ascending)
 
         if self._is_mixed_type and not inplace:
-            if axis == 0:
-                return self.reindex(index=new_axis)
+            ax = 'index' if axis == 0 else 'columns'
+
+            if new_axis.is_unique:
+                d = { ax : new_axis }
             else:
-                return self.reindex(columns=new_axis)
+                d = { ax : indexer, 'takeable' : True }
+            return self.reindex(**d)
 
         if inplace:
             if axis == 1:
diff --git a/pandas/core/index.py b/pandas/core/index.py
--- a/pandas/core/index.py
+++ b/pandas/core/index.py
@@ -2165,8 +2165,17 @@ def reindex(self, target, method=None, level=None, limit=None,
             if self.equals(target):
                 indexer = None
             else:
-                indexer = self.get_indexer(target, method=method,
-                                           limit=limit)
+                if self.is_unique:
+                    indexer = self.get_indexer(target, method=method,
+                                               limit=limit)
+                else:
+                    if takeable:
+                        if method is not None or limit is not None:
+                            raise ValueError("cannot do a takeable reindex with "
+                                             "with a method or limit")
+                        return self[target], target
+
+                    raise Exception("cannot handle a non-takeable non-unique multi-index!")
 
         if not isinstance(target, MultiIndex):
             if indexer is None:


Test patch:
diff --git a/pandas/tests/test_multilevel.py b/pandas/tests/test_multilevel.py
--- a/pandas/tests/test_multilevel.py
+++ b/pandas/tests/test_multilevel.py
@@ -1828,6 +1828,18 @@ def test_duplicate_groupby_issues(self):
         result = s.groupby(s.index).first()
         self.assertEquals(len(result), 3)
 
+    def test_duplicate_mi(self):
+        # GH 4516
+        df = DataFrame([['foo','bar',1.0,1],['foo','bar',2.0,2],['bah','bam',3.0,3],
+                        ['bah','bam',4.0,4],['foo','bar',5.0,5],['bah','bam',6.0,6]],
+                       columns=list('ABCD'))
+        df = df.set_index(['A','B'])
+        df = df.sortlevel(0)
+        result = df.loc[('foo','bar')]
+        expected = DataFrame([['foo','bar',1.0,1],['foo','bar',2.0,2],['foo','bar',5.0,5]],
+                             columns=list('ABCD')).set_index(['A','B'])
+        assert_frame_equal(result,expected)
+
     def test_multiindex_set_index(self):
         # segfault in #3308
         d = {'t1': [2, 2.5, 3], 't2': [4, 5, 6]}