Repository: pandas-dev/pandas

Issue:
MemoryError with more than 1E9 rows
I have 240GB of RAM. Nothing else running on the machine.  I'm trying to create 1.5E9 rows, which I think should create a data frame of around 100GB, but getting this MemoryError.  This works fine with 1E9 but not 1.5E9.   I could understand a limit at about 2^31 (2E9) or 2^32 (4E9) but all 240GB seems exhausted (according to htop) at somewhere between 1E9 and 1.5E9 rows.  Any ideas?  Thanks.

``` python
$ python3
Python 3.4.0 (default, Apr 11 2014, 13:05:11) 
[GCC 4.8.2] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import pandas as pd
>>> import numpy as np
>>> import timeit
>>> pd.__version__
'0.14.1'
>>> def randChar(f, numGrp, N) :
...    things = [f%x for x in range(numGrp)]
...    return [things[x] for x in np.random.choice(numGrp, N)]
... 
>>> def randFloat(numGrp, N) :
...    things = [round(100*np.random.random(),4) for x in range(numGrp)]
...    return [things[x] for x in np.random.choice(numGrp, N)]
... 
>>> N=int(1.5e9)       # N=int(1e9) works fine
>>> K=100
>>> DF = pd.DataFrame({
...   'id1' : randChar("id%03d", K, N),       # large groups (char)
...   'id2' : randChar("id%03d", K, N),       # large groups (char)
...   'id3' : randChar("id%010d", N//K, N),   # small groups (char)
...   'id4' : np.random.choice(K, N),         # large groups (int)
...   'id5' : np.random.choice(K, N),         # large groups (int)
...   'id6' : np.random.choice(N//K, N),      # small groups (int)
...   'v1' :  np.random.choice(5, N),         # int in range [1,5]
...   'v2' :  np.random.choice(5, N),         # int in range [1,5]
...   'v3' :  randFloat(100,N)                # numeric e.g. 23.5749
... })
Traceback (most recent call last):
  File "<stdin>", line 10, in <module>
  File "/usr/lib/python3/dist-packages/pandas/core/frame.py", line 203, in __init__
    mgr = self._init_dict(data, index, columns, dtype=dtype)
  File "/usr/lib/python3/dist-packages/pandas/core/frame.py", line 327, in _init_dict
    dtype=dtype)
  File "/usr/lib/python3/dist-packages/pandas/core/frame.py", line 4630, in _arrays_to_mgr
    return create_block_manager_from_arrays(arrays, arr_names, axes)
  File "/usr/lib/python3/dist-packages/pandas/core/internals.py", line 3235, in create_block_manager_from_arrays
    blocks = form_blocks(arrays, names, axes)
  File "/usr/lib/python3/dist-packages/pandas/core/internals.py", line 3322, in form_blocks
    object_items, np.object_)
  File "/usr/lib/python3/dist-packages/pandas/core/internals.py", line 3346, in _simple_blockify
    values, placement = _stack_arrays(tuples, dtype)
  File "/usr/lib/python3/dist-packages/pandas/core/internals.py", line 3410, in _stack_arrays
    stacked = np.empty(shape, dtype=dtype)
MemoryError
```

``` bash
$ lscpu
Architecture:          x86_64
CPU op-mode(s):        32-bit, 64-bit
Byte Order:            Little Endian
CPU(s):                32
On-line CPU(s) list:   0-31
Thread(s) per core:    2
Core(s) per socket:    8
Socket(s):             2
NUMA node(s):          2
Vendor ID:             GenuineIntel
CPU family:            6
Model:                 62
Stepping:              4
CPU MHz:               2494.070
BogoMIPS:              5054.21
Hypervisor vendor:     Xen
Virtualization type:   full
L1d cache:             32K
L1i cache:             32K
L2 cache:              256K
L3 cache:              25600K
NUMA node0 CPU(s):     0-7,16-23
NUMA node1 CPU(s):     8-15,24-31
$ free -h
             total       used       free     shared    buffers     cached
Mem:          240G       2.3G       237G       364K        66M       632M
-/+ buffers/cache:       1.6G       238G
Swap:           0B         0B         0B
$
```

An earlier question on S.O. is here :  http://stackoverflow.com/questions/25631076/is-this-the-fastest-way-to-group-in-pandas



Discussion:
You can try separately creating Series (with each of the columns first), then putting them into a dict and creating the frame. However you might be having a problem finding contiguous memory.


Resolving patch:
diff --git a/pandas/tools/merge.py b/pandas/tools/merge.py
--- a/pandas/tools/merge.py
+++ b/pandas/tools/merge.py
@@ -666,7 +666,7 @@ def _sort_labels(uniques, left, right):
 
 
 def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
-           keys=None, levels=None, names=None, verify_integrity=False):
+           keys=None, levels=None, names=None, verify_integrity=False, copy=True):
     """
     Concatenate pandas objects along a particular axis with optional set logic
     along the other axes. Can also add a layer of hierarchical indexing on the
@@ -704,6 +704,8 @@ def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
         concatenating objects where the concatenation axis does not have
         meaningful indexing information. Note the the index values on the other
         axes are still respected in the join.
+    copy : boolean, default True
+        If False, do not copy data unnecessarily
 
     Notes
     -----
@@ -716,7 +718,8 @@ def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
     op = _Concatenator(objs, axis=axis, join_axes=join_axes,
                        ignore_index=ignore_index, join=join,
                        keys=keys, levels=levels, names=names,
-                       verify_integrity=verify_integrity)
+                       verify_integrity=verify_integrity,
+                       copy=copy)
     return op.get_result()
 
 
@@ -727,7 +730,7 @@ class _Concatenator(object):
 
     def __init__(self, objs, axis=0, join='outer', join_axes=None,
                  keys=None, levels=None, names=None,
-                 ignore_index=False, verify_integrity=False):
+                 ignore_index=False, verify_integrity=False, copy=True):
         if not isinstance(objs, (list,tuple,types.GeneratorType,dict,TextFileReader)):
             raise TypeError('first argument must be a list-like of pandas '
                             'objects, you passed an object of type '
@@ -846,6 +849,7 @@ def __init__(self, objs, axis=0, join='outer', join_axes=None,
 
         self.ignore_index = ignore_index
         self.verify_integrity = verify_integrity
+        self.copy = copy
 
         self.new_axes = self._get_new_axes()
 
@@ -879,7 +883,9 @@ def get_result(self):
                 mgrs_indexers.append((obj._data, indexers))
 
             new_data = concatenate_block_managers(
-                mgrs_indexers, self.new_axes, concat_axis=self.axis, copy=True)
+                mgrs_indexers, self.new_axes, concat_axis=self.axis, copy=self.copy)
+            if not self.copy:
+                new_data._consolidate_inplace()
 
             return self.objs[0]._from_axes(new_data, self.new_axes).__finalize__(self, method='concat')
 


Test patch:
diff --git a/pandas/tools/tests/test_merge.py b/pandas/tools/tests/test_merge.py
--- a/pandas/tools/tests/test_merge.py
+++ b/pandas/tools/tests/test_merge.py
@@ -1485,6 +1485,38 @@ def test_append_missing_column_proper_upcast(self):
         self.assertEqual(appended['A'].dtype, 'f8')
         self.assertEqual(appended['B'].dtype, 'O')
 
+    def test_concat_copy(self):
+
+        df = DataFrame(np.random.randn(4, 3))
+        df2 = DataFrame(np.random.randint(0,10,size=4).reshape(4,1))
+        df3 = DataFrame({5 : 'foo'},index=range(4))
+
+        # these are actual copies
+        result = concat([df,df2,df3],axis=1,copy=True)
+        for b in result._data.blocks:
+            self.assertIsNone(b.values.base)
+
+        # these are the same
+        result = concat([df,df2,df3],axis=1,copy=False)
+        for b in result._data.blocks:
+            if b.is_float:
+                self.assertTrue(b.values.base is df._data.blocks[0].values.base)
+            elif b.is_integer:
+                self.assertTrue(b.values.base is df2._data.blocks[0].values.base)
+            elif b.is_object:
+                self.assertIsNotNone(b.values.base)
+
+        # float block was consolidated
+        df4 = DataFrame(np.random.randn(4,1))
+        result = concat([df,df2,df3,df4],axis=1,copy=False)
+        for b in result._data.blocks:
+            if b.is_float:
+                self.assertIsNone(b.values.base)
+            elif b.is_integer:
+                self.assertTrue(b.values.base is df2._data.blocks[0].values.base)
+            elif b.is_object:
+                self.assertIsNotNone(b.values.base)
+
     def test_concat_with_group_keys(self):
         df = DataFrame(np.random.randn(4, 3))
         df2 = DataFrame(np.random.randn(4, 4))