Repository: nilearn/nilearn

Issue:
resample_img() might not properly handle file handles
Trying to iterate through 18749 nifti images with individual resample_img calls on a computer cluster.

``` python
transf_data = np.zeros((nfiles, nvox))
for i, fpath in enumerate(paths):
    print i
    cur_nii = resample_img(fpath, target_affine=masknii.get_affine(), target_shape=masknii.shape)
    transf_data[i, :] = masker.transform(cur_nii)[0, :]
```

I got a "too many open files" error with the following traceback:

```
Traceback (most recent call last):
  File "prepdata.py", line 37, in <module>
    niis4d = check_niimg(paths)
  File "/storage/workspace/danilo/nilearn/nilearn/_utils/niimg_conversions.py", line 147, in check_niimg
    return concat_niimgs(niimg, ensure_ndim=ensure_ndim)
  File "/storage/workspace/danilo/nilearn/nilearn/_utils/niimg_conversions.py", line 307, in concat_niimgs
    memory=memory, memory_level=memory_level))):
  File "/storage/workspace/danilo/nilearn/nilearn/_utils/niimg_conversions.py", line 71, in _iter_check_niimg
    niimg, ensure_ndim=ndim_minus_one, atleast_4d=atleast_4d)
  File "/storage/workspace/danilo/nilearn/nilearn/_utils/niimg_conversions.py", line 158, in check_niimg
    data = niimg.get_data().view()
  File "/usr/lib/pymodules/python2.7/nibabel/spatialimages.py", line 341, in get_data
    return np.asanyarray(self._data)
  File "/usr/lib/python2.7/dist-packages/numpy/core/numeric.py", line 512, in asanyarray
    return array(a, dtype, copy=False, order=order, subok=True)
  File "/usr/lib/pymodules/python2.7/nibabel/arrayproxy.py", line 55, in __array__
    self._data = self._read_data()
  File "/usr/lib/pymodules/python2.7/nibabel/arrayproxy.py", line 60, in _read_data
    data = self.header.data_from_fileobj(fileobj)
  File "/usr/lib/pymodules/python2.7/nibabel/analyze.py", line 486, in data_from_fileobj
    data = self.raw_data_from_fileobj(fileobj)
  File "/usr/lib/pymodules/python2.7/nibabel/analyze.py", line 458, in raw_data_from_fileobj
    return array_from_file(shape, dtype, fileobj, offset)
  File "/usr/lib/pymodules/python2.7/nibabel/volumeutils.py", line 474, in array_from_file
    offset=offset)
  File "/usr/lib/python2.7/dist-packages/numpy/core/memmap.py", line 254, in __new__
    mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
mmap.error: [Errno 24] Too many open files
```



Discussion:
Could you try:

``` python
transf_data = np.zeros((nfiles, nvox))
for i, fpath in enumerate(paths):
    print i
    cur_nii = resample_img(fpath, target_affine=masknii.get_affine(), target_shape=masknii.shape)
    transf_data[i, :] = masker.transform(cur_nii)[0, :].copy()
```

Tried to reproduce your bug but couldn't... This may be Mac OS specific.

Just a thought... Is it possible that your file handler is kept open because your niftis are not garbage collected? Could you call `gc.collect()` in your loop and see if it fixes the problem?


Resolving patch:
diff --git a/nilearn/_utils/compat.py b/nilearn/_utils/compat.py
--- a/nilearn/_utils/compat.py
+++ b/nilearn/_utils/compat.py
@@ -15,6 +15,7 @@
     StringIO = io.StringIO
     BytesIO = io.BytesIO
     _urllib = urllib
+    izip = zip
 
     def md5_hash(string):
         m = hashlib.md5()
@@ -27,10 +28,12 @@ def md5_hash(string):
     import urllib2
     import urlparse
     import types
+    import itertools
 
     _basestring = basestring
     cPickle = cPickle
     StringIO = BytesIO = StringIO.StringIO
+    izip = itertools.izip
 
     class _module_lookup(object):
         modules = [urlparse, urllib2, urllib]
diff --git a/nilearn/_utils/niimg_conversions.py b/nilearn/_utils/niimg_conversions.py
--- a/nilearn/_utils/niimg_conversions.py
+++ b/nilearn/_utils/niimg_conversions.py
@@ -11,7 +11,7 @@
 
 from .cache_mixin import cache
 from .niimg import _safe_get_data, load_niimg, new_img_like
-from .compat import _basestring
+from .compat import _basestring, izip
 
 
 def _check_fov(img, affine, shape):
@@ -314,7 +314,7 @@ def concat_niimgs(niimgs, dtype=np.float32, ensure_ndim=None,
     data = np.ndarray(target_shape + (sum(lengths), ),
                       order="F", dtype=dtype)
     cur_4d_index = 0
-    for index, (size, niimg) in enumerate(zip(lengths, _iter_check_niimg(
+    for index, (size, niimg) in enumerate(izip(lengths, _iter_check_niimg(
             iterator, atleast_4d=True, target_fov=target_fov,
             memory=memory, memory_level=memory_level))):
 
diff --git a/nilearn/input_data/base_masker.py b/nilearn/input_data/base_masker.py
--- a/nilearn/input_data/base_masker.py
+++ b/nilearn/input_data/base_masker.py
@@ -18,7 +18,7 @@
 from .. import _utils
 from .._utils.cache_mixin import CacheMixin, cache
 from .._utils.class_inspect import enclosing_scope_name, get_params
-from .._utils.compat import _basestring
+from .._utils.compat import _basestring, izip
 from nilearn._utils.niimg_conversions import _iter_check_niimg
 
 
@@ -190,16 +190,15 @@ def transform_imgs(self, imgs_list, confounds=None, copy=True, n_jobs=1):
                              % self.__class__.__name__)
         params = get_params(self.__class__, self)
 
-        reference_affine = None
+        target_fov = None
         if self.target_affine is None:
-            # Load the first image and use it as a reference for all other
-            # subjects
-            reference_affine = _utils.check_niimg(imgs_list[0]).get_affine()
+            # Force resampling on first image
+            target_fov = 'first'
 
-        fov = (self.mask_img_.get_affine(), self.mask_img_.shape)
         niimg_iter = _iter_check_niimg(imgs_list, ensure_ndim=None,
                                        atleast_4d=False,
-                                       target_fov=fov, memory=self.memory,
+                                       target_fov=target_fov,
+                                       memory=self.memory,
                                        memory_level=self.memory_level,
                                        verbose=self.verbose)
 
@@ -215,7 +214,7 @@ def transform_imgs(self, imgs_list, confounds=None, copy=True, n_jobs=1):
                                 verbose=self.verbose,
                                 confounds=confounds,
                                 copy=copy)
-                          for imgs, confounds in zip(niimg_iter, confounds))
+                          for imgs, confounds in izip(niimg_iter, confounds))
         return list(zip(*data))[0]
 
     def fit_transform(self, X, y=None, confounds=None, **fit_params):


Test patch: