astropy/astropy — pull request #839 by iguananaut (2013-07-19 19:58:15+00:00)
Title: Issue with astropy.io.fits and large arrays

I'm seeing the following issue with astropy.io.fits:

```
In [1]: import numpy as np

In [2]: from astropy.io import fits

In [3]: x = np.random.random((1e9))

In [4]: fits.writeto('test.fits', x, clobber=True)
WARNING: Overwriting existing file 'test.fits'. [astropy.io.fits.hdu.hdulist]

In [5]: d = fits.getdata('test.fits', memmap=False)
WARNING: File may have been truncated: actual file length (3705036224) is smaller than the expected size (8000003520) [astropy.io.fits.file]

In [6]: d = fits.getdata('test.fits', memmap=True)
ERROR: ValueError: mmap length is greater than file size [numpy.core.memmap]
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
in ()
----> 1 d = fits.getdata('test.fits', memmap=True)

/Volumes/Raptor/Library/Python/3.2/lib/python/site-packages/astropy-0.3.dev3272-py3.2-macosx-10.8-x86_64.egg/astropy/io/fits/convenience.py in getdata(filename, *args, **kwargs)
186 hdulist, extidx = _getext(filename, mode, *args, **kwargs)
187 hdu = hdulist[extidx]
--> 188 data = hdu.data
189 if data is None and extidx == 0:
190 try:

/Volumes/Raptor/Library/Python/3.2/lib/python/site-packages/astropy-0.3.dev3272-py3.2-macosx-10.8-x86_64.egg/astropy/utils/misc.py in __get__(self, obj, owner)
259 key = self._fget.__name__
260 if key not in obj.__dict__:
--> 261 val = self._fget(obj)
262 obj.__dict__[key] = val
263 return val

/Volumes/Raptor/Library/Python/3.2/lib/python/site-packages/astropy-0.3.dev3272-py3.2-macosx-10.8-x86_64.egg/astropy/io/fits/hdu/image.py in data(self)
201 return
202
--> 203 data = self._get_scaled_image_data(self._datLoc, self.shape)
204 self._update_header_scale_info(data.dtype)
205

/Volumes/Raptor/Library/Python/3.2/lib/python/site-packages/astropy-0.3.dev3272-py3.2-macosx-10.8-x86_64.egg/astropy/io/fits/hdu/image.py in _get_scaled_image_data(self, offset, shape)
521 code = _ImageBaseHDU.NumCode[self._orig_bitpix]
522
--> 523 raw_data = self._get_raw_data(shape, code, offset)
524 raw_data.dtype = raw_data.dtype.newbyteorder('>')
525

/Volumes/Raptor/Library/Python/3.2/lib/python/site-packages/astropy-0.3.dev3272-py3.2-macosx-10.8-x86_64.egg/astropy/io/fits/hdu/base.py in _get_raw_data(self, shape, code, offset)
366 offset=offset)
367 else:
--> 368 return self._file.readarray(offset=offset, dtype=code, shape=shape)
369
370 # TODO: Rework checksum handling so that it's not necessary to add a

/Volumes/Raptor/Library/Python/3.2/lib/python/site-packages/astropy-0.3.dev3272-py3.2-macosx-10.8-x86_64.egg/astropy/io/fits/py3compat.py in readarray(self, size, offset, dtype, shape)
171 if isinstance(dtype, numpy.dtype):
172 dtype = _fix_dtype(dtype)
--> 173 return old_File.readarray(self, size, offset, dtype, shape)
174 readarray.__doc__ = old_File.readarray.__doc__
175 file._File = _File

/Volumes/Raptor/Library/Python/3.2/lib/python/site-packages/astropy-0.3.dev3272-py3.2-macosx-10.8-x86_64.egg/astropy/io/fits/file.py in readarray(self, size, offset, dtype, shape)
270 return Memmap(self.__file, offset=offset,
271 mode=MEMMAP_MODES[self.mode], dtype=dtype,
--> 272 shape=shape).view(np.ndarray)
273 else:
274 count = reduce(lambda x, y: x * y, shape)

/Volumes/Raptor/Library/Python/3.2/lib/python/site-packages/numpy/core/memmap.py in __new__(subtype, filename, dtype, mode, offset, shape, order)
251 bytes -= start
252 offset -= start
--> 253 mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
254 else:
255 mm = mmap.mmap(fid.fileno(), bytes, access=acc)

ValueError: mmap length is greater than file size
```

This is on MacOS 10.8 with 64Gb of RAM.

@iguananaut - I know you want issues to be opened on the pyfits tracker, but this is an issue that IMHO should be fixed in 0.2.1 (if it's an easy fix) so I need to open it here so we can keep track of it.