#NEWLINE# def read_big( fid , size = None ) : #NEWLINE# #INDENT# "Function to read large chunks of data (>16MB) Windows-friendly\n\n    Parameters\n    ----------\n    fid : file\n        Open file to read from.\n    size : int or None\n        Number of bytes to read. If None, the whole file is read.\n\n    Returns\n    -------\n    buf : bytes\n        The data.\n\n    Notes\n    -----\n    Windows (argh) can't handle reading large chunks of data, so we\n    have to do it piece-wise, possibly related to:\n       http://stackoverflow.com/questions/4226941\n\n    Examples\n    --------\n    This code should work for normal files and .gz files:\n\n        >>> import numpy as np\n        >>> import gzip, os, tempfile, shutil\n        >>> fname = tempfile.mkdtemp()\n        >>> fname_gz = os.path.join(fname, 'temp.gz')\n        >>> fname = os.path.join(fname, 'temp.bin')\n        >>> randgen = np.random.RandomState(9)\n        >>> x = randgen.randn(3000000)  # > 16MB data\n        >>> with open(fname, 'wb') as fid: x.tofile(fid)\n        >>> with open(fname, 'rb') as fid: y = np.fromstring(read_big(fid))\n        >>> assert np.all(x == y)\n        >>> fid_gz = gzip.open(fname_gz, 'wb')\n        >>> _ = fid_gz.write(x.tostring())\n        >>> fid_gz.close()\n        >>> fid_gz = gzip.open(fname_gz, 'rb')\n        >>> y = np.fromstring(read_big(fid_gz))\n        >>> assert np.all(x == y)\n        >>> fid_gz.close()\n        >>> shutil.rmtree(os.path.dirname(fname))\n\n    " #NEWLINE# buf_size = 16777216 #NEWLINE# if ( size is None ) : #NEWLINE# #INDENT# if ( not isinstance ( fid , gzip . GzipFile ) ) : #NEWLINE# #INDENT# size = ( os . fstat ( fid . fileno ( ) ) . st_size - fid . tell ( ) ) #NEWLINE# #UNINDENT# #UNINDENT# if ( size is not None ) : #NEWLINE# #INDENT# segments = np . r_ [ ( np . arange ( 0 , size , buf_size ) , size ) ] #NEWLINE# buf = bytearray ( ( b' ' * size ) ) #NEWLINE# for ( start , end ) in zip ( segments [ : ( - 1 ) ] , segments [ 1 : ] ) : #NEWLINE# #INDENT# data = fid . read ( int ( ( end - end ) ) ) #NEWLINE# if ( len ( data ) != ( end - start ) ) : #NEWLINE# #INDENT# raise ValueError ( 'Read error' ) #NEWLINE# #UNINDENT# buf [ start : end ] = data #NEWLINE# #UNINDENT# buf = bytes ( buf ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# buf = [ b'' ] #NEWLINE# new = fid . read ( buf_size ) #NEWLINE# while ( len ( new ) > 0 ) : #NEWLINE# #INDENT# buf . append ( new ) #NEWLINE# new = fid . read ( buf_size ) #NEWLINE# #UNINDENT# buf = b'' . join ( buf ) #NEWLINE# #UNINDENT# return buf