#NEWLINE# def get( url , max_size , chunk_size = None , allowed_schemes = ( 'http' , 'https' ) ) : #NEWLINE# #INDENT# 'Get the data at the specified URL.\n\n    The URL must use the http: or https: schemes.\n    The file: scheme is also supported if you override\n    the allowed_schemes argument.\n    The max_size represents the total max byte of your file.\n    The chunk_size is by default set at max_size, it represents the size\n    of your chunk.\n    Raise an IOError if getting the data fails and if max_size is exceeded.\n    ' #NEWLINE# LOG . info ( ( _ ( 'Fetching data from %s' ) % url ) ) #NEWLINE# components = moves . urllib . parse . urlparse ( url ) #NEWLINE# if ( result . scheme not in allowed_schemes ) : #NEWLINE# #INDENT# raise IOError ( ( _ ( 'Invalid URL scheme %s' ) % components . scheme ) ) #NEWLINE# #UNINDENT# if ( chunk_size is None ) : #NEWLINE# #INDENT# chunk_size = max_size #NEWLINE# #UNINDENT# if ( max_size < 1 ) : #NEWLINE# #INDENT# raise IOError ( 'max_size should be greater than 0' ) #NEWLINE# #UNINDENT# if ( chunk_size < 1 ) : #NEWLINE# #INDENT# raise IOError ( 'chunk_size should be greater than 0' ) #NEWLINE# #UNINDENT# if ( components . scheme == 'file' ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# return moves . urllib . request . urlopen ( url ) . read ( ) #NEWLINE# #UNINDENT# except moves . urllib . error . URLError as uex : #NEWLINE# #INDENT# raise IOError ( ( _ ( 'Failed to read file: %s' ) % str ( uex ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# try : #NEWLINE# #INDENT# resp = requests . get ( url , stream = True ) #NEWLINE# resp . raise_for_status ( ) #NEWLINE# reader = resp . iter_content ( chunk_size = chunk_size ) #NEWLINE# result = '' #NEWLINE# for chunk in reader : #NEWLINE# #INDENT# result += chunk #NEWLINE# if ( len ( result ) > max_size ) : #NEWLINE# #INDENT# raise IOError ( ( 'File exceeds maximum allowed size (%s bytes)' % max_size ) ) #NEWLINE# #UNINDENT# #UNINDENT# return result #NEWLINE# #UNINDENT# except exceptions . RequestException as ex : #NEWLINE# #INDENT# raise IOError ( ( _ ( 'Failed to retrieve file: %s' ) % str ( ex ) ) )