#NEWLINE# def fetch_url( url , useragent , referer = None , retries = 1 , dimension = False ) : #NEWLINE# #INDENT# cur_try = 0 #NEWLINE# nothing = ( None if is_favicon else ( None , None ) ) #NEWLINE# url = clean_url ( url ) #NEWLINE# if ( not url . startswith ( ( 'http://' , 'https://' ) ) ) : #NEWLINE# #INDENT# return nothing #NEWLINE# #UNINDENT# response = None #NEWLINE# while True : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# response = requests . get ( url , stream = True , timeout = 5 , headers = { #NEWLINE# #INDENT# 'User-Agent' : useragent , #NEWLINE# 'Referer' : referer , #NEWLINE# #UNINDENT# } ) #NEWLINE# if dimension : #NEWLINE# #INDENT# content = response . raw . read ( chunk_size ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# content = response . raw . read ( ) #NEWLINE# #UNINDENT# content_type = response . headers . get ( 'Content-Type' ) #NEWLINE# if ( not content_type ) : #NEWLINE# #INDENT# return nothing #NEWLINE# #UNINDENT# if ( 'image' in content_type ) : #NEWLINE# #INDENT# p = ImageFile . Parser ( ) #NEWLINE# new_data = content #NEWLINE# while ( ( not p . image ) and new_data ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# p . feed ( new_data ) #NEWLINE# #UNINDENT# except IOError : #NEWLINE# #INDENT# traceback . print_exc ( ) #NEWLINE# p = None #NEWLINE# break #NEWLINE# #UNINDENT# except ValueError : #NEWLINE# #INDENT# traceback . print_exc ( ) #NEWLINE# p = None #NEWLINE# break #NEWLINE# #UNINDENT# except Exception as e : #NEWLINE# #INDENT# is_favicon = ( urls . url_to_filetype ( url ) == 'ico' ) #NEWLINE# if is_favicon : #NEWLINE# #INDENT# pass #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# raise e #NEWLINE# #UNINDENT# p = None #NEWLINE# break #NEWLINE# #UNINDENT# new_data = response . raw . read ( chunk_size ) #NEWLINE# content += new_data #NEWLINE# #UNINDENT# if ( p is None ) : #NEWLINE# #INDENT# return nothing #NEWLINE# #UNINDENT# if ( dimension and p . image ) : #NEWLINE# #INDENT# return p . image . size #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if dimension : #NEWLINE# #INDENT# return nothing #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# if dimension : #NEWLINE# #INDENT# return nothing #NEWLINE# #UNINDENT# #UNINDENT# return ( content_type , content ) #NEWLINE# #UNINDENT# except requests . exceptions . RequestException as e : #NEWLINE# #INDENT# cur_try += 1 #NEWLINE# if ( cur_try >= retries ) : #NEWLINE# #INDENT# log . debug ( ( 'error while fetching: %s refer: %s' % ( url , referer ) ) ) #NEWLINE# return nothing #NEWLINE# #UNINDENT# #UNINDENT# finally : #NEWLINE# #INDENT# if ( response is not None ) : #NEWLINE# #INDENT# response . raw . close ( ) #NEWLINE# if response . raw . _connection : #NEWLINE# #INDENT# response . raw . _connection . close ( )