#NEWLINE# def getSourceCode( url , proxy , returnRedirctUrl = False , maxRetries = 1 , waitRetryTime = 1 ) : #NEWLINE# #INDENT# '\n\tLoop to get around server denies for info or minor disconnects.\n\t' #NEWLINE# if ( proxy is not None ) : #NEWLINE# #INDENT# if NO_SOCKS : #NEWLINE# #INDENT# raise FatalError ( 'socks library required to use proxy (e.g. SocksiPy)' ) #NEWLINE# #UNINDENT# proxySettings = proxy . split ( ':' ) #NEWLINE# socks . setdefaultproxy ( socks . PROXY_TYPE_SOCKS4 , gzipper [ 0 ] , int ( proxySettings [ 1 ] ) , True ) #NEWLINE# socket . socket = socks . socksocket #NEWLINE# #UNINDENT# global urlReqHeaders #NEWLINE# ret = None #NEWLINE# request = urllib2 . Request ( url , headers = urlReqHeaders ) #NEWLINE# while ( ret == None ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# f = urllib2 . urlopen ( request ) #NEWLINE# encoding = f . headers . get ( 'Content-Encoding' ) #NEWLINE# if ( encoding == None ) : #NEWLINE# #INDENT# ret = f . read ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( encoding . upper ( ) == 'GZIP' ) : #NEWLINE# #INDENT# compressedstream = io . BytesIO ( f . read ( ) ) #NEWLINE# gzipper = gzip . GzipFile ( fileobj = compressedstream ) #NEWLINE# ret = gzipper . read ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# raise FatalError ( 'Unknown HTTP Encoding returned' ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# except urllib2 . URLError : #NEWLINE# #INDENT# if ( maxRetries == 0 ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# time . sleep ( random . uniform ( ( 0.5 * waitRetryTime ) , ( 1.5 * waitRetryTime ) ) ) #NEWLINE# maxRetries -= 1 #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# if returnRedirctUrl : #NEWLINE# #INDENT# return ( ret , f . geturl ( ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# return ret