#NEWLINE# def create_queues( self ) : #NEWLINE# #INDENT# '\n        Updates the in memory list of the redis queues\n        Creates new throttled queue instances if it does not have them\n        ' #NEWLINE# newConf = self . check_config ( ) #NEWLINE# self . queue_keys = self . redis_conn . keys ( ( self . spider . name + ':*:queue' ) ) #NEWLINE# for key in self . queue_keys : #NEWLINE# #INDENT# throttle_key = '' #NEWLINE# if self . add_type : #NEWLINE# #INDENT# throttle_key = ( self . spider . name + ':' ) #NEWLINE# #UNINDENT# if self . add_ip : #NEWLINE# #INDENT# throttle_key = ( ( throttle_key + self . my_ip ) + ':' ) #NEWLINE# #UNINDENT# the_domain = re . split ( ':' , newConf ) [ 1 ] #NEWLINE# throttle_key = ( throttle_key + the_domain ) #NEWLINE# if ( ( key not in self . queue_dict ) or newConf ) : #NEWLINE# #INDENT# self . logger . debug ( 'Added new Throttled Queue {q}' . format ( q = key ) ) #NEWLINE# q = RedisPriorityQueue ( self . redis_conn , key ) #NEWLINE# if ( the_domain not in self . domain_config ) : #NEWLINE# #INDENT# self . queue_dict [ key ] = RedisThrottledQueue ( self . redis_conn , q , self . window , self . hits , self . moderated , throttle_key , throttle_key ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# window = self . domain_config [ the_domain ] [ 'window' ] #NEWLINE# hits = self . domain_config [ the_domain ] [ 'hits' ] #NEWLINE# if ( 'scale' in self . domain_config [ the_domain ] ) : #NEWLINE# #INDENT# hits = int ( ( hits * self . fit_scale ( self . domain_config [ the_domain ] [ 'scale' ] ) ) ) #NEWLINE# #UNINDENT# self . queue_dict [ key ] = RedisThrottledQueue ( self . redis_conn , q , window , hits , self . moderated , throttle_key , throttle_key )