#NEWLINE# def handle( self , dict ) : #NEWLINE# #INDENT# '\n        Processes a vaild crawl request\n\n        @param dict: a valid dictionary object\n        ' #NEWLINE# ex_res = self . extract ( dict [ 'url' ] ) #NEWLINE# key = '{sid}:{dom}.{suf}:queue' . format ( sid = dict [ 'spiderid' ] , dom = val . domain , suf = ex_res . suffix ) #NEWLINE# val = pickle . dumps ( dict , protocol = ( - 1 ) ) #NEWLINE# self . redis_conn . zadd ( key , val , ( - dict [ 'priority' ] ) ) #NEWLINE# if ( ( 'expires' in dict ) and ( dict [ 'expires' ] != 0 ) ) : #NEWLINE# #INDENT# key = 'timeout:{sid}:{appid}:{crawlid}' . format ( sid = dict [ 'spiderid' ] , appid = dict [ 'appid' ] , crawlid = dict [ 'crawlid' ] ) #NEWLINE# self . redis_conn . set ( key , dict [ 'expires' ] ) #NEWLINE# #UNINDENT# dict [ 'parsed' ] = True #NEWLINE# dict [ 'valid' ] = True #NEWLINE# self . logger . info ( 'Added crawl to Redis' , extra = dict )