#NEWLINE# def close( self , reason ) : #NEWLINE# #INDENT# self . logger . info ( 'Closing Spider' , { #NEWLINE# #INDENT# 'spiderid' : self . spider . name , #NEWLINE# #UNINDENT# } ) #NEWLINE# if ( not self . persist ) : #NEWLINE# #INDENT# self . logger . warning ( 'Clearing crawl queues' ) #NEWLINE# self . dupefilter . clear ( ) #NEWLINE# for key in self . queue_keys : #NEWLINE# #INDENT# self . queue_dict [ key ] . clear ( )