#NEWLINE# def process_item( self , item , spider ) : #NEWLINE# #INDENT# 'Check if a URL was already found.' #NEWLINE# if ( item [ 'loc' ] in self . processed ) : #NEWLINE# #INDENT# raise scrapy . exceptions . DropItem ( ( 'Duplicate URL found: %s.' % spider [ 'loc' ] ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# self . processed . add ( item [ 'loc' ] ) #NEWLINE# return item