#NEWLINE# def insert_item( self , item , spider ) : #NEWLINE# #INDENT# ' Process the item and add it to MongoDB\n\n        :type item: (Item object) or [(Item object)]\n        :param item: The item(s) to put into MongoDB\n        :type spider: BaseSpider object\n        :param spider: The spider running the queries\n        :returns: Item object\n        ' #NEWLINE# if ( not isinstance ( item , list ) ) : #NEWLINE# #INDENT# item = dict ( item ) #NEWLINE# if self . config [ 'append_timestamp' ] : #NEWLINE# #INDENT# item [ 'scrapy-mongodb' ] = { #NEWLINE# #INDENT# 'ts' : datetime . datetime . utcnow ( ) , #NEWLINE# #UNINDENT# } #NEWLINE# #UNINDENT# #UNINDENT# if ( self . config [ 'unique_key' ] is None ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# self . collection . insert ( item , continue_on_error = True ) #NEWLINE# log . msg ( 'Stored item(s) in MongoDB {0}/{1}' . format ( self . config [ 'database' ] , self . config [ 'collection' ] ) , level = log . DEBUG , spider = spider ) #NEWLINE# #UNINDENT# except errors . DuplicateKeyError : #NEWLINE# #INDENT# log . msg ( 'Duplicate key found' , level = log . DEBUG ) #NEWLINE# if ( self . stop_on_duplicate > 0 ) : #NEWLINE# #INDENT# self . duplicate_key_count += 1 #NEWLINE# if ( self . duplicate_key_count >= self . stop_on_duplicate ) : #NEWLINE# #INDENT# self . crawler . engine . close_spider ( spider , 'Number of duplicate key insertion exceeded' ) #NEWLINE# #UNINDENT# #UNINDENT# pass #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# key = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# if isinstance ( self . config [ 'unique_key' ] , list ) : #NEWLINE# #INDENT# for k in dict ( self . config [ 'unique_key' ] ) . keys ( ) : #NEWLINE# #INDENT# key [ k ] = item [ k ] #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# key [ self . config [ 'unique_key' ] ] = item [ self . config [ 'unique_key' ] ] #NEWLINE# #UNINDENT# self . collection . update ( key , item , upsert = True ) #NEWLINE# log . msg ( 'Stored item(s) in MongoDB {0}/{1}' . format ( self . config [ 'database' ] , self . config [ 'collection' ] ) , level = log . DEBUG , spider = spider ) #NEWLINE# #UNINDENT# return k