#NEWLINE# def open_spider( self , spider ) : #NEWLINE# #INDENT# self . load_spider ( spider ) #NEWLINE# self . configure ( ) #NEWLINE# if ( self . config [ 'replica_set' ] is not None ) : #NEWLINE# #INDENT# connection = MongoReplicaSetClient ( self . config [ 'uri' ] , replicaSet = self . config [ 'replica_set' ] , w = self . config [ 'write_concern' ] , fsync = self . config [ 'fsync' ] , read_preference = ReadPreference . PRIMARY_PREFERRED ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# connection = MongoClient ( self . config [ 'uri' ] , fsync = self . config [ 'fsync' ] , read_preference = ReadPreference . PRIMARY ) #NEWLINE# #UNINDENT# database = connection [ self . config [ 'database' ] ] #NEWLINE# self . collection = database [ self . config [ 'collection' ] ] #NEWLINE# log . msg ( 'Connected to MongoDB {0}, using "{1}/{2}"' . format ( spider . config [ 'uri' ] , self . config [ 'database' ] , self . config [ 'collection' ] ) ) #NEWLINE# if self . config [ 'unique_key' ] : #NEWLINE# #INDENT# self . collection . ensure_index ( self . config [ 'unique_key' ] , unique = True ) #NEWLINE# log . msg ( 'Ensuring index for key {0}' . format ( self . config [ 'unique_key' ] ) ) #NEWLINE# #UNINDENT# if self . config [ 'stop_on_duplicate' ] : #NEWLINE# #INDENT# tmpValue = self . config [ 'stop_on_duplicate' ] #NEWLINE# if ( tmpValue < 0 ) : #NEWLINE# #INDENT# log . msg ( 'Negative values are not allowed for MONGODB_STOP_ON_DUPLICATE option.' , level = log . ERROR ) #NEWLINE# raise SyntaxError ( 'Negative values are not allowed for MONGODB_STOP_ON_DUPLICATE option.' ) #NEWLINE# #UNINDENT# self . stop_on_duplicate = self . config [ 'stop_on_duplicate' ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# self . stop_on_duplicate = 0