#NEWLINE# @ click . group ( invoke_without_command = True ) #NEWLINE# @ click . option ( '-c' , '--config' , callback = read_config , type = click . File ( 'r' ) , help = 'a json file with default values for subcommands. {"webui": {"port":5001}}' ) #NEWLINE# @ click . option ( '--logging-config' , default = os . path . join ( os . path . dirname ( __file__ ) , 'logging.conf' ) , help = 'logging config file for built-in python logging module' , show_default = True ) #NEWLINE# @ click . option ( '--debug' , envvar = 'DEBUG' , default = False , is_flag = True , help = 'debug mode' ) #NEWLINE# @ click . option ( '--queue-maxsize' , envvar = 'QUEUE_MAXSIZE' , default = 100 , help = 'maxsize of queue' ) #NEWLINE# @ click . option ( '--taskdb' , envvar = 'TASKDB' , callback = connect_db , help = 'database url for taskdb, default: sqlite' ) #NEWLINE# @ click . option ( '--projectdb' , envvar = 'PROJECTDB' , callback = connect_db , help = 'database url for projectdb, default: sqlite' ) #NEWLINE# @ click . option ( '--resultdb' , envvar = 'RESULTDB' , callback = connect_db , help = 'database url for resultdb, default: sqlite' ) #NEWLINE# @ click . option ( '--message-queue' , envvar = 'AMQP_URL' , help = 'connection url to message queue, default: builtin multiprocessing.Queue' ) #NEWLINE# @ click . option ( '--amqp-url' , help = '[deprecated] amqp url for rabbitmq. please use --message-queue instead.' ) #NEWLINE# @ click . option ( '--beanstalk' , envvar = 'BEANSTALK_HOST' , help = '[deprecated] beanstalk config for beanstalk queue. please use --message-queue instead.' ) #NEWLINE# @ click . option ( '--phantomjs-proxy' , envvar = 'PHANTOMJS_PROXY' , help = 'phantomjs proxy ip:port' ) #NEWLINE# @ click . option ( '--data-path' , default = './data' , help = 'data dir path' ) #NEWLINE# @ click . option ( '--add-sys-path/--not-add-sys-path' , default = True , is_flag = True , help = 'add current working directory to python lib search path' ) #NEWLINE# @ click . version_option ( version = pyspider . __version__ , prog_name = pyspider . __name__ ) #NEWLINE# @ click . pass_context #NEWLINE# def cli( ctx , **kwargs ) : #NEWLINE# #INDENT# '\n    A powerful spider system in python.\n    ' #NEWLINE# if kwargs [ 'add_sys_path' ] : #NEWLINE# #INDENT# sys . path . append ( os . getcwd ( ) ) #NEWLINE# #UNINDENT# logging . config . fileConfig ( kwargs [ 'logging_config' ] ) #NEWLINE# for db in ( 'taskdb' , 'projectdb' , 'resultdb' ) : #NEWLINE# #INDENT# if ( kwargs [ db ] is not None ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if os . environ . get ( 'MYSQL_NAME' ) : #NEWLINE# #INDENT# kwargs [ db ] = utils . Get ( ( lambda db = db : connect_database ( ( 'sqlalchemy+mysql+%s://%s:%s/%s' % ( db , os . environ [ 'MYSQL_PORT_3306_TCP_ADDR' ] , os . environ [ 'MYSQL_PORT_3306_TCP_PORT' ] , db ) ) ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if os . environ . get ( 'MONGODB_NAME' ) : #NEWLINE# #INDENT# kwargs [ db ] = utils . Get ( ( lambda db = db : connect_database ( ( 'mongodb+%s://%s:%s/%s' % ( db , os . environ [ 'MONGODB_PORT_27017_TCP_ADDR' ] , os . environ [ 'MONGODB_PORT_27017_TCP_PORT' ] , db ) ) ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( ctx . invoked_subcommand == 'bench' ) : #NEWLINE# #INDENT# if ( kwargs [ 'data_path' ] == './data' ) : #NEWLINE# #INDENT# kwargs [ 'data_path' ] += '/bench' #NEWLINE# shutil . rmtree ( kwargs [ 'data_path' ] , ignore_errors = True ) #NEWLINE# os . mkdir ( kwargs [ 'data_path' ] ) #NEWLINE# #UNINDENT# if ( db in ( 'taskdb' , 'resultdb' ) ) : #NEWLINE# #INDENT# kwargs [ db ] = utils . Get ( ( lambda db = db : connect_database ( ( 'sqlite+%s://' % db ) ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# kwargs [ db ] = utils . Get ( ( lambda db = db : connect_database ( ( 'sqlite+%s:///%s/%s.db' % ( db , kwargs [ 'data_path' ] , db [ : ( - 2 ) ] ) ) ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# if ( not os . path . exists ( kwargs [ 'data_path' ] ) ) : #NEWLINE# #INDENT# os . mkdir ( kwargs [ 'data_path' ] ) #NEWLINE# #UNINDENT# kwargs [ db ] = utils . Get ( ( lambda db = db : connect_database ( ( 'sqlite+%s:///%s/%s.db' % ( db , kwargs [ 'data_path' ] , db [ : ( - 2 ) ] ) ) ) ) ) #NEWLINE# kwargs [ ( 'is_%s_default' % db ) ] = True #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# if ( not os . path . exists ( kwargs [ 'data_path' ] ) ) : #NEWLINE# #INDENT# os . mkdir ( kwargs [ 'data_path' ] ) #NEWLINE# #UNINDENT# if kwargs . get ( 'message_queue' ) : #NEWLINE# #INDENT# pass #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if kwargs . get ( 'amqp_url' ) : #NEWLINE# #INDENT# kwargs [ 'message_queue' ] = kwargs [ 'amqp_url' ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if os . environ . get ( 'RABBITMQ_NAME' ) : #NEWLINE# #INDENT# kwargs [ 'message_queue' ] = ( 'amqp://guest:guest@%(RABBITMQ_PORT_5672_TCP_ADDR)s:%(RABBITMQ_PORT_5672_TCP_PORT)s/%%2F' % os . environ ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if kwargs . get ( 'beanstalk' ) : #NEWLINE# #INDENT# kwargs [ 'message_queue' ] = ( 'beanstalk://%s/' % kwargs [ 'beanstalk' ] ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# for name in ( 'newtask_queue' , 'status_queue' , 'scheduler2fetcher' , 'fetcher2processor' , 'processor2result' ) : #NEWLINE# #INDENT# if kwargs . get ( 'message_queue' ) : #NEWLINE# #INDENT# kwargs [ name ] = utils . Get ( ( lambda name = name : connect_message_queue ( name , kwargs . get ( 'message_queue' ) , kwargs [ 'queue_maxsize' ] ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# kwargs [ name ] = connect_message_queue ( name , kwargs . get ( 'message_queue' ) , kwargs [ 'queue_maxsize' ] ) #NEWLINE# #UNINDENT# #UNINDENT# if kwargs . get ( 'phantomjs_proxy' ) : #NEWLINE# #INDENT# pass #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if os . environ . get ( 'PHANTOMJS_NAME' ) : #NEWLINE# #INDENT# kwargs [ 'phantomjs_proxy' ] = os . environ [ 'PHANTOMJS_PORT_25555_TCP' ] [ len ( 'tcp://' ) : ] #NEWLINE# #UNINDENT# #UNINDENT# ctx . obj = utils . ObjectDict ( ( ctx . obj or { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } ) ) #NEWLINE# ctx . obj [ 'instances' ] = [ ] #NEWLINE# ctx . obj . update ( kwargs ) #NEWLINE# if ( ( ctx . invoked_subcommand is None ) and ( not ctx . obj . get ( 'testing_mode' ) ) ) : #NEWLINE# #INDENT# ctx . invoke ( all ) #NEWLINE# #UNINDENT# return ctx