#NEWLINE# def bench_test_message_queue( queue ) : #NEWLINE# #INDENT# task = { #NEWLINE# #INDENT# 'fetch' : { #NEWLINE# #INDENT# 'fetch_type' : 'js' , #NEWLINE# 'headers' : { #NEWLINE# #INDENT# 'User-Agent' : 'BaiDuSpider' , #NEWLINE# #UNINDENT# } , #NEWLINE# #UNINDENT# } , #NEWLINE# 'process' : { #NEWLINE# #INDENT# 'callback' : 'detail_page' , #NEWLINE# #UNINDENT# } , #NEWLINE# 'project' : '__bench_test__' , #NEWLINE# 'taskid' : '553300d2582154413b4982c00c34a2d5' , #NEWLINE# 'url' : 'http://www.sciencedirect.com/science/article/pii/S1674200109000704' , #NEWLINE# #UNINDENT# } #NEWLINE# def test_put( n ) : #NEWLINE# #INDENT# logger . info ( 'message queue put %d' , n ) #NEWLINE# start_time = time . time ( ) #NEWLINE# for i in range ( n ) : #NEWLINE# #INDENT# task [ 'url' ] = ( 'http://bench.pyspider.org/?l=%d' % i ) #NEWLINE# task [ 'taskid' ] = md5string ( task [ 'url' ] ) #NEWLINE# queue . put ( task , block = True , timeout = 1 ) #NEWLINE# #UNINDENT# end_time = time . time ( ) #NEWLINE# cost_time = ( end_time - start_time ) #NEWLINE# logger . info ( 'cost %.2fs, %.2f/s %.2fms' , cost_time , ( ( n * 1.0 ) / cost_time ) , ( ( cost_time / n ) * 1000 ) ) #NEWLINE# #UNINDENT# def test_get( n ) : #NEWLINE# #INDENT# logger . info ( 'message queue get %d' , n ) #NEWLINE# start_time = time . time ( ) #NEWLINE# for i in range ( n ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# queue . get ( True , 1 ) #NEWLINE# #UNINDENT# except Queue . Empty : #NEWLINE# #INDENT# logger . error ( 'message queue empty while get %d' , i ) #NEWLINE# raise #NEWLINE# #UNINDENT# #UNINDENT# end_time = time . time ( ) #NEWLINE# cost_time = ( end_time - start_time ) #NEWLINE# logger . info ( 'cost %.2fs, %.2f/s %.2fms' , cost_time , ( ( n * 1.0 ) / cost_time ) , ( ( cost_time / n ) * 1000 ) ) #NEWLINE# #UNINDENT# try : #NEWLINE# #INDENT# test_put ( 1000 ) #NEWLINE# test_get ( 1000 ) #NEWLINE# test_put ( 10000 ) #NEWLINE# test_get ( 10000 ) #NEWLINE# #UNINDENT# except Exception as e : #NEWLINE# #INDENT# logger . exception ( e ) #NEWLINE# #UNINDENT# finally : #NEWLINE# #INDENT# if hasattr ( queue , 'channel' ) : #NEWLINE# #INDENT# queue . channel . queue_purge ( e . name )