#NEWLINE# def test_setup_proxylist( self ) : #NEWLINE# #INDENT# content = ( '%s\n%s\n%s' % ( PROXY1 , PROXY2 , PROXY3 ) ) #NEWLINE# open ( '/tmp/__proxy.txt' , 'w' ) . write ( content ) #NEWLINE# bot = build_spider ( SimpleSpider , thread_number = 1 ) #NEWLINE# bot . load_proxylist ( '/tmp/__proxy.txt' , 'text_file' ) #NEWLINE# bot . setup_queue ( ) #NEWLINE# bot . add_task ( Task ( 'baz' , grab = Grab ( url = 'http://yandex.ru' , debug = True ) ) ) #NEWLINE# bot . run ( ) #NEWLINE# self . assertEqual ( self . server . request [ 'headers' ] [ 'host' ] , 'yandex.ru' ) #NEWLINE# self . assertEqual ( 1 , len ( set ( bot . stat . collections [ 'ports' ] ) ) ) #NEWLINE# bot = build_spider ( SimpleSpider , thread_number = 1 ) #NEWLINE# bot . load_proxylist ( '/tmp/__proxy.txt' , 'text_file' ) #NEWLINE# bot . setup_queue ( ) #NEWLINE# for x in six . moves . range ( 10 ) : #NEWLINE# #INDENT# bot . add_task ( Task ( 'baz' , 'http://yandex.ru' ) ) #NEWLINE# #UNINDENT# bot . run ( ) #NEWLINE# self . assertEqual ( self . server . request [ 'headers' ] [ 'host' ] , 'yandex.ru' ) #NEWLINE# self . assertTrue ( ( len ( set ( bot . stat . collections [ 'ports' ] ) ) > 1 ) ) #NEWLINE# bot = build_spider ( SimpleSpider , thread_number = 1 ) #NEWLINE# bot . load_proxylist ( '/tmp/__proxy.txt' , 'text_file' ) #NEWLINE# bot . setup_queue ( ) #NEWLINE# for x in six . moves . range ( 10 ) : #NEWLINE# #INDENT# bot . add_task ( Task ( 'baz' , 'http://yandex.ru' ) ) #NEWLINE# #UNINDENT# bot . run ( ) #NEWLINE# self . assertEqual ( self . server . request [ 'headers' ] [ 'host' ] , 'yandex.ru' ) #NEWLINE# self . assertTrue ( ( len ( set ( bot . stat . collections [ 'ports' ] ) ) > 1 ) ) #NEWLINE# bot = build_spider ( SimpleSpider , thread_number = 1 ) #NEWLINE# bot . load_proxylist ( '/tmp/__proxy.txt' , 'text_file' , auto_change = False ) #NEWLINE# bot . setup_queue ( ) #NEWLINE# for x in six . moves . range ( 10 ) : #NEWLINE# #INDENT# bot . add_task ( Task ( 'baz' , 'http://yandex.ru' ) ) #NEWLINE# #UNINDENT# bot . run ( ) #NEWLINE# self . assertEqual ( self . server . request [ 'headers' ] [ 'host' ] , 'yandex.ru' ) #NEWLINE# self . assertEqual ( 1 , len ( set ( bot . stat . collections [ 'ports' ] ) ) ) #NEWLINE# bot = build_spider ( SimpleSpider , thread_number = 1 ) #NEWLINE# bot . load_proxylist ( '/tmp/__proxy.txt' , 'text_file' , auto_change = False , auto_init = False ) #NEWLINE# bot . setup_queue ( ) #NEWLINE# for x in six . moves . range ( 10 ) : #NEWLINE# #INDENT# bot . add_task ( Task ( 'baz' , self . server . get_url ( ) ) ) #NEWLINE# #UNINDENT# bot . run ( ) #NEWLINE# self . assertEqual ( self . server . request [ 'headers' ] . get ( 'host' ) , ( '%s:%s' % ( ADDRESS , self . server . port ) ) ) #NEWLINE# self . assertEqual ( 1 , len ( set ( bot . stat . collections [ 'ports' ] ) ) ) #NEWLINE# self . assertEqual ( bot . stat . collections [ 'ports' ] [ 0 ] , self . server . port )