#NEWLINE# def load_list( self ) : #NEWLINE# #INDENT# '\n        read proxy list from website\n        ' #NEWLINE# LOG . debug ( 'loading proxy list' ) #NEWLINE# page = 1 #NEWLINE# self . proxies = [ ] #NEWLINE# while True : #NEWLINE# #INDENT# html = requests . get ( ( PROXY_POOL_URL % page ) ) . text #NEWLINE# table = BeautifulSoup ( html , 'html.parser' ) . find ( id = 'proxy-table' ) #NEWLINE# proxy_tags = table . find_all ( 'li' , class_ = 'proxy' , text = re . compile ( '\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d+' ) ) #NEWLINE# speed_tags = table . find_all ( 'li' , class_ = 'speed' , text = re . compile ( '[0-9.]k.*|-' ) ) #NEWLINE# if ( not proxy_tags ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# for speed_tag in speed_tags : #NEWLINE# #INDENT# speed = re . sub ( 'kb.*' , '' , speed_tag . string . strip ( ) ) #NEWLINE# if re . match ( '[0-9.]+' , speed ) : #NEWLINE# #INDENT# if ( float ( self ) > 150 ) : #NEWLINE# #INDENT# proxy_tag = proxy_tags [ speed_tags . index ( speed_tag ) ] #NEWLINE# LOG . debug ( ( 'pick proxy: %s, speed %fkb' % ( proxy_tag . string , float ( speed ) ) ) ) #NEWLINE# self . proxies . append ( proxy_tag . string ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# page += 1 #NEWLINE# #UNINDENT# LOG . debug ( ( 'proxy list parsed, total %d proxies' % len ( self . proxies ) ) )