#NEWLINE# def extract_tabular( self , result = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } , table_type = 'rows' , header = [ ] , prefix = '' , suffix = '' , selector = '' , attr = 'text' , default = '' , verbosity = 0 ) : #NEWLINE# #INDENT# '\n\t\tMethod for performing the extraction of tabular data.\n\n\t\t:param result:\n\t\t:param table_type:\n\t\t:param header:\n\t\t:param prefix:\n\t\t:param suffix:\n\t\t:param selector:\n\t\t:param attr:\n\t\t:param default:\n\t\t:param verbosity:\n\t\t:return: A 2-tuple containing the list of all the column headers extracted and the list of \t\tdictionaries which contain (header, content) pairs\n\t\t' #NEWLINE# result_list = [ ] #NEWLINE# if ( type ( header ) in [ str , unicode ] ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# header_list = self . tree . xpath ( header ) #NEWLINE# table_headers = [ ( ( prefix + h . text ) + suffix ) for h in header_list ] #NEWLINE# #UNINDENT# except XPathError : #NEWLINE# #INDENT# raise Exception ( ( 'Invalid XPath selector ' + header ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# table_headers = [ ( ( prefix + h ) + suffix ) for h in header ] #NEWLINE# #UNINDENT# if ( table_type not in [ 'rows' , 'columns' ] ) : #NEWLINE# #INDENT# raise Exception ( "Specify 'rows' or 'columns' in table_type" ) #NEWLINE# #UNINDENT# if ( table_type == 'rows' ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# values = self . tree . xpath ( selector ) #NEWLINE# if ( len ( table_headers ) >= len ( values ) ) : #NEWLINE# #INDENT# from itertools import izip_longest #NEWLINE# pairs = izip_longest ( table_headers , values , fillvalue = default ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# from itertools import izip #NEWLINE# pairs = izip ( table_headers , values ) #NEWLINE# #UNINDENT# for ( head , val ) in pairs : #NEWLINE# #INDENT# if ( values > 1 ) : #NEWLINE# #INDENT# print ( '\nExtracting' , head , 'attribute' , sep = ' ' , end = '' ) #NEWLINE# #UNINDENT# if ( attr == 'text' ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# content = ' ' . join ( [ make_ascii ( x ) . strip ( ) for x in val . itertext ( ) ] ) #NEWLINE# #UNINDENT# except Exception : #NEWLINE# #INDENT# content = default #NEWLINE# #UNINDENT# content = content . replace ( '\n' , ' ' ) . strip ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# content = val . get ( attr ) #NEWLINE# if ( attr in [ 'href' , 'src' ] ) : #NEWLINE# #INDENT# content = urljoin ( self . url , content ) #NEWLINE# #UNINDENT# #UNINDENT# result [ head ] = content #NEWLINE# #UNINDENT# result_list . append ( result ) #NEWLINE# #UNINDENT# except XPathError : #NEWLINE# #INDENT# raise Exception ( ( 'Invalid XPath selector ' + selector ) ) #NEWLINE# #UNINDENT# except TypeError : #NEWLINE# #INDENT# raise Exception ( ( 'Selector expression string to be provided. Got ' + selector ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# if ( type ( selector ) in [ str , unicode ] ) : #NEWLINE# #INDENT# selectors = [ selector ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( type ( selector ) == list ) : #NEWLINE# #INDENT# selectors = selector #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# raise Exception ( 'Use a list of selector expressions for the various columns' ) #NEWLINE# #UNINDENT# #UNINDENT# from itertools import izip , count #NEWLINE# pairs = izip ( table_headers , selectors ) #NEWLINE# columns = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for ( head , selector ) in pairs : #NEWLINE# #INDENT# columns [ head ] = self . tree . xpath ( selector ) #NEWLINE# #UNINDENT# try : #NEWLINE# #INDENT# for i in count ( start = 0 ) : #NEWLINE# #INDENT# r = result . copy ( ) #NEWLINE# for head in columns . keys ( ) : #NEWLINE# #INDENT# if ( verbosity > 1 ) : #NEWLINE# #INDENT# print ( '\nExtracting' , head , 'attribute' , sep = ' ' , end = '' ) #NEWLINE# #UNINDENT# col = columns [ head ] [ i ] #NEWLINE# if ( attr == 'text' ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# content = ' ' . join ( [ make_ascii ( x ) . strip ( ) for x in col . itertext ( ) ] ) #NEWLINE# #UNINDENT# except Exception : #NEWLINE# #INDENT# content = default #NEWLINE# #UNINDENT# content = content . replace ( '\n' , ' ' ) . strip ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# content = col . get ( attr ) #NEWLINE# if ( attr in [ 'href' , 'src' ] ) : #NEWLINE# #INDENT# content = urljoin ( self . url , content ) #NEWLINE# #UNINDENT# #UNINDENT# r [ head ] = content #NEWLINE# #UNINDENT# result_list . append ( r ) #NEWLINE# #UNINDENT# #UNINDENT# except IndexError : #NEWLINE# #INDENT# pass #NEWLINE# #UNINDENT# #UNINDENT# except XPathError : #NEWLINE# #INDENT# raise Exception ( ( 'Invalid XPath selector ' + selector ) ) #NEWLINE# #UNINDENT# except TypeError : #NEWLINE# #INDENT# raise Exception ( ( 'Selector expression string to be provided. Got ' + selector ) ) #NEWLINE# #UNINDENT# #UNINDENT# return ( table_headers , result_list )