#NEWLINE# def _ReadCsvDict( self , file_name , cols , required , deprecated ) : #NEWLINE# #INDENT# 'Reads lines from file_name, yielding a dict of unicode values.' #NEWLINE# assert file_name . endswith ( '.txt' ) #NEWLINE# table_name = file_name [ 0 : ( - 4 ) ] #NEWLINE# contents = self . _GetUtf8Contents ( file_name ) #NEWLINE# if ( not contents ) : #NEWLINE# #INDENT# return #NEWLINE# #UNINDENT# eol_checker = util . EndOfLineChecker ( StringIO . StringIO ( contents ) , file_name , self . _problems ) #NEWLINE# reader = csv . reader ( eol_checker , skipinitialspace = True ) #NEWLINE# raw_header = reader . next ( ) #NEWLINE# header_occurrences = util . defaultdict ( ( lambda : 0 ) ) #NEWLINE# header = [ ] #NEWLINE# valid_columns = [ ] #NEWLINE# for ( i , h ) in enumerate ( raw_header ) : #NEWLINE# #INDENT# h_stripped = h . strip ( ) #NEWLINE# if ( not h_stripped ) : #NEWLINE# #INDENT# self . _problems . CsvSyntax ( description = 'The header row should not contain any blank values. The corresponding column will be skipped for the entire file.' , context = ( file_name , 1 , ( [ '' ] * len ( raw_header ) ) , raw_header ) , type = problems . TYPE_ERROR ) #NEWLINE# continue #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( h != h_stripped ) : #NEWLINE# #INDENT# self . _problems . CsvSyntax ( description = 'The header row should not contain any space characters.' , context = ( file_name , 1 , ( [ '' ] * len ( raw_header ) ) , raw_header ) , type = problems . TYPE_WARNING ) #NEWLINE# #UNINDENT# #UNINDENT# header . append ( h_stripped ) #NEWLINE# valid_columns . append ( i ) #NEWLINE# header_occurrences [ h_stripped ] += 1 #NEWLINE# #UNINDENT# for ( name , count ) in header_occurrences . items ( ) : #NEWLINE# #INDENT# if ( count > 1 ) : #NEWLINE# #INDENT# self . _problems . DuplicateColumn ( header = name , file_name = file_name , count = count ) #NEWLINE# #UNINDENT# #UNINDENT# self . _schedule . _table_columns [ table_name ] = header #NEWLINE# header_context = ( file_name , 1 , ( [ '' ] * len ( header ) ) , header ) #NEWLINE# valid_cols = ( cols + [ deprecated_name for ( deprecated_name , _ ) in deprecated ] ) #NEWLINE# unknown_cols = ( set ( header ) - set ( valid_cols ) ) #NEWLINE# if ( len ( unknown_cols ) == len ( header ) ) : #NEWLINE# #INDENT# self . _problems . CsvSyntax ( description = 'The header row did not contain any known column names. The file is most likely missing the header row or not in the expected CSV format.' , context = ( file_name , 1 , ( [ '' ] * len ( raw_header ) ) , raw_header ) , type = problems . TYPE_ERROR ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# for col in unknown_cols : #NEWLINE# #INDENT# self . _problems . UnrecognizedColumn ( file_name , col , header_context ) #NEWLINE# #UNINDENT# #UNINDENT# missing_cols = ( set ( required ) - set ( header ) ) #NEWLINE# for col in missing_cols : #NEWLINE# #INDENT# self . _problems . MissingColumn ( file_name , col , header_context ) #NEWLINE# #UNINDENT# for ( deprecated_name , new_name ) in deprecated : #NEWLINE# #INDENT# if ( deprecated_name in header ) : #NEWLINE# #INDENT# self . _problems . DeprecatedColumn ( file_name , deprecated_name , new_name , header_context ) #NEWLINE# #UNINDENT# #UNINDENT# line_num = 1 #NEWLINE# for raw_row in reader : #NEWLINE# #INDENT# line_num += 1 #NEWLINE# if ( len ( raw_row ) == 0 ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if ( len ( raw_row ) > len ( raw_header ) ) : #NEWLINE# #INDENT# self . _problems . OtherProblem ( ( 'Found too many cells (commas) in line %d of file "%s".  Every row in the file should have the same number of cells as the header (first line) does.' % ( line_num , file_name ) ) , ( file_name , line_num ) , type = problems . TYPE_WARNING ) #NEWLINE# #UNINDENT# if ( len ( raw_row ) < len ( raw_header ) ) : #NEWLINE# #INDENT# self . _problems . OtherProblem ( ( 'Found missing cells (commas) in line %d of file "%s".  Every row in the file should have the same number of cells as the header (first line) does.' % ( line_num , file_name ) ) , ( file_name , line_num ) , type = problems . TYPE_WARNING ) #NEWLINE# #UNINDENT# valid_values = [ ] #NEWLINE# unicode_error_columns = [ ] #NEWLINE# for i in valid_columns : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# valid_values . append ( raw_row [ i ] . decode ( 'utf-8' ) ) #NEWLINE# #UNINDENT# except UnicodeDecodeError : #NEWLINE# #INDENT# valid_values . append ( codecs . getdecoder ( 'utf8' ) ( raw_row [ i ] , errors = 'replace' ) [ 0 ] ) #NEWLINE# unicode_error_columns . append ( ( len ( valid_values ) - 1 ) ) #NEWLINE# #UNINDENT# except IndexError : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# #UNINDENT# for i in unicode_error_columns : #NEWLINE# #INDENT# self . _problems . InvalidValue ( header [ i ] , valid_values [ i ] , 'Unicode error' , ( unknown_cols , line_num , valid_values , header ) ) #NEWLINE# #UNINDENT# valid_values = [ value . strip ( ) for value in valid_values ] #NEWLINE# d = dict ( zip ( header , valid_values ) ) #NEWLINE# ( yield ( d , line_num , header , valid_values ) )