#NEWLINE# def loadDatasetWithMappedCategories( dsetJsonPath , mappedCategoryPath ) : #NEWLINE# #INDENT# '\n    Load a dataset while mapping specific categories to a more general,\n    and common categories.\n    ' #NEWLINE# messageList = getFullMessagesFromJson ( dsetJsonPath ) #NEWLINE# for msg in messageList : #NEWLINE# #INDENT# msg [ 'categories' ] = list ( set ( msg [ 'categories' ] ) ) #NEWLINE# #UNINDENT# catMap = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# with open ( mappedCategoryPath , 'rb' ) as inf : #NEWLINE# #INDENT# csvReader = UnicodeDictReader ( inf ) #NEWLINE# for row in csvReader : #NEWLINE# #INDENT# engCat = row [ 'Category (English)' ] #NEWLINE# superCat = row [ 'Super Category' ] #NEWLINE# assert ( ( superCat is not None ) and ( superCat != '' ) ) #NEWLINE# catMap [ engCat ] = superCat #NEWLINE# #UNINDENT# #UNINDENT# for msg in messageList : #NEWLINE# #INDENT# catList = msg [ 'categories' ] #NEWLINE# newCatSet = set ( ) #NEWLINE# for cat in catList : #NEWLINE# #INDENT# mappedCat = catMap [ cat ] #NEWLINE# if ( mappedCat not in [ 'Other' , '?' ] ) : #NEWLINE# #INDENT# newCatSet . add ( mappedCat ) #NEWLINE# #UNINDENT# #UNINDENT# msg [ 'categories' ] = list ( newCatSet ) #NEWLINE# #UNINDENT# return messageList