#NEWLINE# def test_autoclean_cv_with_nans_with_strings( ) : #NEWLINE# #INDENT# 'Test autoclean_cv() with a data set that has some string-encoded categorical values and some NaNs' #NEWLINE# data = pd . DataFrame ( { #NEWLINE# #INDENT# 'A' : np . random . rand ( 1000 ) , #NEWLINE# 'B' : np . random . rand ( 1000 ) , #NEWLINE# 'C' : np . random . randint ( 0 , 3 , 1000 ) , #NEWLINE# #UNINDENT# } ) #NEWLINE# string_map = { #NEWLINE# #INDENT# 0 : 'oranges' , #NEWLINE# 1 : 'apples' , #NEWLINE# 2 : 'bananas' , #NEWLINE# #UNINDENT# } #NEWLINE# data [ 'C' ] = data [ 'C' ] . apply ( ( lambda x : string_map [ x ] ) ) #NEWLINE# training_data = data [ : 500 ] . copy ( ) #NEWLINE# testing_data = data [ 500 : ] . copy ( ) #NEWLINE# training_data . loc [ 10 : 20 , 'A' ] = np . nan #NEWLINE# training_data . loc [ 50 : 70 , 'C' ] = np . nan #NEWLINE# testing_data . loc [ 70 : 80 , 'A' ] = np . nan #NEWLINE# testing_data . loc [ 10 : 40 , 'C' ] = np . nan #NEWLINE# hand_cleaned_training_data = training_data . copy ( ) #NEWLINE# hand_cleaned_testing_data = testing_data . copy ( ) #NEWLINE# training_A_median = hand_cleaned_training_data [ 'A' ] . median ( ) #NEWLINE# training_C_mode = hand_cleaned_training_data [ 'C' ] . mode ( ) [ 0 ] #NEWLINE# hand_cleaned_training_data [ 'A' ] . fillna ( training_A_median , inplace = True ) #NEWLINE# hand_cleaned_training_data [ 'C' ] . fillna ( training_C_mode , inplace = True ) #NEWLINE# hand_cleaned_testing_data [ 'A' ] . fillna ( training_A_median , inplace = True ) #NEWLINE# hand_cleaned_testing_data [ 'C' ] . fillna ( training_C_mode , inplace = True ) #NEWLINE# encoder = LabelEncoder ( ) #NEWLINE# hand_cleaned_training_data [ 'C' ] = encoder . fit_transform ( hand_cleaned_training_data [ 'C' ] . values ) #NEWLINE# hand_cleaned_testing_data [ 'C' ] = encoder . transform ( hand_cleaned_testing_data [ 'C' ] . values ) #NEWLINE# ( cleaned_training_data , cleaned_testing_data ) = autoclean_cv ( training_data , testing_data ) #NEWLINE# assert cleaned_training_data . equals ( hand_cleaned_training_data ) #NEWLINE# assert cleaned_testing_data . equals ( hand_cleaned_testing_data )