#NEWLINE# def test_autoclean_cv_real_data( ) : #NEWLINE# #INDENT# 'Test autoclean_cv() with the adult data set' #NEWLINE# adult_data = pd . read_csv ( 'adult.csv.gz' , sep = '\t' , compression = 'gzip' ) #NEWLINE# training_adult_data = adult_data [ : int ( ( len ( adult_data ) / 2.0 ) ) ] . copy ( ) #NEWLINE# testing_adult_data = adult_data [ int ( ( len ( adult_data ) / 2.0 ) ) : ] . copy ( ) #NEWLINE# training_adult_data . loc [ 30 : 60 , 'age' ] = np . nan #NEWLINE# training_adult_data . loc [ 90 : 100 , 'education' ] = np . nan #NEWLINE# testing_adult_data . loc [ 90 : 110 , 'age' ] = np . nan #NEWLINE# testing_adult_data . loc [ 20 : 40 , 'education' ] = np . nan #NEWLINE# hand_cleaned_training_adult_data = training_adult_data . copy ( ) #NEWLINE# hand_cleaned_testing_adult_data = testing_adult_data . copy ( ) #NEWLINE# training_age_median = hand_cleaned_training_adult_data [ 'age' ] . median ( ) #NEWLINE# training_education_mode = hand_cleaned_training_adult_data [ 'education' ] . mode ( ) [ 0 ] #NEWLINE# hand_cleaned_training_adult_data [ 'age' ] . fillna ( training_age_median , inplace = True ) #NEWLINE# hand_cleaned_training_adult_data [ 'education' ] . fillna ( training_education_mode , inplace = True ) #NEWLINE# hand_cleaned_testing_adult_data [ 'age' ] . fillna ( training_age_median , inplace = True ) #NEWLINE# hand_cleaned_testing_adult_data [ 'education' ] . fillna ( training_education_mode , inplace = True ) #NEWLINE# for column in [ 'workclass' , 'education' , 'marital-status' , 'occupation' , 'relationship' , 'race' , 'sex' , 'native-country' , 'label' ] : #NEWLINE# #INDENT# encoder = LabelEncoder ( ) #NEWLINE# hand_cleaned_training_adult_data [ column ] = encoder . fit_transform ( hand_cleaned_training_adult_data [ column ] . values ) #NEWLINE# hand_cleaned_testing_adult_data [ column ] = encoder . transform ( hand_cleaned_testing_adult_data [ column ] . values ) #NEWLINE# #UNINDENT# ( cleaned_adult_training_data , cleaned_adult_testing_data ) = autoclean_cv ( training_adult_data , testing_adult_data ) #NEWLINE# assert cleaned_adult_training_data . equals ( hand_cleaned_training_adult_data ) #NEWLINE# assert cleaned_adult_testing_data . equals ( hand_cleaned_testing_adult_data )