#NEWLINE# def preprocess_4( article_file , lable_file ) : #NEWLINE# #INDENT# features = pickle . load ( open ( article_file ) ) #NEWLINE# features = np . array ( features ) #NEWLINE# lables = pickle . load ( open ( lable_file ) ) #NEWLINE# le = preprocessing . LabelEncoder ( ) #NEWLINE# le . fit ( lables ) #NEWLINE# lables = le . transform ( lables ) #NEWLINE# ( features_train , features_test , labels_train , labels_test ) = cross_validation . train_test_split ( le , lables , test_size = 0.1 , random_state = 42 ) #NEWLINE# t0 = time ( ) #NEWLINE# vectorizer = TfidfVectorizer ( sublinear_tf = True , max_df = 0.5 , min_df = 1 , stop_words = 'english' ) #NEWLINE# features_train_transformed = vectorizer . fit_transform ( features_train ) #NEWLINE# features_test_transformed = vectorizer . transform ( features_test ) #NEWLINE# t0 = time ( ) #NEWLINE# selector = SelectPercentile ( f_classif , percentile = 30 ) #NEWLINE# selector . fit ( features_train_transformed , labels_train ) #NEWLINE# features_train_transformed = selector . transform ( features_train_transformed ) . toarray ( ) #NEWLINE# features_test_transformed = selector . transform ( features_test_transformed ) . toarray ( ) #NEWLINE# return ( features_train_transformed , features_test_transformed , labels_train , labels_test )