#NEWLINE# def get_vocab( self , input_text , input_scores , max_features ) : #NEWLINE# #INDENT# train_mat = self . vectorizer1 . transform ( input_text ) #NEWLINE# input_score_med = np . median ( input_scores ) #NEWLINE# new_scores = [ ( 0 if ( i <= input_score_med ) else 1 ) for i in input_scores ] #NEWLINE# ind_max_features = math . floor ( ( max_features / max ( input_scores ) ) ) #NEWLINE# all_vocab = [ ] #NEWLINE# all_cols = [ np . asarray ( train_mat . getcol ( i ) . todense ( ) . transpose ( ) ) [ 0 ] for i in xrange ( 0 , train_mat . shape [ 1 ] ) ] #NEWLINE# for s in xrange ( 0 , max ( input_scores ) ) : #NEWLINE# #INDENT# sel_inds = [ i for i in xrange ( 0 , len ( input_scores ) ) if ( input_scores [ i ] == s ) ] #NEWLINE# out_inds = [ i for i in xrange ( 0 , len ( input_scores ) ) if ( input_scores [ i ] != s ) ] #NEWLINE# pvalues = [ ] #NEWLINE# for i in xrange ( 0 , len ( all_cols ) ) : #NEWLINE# #INDENT# lcol = all_cols [ i ] #NEWLINE# good_lcol = lcol [ sel_inds ] #NEWLINE# bad_lcol = lcol [ out_inds ] #NEWLINE# good_lcol_present = len ( good_lcol [ ( good_lcol > 0 ) ] ) #NEWLINE# good_lcol_missing = len ( good_lcol [ ( good_lcol == 0 ) ] ) #NEWLINE# bad_lcol_present = len ( bad_lcol [ ( bad_lcol > 0 ) ] ) #NEWLINE# bad_lcol_missing = len ( bad_lcol [ ( bad_lcol == 0 ) ] ) #NEWLINE# pval = pvalue ( good_lcol_present , bad_lcol_present , good_lcol_missing , bad_lcol_missing ) #NEWLINE# pvalues . append ( pval . two_tail ) #NEWLINE# #UNINDENT# col_inds = list ( xrange ( 0 , train_mat . shape [ 1 ] ) ) #NEWLINE# p_frame = pd . DataFrame ( np . array ( [ col_inds , pvalues ] ) . transpose ( ) , columns = [ 'inds' , 'pvalues' ] ) #NEWLINE# p_frame = p_frame . sort ( [ 'pvalues' ] , ascending = True ) #NEWLINE# getVar = ( lambda searchList , ind : [ searchList [ int ( i ) ] for i in ind ] ) #NEWLINE# vocab = getVar ( self . vectorizer1 . get_feature_names ( ) , p_frame [ 'inds' ] [ : ( ind_max_features + 2 ) ] ) #NEWLINE# all_vocab . append ( vocab ) #NEWLINE# #UNINDENT# return list ( set ( list ( chain . from_iterable ( all_vocab ) ) ) )