Kaggle competition: Quora Insincere Questions Classification
lgbm_clf = LGBMClassifier(n_estimators = 400)
 lgbm_clf.fit(train, target)
 lgbm_pred = lgbm_clf.predict(test)
 print('Accuracy Score:',round(lgbm_clf.score(train, target)*100,2))<train_model>
train = pd.read_csv('.. /input/quora-insincere-questions-classification/train.csv', encoding='utf-8')
 test = pd.read_csv('.. /input/quora-insincere-questions-classification/test.csv', encoding='utf-8')
 df = pd.concat([train,test],axis=0)
 print(f )
xgb_clf = XGBClassifier(n_estimators = 200)
 xgb_clf.fit(train, target)
 xgb_pred = xgb_clf.predict(test)
 print('Accuracy Score:',round(xgb_clf.score(train, target)*100,2))<train_model>
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float16')
 if file == '.. /input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='utf8')if len(o)>100)
 elif file == '.. /input/quora-insincere-questions-classification/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin':
embeddings_index = KeyedVectors.load_word2vec_format(file, binary=True)
 elif file == '.. /input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin')if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
glove = '.. /input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
base_estimators = [('lgbm',lgbm_clf), 
('rf',rf_clf)]
stc_clf = StackingClassifier(estimators = base_estimators, final_estimator = svc_clf)
 stc_clf.fit(train, target)
 stc_pred = stc_clf.predict(test)
 print('Accuracy Score:',round(stc_clf.score(train, target)*100,2))<find_best_model_class>
print("Extracting Paragram embedding")
 embed_paragram = load_embed(paragram)
estimators = [('rf',rf_clf), 
('xgb',xgb_clf), 
('lgbm',lgbm_clf)]
vot_clf = VotingClassifier(estimators = estimators, voting = 'soft')
 vot_clf.fit(train, target)
 vot_pred = vot_clf.predict(test)
 print('Accuracy Score:',round(vot_clf.score(train, target)*100,2))<load_from_csv>
def check_coverage(vocab,embeddings_index):
known_words={}
unknown_words={}
nb_known_words=0
nb_unknown_words=0
for word in vocab.keys() :
if word in embeddings_index.keys() :
known_words[word]=embeddings_index[word]
nb_known_words += vocab[word]
elif word.capitalize() in embeddings_index.keys() :
known_words[word] = embeddings_index[word.capitalize() ]
nb_known_words += vocab[word]
elif word.lower() in embeddings_index.keys() :
known_words[word] = embeddings_index[word.lower() ]
nb_known_words += vocab[word]
elif word.upper() in embeddings_index.keys() :
known_words[word] = embeddings_index[word.upper() ]
nb_known_words += vocab[word]
else:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
print(f'Found embeddings for {round(( len(known_words)/len(vocab)) *100,5)}% of the vocab
Found embeddings for {round(( nb_known_words/(nb_known_words+nb_unknown_words)) *100,5)}% of all text')
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1),reverse=True)
 return unknown_words
submit_data = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 submit_data.head()<prepare_output>
contraction_mapping = {
"Trump's" : 'trump is',"'cause": 'because','â€™': "'",',cause': 'because',';cause': 'because',"ain't": 'am not','ain,t': 'am not',
'ain;t': 'am not','ain´t': 'am not','ain’t': 'am not',"aren't": 'are not','â€“': '-','â€œ':'"',
'aren,t': 'are not','aren;t': 'are not','aren´t': 'are not','aren’t': 'are not',"can't": 'cannot',"can't've": 'cannot have','can,t': 'cannot','can,t,ve': 'cannot have',
'can;t': 'cannot','can;t;ve': 'cannot have',
'can´t': 'cannot','can´t´ve': 'cannot have','can’t': 'cannot','can’t’ve': 'cannot have',
"could've": 'could have','could,ve': 'could have','could;ve': 'could have',"couldn't": 'could not',"couldn't've": 'could not have','couldn,t': 'could not','couldn,t,ve': 'could not have','couldn;t': 'could not',
'couldn;t;ve': 'could not have','couldn´t': 'could not',
'couldn´t´ve': 'could not have','couldn’t': 'could not','couldn’t’ve': 'could not have','could´ve': 'could have',
'could’ve': 'could have',"didn't": 'did not','didn,t': 'did not','didn;t': 'did not','didn´t': 'did not',
'didn’t': 'did not',"doesn't": 'does not','doesn,t': 'does not','doesn;t': 'does not','doesn´t': 'does not',
'doesn’t': 'does not',"don't": 'do not','don,t': 'do not','don;t': 'do not','don´t': 'do not','don’t': 'do not',
"hadn't": 'had not',"hadn't've": 'had not have','hadn,t': 'had not','hadn,t,ve': 'had not have','hadn;t': 'had not',
'hadn;t;ve': 'had not have','hadn´t': 'had not','hadn´t´ve': 'had not have','hadn’t': 'had not','hadn’t’ve': 'had not have',"hasn't": 'has not','hasn,t': 'has not','hasn;t': 'has not','hasn´t': 'has not','hasn’t': 'has not',
"haven't": 'have not','haven,t': 'have not','haven;t': 'have not','haven´t': 'have not','haven’t': 'have not',"he'd": 'he would',
"he'd've": 'he would have',"he'll": 'he will',
"he's": 'he is','he,d': 'he would','he,d,ve': 'he would have','he,ll': 'he will','he,s': 'he is','he;d': 'he would',
'he;d;ve': 'he would have','he;ll': 'he will','he;s': 'he is','he´d': 'he would','he´d´ve': 'he would have','he´ll': 'he will',
'he´s': 'he is','he’d': 'he would','he’d’ve': 'he would have','he’ll': 'he will','he’s': 'he is',"how'd": 'how did',"how'll": 'how will',
"how's": 'how is','how,d': 'how did','how,ll': 'how will','how,s': 'how is','how;d': 'how did','how;ll': 'how will',
'how;s': 'how is','how´d': 'how did','how´ll': 'how will','how´s': 'how is','how’d': 'how did','how’ll': 'how will',
'how’s': 'how is',"i'd": 'i would',"i'll": 'i will',"i'm": 'i am',"i've": 'i have','i,d': 'i would','i,ll': 'i will',
'i,m': 'i am','i,ve': 'i have','i;d': 'i would','i;ll': 'i will','i;m': 'i am','i;ve': 'i have',"isn't": 'is not',
'isn,t': 'is not','isn;t': 'is not','isn´t': 'is not','isn’t': 'is not',"it'd": 'it would',"it'll": 'it will',"It's":'it is',
"it's": 'it is','it,d': 'it would','it,ll': 'it will','it,s': 'it is','it;d': 'it would','it;ll': 'it will','it;s': 'it is','it´d': 'it would','it´ll': 'it will','it´s': 'it is',
'it’d': 'it would','it’ll': 'it will','it’s': 'it is',
'i´d': 'i would','i´ll': 'i will','i´m': 'i am','i´ve': 'i have','i’d': 'i would','i’ll': 'i will','i’m': 'i am',
'i’ve': 'i have',"let's": 'let us','let,s': 'let us','let;s': 'let us','let´s': 'let us',
'let’s': 'let us',"ma'am": 'madam','ma,am': 'madam','ma;am': 'madam',"mayn't": 'may not','mayn,t': 'may not','mayn;t': 'may not',
'mayn´t': 'may not','mayn’t': 'may not','ma´am': 'madam','ma’am': 'madam',"might've": 'might have','might,ve': 'might have','might;ve': 'might have',"mightn't": 'might not','mightn,t': 'might not','mightn;t': 'might not','mightn´t': 'might not',
'mightn’t': 'might not','might´ve': 'might have','might’ve': 'might have',"must've": 'must have','must,ve': 'must have','must;ve': 'must have',
"mustn't": 'must not','mustn,t': 'must not','mustn;t': 'must not','mustn´t': 'must not','mustn’t': 'must not','must´ve': 'must have',
'must’ve': 'must have',"needn't": 'need not','needn,t': 'need not','needn;t': 'need not','needn´t': 'need not','needn’t': 'need not',"oughtn't": 'ought not','oughtn,t': 'ought not','oughtn;t': 'ought not',
'oughtn´t': 'ought not','oughtn’t': 'ought not',"sha'n't": 'shall not','sha,n,t': 'shall not','sha;n;t': 'shall not',"shan't": 'shall not',
'shan,t': 'shall not','shan;t': 'shall not','shan´t': 'shall not','shan’t': 'shall not','sha´n´t': 'shall not','sha’n’t': 'shall not',
"she'd": 'she would',"she'll": 'she will',"she's": 'she is','she,d': 'she would','she,ll': 'she will',
'she,s': 'she is','she;d': 'she would','she;ll': 'she will','she;s': 'she is','she´d': 'she would','she´ll': 'she will',
'she´s': 'she is','she’d': 'she would','she’ll': 'she will','she’s': 'she is',"should've": 'should have','should,ve': 'should have','should;ve': 'should have',
"shouldn't": 'should not','shouldn,t': 'should not','shouldn;t': 'should not','shouldn´t': 'should not','shouldn’t': 'should not','should´ve': 'should have',
'should’ve': 'should have',"that'd": 'that would',"that's": 'that is','that,d': 'that would','that,s': 'that is','that;d': 'that would',
'that;s': 'that is','that´d': 'that would','that´s': 'that is','that’d': 'that would','that’s': 'that is',"there'd": 'there had',
"there's": 'there is','there,d': 'there had','there,s': 'there is','there;d': 'there had','there;s': 'there is',
'there´d': 'there had','there´s': 'there is','there’d': 'there had','there’s': 'there is',
"they'd": 'they would',"they'll": 'they will',"they're": 'they are',"they've": 'they have',
'they,d': 'they would','they,ll': 'they will','they,re': 'they are','they,ve': 'they have','they;d': 'they would','they;ll': 'they will','they;re': 'they are',
'they;ve': 'they have','they´d': 'they would','they´ll': 'they will','they´re': 'they are','they´ve': 'they have','they’d': 'they would','they’ll': 'they will',
'they’re': 'they are','they’ve': 'they have',"wasn't": 'was not','wasn,t': 'was not','wasn;t': 'was not','wasn´t': 'was not',
'wasn’t': 'was not',"we'd": 'we would',"we'll": 'we will',"we're": 'we are',"we've": 'we have','we,d': 'we would','we,ll': 'we will',
'we,re': 'we are','we,ve': 'we have','we;d': 'we would','we;ll': 'we will','we;re': 'we are','we;ve': 'we have',
"weren't": 'were not','weren,t': 'were not','weren;t': 'were not','weren´t': 'were not','weren’t': 'were not','we´d': 'we would','we´ll': 'we will',
'we´re': 'we are','we´ve': 'we have','we’d': 'we would','we’ll': 'we will','we’re': 'we are','we’ve': 'we have',"what'll": 'what will',"what're": 'what are',"what's": 'what is',
"what've": 'what have','what,ll': 'what will','what,re': 'what are','what,s': 'what is','what,ve': 'what have','what;ll': 'what will','what;re': 'what are',
'what;s': 'what is','what;ve': 'what have','what´ll': 'what will',
'what´re': 'what are','what´s': 'what is','what´ve': 'what have','what’ll': 'what will','what’re': 'what are','what’s': 'what is',
'what’ve': 'what have',"where'd": 'where did',"where's": 'where is','where,d': 'where did','where,s': 'where is','where;d': 'where did',
'where;s': 'where is','where´d': 'where did','where´s': 'where is','where’d': 'where did','where’s': 'where is',
"who'll": 'who will',"who's": 'who is','who,ll': 'who will','who,s': 'who is','who;ll': 'who will','who;s': 'who is',
'who´ll': 'who will','who´s': 'who is','who’ll': 'who will','who’s': 'who is',"won't": 'will not','won,t': 'will not','won;t': 'will not',
'won´t': 'will not','won’t': 'will not',"wouldn't": 'would not','wouldn,t': 'would not','wouldn;t': 'would not','wouldn´t': 'would not',
'wouldn’t': 'would not',"you'd": 'you would',"you'll": 'you will',"you're": 'you are','you,d': 'you would','you,ll': 'you will',
'you,re': 'you are','you;d': 'you would','you;ll': 'you will',
'you;re': 'you are','you´d': 'you would','you´ll': 'you will','you´re': 'you are','you’d': 'you would','you’ll': 'you will','you’re': 'you are',
'´cause': 'because','’cause': 'because',"you've": "you have","could'nt": 'could not',
"havn't": 'have not',"here’s": "here is",'i""m': 'i am',"i'am": 'i am',"i'l": "i will","i'v": 'i have',"wan't": 'want',"was'nt": "was not","who'd": "who would",
"who're": "who are","who've": "who have","why'd": "why would","would've": "would have","y'all": "you all","y'know": "you know","you.i": "you i",
"your'e": "you are","arn't": "are not","agains't": "against","c'mon": "common","doens't": "does not",'don""t': "do not","dosen't": "does not",
"dosn't": "does not","shoudn't": "should not","that'll": "that will","there'll": "there will","there're": "there are",
"this'll": "this all","u're": "you are", "ya'll": "you all","you'r": "you are","you’ve": "you have","d'int": "did not","did'nt": "did not","din't": "did not","dont't": "do not","gov't": "government",
"i'ma": "i am","is'nt": "is not","‘I":'I','ᴀɴᴅ':'and','ᴛʜᴇ':'the','ʜᴏᴍᴇ':'home','ᴜᴘ':'up','ʙʏ':'by','ᴀᴛ':'at','…and':'and','civilbeat':'civil beat','TrumpCare':'Trump care','Trumpcare':'Trump care', 'OBAMAcare':'Obama care','ᴄʜᴇᴄᴋ':'check','ғᴏʀ':'for','ᴛʜɪs':'this','ᴄᴏᴍᴘᴜᴛᴇʀ':'computer','ᴍᴏɴᴛʜ':'month','ᴡᴏʀᴋɪɴɢ':'working','ᴊᴏʙ':'job','ғʀᴏᴍ':'from','Sᴛᴀʀᴛ':'start','gubmit':'submit','CO₂':'carbon dioxide','ғɪʀsᴛ':'first','ᴇɴᴅ':'end','ᴄᴀɴ':'can','ʜᴀᴠᴇ':'have','ᴛᴏ':'to','ʟɪɴᴋ':'link','ᴏғ':'of','ʜᴏᴜʀʟʏ':'hourly','ᴡᴇᴇᴋ':'week','ᴇɴᴅ':'end','ᴇxᴛʀᴀ':'extra','Gʀᴇᴀᴛ':'great','sᴛᴜᴅᴇɴᴛs':'student','sᴛᴀʏ':'stay','ᴍᴏᴍs':'mother','ᴏʀ':'or','ᴀɴʏᴏɴᴇ':'anyone','ɴᴇᴇᴅɪɴɢ':'needing','ᴀɴ':'an','ɪɴᴄᴏᴍᴇ':'income',
'ʀᴇʟɪᴀʙʟᴇ':'reliable','ғɪʀsᴛ':'first','ʏᴏᴜʀ':'your','sɪɢɴɪɴɢ':'signing','ʙᴏᴛᴛᴏᴍ':'bottom','ғᴏʟʟᴏᴡɪɴɢ':'following','Mᴀᴋᴇ':'make',
'ᴄᴏɴɴᴇᴄᴛɪᴏɴ':'connection','ɪɴᴛᴇʀɴᴇᴛ':'internet','financialpost':'financial post', 'ʜaᴠᴇ':' have ', 'ᴄaɴ':' can ', 'Maᴋᴇ':' make ', 'ʀᴇʟɪaʙʟᴇ':' reliable ', 'ɴᴇᴇᴅ':' need ',
'ᴏɴʟʏ':' only ', 'ᴇxᴛʀa':' extra ', 'aɴ':' an ', 'aɴʏᴏɴᴇ':' anyone ', 'sᴛaʏ':' stay ', 'Sᴛaʀᴛ':' start', 'SHOPO':'shop',
}
submit_data['Survived'] = rf_pred
submit_data.head()<save_to_csv>
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
submit_data.to_csv('output.csv', index = False )<import_modules>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'lnmiit':'limit', 'quorans': 'users of quora', 'mhcet':'competitive examination', 'swachh':'clean', 'chapterwise':'chapter wise', 'bitconnect':'cryptocurrency', 'bittrex':'cryptocurrency exchange','koinex':'cryptocurrency exchange', 'zebpay':'cryptocurrency exchange', 'binance':'cryptocurrency exchange', 'coinbase':'cryptocurrency exchange', 'brexit':'exit of UK from European Union', 'cryptocurrencies':'multiple cryptocurrency','redmi':'mobile phone', 'oneplus':'mobile phone company', 'pokémon':'anime', 'boruto':'anime', 'bhakts':'loyal followers', 'litecoin':'cryptocurreny','qoura':'quora','altcoin':'cryptocurreny','blockchains':'blockchain', 'airpods':'wireless earphones','zenfone':'mobile phone','altcoins':'multiple cryptocurrency','electroneum':'cryptocurreny','cryptocurreny':'blockchain based currency','iitians':'students','iitian':'student','gdpr':'data protection law'}
from sklearn.ensemble import\
GradientBoostingRegressor, RandomForestRegressor, ExtraTreesRegressor
callbacks, layers, optimizers, regularizers, Sequential
sns.set(style="white")
 warnings.filterwarnings("ignore")
 %matplotlib inline<load_from_csv>
train['treated_question'] = train['question_text'].apply(lambda x: x.lower())
 test['treated_question'] = test['question_text'].apply(lambda x: x.lower())
 train['treated_question'] = train['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
test['treated_question'] = test['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
train['treated_question'] = train['treated_question'].apply(lambda x: clean_puncts(x, punct_mapping, punct))
test['treated_question'] = test['treated_question'].apply(lambda x: clean_puncts(x, punct_mapping, punct))
train['treated_question'] = train['treated_question'].apply(lambda x: correct_spell(x, mispell_dict))
test['treated_question'] = test['treated_question'].apply(lambda x: correct_spell(x, mispell_dict))
def reload(df="train.csv", dropped_columns=['Id'], msg=True):
data_path = ".. /input/house-prices-advanced-regression-techniques/"
if df == "all_data":
train = pd.read_csv(data_path + "train.csv")
 test = pd.read_csv(data_path + "test.csv")
 data = train.drop(columns="SalePrice" ).append(test)
 else:
data = pd.read_csv(data_path + df)
 if msg:
print(df + " loaded successfully!")
 return data.reset_index(drop=True ).drop(columns=dropped_columns)
 train = reload()
test = reload("test.csv")
 all_data = reload("all_data" )<categorify>
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model, Sequential
from keras.layers import Flatten,Dense, Embedding, Conv1D, MaxPooling1D, GlobalAveragePooling1D, GlobalMaxPooling1D, Concatenate, Input, Dropout,Bidirectional, CuDNNGRU, GlobalAveragePooling1D, GlobalMaxPooling1D, LeakyReLU, Activation, CuDNNLSTM, SpatialDropout1D
from keras.optimizers import Adam
from keras import backend as K
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_fscore_support
train = reload()
class PreProcessor(BaseEstimator, TransformerMixin):
def __init__(self, old_new, numeric_to_object):
self.numeric_to_object = numeric_to_object
self.old_new = old_new
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X = X.copy()
sold_time = X[["MoSold", "YrSold"]].astype(str ).agg("/".join, axis=1)
 sold_time = pd.to_datetime(sold_time)
 for old in self.old_new:
X[self.old_new[old]] =(sold_time - pd.to_datetime(X[old], format="%Y")) \
/ pd.Timedelta(days=365.25)
 X[self.numeric_to_object] = X[self.numeric_to_object].astype(str)
 X.drop(columns=list(self.old_new.keys()), inplace=True)
 return X
class RelationImputer(BaseEstimator, TransformerMixin):
def __init__(self, related, threshold=None, strategy="most_frequent", fill_value=None
, missing_num=np.nan, missing_obj=np.nan
, final_num=0, final_obj="None"):
self.related = related if isinstance(related[0], list)else [related]
self.threshold = threshold
if strategy == 'constant':
self.fill_value = fill_value
elif strategy not in ['mean', 'median', 'most_frequent']:
raise Exception("Wrong strategy type! --{}".format(strategy))
self.strategy = strategy
self.missings = [missing_num, missing_obj, final_num, final_obj]
self.missing_num, self.missing_obj, self.final_num, self.final_obj =\
self.missings
def fit(self, X, y=None):
if self.strategy == 'constant':
self.statistics_ = pd.Series(data=self.fill_value, index=X.columns)
 else:
self.statistics_ = X[X.select_dtypes(object ).columns].mode().T[0]
num_cols = X.select_dtypes("number" ).columns
if self.strategy == "most_frequent":
self.statistics_ = self.statistics_.append(X[num_cols].mode().T[0])
 elif self.strategy in ["mean", "median"]:
strat = eval("X[num_cols].{}() ".format(self.strategy))
self.statistics_ = self.statistics_.append(strat)
 return self
def _indexes(self, X, cols):

 if not self.threshold:
l = len(cols)
 elif(self.threshold > 0)&(self.threshold < 1):
l = round(len(cols)*self.threshold)
 else:
l = self.null_threshold
missing = X[cols].isin(self.missings ).sum(axis=1)
 index =(missing > 0)&(missing < l)
 return index
def transform(self, X, y=None):
X = X.copy()
obj_cols = list(X.select_dtypes(object ).columns)
 num_cols = list(X.select_dtypes("number" ).columns)
 obj_len = len(obj_cols)
 num_len = len(num_cols)
 to_replace = pd.Series(
 data=[self.missing_obj]*obj_len + [self.missing_num]*num_len
, index=obj_cols + num_cols)
 final_value = pd.Series(
 data=[self.final_obj]*obj_len + [self.final_num]*num_len
, index=obj_cols + num_cols)
 for cols in self.related:
index = self._indexes(X, cols)
 X.loc[index, cols] = X.loc[index, cols].replace(
 to_replace=to_replace, value=self.statistics_[cols])
 X[cols] = X[cols].replace(to_replace=to_replace, value=final_value)
 flat_related = [c for sublist in self.related for c in sublist]
remaining = [c for c in X.columns if c not in flat_related]
X[remaining] = X[remaining].replace(to_replace=to_replace
, value=self.statistics_[remaining])
 return X
RELATED = [["Alley"]
, ["Condition1", "Condition2"]
, ["Exterior2nd"]
, ["BsmtQual", "BsmtCond", "BsmtExposure", "BsmtFinType1"
, "BsmtFinSF1", "BsmtUnfSF", "TotalBsmtSF"
, "BsmtFullBath", "BsmtHalfBath"]
, ["BsmtFinType2", "BsmtFinSF2"]
, ["Heating", "HeatingQC"]
, ["KitchenQual", "KitchenAbvGr"]
, ["FireplaceQu", "Fireplaces"]
, ["GarageType", "YrSinceGarageBlt", "GarageFinish"
, "GarageCars", "GarageArea", "GarageQual", "GarageCond"]
, ["MasVnrType", "MasVnrArea"]
, ["PoolArea", "PoolQC"]
, ["Fence"]
, ["MiscFeature", "MiscVal"]
]
nonrelated = [
"MSSubClass", "MSZoning", "LotArea", "Street", "Alley"
, "LotShape", "LandContour", "Utilities", "LotConfig", "LandSlope"
, "Neighborhood", "Condition1", "Condition2", "BldgType", "HouseStyle"
, "OverallQual", "OverallCond", "RoofStyle", "RoofMatl", "Exterior1st"
, "Exterior2nd", "ExterQual", "ExterCond", "Foundation", "CentralAir"
, "Electrical", "1stFlrSF", "2ndFlrSF", "LowQualFinSF", "GrLivArea"
, "BedroomAbvGr", "TotRmsAbvGrd", "Functional", "PavedDrive"
, "WoodDeckSF", "Fence", "MoSold", "YrSold", "SaleType", "LotFrontage"
, "SaleCondition", "Age", "YrSinceRemod", "YrSinceGarageBlt"
]
flat_related = [c for cols in RELATED for c in cols]
old_new = {"YearBuilt":"Age", "YearRemodAdd":"YrSinceRemod"
, "GarageYrBlt": "YrSinceGarageBlt"}
num_to_obj = ["MSSubClass", "MoSold", "YrSold"]
categorical_columns = list(all_data.select_dtypes(object ).columns)+ num_to_obj
cat_to_transform = [c for c in categorical_columns if c in flat_related]
PreI = PreProcessor(old_new, num_to_obj)
 RI = RelationImputer(related = RELATED, final_obj="None")
 train = PreI.fit_transform(train)
 train = RI.fit_transform(train)
 numerical_columns = [col for col in all_data.columns if col not in categorical_columns]<count_values>
length_to_keep = 100000
max_len = 75
train.BsmtHalfBath.value_counts()<feature_engineering>
train["BsmtFinSF1_UnfSF"] = train.BsmtFinSF1 + train.BsmtUnfSF
train["BsmtFinSF"] = train.BsmtFinSF1 + train.BsmtFinSF2
train["BsmtBathPerSF"] =(train.BsmtFullBath + train.BsmtHalfBath/2)/ train.TotalBsmtSF
train["BsmtBathPerFinSF"] =(train.BsmtFullBath + train.BsmtHalfBath/2)\
/(train.TotalBsmtSF - train.BsmtUnfSF )<categorify>
def tokenize_treated_data(text, text_test):
t = Tokenizer(num_words=length_to_keep, filters='')
 t.fit_on_texts(text)
 text = t.texts_to_sequences(text)
 text_test = t.texts_to_sequences(text_test)
 text = pad_sequences(text, maxlen=max_len)
 text_test = pad_sequences(text_test, maxlen=max_len)
 return text, t.word_index, text_test
X_treated, word_index_treated, X_test_treated = tokenize_treated_data(train['treated_question'], test['treated_question'] )
Score_map = {"Ex":5, "Gd":4, "TA":3, "Fa":2, "None":1, "Po":0}
BsmtFin_map = {"GLQ":5, "ALQ":4, "BLQ":3, "Rec":2, "LwQ":1, "Unf":0, "None":0}
BsmtEx_map = {"Gd":3, "Av":2, "Mn":1, "No":0, "None":0}
train["BsmtFinType1"] = train.BsmtFinType1.replace(BsmtFin_map)
 train["BsmtFinType2"] = train.BsmtFinType2.replace(BsmtFin_map)
 train["BsmtFinTypeAvg"] =(train["BsmtFinType1"] + train["BsmtFinType2"])/ 2
train["BsmtFinTypeProd"] = train["BsmtFinType1"] * train["BsmtFinType2"]<define_variables>
vocab_len_treated = len(word_index_treated)+1
def embed_matrix(embed_paragram, word_index, length_to_keep):
embeddings = np.stack(embed_paragram.values())
 embeddings_shape = embeddings.shape[1]
embedding_matrix = np.zeros(( length_to_keep, 300))
for word, i in word_index.items() :
if i >= length_to_keep:
continue
embeddings_vector = embed_paragram.get(word)
 if embeddings_vector is not None:
embedding_matrix[i] = embeddings_vector
return embedding_matrix
embedding_treated = embed_matrix(embed_paragram, word_index_treated, length_to_keep)
test_features = [["BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF"
,"BsmtFinType1", "BsmtFinType2", "BsmtUnfExp"]]<concatenate>
del word_index_treated
gc.collect()
test_features.append(
 ["AllBath", "BsmtHalfBath", "BsmtFullBath", "HalfBath", "FullBath"]
)<concatenate>
y = train['target'].values
train.drop(columns='target',inplace=True)
test_features.append(["GarageCars", "GarageArea"] )<count_values>
X_t_train, X_t_chk, y_train_treated, y_chk_treated = train_test_split(X_treated, y, test_size=0.1, random_state=0)
 print(f'Training on {X_t_train.shape[0]} texts' )
train.KitchenAbvGr.value_counts()<concatenate>
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
test_features.append(["KitchenAbvGr"] )<count_values>
train.PoolArea.value_counts()<concatenate>
test_features.append(["PoolArea", "PoolQC"] )<concatenate>
test_features.append(["MiscFeature", "MiscVal"] )<count_values>
print(train.Utilities.value_counts() , end="
")
 print(train.Electrical.value_counts() )<categorify>
def make_model(embedding_matrix, embed_size=300, loss='binary_crossentropy'):
inp = Input(shape=(max_len,))
embed= Embedding(length_to_keep, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 spatial_drop = SpatialDropout1D(0.3 )(embed)
 lstm1=(Bidirectional(CuDNNLSTM(256,return_sequences=True)) )(spatial_drop)
 lstm2=(Bidirectional(CuDNNGRU(256,return_sequences=True)) )(lstm1)
 maxpool1 = GlobalMaxPooling1D()(lstm1)
 maxpool2 = GlobalMaxPooling1D()(lstm2)
 conc = Concatenate(axis=1 )([maxpool1,maxpool2])
 dense1 = Dense(128,activation='relu' )(conc)
 output=Dense(1,activation='sigmoid' )(dense1)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss=loss, optimizer=Adam(lr=0.001), metrics=['accuracy', f1])
 return model
class FeatureEngineer(BaseEstimator, TransformerMixin):

 Score_map = {"Ex":5, "Gd":4, "TA":3, "Fa":2, "None":1, "Po":0}
BsmtFin_map = {"GLQ":5, "ALQ":4, "BLQ":3, "Rec":2, "LwQ":1, "Unf":0, "None":0}
BsmtEx_map = {"Gd":3, "Av":2, "Mn":1, "No":0, "None":0}
GargFin_map = {"Fin": 3, "RFn": 2, "Unf": 1, "None": 0}
GargType_map = {"BuiltIn": "Attchd", "Basment": "Attchd"
, "CarPort": "Detchd",}
StreetAlley_map = {"Pave":2, "Grvl":1, "None":0}
CentralAir_map = {"Y":1, "N":0}
LotShape_map = {"Reg":0, "IR1":1, "IR2":2, "IR3":3}
LandSlope_map = {"Gtl":0, "Mod":1, "Sev":2}
Electrical_map = {"SBrkr":3, "FuseA":2, "FuseF":1, "FuseP":0}
ElectricMix_map = {"Mix":1}
Functional_map = {"Typ":5, "Min1":4, "Min2":4, "Mod":3
, "Maj1":2, "Maj2":2, "Sev":1, "Sal":0}
PavedDrive_map = {"Y":2, "P":1, "N":0}
Fence_map = {"GdPrv":4, "MnPrv":3, "GdWo":2, "MnWw":1, "None":0}
CondStr_map = {"Feedr":1, "Artery":2}
CondNSRR_map = {"RRNn":1, "RRAn":2}
CondEWRR_map = {"RRNe":1, "RRAe":2}
CondOff_map = {"PosN":1, "PosA":2}
to_drop_ = ["MSSubClass", "GarageCond", "1stFlrSF", "2ndFlrSF"]
test_cols_ = [["BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF"
,"BsmtFinType1", "BsmtFinType2", "BsmtUnfExp"]
, ["AllBath", "BsmtFullBath", "BsmtHalfBath"
, "FullBath", "HalfBath"]
, ["GarageCars", "GarageArea"]
, ['KitchenAbvGr']
, ["PoolArea", "PoolQC"]
, ["MiscFeature", "MiscVal"]
, ["Utilities", "Electrical", "OverallCond", "LowQualFinSF"]
, ["Age", "YrSinceRemod"]
]
def __init__(self, execute=True
, cols_to_keep_or_drop=None
, keep_or_drop=None):

 self.execute = execute
self.cols_to_keep_or_drop = cols_to_keep_or_drop
self.keep_or_drop=keep_or_drop
def map_val(self, x, mp):
return mp[x] if x in mp else 0
def map_s(self, s, mp):
return s.apply(lambda x: self.map_val(x, mp))
def _feature_engineer(self, X):
X = X.copy()
for c in ["BsmtQual", "BsmtCond", "GarageQual"
, "HeatingQC", "KitchenQual", "FireplaceQu"
, "PoolQC", "ExterQual", "ExterCond"]:
X[c] = self.map_s(X[c], self.Score_map)
 for c, mpping in(
 ["BsmtFinType1", self.BsmtFin_map]
, ["BsmtFinType2", self.BsmtFin_map]
, ["BsmtExposure", self.BsmtEx_map]
, ["GarageFinish", self.GargFin_map]
, ["Alley", self.StreetAlley_map]
, ["Street", self.StreetAlley_map]
, ["CentralAir", self.CentralAir_map]
, ["LotShape", self.LotShape_map]
, ["LandSlope", self.LandSlope_map]
, ["Electrical", self.Electrical_map]
, ["Functional", self.Functional_map]
, ["PavedDrive", self.PavedDrive_map]
, ["Fence", self.Fence_map] 
):
if c == "Electrical":
X[c + "Mix"] = self.map_s(X[c], self.ElectricMix_map)
 X[c] = self.map_s(X[c], mpping)
 X["GarageType"] = X.GarageType.replace(self.GargType_map)
 for c, m in [
["Str", self.CondStr_map]
, ["NSRR", self.CondNSRR_map]
, ["EWRR", self.CondEWRR_map]
, ["Offsite", self.CondOff_map]
]:
cond1 = self.map_s(X.Condition1, m)
 cond2 = self.map_s(X.Condition2, m)
 X[c] = pd.concat([cond1, cond2], axis=1 ).max(axis=1)
 X.drop(columns = ["Condition1", "Condition2"], inplace=True)
 X["BsmtHfBthDummy"] = np.where(X.BsmtHalfBath == 0, 0, 1)
 X["BsmtFinType_SF1"] = X.BsmtFinType1 * X.BsmtFinSF1
X["BsmtFinType_SF2"] = X.BsmtFinType2 * X.BsmtFinSF2
X["BsmtUnfExp"] = X.BsmtUnfSF * X.BsmtExposure
X["AllBath"] = X.BsmtFullBath + X.FullBath +\ 
(X.BsmtHalfBath + X.HalfBath)/2
X["KitchenScore"] = X.KitchenQual * X.KitchenAbvGr
X.drop(columns = self.to_drop_, inplace=True)
 return X
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
if self.execute:
X = self._feature_engineer(X)
 if self.cols_to_keep_or_drop:
if self.keep_or_drop == "keep":
X = X[self.cols_to_keep_or_drop]
elif self.keep_or_drop == "drop":
X = X.drop(columns=self.cols_to_keep_or_drop, inplace=True)
 return X
class CustomizedOHE(BaseEstimator, TransformerMixin):

 def __init__(self, columns="categorical"
, OHE=OneHotEncoder(handle_unknown="ignore")
 , keep_header=True):
self.columns = columns
self.OHE = OHE
self.keep_header = keep_header
def fit(self, X, y=None):
self.header_ = list(X.columns)
 if self.columns == "categorical":
self.transformed_cols_ = X.select_dtypes(object ).columns
else:
self.transformed_cols_ = self.columns
self.not_transformed_cols_ = [
c for c in self.header_ if c not in self.transformed_cols_
]
if self.keep_header:
self.CT = ColumnTransformer(
 [("OHE", self.OHE, self.transformed_cols_)]
, remainder="passthrough"
, sparse_threshold=0 
)
 self.CT.fit(X)
 if len(self.transformed_cols_)> 0:
self.transformed_cols_new_ = \
self.CT.transformers_[0][1].get_feature_names()
else:
self.transformed_cols_new_ = []
self.header_new_ = np.concatenate(
(self.transformed_cols_new_,
self.not_transformed_cols_)
)
 else:
self.CT = ColumnTransformer(
 [("OHE", self.OHE, self.transformed_cols_)]
, remainder="passthrough" 
)
 self.CT.fit(X)
 return self
def transform(self, X, y=None):
transformed = self.CT.transform(X)
 self.a_ = transformed
if self.keep_header:
return pd.DataFrame(data=transformed, columns=self.header_new_)
 else:
return transformed
class CustomizedScaler(BaseEstimator, TransformerMixin):

 def __init__(self, scaler=StandardScaler() , keep_header=True):
self.scaler,self.keep_header = scaler, keep_header
def fit(self, X, y=None):
self.header = list(X.columns)
 self.scaler.fit(X, y)
 return self
def transform(self, X, y=None):
transformed = self.scaler.transform(X)
 if self.keep_header:
return pd.DataFrame(data=transformed, columns=self.header)
 else:
return transformed<define_variables>
early_stopping = EarlyStopping(patience=5, verbose=1, monitor='val_loss', mode='min')
 checkpoints_treated = ModelCheckpoint('treated_weights.hdf5', monitor="val_loss", mode="min", verbose=1, save_best_only=True)
 reduce_lr_treated = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2, verbose=1, min_lr=0.00001, mode='min' )
ADD_NN = False
ALPHAS = [0.1, 0.3, 1, 3, 10]
RS = 713
KFOLDS = 3
test_features = [['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF'
,'BsmtFinType1', 'BsmtFinType2', 'BsmtUnfExp']
, ['AllBath', 'BsmtHalfBath', 'BsmtFullBath'
, 'HalfBath', 'FullBath']
, ['GarageCars', 'GarageArea']
, ['KitchenAbvGr']
, ['PoolArea', 'PoolQC']
, ['MiscFeature', 'MiscVal']
, ['Utilities', 'Electrical', 'OverallCond', 'LowQualFinSF']
, ['Age', 'YrSinceRemod']]
def get_pipe(additional_transform=[]):
transform = [("PreI", PreProcessor(old_new, num_to_obj))
,("RI", RelationImputer(related = RELATED)) ]
transform += additional_transform
return Pipeline(transform)
 def get_GBR() :
return GradientBoostingRegressor(n_estimators=500
, max_depth=2
, max_features=0.8
, random_state=RS)
 def get_ETR() :
return ExtraTreesRegressor(n_estimators=700
, max_depth=3
, max_features=0.8
, random_state=RS)
 def get_RFR() :
return RandomForestRegressor(n_estimators=700
, max_depth=3
, max_features=0.8
, max_samples=0.8
, random_state=RS)
 def get_XGBR() :
return XGBRegressor(n_estimators=500
, learning_rate=0.1
, max_depth=3
, colsample_bytree=0.8
, n_jobs=-1, random_state=RS)
 def get_Ridge() :
return RidgeCV(alphas=ALPHAS, normalize=False, store_cv_values=True)
 def get_Enet() :
return ElasticNetCV(l1_ratio=0.5
, alphas=ALPHAS
, normalize=False
, max_iter=1000
, random_state=RS 
)
 def get_NN(hidden_layers=1, compiled=True):
NN = Sequential()
regu = regularizers.l1_l2(l1=0.01, l2=0.01)
 for _ in range(hidden_layers):
NN.add(layers.Dense(100
, activation="relu"
, kernel_regularizer=regu
, bias_regularizer=regu))
NN.add(layers.Dense(1))
if compiled:
optimizer = optimizers.SGD(0.001)
 NN.compile(optimizer=optimizer, loss="mse", metrics=["MSE"])
 return NN
def get_SVM() :
return LinearSVR(random_state=RS, max_iter=1000)
 def get_LGBR() :
return LGBMRegressor(max_depth=3, n_estimators=500
, random_state=RS, n_jobs=-1)
 estimators = [get_GBR, get_ETR, get_RFR, get_XGBR, get_Ridge
, get_Enet, get_SVM, get_LGBR]
if ADD_NN:
estimators.add(get_NN)
 early_stop = callbacks.EarlyStopping(monitor='val_loss', patience=5)
 NN_kwargs = dict(epochs=100
, batch_size=10
, validation_split=0.2
, verbose=0
, callbacks=[early_stop])
 def RMSE(y_true, y_pred):
return math.sqrt(mean_squared_error(y_true, y_pred))
def reload_xy(msg=False):
df = reload(msg=msg)
 X = df[[c for c in df.columns if c != "SalePrice"]].copy()
y = df["SalePrice"].copy()
return X, np.array(np.log1p(y))
X, y_trans = reload_xy()
def new_output(bl):
cols = ["Est", bl, "Score_mean", "Score_delt(%)"
, "Score_std", "Time_run", "Time_delt(%)"]
return pd.DataFrame(columns=cols)
 def fit_timer(get_est, X, y, params={}):
e = get_est()
t = dt.now()
e.fit(X, y, **params)
 return timer(t)
 def scoring(get_est, additional_transform
, X=X, y=y_trans, scorer=RMSE, splitter=KFold, cv=KFOLDS):
scores = []
splitter = splitter(cv, shuffle=True, random_state=713)
 kw = {}
if get_est.__name__ == 'get_NN':
kw = NN_kwargs
for tr, te in splitter.split(X):
X_tr, X_te = X.iloc[tr, :], X.iloc[te, :]
y_tr, y_te = y[tr], y[te]
p = get_pipe(additional_transform)
 e = get_est()
X_tr = p.fit_transform(X_tr)
 X_te = p.transform(X_te)
 e.fit(X_tr, y_tr, **kw)
 y_pre = e.predict(X_te)
 scores.append(RMSE(y_te, y_pre))
pipe = get_pipe(additional_transform)
 X_trans = pipe.fit_transform(X)
 t = fit_timer(get_est, X_trans, y_trans, params=kw)
 return scores, t
def new_row(df, tactics, i, get_est, bl, scores, time):
scr_m, scr_std = np.mean(scores), np.std(scores)
 est = get_est()
if(i % len(tactics)) == 0:
scr_d, time_d = np.nan, np.nan
else:
last_scr, last_t = df.loc[i-1, ["Score_mean", "Time_run"]]
scr_d =(scr_m - last_scr)/(last_scr)* 100
time_d =(t - last_t)/(last_t)* 100
output.loc[i] = [
type(est ).__name__, bl, scr_m, scr_d, scr_std, time, time_d
]
def timer(t):
t = dt.now() - t
return t.seconds + t.microseconds/1e6
def for_loop(bl, estimators=estimators):
l = list(product(estimators, bl))
return tqdm(l, position=0 )<normalization>
epochs = 4
batch_size = 128
add_trans = [("FE", FeatureEngineer())
 ,("CusOHE", CustomizedOHE())
 ,("Stdize", CustomizedScaler())]
pipe = get_pipe(add_trans)
 X_trans = pipe.fit_transform(X, y_trans)
 def col_name(col, ohe_cols=pipe.steps[3][1].transformed_cols_):
mat = re.match(r"(^x\d{1,2})_ (.+)", col)
 if mat:
return ohe_cols[int(mat.group(1)[1:])]
else:
return col
get_ests = [get_GBR, get_ETR, get_RFR, get_XGBR]
feat_imp = pd.DataFrame(
 columns = ["OHE_feat", "Features"]
+ [type(e() ).__name__ for e in get_ests]
+ ["Average"] 
)
 feat_imp.OHE_feat = list(X_trans.columns)
 feat_imp.Features = feat_imp.OHE_feat.apply(lambda x: col_name(x))
for get_est in get_ests:
est = get_est()
est.fit(X_trans, y_trans)
 feat_imp[type(est ).__name__[:-9]] = est.feature_importances_
feat_imp = feat_imp.groupby(["Features"] ).sum().reset_index()
feat_imp["Average"] = feat_imp.iloc[:,[1,2,3,4]].mean(axis=1)
 feat_imp = feat_imp.sort_values(by="Average", ascending=False)
 feat_imp = feat_imp.reset_index(drop=True)
 n = len(feat_imp)
 all_columns = list(feat_imp.Features)
 weak_cols = list(feat_imp[feat_imp.Average <(1/n)].Features)
 insig_cols = list(feat_imp[ 
(feat_imp.iloc[:,1:5] >(1/n)).sum(axis=1)== 0
].Features)
 zero_cols = list(feat_imp[feat_imp.Average == 0].Features)
 print(f"Features with AVERAGE importance score <(1/n_features):\
{len(weak_cols)}/{n}")
 print(*weak_cols[:5], sep=", ", end=", etc.
 ")
 print(f"Features with ALL importance scores <(1/len(features)) :\
{len(insig_cols)}/{n}")
 print(*insig_cols[:5], sep=", ", end=", etc.
 ")
 print(f"Features with 0 importance scores: {len(zero_cols)}/{n}")
 print(*zero_cols, sep=", ")
 feat_imp.to_csv("Column_importances.csv", index=False)
 a = feat_imp.iloc[:25,:]
ax = a.plot(y=[1,2,3,4], kind="line", figsize=(13,7), rot=90)
 a.plot(x="Features", y="Average"
, kind="bar", ax=ax, color="lightsteelblue"
, title="Top 25 Most Important Features" )<compute_test_metric>
a=3
print(f"Average importance: {round(1/n,3)}")
 for c in test_features:
print("
" + "="*20)
 print(feat_imp[feat_imp.Features.isin(c)].round(3))<load_from_csv>
history = model_treated.fit(X_treated, y, batch_size=batch_size,
epochs=epochs,
validation_data=[X_t_chk, y_chk_treated],
verbose=2, callbacks=[checkpoints_treated, reduce_lr_treated, early_stopping] )
class ColumnDropper(BaseEstimator, TransformerMixin):
col_imp = pd.read_csv("/kaggle/working/Column_importances.csv")
 all_cols = list(col_imp.Features)
 n = len(all_cols)
 weak_cols = list(col_imp[col_imp.Average <(1/n)].Features)
 insig_cols = list(col_imp[ 
(col_imp.iloc[:,1:5] >(1/n)).sum(axis=1)== 0
].Features)
 insig2_cols = list(col_imp[ 
(col_imp.iloc[:,1:5] >(1/n/10)).sum(axis=1)== 0
].Features)
 zero_cols = list(col_imp[col_imp.Average == 0].Features)
 def __init__(self, strategy=None, others=None):

 self.strategy = strategy
self.others = others
if isinstance(strategy,(int,np.integer)) :
self.columns = self.all_cols[strategy:]
elif strategy == "weak":
self.columns = self.weak_cols
elif strategy == "insignificant":
self.columns = self.insig_cols
elif strategy == "insignificant2":
self.columns = self.insig2_cols
elif strategy == "zero":
self.columns = self.zero_cols
elif strategy is None:
self.columns = []
else:
raise Exception(f"Wrong 'strategy' input: "{strategy}"")
 if others:
self.columns += others
self.columns = list(set(self.columns))
def fit(self, X, y=None):
if X.shape[1] != self.n:
msg = f"Wrong number of features: expecting {self.n}, got {X.shape[1]}"
raise Exception(msg)
 missing_cols = [c for c in X.columns if c not in self.all_cols]
if len(missing_cols)> 0:
msg = "Columns not existing: " + ", ".join(missing_cols)
 raise Exception(msg)
 else:
return self
def transform(self, X, y=None):
if len(self.columns)> 0:
X = X.drop(columns=self.columns)
 return X<drop_column>
y_hat = model_treated.predict(X_test_treated,verbose=True )
cols_to_drop = ["BsmtUnfExp", "BsmtUnfSF", "BsmtFinType2"
, "BsmtFinSF2", "BsmtHalfBath", "BsmtFullBath"
, "HalfBath", "KitchenAbvGr", "PoolQC"
, "MiscFeature", "MiscVal", "Electrical"
, "LowQualFinSF", "Utilities"]
output = new_output("Drop?")
 i = 0
tactics = [None, cols_to_drop]
for get_est, drop in for_loop(tactics):
add_trans = [("FE", FeatureEngineer())
 ,("Drop", ColumnDropper(others=drop))
,("CusOHE", CustomizedOHE())
 ,("Stdize", CustomizedScaler(keep_header=False)) ]
scores, t = scoring(get_est, add_trans)
 d = True if drop else False
new_row(output, tactics, i, get_est, d, scores, t)
 i += 1
output.round(3 )<statistical_test>
pred_val_y = model_treated.predict([X_t_chk], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = f1_score(y_chk_treated,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_1 = thresholds[0][0]
print("Best threshold: ", best_thresh_1)
scrs = {}
time = {}
tactics = np.arange(84, 0, -1)
 for get_est, tac in for_loop(tactics):
dropper = ColumnDropper(strategy=tac, others=cols_to_drop)
 add_trans = [("FE", FeatureEngineer())
 ,("Drop", dropper)
 ,("CusOHE", CustomizedOHE())
 ,("Stdize", CustomizedScaler(keep_header=False)) ]
scores, t = scoring(get_est, add_trans)
 est_name = type(get_est() ).__name__
if est_name in scrs:
scrs[est_name].append(np.mean(scores))
time[est_name].append(t)
 else:
scrs[est_name] = [np.mean(scores)]
time[est_name] = [t]<prepare_x_and_y>
submission = pd.read_csv('.. /input/quora-insincere-questions-classification/sample_submission.csv', encoding='utf-8' )
<save_to_csv><EOS> 
y_pred =(y_hat > best_thresh_1 ).astype(int)
 submission['prediction'] = y_pred
submission.to_csv('submission.csv', index=False )