Kaggle competition: Quora Insincere Questions Classification
X = df
y = df.SalePrice
metric = 'neg_mean_squared_log_error'
clf = DummyRegressor("median")
 kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f} Log Error" )<compute_train_metric>
import pandas as pd
import numpy as np
import operator
import re
import gc
import pickle
from keras.preprocessing.text import Tokenizer
X = df[["GrLivArea"]]
y = df.SalePrice
metric = 'neg_mean_squared_log_error'
clf = LinearRegression()
kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f} Log Error" )<compute_train_metric>
np.random.seed(42 )
X = df[["OverallCond"]]
y = df.SalePrice
metric = 'neg_mean_squared_log_error'
clf = LinearRegression()
kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f} Log Error" )<compute_train_metric>
def load_embed(file):
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)> 100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
try:
X = df[["OverallQual"]]
y = df.SalePrice
metric = 'neg_mean_squared_log_error'
clf = LinearRegression()
kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f} Log Error")
 except Exception as e:
print("Oh no...An error has occured...")
 print(e )<compute_train_metric>
def index_embs(embeddings_index, word_index, NUM_WORDS, fileName):
all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(NUM_WORDS, embed_size))
for word, i in word_index.items() :
if i >= NUM_WORDS: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
np.save(fileName, embedding_matrix)
 print(fileName + " embedding matrix saved!")
 del(embeddings_index)
 del(word_index)
 del(embedding_matrix)
 gc.collect()
X = df[df.OverallQual >= 3][["OverallQual"]]
y = df[df.OverallQual >= 3].SalePrice
metric = 'neg_mean_squared_log_error'
clf = LinearRegression()
kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f} Log Error" )<compute_test_metric>
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
X = df.select_dtypes("object")
 y = df.SalePrice
metric = 'neg_mean_squared_log_error'
clf = make_pipeline(SimpleImputer(strategy='most_frequent', fill_value='missing'),
OneHotEncoder(handle_unknown="ignore"), LinearRegression())
 kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f} Log Error" )<compute_train_metric>
def build_vocab(texts, num_words):
tokenizer = Tokenizer(num_words=NUM_WORDS, filters="")
 tokenizer.fit_on_texts(texts)
 with open('tokenizer.pickle', 'wb')as handle:
pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 print("
Keras text tokenizer has been saved as tokenizer.pickle")
 return tokenizer.word_index
df_fixed = df[df.OverallQual >= 3]
X = df_fixed.select_dtypes("number" ).drop(["SalePrice"], axis=1)
 y = df_fixed.SalePrice
metric = 'neg_mean_squared_log_error'
clf = make_pipeline(SimpleImputer() , StandardScaler() , LinearRegression())
 kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f} Log Error" )<compute_train_metric>
def add_lower(embedding, vocab):

 count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )
X = df.select_dtypes("number" ).drop(["SalePrice"], axis=1)
 y = np.log1p(df.SalePrice)
 metric = 'neg_mean_squared_error'
clf = make_pipeline(SimpleImputer() , StandardScaler() , LinearRegression())
 kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f} Log Error" )<choose_model_class>
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
classifiers = [
Ridge() ,
Lasso() ,
ElasticNet() ,
KernelRidge() ,
SVR() ,
RandomForestRegressor() ,
GradientBoostingRegressor() ,
lgb.LGBMRegressor() ,
xgb.XGBRegressor(objective="reg:squarederror"),
]
clf_names = [
"ridge ",
"lasso ",
"elastic ",
"kernlrdg ",
"svr ",
"rndmforest ",
"gbm ",
"lgbm ",
"xgboost ",
]<compute_train_metric>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
X = dtrain.drop(["SalePrice"], axis=1)
 y = np.log1p(dtrain.SalePrice)
 metric = 'neg_mean_squared_error'
for clf_name, clf in zip(clf_names, classifiers):
kfold = KFold(n_splits=5, shuffle=True, random_state=1)
 print(f"{clf_name} {np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f}" )<compute_train_metric>
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1)
 y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 metric = 'neg_mean_squared_error'
for clf_name, clf in zip(clf_names, classifiers):
kfold = KFold(n_splits=5, shuffle=True, random_state=1)
 print(f"{clf_name} {np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f}" )<compute_train_metric>
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1)
 pca = PCA(n_components=250)
 X_pca = pca.fit_transform(X)
 y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 metric = 'neg_mean_squared_error'
for clf_name, clf in zip(clf_names, classifiers):
kfold = KFold(n_splits=5, shuffle=True, random_state=1)
 print(f"{clf_name} {np.sqrt(-cross_val_score(clf, X_pca, y, cv=kfold, scoring=metric)).mean() :.4f}" )<compute_train_metric>
def get_mappings() :

 mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
punct = ": )(-!?|;'$&/[]>%=
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
return mispell_dict, contraction_mapping, punct
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1 ).copy()
sk = pd.DataFrame(X.iloc[:, :60].skew() , columns=["skewness"])
 sk = sk[sk.skewness >.75]
for feature_ in sk.index:
X[feature_] = boxcox1p(X[feature_], 0.15)
 y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 metric = 'neg_mean_squared_error'
for clf_name, clf in zip(clf_names, classifiers):
kfold = KFold(n_splits=5, shuffle=True, random_state=1)
 print(f"{clf_name} {np.sqrt(-cross_val_score(clf, X, y, cv=kfold, scoring=metric)).mean() :.4f}" )<prepare_x_and_y>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1 ).copy()
y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 sk = pd.DataFrame(X.iloc[:, :60].skew() , columns=["skewness"])
 sk = sk[sk.skewness >.75]
for feature_ in sk.index:
X[feature_] = boxcox1p(X[feature_], 0.15)
 clf = xgb.XGBRegressor(objective="reg:squarederror", n_jobs=-1)
 coeffs = clf.fit(X, y ).feature_importances_
df_co = pd.DataFrame(coeffs, columns=["importance_"])
 df_co.index = X.columns
df_co.sort_values("importance_", ascending=True, inplace=True)
 plt.figure(figsize=(16,9))
df_co.iloc[250:, :].importance_.plot(kind="barh")
 plt.title(f"XGBoost feature importance")
 plt.show()<choose_model_class>
def run_text_preprocessing(df, textCol, outputFileName):
mispell_map, contraction_map, punct = get_mappings()
Working on: " + outputFileName)
 ****** WORD COVERAGE BEFORE PREPROCESSING ******")
 df['lowered_question'] = df[textCol].apply(lambda x: x.lower())
 df['treated_question'] = df['lowered_question'].apply(lambda x: clean_contractions(x, contraction_map))
df['treated_question'] = df['treated_question'].apply(lambda x: clean_special_chars(x, punct))
****** FINAL WORD COVERAGE AFTER PREPROCESSING ******")
 print("
DATAFRAME SAVED TO:")
 print("{0}_processed_text.csv".format(outputFileName))
df.to_csv("{0}_processed_text.csv".format(outputFileName))
del(df)
 del(mispell_map)
 del(contraction_map)
 del(punct)
 gc.collect()
classifiers = [
Ridge() ,
Lasso() ,
ElasticNet() ,
KernelRidge() ,
GradientBoostingRegressor() ,
lgb.LGBMRegressor() ,
xgb.XGBRegressor(objective="reg:squarederror"),
]
clf_names = [
"ridge ",
"lasso ",
"elastic ",
"kernlrdg ",
"gbm ",
"lgbm ",
"xgboost ",
]<prepare_x_and_y>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1 ).copy()
y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 X_test = dtest.copy()
sk = pd.DataFrame(X.iloc[:, :60].skew() , columns=["skewness"])
 sk = sk[sk.skewness >.75]
for feature_ in sk.index:
X[feature_] = boxcox1p(X[feature_], 0.15)
 X_test[feature_] = boxcox1p(X_test[feature_], 0.15 )<compute_test_metric>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
<predict_on_test>
run_text_preprocessing(train, "question_text", "train")
 run_text_preprocessing(test, "question_text", "test" )
predictions = []
for clf_name, clf in zip(clf_names, classifiers):
clf.fit(X, y)
 preds = clf.predict(X_test)
 predictions.append(np.expm1(preds))<data_type_conversions>
df_texts = pd.concat([train.drop('target', axis=1),test])
 df_texts.to_csv("texts_processed.csv" )
for idx, clf_name in enumerate(clf_names):
p = pd.DataFrame(predictions[idx])
 p_out = p[p[0] > 755_000].astype(int)
 if len(p_out)> 0:
p_out.columns = [f"{clf_name.strip().capitalize() } _ predicted SalePrice"]
display(p_out )<filter>
del(train)
 del(test)
 gc.collect()
dtest.loc[1089][["YearBuilt", "YrSold", "YrActualAge"]]<feature_engineering>
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
wiki_news = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
NUM_WORDS = None
word_idx = build_vocab(df_texts["treated_question"], NUM_WORDS)
 NUM_WORDS = len(word_idx)+ 1
print("
****** LOAD EMBEDDINGS ******")
 print("
---Extracting GloVe embedding---")
 embed_glove = load_embed(glove)
 index_embs(embed_glove, word_idx, NUM_WORDS, fileName="glove")
 del(embed_glove)
 gc.collect()
---Extracting Paragram embedding---")
 print("
---Extracting FastText embedding---")
 embed_fasttext = load_embed(wiki_news)
 index_embs(embed_fasttext, word_idx, NUM_WORDS, fileName="fasttext")
 del(embed_fasttext)
 gc.collect()
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1)
 sk = pd.DataFrame(X.iloc[:, :60].skew() , columns=["skewness"])
 sk = sk[sk.skewness >.75]
for feature_ in sk.index:
X[feature_] = boxcox1p(X[feature_], 0.15)
 y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 metric = 'neg_mean_squared_error'
predictions = []
pred_stats = [df.SalePrice.describe() ]
for clf_name, clf in zip(clf_names, classifiers):
clf.fit(X, y)
 preds = clf.predict(X_test)
 predictions.append(np.expm1(preds))
pred_stats.append(pd.DataFrame(np.expm1(preds)).describe())
 pr_corr = pd.DataFrame(predictions ).T.corr()
pr_corr.columns = [x.strip() for x in clf_names]
pr_corr.index = [x.strip() for x in clf_names]
plt.figure(figsize=(16,9))
sns.heatmap(pr_corr, cmap="RdBu", square=True, cbar_kws={"shrink":.8})
 plt.title("Correlation of predictions of trained models")
 plt.tight_layout()
plt.show()<prepare_x_and_y>
gc.collect()
%reset -f
<compute_train_metric>
startTimePt2 = time.time()
<prepare_x_and_y>
NUM_WORDS = None
EMBEDDING_DIM = 300
NUM_FILTERS = 50
MAX_LEN = 64
BATCH_SIZE = 2560
RANDOM_STATE = 42
NUM_EPOCH = 11
LR = 0.001
LR_MAX = LR * 6
WD = 0.011 *(BATCH_SIZE / 979591 / NUM_EPOCH)**0.5
STEP_SIZE_CLR = 2 *(979591 / BATCH_SIZE)
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1)
 y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 X_test = dtest
X_test.loc[1089]["YrSold"] = 2009
X_test.loc[1089]["YrActualAge"] = 0
metric = 'neg_mean_squared_error'<feature_engineering>
np.random.seed(RANDOM_STATE)
 set_random_seed(RANDOM_STATE )
sk = pd.DataFrame(X.iloc[:, :60].skew() , columns=["skewness"])
 sk = sk[sk.skewness >.75]
for feature_ in sk.index:
X[feature_] = boxcox1p(X[feature_], 0.15)
 X_test[feature_] = boxcox1p(X_test[feature_], 0.15 )<define_search_model>
df_train = pd.read_csv("train_processed_text.csv")
 df_test = pd.read_csv("test_processed_text.csv")
 print("Train shape : ", df_train.shape)
 print("Test shape : ", df_test.shape )
ridge = make_pipeline(RobustScaler() , Ridge(alpha=15, random_state=1))
lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.0005, random_state=1))
elastic = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005,
l1_ratio=1, random_state=1))
kernel = KernelRidge(alpha=1.0)
 gbm = GradientBoostingRegressor(n_estimators=2500, learning_rate=0.04,
max_depth=2, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=1)
 lgbm = lgb.LGBMRegressor(objective='regression', num_leaves=5,
learning_rate=0.03, n_estimators=8000,
max_bin=55, bagging_fraction=0.8,
bagging_freq=5, feature_fraction=0.23,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=6, min_sum_hessian_in_leaf=11)
 xgbreg = xgb.XGBRegressor(objective="reg:squarederror",
colsample_bytree=0.46, gamma=0.047,
learning_rate=0.04, max_depth=2,
min_child_weight=0.5, n_estimators=2000,
reg_alpha=0.46, reg_lambda=0.86,
subsample=0.52, random_state=1, n_jobs=-1 )<compute_train_metric>
X_train = df_train["treated_question"].fillna("_na_" ).values
X_test = df_test["treated_question"].fillna("_na_" ).values
y_train = df_train['target'].values
classifiers = [ridge, lasso, elastic, kernel, gbm, lgbm, xgbreg]
clf_names = ["ridge ", "lasso ", "elastic", "kernel ", "gbm ", "lgbm ", "xgbreg "]
predictions_exp = []
for clf_name, clf in zip(clf_names, classifiers):
print(f"{clf_name} {np.sqrt(-cross_val_score(clf, X, y, scoring=metric ).mean()):.5f}")
 clf.fit(X, y)
 preds = clf.predict(X_test)
 predictions_exp.append(np.expm1(preds))<save_to_csv>
with open('tokenizer.pickle', 'rb')as handle:
tokenizer = pickle.load(handle )
prediction_final = pd.DataFrame(predictions_exp ).mean().T.values
submission = pd.DataFrame({'Id': df_test.Id.values, 'SalePrice': prediction_final})
 submission.to_csv(f"submission.csv", index=False )<set_options>
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 print("hello word" )
warnings.filterwarnings('ignore' )<load_from_csv>
NUM_WORDS = len(tokenizer.word_index)
 NUM_WORDS
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train.head()<load_from_csv>
X_train = pad_sequences(X_train, maxlen=MAX_LEN)
 X_test = pad_sequences(X_test, maxlen=MAX_LEN)
 print("hello word" )
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 test.head()<sort_values>
del(df_train)
 del(df_test)
 del(tokenizer )
cor_target = abs(train.corr() ["SalePrice"] ).sort_values(ascending=False)
 cor_target<sort_values>
embedding_matrix_1 = np.load("glove.npy")
 embedding_matrix_2 = np.load("fasttext.npy")
 print("Done..." )
train.sort_values(by='TotalBsmtSF', ascending=False ).head()
train.sort_values(by='GrLivArea', ascending=False ).head()<drop_column>
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2], axis = 0)
 np.shape(embedding_matrix )
train = train.drop(train[train['Id'] == 524].index)
 train = train.drop(train[train['Id'] == 1299].index)
 train.shape, test.shape<prepare_x_and_y>
del(embedding_matrix_1)
 del(embedding_matrix_2)
y = train['SalePrice'].reset_index(drop=True)
 all_data = pd.concat([train, test] ).reset_index(drop=True )<drop_column>
gc.collect()
print("done" )
all_data.drop(['Id','SalePrice'], axis=1, inplace=True)
 all_data.shape<sort_values>
def SliceDataSRNN(data):
slicedData = []
for i in range(data.shape[0]):
split1 = np.split(data[i], 2)
 a=[]
for j in range(2):
s=np.split(split1[j], 2)
 a.append(s)
 slicedData.append(a)
 arr = np.array(slicedData, dtype=np.int32)
 del(slicedData)
 del(a)
 del(s)
 del(split1)
 gc.collect()
return arr
missingPercentage =(all_data.isnull().sum() * 100 / all_data.shape[0] ).sort_values(ascending=False)
 missingPercentage = missingPercentage[missingPercentage > 0]<drop_column>
x_train_padded_seqs_split = SliceDataSRNN(X_train)
 x_test_padded_seqs_split = SliceDataSRNN(X_test )
all_data.drop(columns=["PoolQC", "MiscFeature", "Alley", "Fence", "FireplaceQu"], inplace=True )<groupby>
del(X_test)
 del(X_train)
 gc.collect()
print("done" )
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
K.set_value(self.model.optimizer.lr, self.clr() )
columns_zero = ['BsmtFullBath', 'BsmtHalfBath', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'GarageYrBlt',
'GarageArea', 'GarageCars', 'MasVnrArea']
all_data[columns_zero] = all_data[columns_zero].fillna(0 )<data_type_conversions>
class AdamW(Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.init_lr = lr
self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
eta_t = lr / self.init_lr
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- eta_t * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
columns_none = ['BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','BsmtQual','GarageCond','GarageFinish',
'GarageType','GarageQual']
all_data[columns_none] = all_data[columns_none].fillna("None" )<data_type_conversions>
glorotInit = initializers.glorot_uniform(seed=RANDOM_STATE)
 orthoInit = initializers.Orthogonal(seed=RANDOM_STATE )
all_data['Functional'] = all_data['Functional'].fillna("Typ" )<data_type_conversions>
def dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel)
 class Attention(Layer):
def __init__(self,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True,
return_attention=False,
**kwargs):

 self.supports_masking = True
self.return_attention = return_attention
self.init = glorotInit
self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
eij = dot_product(x, self.W)
 if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 weighted_input = x * K.expand_dims(a)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, a]
return result
def compute_output_shape(self, input_shape):
if self.return_attention:
return [(input_shape[0], input_shape[-1]), 
(input_shape[0], input_shape[1])]
else:
return input_shape[0], input_shape[-1]
columns_mode = ['MSZoning', 'MasVnrType', 'Electrical', 'Utilities', 'Exterior1st', 'Exterior2nd', 'SaleType', 'KitchenQual']
for i in columns_mode:
all_data[i] = all_data[i].fillna(all_data[i].mode() [0] )<drop_column>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data.drop(columns=['MoSold'], inplace=True )<count_missing_values>
glorotInit = initializers.glorot_uniform(seed=RANDOM_STATE)
 orthoInit = initializers.Orthogonal(seed=RANDOM_STATE )
print('Total missing data points: {}'.format(all_data.isna().sum().sum()))<sort_values>
def srnn_model() :
embedding_layer = Embedding(NUM_WORDS,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=16,
trainable=False)
 input1 = Input(shape=(16,), dtype='int32')
 embed = embedding_layer(input1)
 drop1 = SpatialDropout1D(0.2 )(embed)
 gru1 = Bidirectional(CuDNNLSTM(NUM_FILTERS, return_sequences=True, kernel_initializer=glorotInit,
recurrent_initializer=glorotInit))(drop1)
 gru1_2 = Bidirectional(CuDNNLSTM(NUM_FILTERS, return_sequences=True, kernel_initializer=glorotInit,
recurrent_initializer=glorotInit))(gru1)
 atten_1 = Attention()(gru1)
 atten_1_2 = Attention()(gru1_2)
 max_pool1 = GlobalMaxPooling1D()(gru1_2)
 avg_pool1 = GlobalAveragePooling1D()(gru1_2)
 conc1 = concatenate([atten_1, atten_1_2, max_pool1, avg_pool1])
 Encoder1 = Model(input1, conc1)
 input2 = Input(shape=(2, 16,), dtype='int32')
 embed2 = TimeDistributed(Encoder1 )(input2)
 gru2 = Bidirectional(CuDNNGRU(NUM_FILTERS, return_sequences=False, kernel_initializer=glorotInit,
recurrent_initializer=glorotInit))(embed2)
 Encoder2 = Model(input2, gru2)
 input3 = Input(shape=(2, 2, 16), dtype='int32')
 embed3 = TimeDistributed(Encoder2 )(input3)
 gru3 = Bidirectional(CuDNNGRU(NUM_FILTERS, return_sequences=False, kernel_initializer=glorotInit,
recurrent_initializer=glorotInit))(embed3)
 dense = Dense(32, activation="relu", kernel_initializer=glorotInit )(gru3)
 drop2 = Dropout(0.1 )(dense)
 output = Dense(1, activation="sigmoid" )(drop2)
 model = Model(input3, output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(lr=LR, weight_decay=0.) , metrics=["acc", f1])
 return model
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skewed_feats = skewed_feats[abs(skewed_feats)> 0.5].index
high_skewed_feats<feature_engineering>
for feature in high_skewed_feats:
all_data[feature] = np.log1p(all_data[feature] )<categorify>
clr = CyclicLR(base_lr=0.001, max_lr=0.004,
step_size=300., mode='exp_range',
gamma=0.99994 )
all_data = pd.get_dummies(all_data ).reset_index(drop=True)
 all_data.shape<import_modules>
def train_pred(model, train_X, train_y, val_X, val_y, test_data, callbacks=None):
st = time.time()
model.fit(np.array(train_X), train_y, batch_size=BATCH_SIZE, epochs=NUM_EPOCH, validation_data=(np.array(val_X), val_y),
callbacks = callbacks, verbose=2)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = threshold_search(val_y, pred_val_y)
 print("\tVal F1 Score: {:.4f}\tThresh: {:.2f}".format(best_thresh["f1"], best_thresh["threshold"]))
best_score = best_thresh["f1"]
pred_test_y = model.predict(test_data, batch_size=1024, verbose=0)
 print("Training time was: " + str(time.time() - st))
print('=' * 60)
 return pred_val_y, pred_test_y, best_score
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import cross_val_score, KFold<choose_model_class>
train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(len(x_test_padded_seqs_split))
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=RANDOM_STATE ).split(x_train_padded_seqs_split, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train_tmp = x_train_padded_seqs_split[train_idx]
y_train_tmp = y_train[train_idx]
X_val_tmp = x_train_padded_seqs_split[valid_idx]
y_val_tmp = y_train[valid_idx]
model = srnn_model()
pred_val_y, pred_test_y, best_score = train_pred(model, X_train_tmp, y_train_tmp, X_val_tmp, y_val_tmp, x_test_padded_seqs_split,
callbacks=[clr])
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits)
 del(X_train_tmp)
 del(y_train_tmp)
 del(X_val_tmp)
 del(y_val_tmp)
 gc.collect()
kf = KFold(n_splits=10, shuffle=True, random_state=123 )<define_search_model>
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_meta > search_result['threshold']
sub.to_csv("submission.csv", index=False )
<define_search_model><EOS> 
( time.time() - startTimePt2)/ 60