Kaggle competition: Quora Insincere Questions Classification
model_lasso = LassoCV(alphas = np.logspace(-4, -2, 20), cv = 4, n_jobs=4, max_iter=500)
 model_lasso.fit(df_train_skewed, target_skewed)
 scores = cross_val_score(model_lasso, df_train_skewed, target_skewed, cv=cv_num, scoring='neg_mean_squared_error')
 print("cross val scores(lasso):" , np.sqrt(-scores))
print('average root mean squared log error(lasso)=', np.mean(np.sqrt(-scores)))
 preds_lasso_skewed = np.expm1(model_lasso.predict(df_test_skewed)) 
<choose_model_class>
tqdm.pandas()
reoptimize=False
nestim = 1921
if(reoptimize):
learning_rate_grid = np.linspace(0.05, 0.05, 1)
 max_depth_grid = np.linspace(1, 5, 5, dtype='int')
 n_estimators_grid = np.linspace(1000, 2500, 250, dtype='int')
 xgbgrid={
'n_estimators' : n_estimators_grid }
xgb_cv=GridSearchCV(XGBRegressor(colsample_bytree=0.2), xgbgrid, cv=cv_num, refit=True, n_jobs=-1, scoring='neg_mean_squared_log_error')
 xgb_cv.fit(df_train_skewed, target_skewed)
 print(xgb_cv.best_params_)
 nestim = xgb_cv.best_params_['n_estimators']<predict_on_test>
emb_size = 300
max_features = 200000
maxlen = 100
class CustomEnsembleRegressor(BaseEstimator, RegressorMixin):
def __init__(self, regressors=None):
self.regressors = regressors
def fit(self, X, y):
for regressor in self.regressors:
regressor.fit(X, y)
 def predict(self, X):
self.predictions_ = list()
for regressor in self.regressors:
self.predictions_.append(( regressor.predict(X ).ravel()))
return(np.mean(self.predictions_, axis=0))
xgb1 = XGBRegressor(colsample_bytree=0.02,
learning_rate=0.05,
max_depth=3,
n_estimators=nestim 
)
 xgb2 = XGBRegressor(colsample_bytree=0.2,
learning_rate=0.05,
max_depth=3,
n_estimators=nestim,
seed = 1234 
)
 xgb3 = XGBRegressor(colsample_bytree=0.2,
learning_rate=0.05,
max_depth=3,
n_estimators=nestim,
seed = 1337 
)
 xgb_ens = CustomEnsembleRegressor([xgb1,xgb2,xgb3])
 xgb_ens.fit(df_train_skewed, target_skewed)
 scores = cross_val_score(cv=cv_num,estimator=xgb_ens,X = df_train_skewed,y = target_skewed, n_jobs = -1, scoring='neg_mean_squared_error')
 print("cross val scores(xgboost ensemble):" , np.sqrt(-scores))
print('average root mean squared log error(xgboost ensemble)=', np.mean(np.sqrt(-scores)))
 preds_xgb_skewed = np.expm1(xgb_ens.predict(df_test_skewed)) 
<save_to_csv>
def clean_memory(*args):
for arg in args:
del arg
gc.collect()
time.sleep(10 )
preds_avg=(preds_lasso_skewed+preds_xgb_skewed)/2
my_submission_avg = pd.DataFrame({'Id': df_test_skewed.index, 'SalePrice': preds_avg})
 my_submission_avg.to_csv('submission_avg.csv', index=False )<import_modules>
path = '/kaggle/input/quora-insincere-questions-classification'
train_df = pd.read_csv(path + "/train.csv")
 test_df = pd.read_csv(path + "/test.csv")
 train_df["question_text"].fillna("_na_", inplace=True)
 test_df["question_text"].fillna("_na_", inplace=True )
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
import numpy as np
import pandas as pd
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import seaborn as sns
from sklearn.preprocessing import Imputer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression<load_from_csv>
train_df["num_words"] = train_df["question_text"].progress_apply(lambda x: len(str(x ).split()))
train_df["num_unique_words"] = train_df["question_text"].progress_apply(lambda x: len(set(str(x ).split())))
 train_df["num_chars"] = train_df["question_text"].progress_apply(lambda x: len(str(x)))
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_train.head()
df_test=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 df_test.shape<define_variables>
test_df["num_words"] = test_df["question_text"].progress_apply(lambda x: len(str(x ).split()))
test_df["num_unique_words"] = test_df["question_text"].progress_apply(lambda x: len(set(str(x ).split())))
 test_df["num_chars"] = test_df["question_text"].progress_apply(lambda x: len(str(x)) )
train_ID= df_train['Id']
train_ID
test_ID= df_test['Id']
test_ID<prepare_x_and_y>
def build_vocab(sentences, verbose = True):
vocab = {}
for sentence in tqdm(sentences, disable =(not verbose)) :
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
df_test.drop('Id',axis=1,inplace=True)
 df_test.drop('Utilities',axis=1,inplace=True)
 df_test.shape
y_train=df_train.SalePrice.values
y_train<feature_engineering>
emb_path = '/kaggle/input/quora-insincere-questions-classification/embeddings'
def load_all() :
word2vec_format = {}
glove = [o.split(" ")[0] for o in tqdm(open(emb_path + '/glove.840B.300d/glove.840B.300d.txt')) ]
for word in tqdm(glove):
word2vec_format[word] = 1
clean_memory(glove)
 paragram = [o.split(" ")[0] for o in open(emb_path + '/paragram_300_sl999/paragram_300_sl999.txt', encoding="utf8", errors='ignore')if len(o)>100]
para = 0
for word in tqdm(paragram):
word2vec_format[word] = 1
clean_memory(paragram)
 return word2vec_format
emb_all = load_all()
df_train.SalePrice = np.log1p(df_train.SalePrice )<prepare_x_and_y>
def check_coverage(vocab, embeddings_index):
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x
y_train = df_train.SalePrice.values<sort_values>
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]
train_miss = df_train.isnull().sum()
print(train_miss)
 train_miss.sort_values(ascending=False)
 print(train_miss )<sort_values>
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
def clean_special_chars(text, punct=punct, mapping=punct_mapping):
for p in mapping:
if p in text:
text = text.replace(p, mapping[p])
 for p in punct:
if p in text:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
if s in text:
text = text.replace(s, specials[s])
 return text
test_miss = df_test.isnull().sum()
print(test_miss)
 test_miss.sort_values(ascending=False)
 print(test_miss )<define_variables>
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_special_chars(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_special_chars(x))
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab,emb_all)
 oov[:10]
commonNa = [
'MSZoning','Electrical','KitchenQual',
'Exterior1st','Exterior2nd','SaleType',
'LotFrontage','Functional'
]
toZero = [
'MasVnrArea','GarageYrBlt','BsmtHalfBath',
'BsmtFullBath','GarageArea','GarageCars',
'BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF'
]
toNoData = [
'PoolQC','MiscFeature','Alley',
'Fence','FireplaceQu','GarageType',
'GarageFinish','GarageQual',
'GarageCond','BsmtQual','BsmtCond',
'BsmtExposure','BsmtFinType1','BsmtFinType2',
'MasVnrType'
] 
<data_type_conversions>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}
def correct_spelling(x, dic):
for word in dic.keys() :
if word in x:
x = x.replace(word, dic[word])
 return x
def fillNan(df):
df['Functional']=df['Functional'].fillna('Typ')
 for i in commonNa:
df[i]=df[i].fillna(df[i].mode() [0])
 for i in toNoData:
df[i]=df[i].fillna('None')
 for i in toZero:
df[i]=df[i].fillna(0)
 return df<count_missing_values>
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: correct_spelling(x, mispell_dict))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: correct_spelling(x, mispell_dict))
sentences = train_df["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]
df_train =fillNan(df_train)
 df_train.isnull().sum()<count_missing_values>
go_to_more_common_words = {
'Redmi': 'Mobile',
'OnePlus': 'Mobile',
'Quorans': 'Quoran',
'cryptocurrencies': 'technology',
'Cryptocurrency': 'Technology',
'Blockchain': 'Technology',
'Upwork': 'Technology',
'HackerRank': 'Programming',
}
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: correct_spelling(x, go_to_more_common_words))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: correct_spelling(x, go_to_more_common_words))
sentences = train_df["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]
df_test =fillNan(df_test)
 df_test.isnull().sum()<feature_engineering>
clean_memory(oov, vocab, sentences, mispell_dict, go_to_more_common_words, punct, punct_mapping, emb_all )
def skew_(df):
colls = [col for col in df.columns if df[col].dtype in ['int64','float']]
skews_df = [col for col in df[colls].columns if df[col].skew() >.7]
def skewfix(data, data2):
for i in data2:
data[i] = np.log1p(data[i])
 return data
return skewfix(df, skews_df)
 df_train, df_test = skew_(df_train), skew_(df_test)
<categorify>
def load_all_emb(word_index):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 glove = dict(get_coefs(*o.split(" ")) for o in tqdm(open(emb_path + '/glove.840B.300d/glove.840B.300d.txt')) if o.split(" ")[0] in word_index)
 paragram = dict(get_coefs(*o.split(" ")) for o in open(emb_path + '/paragram_300_sl999/paragram_300_sl999.txt', encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 global max_features
all_embs = np.stack(glove.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
clean_memory(all_embs)
 nb_words = min(max_features, len(word_index))
max_features = nb_words
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, emb_size))
for word, i in tqdm(word_index.items()):
if i >= nb_words: continue
glove_vector = glove.get(word)
 paragram_vector = paragram.get(word)
 if glove_vector is not None and paragram_vector is not None:
embedding_matrix[i] = 0.7 * glove_vector + 0.3 * paragram_vector
elif glove_vector is not None:
embedding_matrix[i] = glove_vector
elif paragram_vector is not None:
embedding_matrix[i] = paragram_vector
clean_memory(glove_vector, paragram_vector)
 return embedding_matrix
encoder = LabelEncoder()
def encode(df):
cat_df = [col for col in df.columns if df[col].dtype not in ['int','float']]
for col in cat_df:
df[col] = encoder.fit_transform(df[col])
 return df
df_test,df_train=encode(df_test),encode(df_train )<prepare_x_and_y>
emb_path = '/kaggle/input/quora-insincere-questions-classification/embeddings'
tokenizer = Tokenizer(num_words=max_features, filters="")
 tokenizer.fit_on_texts(np.concatenate(( train_df["question_text"].values, test_df["question_text"].values)))
 embeddings = load_all_emb(tokenizer.word_index )
X=df_train.drop(['Id', 'SalePrice'], axis=1)
 y=df_train.SalePrice.values 
<split>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=47)
 train_X = train_df["question_text"].values
val_X = val_df["question_text"].values
test_X = test_df["question_text"].values
train_X2 = train_df[['num_words', 'num_unique_words', 'num_chars']]
val_X2 = val_df[['num_words', 'num_unique_words', 'num_chars']]
test_X2 = test_df[['num_words', 'num_unique_words', 'num_chars']]
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
print('Train shape: {}'.format(train_X.shape))
print('Validation shape: {}'.format(val_X.shape))
print('Test shape: {}'.format(test_X.shape))
clean_memory(train_df, val_df, test_df )
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.35,random_state=100 )<train_model>
early_stopping = EarlyStopping(monitor="val_f1", patience=2, restore_best_weights=True, mode="max")
 reduce_lr = ReduceLROnPlateau(monitor="val_f1", mode="max")
 callbacks = [early_stopping, reduce_lr]
regressor = SVR()
regressor.fit(X_train,y_train)
 svrpredict=regressor.predict(X_test )<compute_test_metric>
def f1(y_true, y_pred):

 def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
for i in range(0,433):
print("Error in value number",i,(y_test[i]-svrpredict[i]))<compute_test_metric>
inp1 = Input(shape=(maxlen,))
x = Embedding(max_features, emb_size, weights=[embeddings], trainable=False )(inp1)
 x = SpatialDropout1D(0.4 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Conv1D(64, 1 )(x)
 max_pool = GlobalMaxPooling1D()(x)
 inp2 = Input(shape=(3,))
y = Dense(64 )(inp2)
 xy = concatenate([max_pool, y], axis=1)
 xy = Dense(128, activation="relu" )(xy)
 xy = Dropout(0.1 )(xy)
 xy = BatchNormalization()(xy)
 outp = Dense(1, activation="sigmoid" )(xy)
 model = Model(inputs=[inp1, inp2], outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1] )
mean_squared_error(y_test,svrpredict )<choose_model_class>
model.fit([train_X, train_X2], train_y, batch_size=512, epochs=20, validation_data=([val_X, val_X2], val_y), callbacks=callbacks, verbose=True )
model = xgboost.XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
 model.fit(X_train,y_train)
 xgb_prediction = model.predict(X_test)
 def inv_y(transformed_y):
return np.exp(transformed_y)
<compute_test_metric>
clean_memory(train_X, train_y )
for i in range(0,433):
print("Error in value number",i,(y_test[i]-xgb_prediction[i]))<compute_test_metric>
pred_val_y = model.predict([val_X, val_X2], batch_size=512, verbose=1)
 def scoring(y_true, y_proba, verbose=True):
def threshold_search(y_true, y_proba):
precision , recall, thresholds = precision_recall_curve(y_true, y_proba)
 thresholds = np.append(thresholds, 1.001)
 F = 2 /(1/precision + 1/recall)
 best_score = np.max(F)
 best_th = thresholds[np.argmax(F)]
return best_th
rkf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10)
 scores = []
ths = []
for train_index, test_index in rkf.split(y_true, y_true):
y_prob_train, y_prob_test = y_proba[train_index], y_proba[test_index]
y_true_train, y_true_test = y_true[train_index], y_true[test_index]
best_threshold = threshold_search(y_true_train, y_prob_train)
 sc = f1_score(y_true_test,(y_prob_test >= best_threshold ).astype(int))
scores.append(sc)
 ths.append(best_threshold)
 best_th = np.mean(ths)
 score = np.mean(scores)
 if verbose: print(f'Best threshold: {np.round(best_th, 4)}, Score: {np.round(score,5)}')
 return best_th, score
optimal_point1, optimal1 = scoring(val_y, pred_val_y)
 print("Optimal F1 {} on threshhold {}".format(optimal1, optimal_point1))
mean_squared_error(y_test,xgb_prediction )<prepare_x_and_y>
clean_memory(val_X, val_y, pred_val_y )
X=df_train.drop(['Id', 'SalePrice'], axis=1)
 X4=X.iloc[:-1,:]
X5=X4.drop('Utilities',axis=1)
 y=df_train.iloc[:-1,:].SalePrice.values
X5.shape
df_test.shape
y.shape
model = xgboost.XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
 model.fit(X5,y)
 xgb_prediction = model.predict(df_test)
 def inv_y(transformed_y):
return np.exp(transformed_y)
<save_to_csv>
all_preds = model.predict([test_X, test_X2], batch_size=512, verbose=1)
 pred_test_y =(np.array(all_preds)> optimal_point1 ).astype(np.int )
<set_options><EOS> 
submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": pred_test_y1})
 submit_df.to_csv("submission.csv", index=False )