Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<save_to_csv>
import numpy as np
import pandas as pd
import os
from multiprocessing import Pool
clf = RandomForestClassifier(n_estimators=100, max_depth=6,max_features=11, min_samples_leaf=0.0001,criterion="gini",n_jobs=-1, random_state=42)
 clf.fit(train_encoded, y_data)
 test_preds = clf.predict(test_encoded)
 submission.Survived = test_preds
submission.to_csv('rf_submission.csv', index=False )<set_options>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
 plt.rc('font', size=18)
 plt.rc('axes', titlesize=22)
 plt.rc('axes', labelsize=18)
 plt.rc('xtick', labelsize=12)
 plt.rc('ytick', labelsize=12)
 plt.rc('legend', fontsize=12)
 plt.rcParams['font.sans-serif'] = ['Verdana']
def format_1000(value, tick_number):
return int(value / 1_000)
 pd.options.mode.chained_assignment = None
pd.options.display.max_seq_items = 500
pd.options.display.max_rows = 500
pd.set_option('display.float_format', lambda x: '%.5f' % x)
 BASE_PATH = "/kaggle/input/house-prices-advanced-regression-techniques/"<load_from_csv>
def parallelize_apply(df,func,colname,num_process,newcolnames):
pool =Pool(processes=num_process)
 arraydata = pool.map(func,tqdm(df[colname].values))
pool.close()
newdf = pd.DataFrame(arraydata,columns = newcolnames)
 df = pd.concat([df,newdf],axis=1)
 return df
def parallelize_dataframe(df, func):
df_split = np.array_split(df, 4)
 pool = Pool(4)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
def add_features(df):
df['question_text'] = df['question_text'].apply(lambda x:str(x))
df['total_length'] = df['question_text'].apply(len)
 df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
df = pd.read_csv(f"{BASE_PATH}train.csv")
 df_test = pd.read_csv(f"{BASE_PATH}test.csv")
 df.head()<create_dataframe>
train_df = parallelize_dataframe(train_df, add_features)
 test_df = parallelize_dataframe(test_df, add_features )
missing = [(c, df[c].isna().mean() *100)for c in df]
missing = pd.DataFrame(missing, columns=["column_name", "percentage"])
 missing = missing[missing.percentage > 0]
display(missing.sort_values("percentage", ascending=False))<filter>
max_features = 100000
tokenizer = Tokenizer(num_words=max_features, lower=True)
 full_text = list(train_df['question_text'].values)
 tokenizer.fit_on_texts(full_text)
 train_tokenized = tokenizer.texts_to_sequences(train_df['question_text'].fillna('missing'))
test_tokenized = tokenizer.texts_to_sequences(test_df['question_text'].fillna('missing'))
df[df.SalePrice > upper]<sort_values>
max_len = 72
train_X = np.column_stack(( pad_sequences(train_tokenized, maxlen=max_len), train_meta))
test_X = np.column_stack(( pad_sequences(test_tokenized, maxlen=max_len), test_meta))
train_y = train_df.target.values
display(df.select_dtypes("number" ).skew().sort_values(ascending=False))<load_from_csv>
news_path = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings = KeyedVectors.load_word2vec_format(news_path, binary=True )
df = pd.read_csv(f"{BASE_PATH}train.csv")
 df_test = pd.read_csv(f"{BASE_PATH}test.csv")
 feat = pd.concat([df, df_test] ).reset_index(drop=True ).copy()<features_selection>
mean = np.mean(embeddings.vectors)
 std = np.std(embeddings.vectors)
 _, size = embeddings.vectors.shape
num_words = min(len(tokenizer.word_index), max_features)
 embedding_matrix = np.random.normal(mean, std,(num_words, size))
missing = feat.columns[feat.isna().any() ]
print(missing )<data_type_conversions>
for word, index in tokenizer.word_index.items() :
if index >= max_features: continue
try:
word_vector = embeddings.get_vector(word)
 embedding_matrix[index] = word_vector
except:
pass
print(embedding_matrix.shape )
feat.Alley = feat.Alley.fillna("None")
 bsmt_cols = ['BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'BsmtQual']
for col in bsmt_cols:
feat[col] = feat[col].fillna("None")
 for col in ['BsmtFullBath', 'BsmtHalfBath', 'BsmtUnfSF', 'BsmtFinSF1', 'BsmtFinSF2', 'TotalBsmtSF']:
feat[col] = feat[col].fillna(0 ).astype(int)
 feat.Electrical = feat.Electrical.fillna("SBrkr")
 feat.Exterior1st = feat.Exterior1st.fillna("VinylSd")
 feat.Exterior2nd = feat.Exterior2nd.fillna("None")
 feat.Fence = feat.Fence.fillna("None")
 feat.FireplaceQu = feat.FireplaceQu.fillna("None")
 feat.Functional = feat.Functional.fillna("Typ")
 grg_cols = ['GarageCond', 'GarageFinish', 'GarageQual', 'GarageType']
for col in grg_cols:
feat[col] = feat[col].fillna("None")
 for col in ['GarageArea', 'GarageCars', 'GarageYrBlt']:
feat[col] = feat[col].fillna(0 ).astype(int)
 to_fix = feat[feat.GarageYrBlt == 2207].index
feat.loc[to_fix, "GarageYrBlt"] = int(feat.GarageYrBlt.mean())
 feat.KitchenQual = feat.KitchenQual.fillna("TA")
 feat.LotFrontage = feat.LotFrontage.fillna(0 ).astype(int)
 feat.MSZoning = feat.MSZoning.fillna("RL")
 feat.MSSubClass = feat.MSSubClass.astype(str)
 feat.MasVnrType = feat.MasVnrType.fillna("None")
 feat.MasVnrArea = feat.MasVnrArea.fillna(0 ).astype(int)
 feat.MiscFeature = feat.MiscFeature.fillna("None")
 feat.PoolQC = feat.PoolQC.fillna("None")
 feat.SaleType = feat.SaleType.fillna("WD")
 feat.Utilities = feat.Utilities.fillna("AllPub" )<categorify>
tqdm.pandas()
def seed_torch(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed = 0
seed_torch(seed)
 def sigmoid(x):
return 1 /(1 + np.exp(-x))
class BiLSTM(nn.Module):
def __init__(self, embedding_matrix = embedding_matrix, embed_size = 300, hidden_size = 128):
super().__init__()
self.embedding = nn.Embedding(max_features, size)
 if embedding_matrix is not None:
self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional = True, batch_first = True, num_layers = 2, dropout = 0.1)
 self.fc = nn.Linear(hidden_size * 2 + 6, 1)
 def forward(self, x):
embedded = self.embedding(x[:,:72])
 f = torch.tensor(x[:,72:], dtype=torch.float32 ).cuda()
h_lstm,(h,c)= self.lstm(embedded)
 h_fc = torch.cat(( h[-1],h[-2],f), dim = 1)
 return self.fc(h_fc )
feat = feat.replace({ "Alley": {"None" : 0, "Grvl" : 1, "Pave" : 2},
"BsmtCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"BsmtExposure": {"None" : 0, "No" : 2, "Mn" : 2, "Av": 3,
"Gd" : 4},
"BsmtFinType1": {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtFinType2": {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd" : 4, "Ex" : 5},
"CentralAir": {"None" : 0, "N" : 1, "Y" : 2},
"ExterCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd": 4, "Ex" : 5},
"ExterQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd": 4, "Ex" : 5},
"Fence": {"None" : 0, "MnWw" : 1, "GdWo" : 2, "MnPrv": 3,
"GdPrv" : 4},
"FireplaceQu": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"Functional": {"None" : 0, "Sal" : 1, "Sev" : 2, "Maj2" : 3,
"Maj1" : 4, "Mod": 5, "Min2" : 6, "Min1" : 7,
"Typ" : 8},
"GarageCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"GarageQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"GarageFinish": {"None" : 0, "Unf" : 1, "RFn" : 2, "Fin" : 3},
"HeatingQC": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"KitchenQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"LandContour": {"None" : 0, "Low" : 1, "HLS" : 2, "Bnk" : 3,
"Lvl" : 4},
"LandSlope": {"None" : 0, "Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape": {"None" : 0, "IR3" : 1, "IR2" : 2, "IR1" : 3,
"Reg" : 4},
"PavedDrive": {"None" : 0, "N" : 0, "P" : 1, "Y" : 2},
"PoolQC": {"None" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3,
"Ex" : 4},
"Street": {"None" : 0, "Grvl" : 1, "Pave" : 2},
"Utilities": {"None" : 0, "ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3,
"AllPub" : 4}} 
)
 feat.BsmtCond = feat.BsmtCond.astype(int )<categorify>
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=0 ).split(train_X, train_y))
train_epochs = 5
batch_size = 64
print(type(train_X), train_X.shape)
 print(type(train_y), train_y.shape)
 train_X = torch.Tensor(train_X)
 train_y = torch.Tensor(train_y)
 train_meta = torch.Tensor(train_meta)
 test_meta = torch.Tensor(test_meta)
 train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
print('Fold:{0}'.format(i))
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
train_data = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid_data = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size, shuffle=False)
 model = BiLSTM()
model.cuda()
loss_function = nn.BCEWithLogitsLoss()
optimiser = optim.Adam(model.parameters())
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.0
for x_batch, y_batch in tqdm(train_loader, disable=True):
y_predicted = model(x_batch)
 loss = loss_function(y_predicted, y_batch)
 optimiser.zero_grad()
loss.backward()
optimiser.step()
avg_loss += loss.item() /len(train_loader)
 model.eval()
val_predicted_fold = np.zeros(x_val_fold.size(0))
test_predicted_fold = np.zeros(len(test_X))
avg_val_loss =0.0
for i,(x_batch, y_batch)in enumerate(valid_loader):
y_predicted = model(x_batch ).detach()
avg_val_loss += loss_function(y_predicted, y_batch ).item() /len(valid_loader)
 val_predicted_fold[i*batch_size:(i+1)*batch_size] = sigmoid(y_predicted.cpu().numpy())[:,0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1,
train_epochs, avg_loss, avg_val_loss, elapsed_time))
test_predicted_fold = np.zeros(len(test_X))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_predicted_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = val_predicted_fold
test_preds += test_predicted_fold / len(splits )
def onehot_encode(data):
df_numeric = data.select_dtypes(exclude=['object'])
 df_obj = data.select_dtypes(include=['object'] ).copy()
cols = []
for c in df_obj:
dummies = pd.get_dummies(df_obj[c])
 dummies.columns = [c + "_" + str(x)for x in dummies.columns]
cols.append(dummies)
 df_obj = pd.concat(cols, axis=1)
 data = pd.concat([df_numeric, df_obj], axis=1)
 data.reset_index(inplace=True, drop=True)
 return data
feat = onehot_encode(feat )<categorify>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
seasons = {12 : 0, 1 : 0, 2 : 0,
3 : 1, 4 : 1, 5 : 1,
6 : 2, 7 : 2, 8 : 2,
9 : 3, 10 : 3, 11 : 3}
feat["SeasonSold"] = feat["MoSold"].map(seasons)
 feat["YrActualAge"] = feat["YrSold"] - feat["YearBuilt"]
feat['TotalSF1'] = feat['TotalBsmtSF'] + feat['1stFlrSF'] + feat['2ndFlrSF']
feat['TotalSF2'] =(feat['BsmtFinSF1'] + feat['BsmtFinSF2'] + feat['1stFlrSF'] + feat['2ndFlrSF'])
 feat["AllSF"] = feat["GrLivArea"] + feat["TotalBsmtSF"]
feat["AllFlrsSF"] = feat["1stFlrSF"] + feat["2ndFlrSF"]
feat["AllPorchSF"] = feat["OpenPorchSF"] + feat["EnclosedPorch"] + feat["3SsnPorch"] + feat["ScreenPorch"]
feat['TotalBath'] = 2 *(feat['FullBath'] +(0.5 * feat['HalfBath'])+ feat['BsmtFullBath'] +(0.5 * feat['BsmtHalfBath']))
feat["TotalBath"] = feat["TotalBath"].astype(int)
 feat['TotalPorch'] =(feat['OpenPorchSF'] + feat['3SsnPorch'] + feat['EnclosedPorch'] + feat['ScreenPorch'] + feat['WoodDeckSF'])
 feat["OverallScore"] = feat["OverallQual"] * feat["OverallCond"]
feat["GarageScore"] = feat["GarageQual"] * feat["GarageCond"]
feat["ExterScore"] = feat["ExterQual"] * feat["ExterCond"]
feat["KitchenScore"] = feat["KitchenAbvGr"] * feat["KitchenQual"]
feat["FireplaceScore"] = feat["Fireplaces"] * feat["FireplaceQu"]
feat["GarageScore"] = feat["GarageArea"] * feat["GarageQual"]
feat["PoolScore"] = feat["PoolArea"] * feat["PoolQC"]
feat['hasPool'] = feat['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 feat['has2ndFloor'] = feat['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasGarage'] = feat['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasBsmt'] = feat['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasFireplace'] = feat['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<load_from_csv>
search_result = threshold_search(train_y, train_preds)
 search_result
df = pd.read_csv(f"{BASE_PATH}train.csv")
 df_test = pd.read_csv(f"{BASE_PATH}test.csv")
 dtrain = feat[feat.SalePrice.notnull() ].copy()
dtest = feat[feat.SalePrice.isnull() ].copy()
dtest = dtest.drop("SalePrice", axis=1 ).reset_index(drop=True)
 print(f"Raw data shape : {df.shape} {df_test.shape}")
 print(f"Clean data shape : {dtrain.shape} {dtest.shape}" )<compute_train_metric>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )