Kaggle competition: Titanic - Machine Learning from Disaster
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = '.. /input/panda-public-models'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:100]
tile_size = 256
image_size = 256
n_tiles = 36
batch_size = 8
num_workers = 4
device = torch.device('cuda')
 print(image_folder )<define_search_model>
import matplotlib.pyplot as plt
import sklearn
import seaborn as sb
class enetv2(nn.Module):
def __init__(self, backbone, out_dim):
super(enetv2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(model_dir, model_f)
 backbone = 'efficientnet-b0'
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
model_files = [
'cls_effnet_b0_Rand36r36tiles256_big_bce_lr0.3_augx2_30epo_model_fold0.pth'
]
models = load_models(model_files )<categorify>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 train.head()
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
idxes = np.asarray(idxes)+ self.n_tiles if self.sub_imgs else idxes
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images)
<load_pretrained>
combine = [train,test]
dataset = PANDADataset(df, image_size, n_tiles, 0)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False )<save_to_csv>
for c in train.columns:
print(c, str(100*train[c].isnull().sum() /len(train)))
LOGITS = []
LOGITS2 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() + torch.cat(LOGITS2 ).sigmoid().cpu())/ 2
PREDS = LOGITS.sum(1 ).round().numpy()
df['isup_grade'] = PREDS.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )<define_variables>
train['Age'] = train['Age'].fillna(train['Age'].mean() )
DEBUG = False<define_variables>
for c in train.columns:
print(c, str(100*train[c].isnull().sum() /len(train)) )
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path<import_modules>
dependencies_sex = train[['Sex', 'Survived']].groupby(['Sex'],as_index=False ).mean()
dependencies_Pclass = train[['Pclass', 'Survived']].groupby(['Pclass'],as_index=False ).mean()
print(dependencies_Pclass)
 print(dependencies_sex )
import skimage.io
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from efficientnet_pytorch import model as enet
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm 
<load_from_csv>
for dats in combine:
dats['Title'] = dats.Name.str.extract('([A-Za-z]+)\.',expand=False )
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = '.. /input/panda-public-models'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:100]
tile_size = 256
image_size = 256
n_tiles = 36
batch_size = 8
num_workers = 4
device = torch.device('cuda')
 print(image_folder )<define_search_model>
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs' )
class enetv2(nn.Module):
def __init__(self, backbone, out_dim):
super(enetv2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(model_dir, model_f)
 backbone = 'efficientnet-b0'
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
model_files = [
'cls_effnet_b0_Rand36r36tiles256_big_bce_lr0.3_augx2_30epo_model_fold0.pth'
]
models = load_models(model_files )<categorify>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0 )
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
idxes = np.asarray(idxes)+ self.n_tiles if self.sub_imgs else idxes
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images)
<load_pretrained>
title_dependencies=train[['Title','Survived','Sex']].groupby(['Title','Sex'],as_index=False ).mean()
title_dependencies
dataset = PANDADataset(df, image_size, n_tiles, 0)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False )<save_to_csv>
train = train.drop(['Name', 'PassengerId', 'Cabin', 'Embarked','Ticket'], axis=1)
 test = test.drop(['Name', 'PassengerId', 'Cabin', 'Embarked','Ticket'], axis=1)
 combine=[train,test]
print(train.head())
 for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 print(train.head())
 print(test.head())
LOGITS = []
LOGITS2 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() + torch.cat(LOGITS2 ).sigmoid().cpu())/ 2
PREDS = LOGITS.sum(1 ).round().numpy()
df['isup_grade'] = PREDS.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )<set_options>
X_train, X_test , Y_train, Y_test = train_test_split(train.drop(['Survived'],axis=1),train['Survived'],test_size=0.10,random_state=None )
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.simplefilter(action='ignore', category=SettingWithCopyWarning)
 warnings.simplefilter(action='ignore', category=FutureWarning )<compute_test_metric>
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
FEATS_EXCLUDED = ['first_active_month', 'target', 'card_id', 'outliers',
'hist_purchase_date_max', 'hist_purchase_date_min', 'hist_card_id_size',
'new_purchase_date_max', 'new_purchase_date_min', 'new_card_id_size',
'OOF_PRED', 'month_0']
@contextmanager
def timer(title):
t0 = time.time()
yield
print("{} - done in {:.0f}s".format(title, time.time() - t0))
def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns
def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance", ascending=False)[:40].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.savefig('lgbm_importances.png')
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<load_from_csv>
modelLR= LogisticRegression(solver='liblinear',C=0.21,random_state=1)
 modelLR.fit(X_train,Y_train)
%%time
def load_data() :
train_df = pd.read_csv('.. /input/elo-blending/train_feature.csv')
 test_df = pd.read_csv('.. /input/elo-blending/test_feature.csv')
 display(train_df.head())
 display(test_df.head())
 print(train_df.shape,test_df.shape)
 train = pd.read_csv('.. /input/elo-merchant-category-recommendation/train.csv', index_col=['card_id'])
 test = pd.read_csv('.. /input/elo-merchant-category-recommendation/test.csv', index_col=['card_id'])
 train_df['card_id'] = train.index
test_df['card_id'] = test.index
train_df.index = train_df['card_id']
test_df.index = test_df['card_id']
display(train_df.head())
 display(test.head())
 print(train.shape,test.shape)
 del train,test
return(train_df,test_df)
 print(gc.collect() )<define_variables>
Y_pred_log=modelLR.predict(X_test)
 acc_LR = metrics.accuracy_score(Y_test, Y_pred_log )
boosting = ["goss","dart"]
boosting[0],boosting[1]<load_pretrained>
print(Y_pred_log)
 print("We see that Logistic regression gives an Accuracy of ",acc_LR*100,"% on the traing set.")
%%time
boosting = ["goss","dart"]
def kfold_lightgbm(train_df, test_df, num_folds, stratified = False, boosting = boosting[0]):
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=326)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=2045)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in FEATS_EXCLUDED]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['outliers'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['target'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['target'].iloc[valid_idx]
lgb_train = lgb.Dataset(train_x,label=train_y,free_raw_data=False)
 lgb_test = lgb.Dataset(valid_x,label=valid_y,free_raw_data=False)
 params ={
'task': 'train',
'boosting': 'goss',
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.005,
'subsample': 0.9855232997390695,
'max_depth': 8,
'top_rate': 0.9064148448434349,
'num_leaves': 87,
'min_child_weight': 41.9612869171337,
'other_rate': 0.0721768246018207,
'reg_alpha': 9.677537745007898,
'colsample_bytree': 0.5665320670155495,
'min_split_gain': 9.820197773625843,
'reg_lambda': 8.2532317400459,
'min_data_in_leaf': 21,
'verbose': -1,
'seed':int(2**n_fold),
'bagging_seed':int(2**n_fold),
'drop_seed':int(2**n_fold)
 }
reg = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_train, lgb_test],
valid_names=['train', 'test'],
num_boost_round=10000,
early_stopping_rounds= 200,
verbose_eval=100 
)
 oof_preds[valid_idx] = reg.predict(valid_x, num_iteration=reg.best_iteration)
 sub_preds += reg.predict(test_df[feats], num_iteration=reg.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = np.log1p(reg.feature_importance(importance_type='gain', iteration=reg.best_iteration))
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d RMSE : %.6f' %(n_fold + 1, rmse(valid_y, oof_preds[valid_idx])))
 del reg, train_x, train_y, valid_x, valid_y
gc.collect()
display_importances(feature_importance_df)
 submission = pd.read_csv(".. /input/elo-merchant-category-recommendation/sample_submission.csv")
 submission['target'] = sub_preds
submission.to_csv(boosting+".csv", index=False)
 display(submission.head())
 return(submission )<split>
from sklearn.tree import DecisionTreeClassifier
%%time
train_df,test_df = load_data()
print(gc.collect())
 submission = kfold_lightgbm(train_df, test_df, num_folds=7, stratified=False, boosting=boosting[0] )<concatenate>
dtree = DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=1,
max_features=7, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0,
random_state=None, splitter='best')
 dtree.fit(X_train, Y_train)
 y_pred_tree = dtree.predict(X_test )
submission1 = kfold_lightgbm(train_df, test_df, num_folds=7, stratified=False, boosting=boosting[1] )<save_to_csv>
acc_DT = metrics.accuracy_score(y_pred_tree, Y_test)
 print(y_pred_tree)
 print("We see that Decision Tree gives an Accuracy of ",acc_DT*100,"% on the traing set." )
final = pd.read_csv(".. /input/elo-merchant-category-recommendation/sample_submission.csv")
 final['target'] = submission['target'] * 0.5 + submission1['target'] * 0.5
final.to_csv("blend.csv",index = False )<set_options>
from sklearn.ensemble import RandomForestClassifier
%matplotlib inline 
<choose_model_class>
rforest = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
max_depth=None, max_features=7, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=20, n_jobs=1,
oob_score=False, random_state=42, verbose=0, warm_start=False)
 rforest.fit(X_train, Y_train)
 y_pred_forest = rforest.predict(X_test )
max_seq_len = 60
embed_size = 300
max_features = 50000
EMBEDDING = 'glove.840B.300d'
MODEL = 'attention'
embedding_matrix = 'None'
embeddings_idx = 'None'
checkpoint = ModelCheckpoint('./checkpoints/', monitor='val_acc', verbose=0, save_best_only=True)
 earlystop = EarlyStopping(monitor='val_acc', min_delta=0, patience=1, verbose=0)
 tensorboard = TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=True, write_images=True)
 reducelr = ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=3, verbose=0, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)
 thresh = 0.4<load_from_csv>
acc_RF = metrics.accuracy_score(y_pred_forest, Y_test)
 print(y_pred_forest)
 print("We see that Random Forest classifier gives an Accuracy of ",acc_RF*100,"% on the traing set." )
train_set = pd.read_csv('.. /input/train.csv')
 test_set = pd.read_csv('.. /input/test.csv')
 train_set.head()<count_values>
from sklearn.svm import SVC
x = train_set['target'].value_counts(dropna=False)
 print(x)
 sincere_examples = x[0]
insincere_examples = x[1]<define_variables>
svc = SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',
max_iter=-1, probability=False, random_state=None, shrinking=True,
tol=0.001, verbose=False)
 svc.fit(X_train, Y_train)
 y_pred_svc = svc.predict(X_test )
print(len(lengths_without_puncs)- np.count_nonzero(lengths_without_puncs))<define_variables>
acc_SVC = metrics.accuracy_score(y_pred_svc, Y_test)
 print(y_pred_svc)
 print("We see that SVC classifier gives an Accuracy of ",acc_SVC*100,"% on the traing set." )
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def clean_contractions(text, mapping=contraction_mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def clean_special_chars(text, punct=punct, mapping=punct_mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def correct_spelling(x, dictionary=mispell_dict):
for word in dictionary.keys() :
x = x.replace(word, dictionary[word])
 return x
def clean(text):
text = text.lower()
text = clean_contractions(text)
 text = clean_special_chars(text)
 text = correct_spelling(text)
 return text<count_values>
from xgboost import XGBClassifier
sincere_counts = Counter()
insincere_counts = Counter()
word_dict = Counter()
sincere_to_insincere_ratio = Counter()
def prepare_dicts() :
qs = [clean(i)for i in train_set['question_text']]
lbl = [j for j in train_set['target']]
for i,j in zip(qs,lbl):
words = i.split()
for word in words:
word_dict[word] += 1
if j == 0:
sincere_counts[word] += 1
elif j == 1:
insincere_counts[word] += 1
tst_qs = [clean(i)for i in test_set['question_text']]
for i in tst_qs:
i = i.split()
for j in i:
word_dict[j] += 1
print('Words in sincere Questions: {}'.format(len(sincere_counts)))
 print('Words in insincere Questions: {}'.format(len(insincere_counts)))
 print('Total Words in corpus: {}'.format(len(word_dict)))
 print('Most Common Words in Sincere Questions : ')
 print(sincere_counts.most_common() [:10])
 print('Most Common Words in Insincere Questions : ')
 print(insincere_counts.most_common() [:10])
 for i in sincere_counts:
if sincere_counts[i] >= 100:
sincere_to_insincere_ratio[i] = np.log(sincere_counts[i]/(insincere_counts[i] + 1))
print('The Most Sincere Words : ')
 print(sincere_to_insincere_ratio.most_common() [:10])
 print('The Most Insincere Words : ')
 print(list(reversed(sincere_to_insincere_ratio.most_common())) [:10])
<define_variables>
xgb=XGBClassifier(learning_rate=0.05, n_estimators=500)
 xgb.fit(X_train,Y_train)
 y_pred_xgb=xgb.predict(X_test )
prepare_dicts()<split>
acc_XGB= metrics.accuracy_score(y_pred_xgb,Y_test)
 print(y_pred_xgb)
 print("We see that XGB classifier gives an Accuracy of ",acc_XGB*100,"% on the traing set." )
train_x = list(train_set['question_text'].fillna("_na_" ).values)
 train_y = train_set['target'].values
test_x = list(test_set['question_text'].fillna("_na_" ).values)
 train_x, val_x, train_y, val_y = train_test_split(train_x, train_y, test_size=0.2)
 train_x = [clean(i)for i in train_x]
val_x = [clean(i)for i in val_x]
test_x = [clean(i)for i in test_x]
print('An Example From Train Set: ')
 print(train_x[0])
 tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_x))
train_X = tokenizer.texts_to_sequences(train_x)
 val_X = tokenizer.texts_to_sequences(val_x)
 test_X = tokenizer.texts_to_sequences(test_x)
 print('After Tokenizing: ')
 print(train_X[0])
 train_X = seq.pad_sequences(train_X, maxlen=max_seq_len)
 val_X = seq.pad_sequences(val_X, maxlen=max_seq_len)
 test_X = seq.pad_sequences(test_X, maxlen=max_seq_len)
 print('After Padding: ')
 print(train_X[0])
 print(np.shape(train_X), np.shape(train_y), np.shape(val_X), np.shape(val_y))<categorify>
import keras
from keras.layers import Dense
from keras.models import Sequential
from sklearn.metrics import classification_report
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def get_embeddings(embedding_name, mode='new'):
filePath = '.. /input/embeddings/{0}/{0}.txt'.format(embedding_name)
 if mode == 'new':
embeddings_idx = dict(get_coefs(*i.split(" ")) for i in open(filePath))
all_embs = np.stack(embeddings_idx.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_idx.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embeddings_idx, embedding_matrix<categorify>
nn = Sequential()
nn.add(Dense(units= 14, activation = 'relu', input_dim=7, kernel_initializer="uniform"))
nn.add(Dense(units= 14, activation = 'relu',kernel_initializer="uniform"))
nn.add(Dense(units= 1, activation = 'sigmoid',kernel_initializer="uniform"))
nn.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words<categorify>
nn.fit(X_train,Y_train, batch_size=32,epochs=50,verbose= 0)
 nn_pred = nn.predict(X_test)
 nn_pred = [ 1 if y>=0.5 else 0 for y in nn_pred]
print(nn_pred)
 acc_NN = metrics.accuracy_score(Y_test, nn_pred )
embedding_idxs, embedding_mtx = get_embeddings(EMBEDDING, 'new')
 unk_wrds = check_coverage(word_dict, embedding_idxs )<choose_model_class>
print("We can see that the neural network gives an Accuracy of ",acc_NN*100 , "% on the training set." )
def get_model(model_type):
if model_type == 'nb':
model = NaiveBayes()
elif model_type == 'svm':
model = __SVC__()
elif model_type == 'rnn':
inp = Input(shape=(max_seq_len,))
layer = Embedding(max_features, embed_size, weights=[embedding_mtx], trainable=False )(inp)
 layer = SimpleRNN(32, return_sequences=True )(layer)
 layer = GlobalMaxPool1D()(layer)
 layer = Dense(16, activation='relu' )(layer)
 layer = Dropout(0.1 )(layer)
 layer = Dense(1, activation='sigmoid' )(layer)
 model = Model(inputs=inp, outputs=layer)
 elif model_type == 'lstm':
inp = Input(shape=(max_seq_len,))
layer = Embedding(max_features, embed_size, weights=[embedding_mtx], trainable=False )(inp)
 layer = Bidirectional(CuDNNLSTM(32, return_sequences=True))(layer)
 layer = GlobalMaxPool1D()(layer)
 layer = Dense(16, activation='relu' )(layer)
 layer = Dropout(0.1 )(layer)
 layer = Dense(1, activation='sigmoid' )(layer)
 model = Model(inputs=inp, outputs=layer)
 elif model_type == 'gru':
inp = Input(shape=(max_seq_len,))
layer = Embedding(max_features, embed_size, weights=[embedding_mtx], trainable=False )(inp)
 layer = Bidirectional(CuDNNGRU(32, return_sequences=True))(layer)
 layer = GlobalMaxPool1D()(layer)
 layer = Dense(16, activation='relu' )(layer)
 layer = Dropout(0.1 )(layer)
 layer = Dense(1, activation='sigmoid' )(layer)
 model = Model(inputs=inp, outputs=layer)
 elif model_type == 'attention':
inp = Input(shape=(max_seq_len,))
layer = Embedding(max_features, embed_size, weights=[embedding_mtx], trainable=False )(inp)
 layer = Bidirectional(CuDNNLSTM(32, return_sequences=True))(layer)
 layer = Attention(max_seq_len )(layer)
 layer = Dense(16, activation='relu' )(layer)
 layer = Dropout(0.1 )(layer)
 layer = Dense(1, activation='sigmoid' )(layer)
 model = Model(inputs=inp, outputs=layer)
 return model 
<init_hyperparams>
best=best_model['Model'].iloc[0]
print(str(best))
class NaiveBayes() :
def __init__(self):
self.sincere_example_count = sincere_examples
self.insincere_example_count = insincere_examples
self.total_examples = x[0]+x[1]
self.sincere_dict = sincere_counts
self.insincere_dict = insincere_counts
self.word_dict= word_dict
self.sincere_word_count = np.sum(list(sincere_counts.values()))
self.insincere_word_count = np.sum(list(insincere_counts.values()))
self.sincere_prob = self.sincere_example_count / self.total_examples
self.insincere_prob = self.insincere_example_count / self.total_examples
def summary(self):
print('Positive Examples : {}, Negative Examples : {}, Total Examples : {}'.format(self.sincere_example_count, self.insincere_example_count, self.total_examples))
def predict(self, x_test):
print('Predicting...')
 predictions = []
for example in x_test:
p_words = np.prod([word_dict[j]/np.sum(list(word_dict.values())) for j in example.split() ])
 p_words += 2
sincere_prob_num = np.prod([sincere_counts[j]/self.sincere_word_count for j in example.split() ])* self.sincere_prob
insincere_prob_num = np.prod([insincere_counts[j]/self.insincere_word_count for j in example.split() ])* self.insincere_prob
sincere_prob = sincere_prob_num/p_words
insincere_prob = insincere_prob_num/p_words
predictions.append(np.argmax([sincere_prob, insincere_prob]))
return predictions
class __SVC__(SVC):
def __init__(self):
super(__SVC__,self ).__init__(verbose=True)
 print('initializing...')
 def summary(self):
print(self.__dict__)
 def prepare_data(self):
self.X_train = [embedding_mtx[i] for example in train_X for i in example]
self.X_val = [embedding_mtx[i] for example in val_X for i in example]
self.X_test = [embedding_mtx[i] for example in test_X for i in example]<choose_model_class>
test.head()
for c in test.columns:
print(c, str(100*test[c].isnull().sum() /len(test)))
 print(".............. Before")
 test['Age'] = test['Age'].fillna(test['Age'].mean())
 test['Fare'] = test['Fare'].fillna(test['Fare'].mean())
 print("
")
 for c in test.columns:
print(c, str(100*test[c].isnull().sum() /len(test)))
 print(".............. After" )
class Attention(Layer):
def __init__(self, step_dim, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.features_dim = 0
self.step_dim = step_dim
self.bias = True
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name))
self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name), 
)
 else:
self.b = None
self.built = True
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<compute_test_metric>
if best == 'Logistic Regression':
modelLR.fit(train.drop(['Survived'],axis=1),train['Survived'])
 test_pred=modelLR.predict(test)
 print(test_pred)
 print('Logistic Regression')
 if best == 'Decision Tree':
dtree.fit(train.drop(['Survived'],axis=1),train['Survived'])
 test_pred=modelLR.predict(test)
 print(test_pred)
 print('Decision Tree')
 if best =='Random Forest':
rforest.fit(train.drop(['Survived'],axis=1),train['Survived'])
 test_pred=rforest.predict(test)
 print(test_pred)
 print('Random Forest')
 if best == 'Support Vector Machine':
svc.fit(train.drop(['Survived'],axis=1),train['Survived'])
 test_pred=svc.predict(test)
 print(test_pred)
 print('Support Vector Machines')
 if best == 'Gradient Boosting Classifier':
xgb.fit(train.drop(['Survived'],axis=1),train['Survived'])
 test_pred=xgb.predict(test)
 print(test_pred)
 print('Gradient Boosting Classifier')
 if best == 'Artificial Neural Network':
nn.fit(train.drop(['Survived'],axis=1),train['Survived'], epochs=500 , verbose=0)
 test_pred=nn.predict(test)
 test_pred = [ 1 if y>=0.5 else 0 for y in test_pred]
print(test_pred)
 print('Artificial Neural Network')
def print_f1s(predictions):
for threshold in np.arange(0.1, 0.501, 0.01):
threshold = np.round(threshold, 2)
 print("F1 score at threshold {0} is {1}".format(threshold, f1_score(val_y,(predictions>threshold ).astype(int))))<prepare_output>
test_data=pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 test_data = test_data.drop(['PassengerId'], axis=1)
 test_data.head()
<find_best_model_class>
test_data.values.tolist()
test_acc = metrics.accuracy_score(test_data, test_pred)
 print("Here we see that the test data has an Accuracy of ",test_acc*100,"% " )
rnn = get_model('rnn')
 rnn.summary()
rnn.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 rnn.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y), callbacks=[earlystop, reducelr])
 predictions_rnn_real = rnn.predict(test_X)
 predictions_rnn =(predictions_rnn_real >= thresh ).astype(int)
 predictions_val_rnn = rnn.predict(val_X, batch_size=1024)
 print_f1s(predictions_val_rnn)
 del rnn
gc.collect()
time.sleep(10 )<save_to_csv>
test_predict = pd.DataFrame(test_pred, columns= ['Survived'])
 test_new= pd.read_csv('/kaggle/input/titanic/test.csv')
 new_test = pd.concat([test_new, test_predict], axis=1, join='inner' )
<find_best_model_class><EOS> 
submit=new_test[['PassengerId','Survived']]
submit.to_csv('predictions.csv',index=False )