Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
%matplotlib inline
tstmdf = pd.read_csv('test_metadata.csv' )<feature_engineering>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 combine = [train_df, test_df]
%%time
tstmdf['SliceID'] = tstmdf[['SeriesInstanceUID', 'StudyInstanceUID']].apply(
 lambda x: '{}__{}'.format(*x.tolist()), 1 )<create_dataframe>
train_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
%%time
poscols = ['ImagePos{}'.format(i)for i in range(1, 4)]
tstmdf[poscols] = pd.DataFrame(tstmdf['ImagePositionPatient']\ 
.apply(lambda x: list(map(float, ast.literal_eval(x)))).tolist())
 tstmdf = tstmdf.sort_values(['SliceID']+poscols)\
[['StudyInstanceUID', 'SliceID', 'SOPInstanceUID']+poscols].reset_index(drop=True )<groupby>
train_df[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
tstmdf['seq'] =(tstmdf.groupby(['SliceID'] ).cumcount() + 1 )<drop_column>
train_df[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
keepcols = ['StudyInstanceUID', 'SliceID', 'SOPInstanceUID', 'seq']
tstmdf = tstmdf[keepcols]<create_dataframe>
train_df[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
%%time
test_emb_f = np.load("emb_test_embdim512.npz")
 test_emb = test_emb_f["embeddings"]
tstdf = pd.DataFrame({
"StudyInstanceUID": test_emb_f["StudyInstanceUID"],
"SeriesInstanceUID": test_emb_f["SeriesInstanceUID"],
"SOPInstanceUID": test_emb_f["SOPInstanceUID"]} )<merge>
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )
tstdf = tstdf[["SOPInstanceUID"]].merge(tstmdf, on="SOPInstanceUID", how="left" )<feature_engineering>
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Jonkheer', 'Dona'], 'Lady')
 dataset['Title'] = dataset['Title'].replace(['Capt', 'Don', 'Major', 'Sir'], 'Sir')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
tstdf['embidx'] = range(tstdf.shape[0] )<set_options>
title_mapping = {"Col": 1, "Dr": 2, "Lady": 3, "Master": 4, "Miss": 5, "Mr": 6, "Mrs": 7, "Rev": 8, "Sir": 9}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df.head()
del test_df, tstmdf
gc.collect()<categorify>
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape
def collatefn(batch):
maxlen = max([l['emb'].shape[0] for l in batch])
 embdim = batch[0]['emb'].shape[1]
withlabel = 'labels' in batch[0]
for b in batch:
masklen = maxlen-len(b['emb'])
 b['emb'] = np.vstack(( np.zeros(( masklen, embdim)) , b['emb']))
b['embidx'] = torch.cat(( torch.ones(( masklen),dtype=torch.long)*-1, b['embidx']))
b['mask'] = np.ones(( maxlen))
b['mask'][:masklen] = 0.
 outbatch = {'emb' : torch.tensor(np.vstack([np.expand_dims(b['emb'], 0)\
for b in batch])).float() }
outbatch['mask'] = torch.tensor(np.vstack([np.expand_dims(b['mask'], 0)\
for b in batch])).float()
outbatch['embidx'] = torch.tensor(np.vstack([np.expand_dims(b['embidx'], 0)\
for b in batch])).float()
if withlabel:
outbatch["labels"] = torch.tensor(np.vstack([b["labels"] for b in batch])).float()
outbatch["StudyInstanceUID"] = [b["StudyInstanceUID"] for b in batch]
return outbatch<create_dataframe>
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()
test_dataset = PEDataset(tstdf, test_emb, labels=False)
 test_loader = data.DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
collate_fn=collatefn )<categorify>
guess_ages = np.zeros(( 2,3))
guess_ages
class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.pe[:x.size(0), :]
return self.dropout(x )<categorify>
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 train_df.head()
class PETransformerModel(nn.Module):
def __init__(self, n_classes, ninp, nhead, nhid, nlayers, dropout=0.5):
super(PETransformerModel, self ).__init__()
self.pos_encoder = PositionalEncoding(ninp, dropout)
 encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
 self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
 self.ninp = ninp
self.decoder = nn.Linear(ninp, n_classes)
 self.init_weigths()
def init_weigths(self):
initrange = 0.1
self.decoder.bias.data.zero_()
self.decoder.weight.data.uniform_(-initrange, initrange)
 def forward(self, x):
x = x * math.sqrt(self.ninp)
 x = self.pos_encoder(x)
 hidden = self.transformer_encoder(x)
 output = self.decoder(hidden.mean(1))
return output<define_variables>
train_df['AgeBand'] = pd.cut(train_df['Age'], 5)
 train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )
nhid = 768
nlayers = 2
nhead = 2
dropout = 0.2<load_pretrained>
for dataset in combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
train_df.head()
embed_size = test_emb.shape[-1] * 3
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 model = PETransformerModel(n_classes, embed_size, nhead, nhid, nlayers, dropout=dropout)
 checkpoint = torch.load(".. /input/pe-transformer-model/transformer_epoch15.pth", map_location=device)
 model.load_state_dict(checkpoint)
 model.to(device )<define_variables>
train_df = train_df.drop(['AgeBand'], axis=1)
 combine = [train_df, test_df]
train_df.head()
classes = [
"{}_negative_exam_for_pe",
"{}_rv_lv_ratio_gte_1",
"{}_rv_lv_ratio_lt_1",
"{}_leftsided_pe",
"{}_chronic_pe",
"{}_rightsided_pe",
"{}_acute_and_chronic_pe",
"{}_central_pe",
"{}_indeterminate",
]<load_from_csv>
for dataset in combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
sub = pd.read_csv(".. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv")
 print(sub.shape )<categorify>
for dataset in combine:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()
%%time
preds_all_df = []
if do_all:
tqdm_loader = tqdm(test_loader)
 for step, batch in enumerate(tqdm_loader):
mask = batch['mask'].to(device, dtype=torch.int)
 inputs = batch["emb"]
inputs = inputs.to(device, dtype=torch.float, non_blocking=True)
 StudyInstanceUIDs = batch["StudyInstanceUID"]
with torch.no_grad() :
logits = model(inputs)
 preds = torch.sigmoid(logits ).detach().cpu().numpy()
for i, pred in enumerate(preds):
StudyInstanceUID = StudyInstanceUIDs[i]
id_names = [c.format(StudyInstanceUID)for c in classes]
tmp_df = pd.DataFrame({"id": id_names, "pred": pred.tolist() })
 preds_all_df.append(tmp_df)
 preds_all_df.append(preds_df)
 pred_sub = pd.concat(preds_all_df, 0)
 sub = sub.merge(pred_sub, on="id", how="left")
 sub = sub[["id", "pred"]]
sub.columns = ["id", "label"]
print(sub.isna().sum())
 sub = sub.fillna(0.5)
 sub.to_csv("submission.csv", index=False)
 else:
sub.to_csv("submission.csv", index=False )<load_from_csv>
train_df = train_df.drop(['Parch',], axis=1)
 test_df = test_df.drop(['Parch'], axis=1)
 combine = [train_df, test_df]
train_df.head()
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 del train_transaction, train_identity
gc.collect()<load_from_csv>
for dataset in combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
train_df.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10 )
%%time
test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 del test_transaction, test_identity
gc.collect()<define_variables>
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port
selected_features = [
'TransactionAmt', 'ProductCD',
'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain',
'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13', 'C14',
'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15',
'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9',
'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12',
'V13', 'V17', 'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36',
'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48', 'V49', 'V51',
'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63',
'V64', 'V69', 'V70', 'V71', 'V72', 'V73', 'V74', 'V75', 'V76', 'V78',
'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128',
'V130', 'V131', 'V138', 'V139', 'V140', 'V143', 'V145', 'V146', 'V147', 'V149',
'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161', 'V162',
'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173',
'V175', 'V176', 'V177', 'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189',
'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204', 'V205', 'V206', 'V207',
'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219',
'V220', 'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229',
'V231', 'V233', 'V234', 'V238', 'V239', 'V242', 'V243', 'V244', 'V245', 'V246',
'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261', 'V262',
'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273',
'V274', 'V275', 'V276', 'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285',
'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303', 'V304', 'V306', 'V307',
'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324',
'V326', 'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338',
'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09', 'id_11', 'id_12', 'id_13', 'id_14',
'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33', 'id_36', 'id_37',
'id_38', 'DeviceType', 'DeviceInfo'
]
len(selected_features )<define_variables>
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
columns_to_drop = list(set(train.columns)- set(selected_features)- set(['isFraud', 'TransactionDT']))
len(columns_to_drop )<categorify>
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()
train = train.fillna(-999)
 test = test.fillna(-999 )<count_values>
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()
many_same_values_columns_train = [c for c in train.columns if train[c].value_counts(normalize=True ).values[0] > 0.9]
many_same_values_columns_test = [c for c in test.columns if test[c].value_counts(normalize=True ).values[0] > 0.9]
columns_to_drop = list(set(many_same_values_columns_train + many_same_values_columns_test))
columns_to_drop.remove('isFraud')
 len(columns_to_drop )<feature_engineering>
train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
 train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False ).mean().sort_values(by='FareBand', ascending=True )
os = ['Windows', 'iOS', 'Android', 'Mac OS', 'Linux']
browser = ['chrome', 'mobile safari', 'ie', 'safari', 'edge', 'firefox']
device = ['Windows', 'iOS', 'MacOS', 'SM', 'SAMSUNG', 'Moto', 'LG']
def to_pattern(x: str, patterns):
for p in patterns:
t = re.compile('^(' + p + ' ).*')
 if t.match(x):
return p
return 'other'
def make_os_feature(df):
return df['id_30'].map(lambda x: to_pattern(str(x), os))
def make_browser_feature(df):
return df['id_31'].map(lambda x: to_pattern(str(x), browser))
def make_device_feature(df):
return df['DeviceInfo'].map(lambda x: to_pattern(str(x), device)) 
<data_type_conversions>
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 combine = [train_df, test_df]
train_df.head(10 )
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int )<merge>
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
X_train.head(10 )
def make_aggregates(df, feature_to_group_by, feature):
out = pd.DataFrame(index=df.index)
 agg = df[df[feature] != -999].groupby([feature_to_group_by])[feature]
new_feature = feature + '_' + feature_to_group_by
out[new_feature + '_mean'] = df[feature] / agg.transform('mean')
 out[new_feature + '_std' ] = df[feature] / agg.transform('std')
 return out
def merge_aggregates(df, feature_to_group_by, feature):
return df.merge(make_aggregates(df, feature_to_group_by, feature), how='left', left_index=True, right_index=True)
<count_values>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log
def make_count_1(feature, df):
temp = df[feature].value_counts(dropna=False)
 new_feature = feature + '_count'
df[new_feature] = df[feature].map(temp)
 def make_count_2(feature, train, test):
temp = pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False)
 new_feature = feature + '_count'
train[new_feature] = train[feature].map(temp)
 test[new_feature] = test[feature].map(temp)
 for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
make_count_1(feature, train)
 make_count_1(feature, test)
 for feature in ['card1', 'id_36']:
make_count_2(feature, train, test )<categorify>
svc = SVC()
svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, Y_train)* 100, 2)
 acc_svc
def features_interaction(df, feature_1, feature_2):
return df[feature_1].astype(str)+ '_' + df[feature_2].astype(str)
 features_interactions = [
'id_02__id_20',
'id_02__D8',
'D11__DeviceInfo',
'DeviceInfo__P_emaildomain',
'P_emaildomain__C2',
'card2__dist1',
'card1__card5',
'card2__id_20',
'card5__P_emaildomain',
'addr1__card1'
]
for new_feature in features_interactions:
feature_1, feature_2 = new_feature.split('__')
 train[new_feature] = features_interaction(train, feature_1, feature_2)
 test[new_feature] = features_interaction(test, feature_1, feature_2 )<import_modules>
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn
import matplotlib.pyplot as plt<categorify>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 acc_gaussian
def make_day_feature(df, offset=0.58, tname='TransactionDT'):

 days = df[tname] /(3600 * 24)
 encoded_days = np.floor(days - 1 + offset)% 7
return encoded_days
def make_hour_feature(df, tname='TransactionDT'):

 hours = df[tname] /(3600)
 encoded_hours = np.floor(hours)% 24
return encoded_hours<feature_engineering>
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 acc_perceptron
train['Weekday'] = make_day_feature(train )<feature_engineering>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 acc_decision_tree
train['Hour'] = make_hour_feature(train )<feature_engineering>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest
test['Weekday'] = make_day_feature(test)
 test['Hour'] = make_hour_feature(test )<define_variables>
grad_boost = GradientBoostingClassifier(n_estimators = 100)
 grad_boost.fit(X_train, Y_train)
 Y_pred = grad_boost.predict(X_test)
 grad_boost.score(X_train, Y_train)
 acc_grad_boost = round(grad_boost.score(X_train, Y_train)* 100, 2)
 acc_grad_boost
email_vendors = {
'gmail': 'google',
'att.net': 'att',
'twc.com': 'spectrum',
'scranton.edu': 'other',
'optonline.net': 'other',
'hotmail.co.uk': 'microsoft',
'comcast.net': 'other',
'yahoo.com.mx': 'yahoo',
'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo',
'charter.net': 'spectrum',
'live.com': 'microsoft',
'aim.com': 'aol',
'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink',
'gmail.com': 'google',
'me.com': 'apple',
'earthlink.net': 'other',
'gmx.de': 'other',
'web.de': 'other',
'cfl.rr.com': 'other',
'hotmail.com': 'microsoft',
'protonmail.com': 'other',
'hotmail.fr': 'microsoft',
'windstream.net': 'other',
'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo',
'servicios-ta.com': 'other',
'netzero.net': 'other',
'suddenlink.net': 'other',
'roadrunner.com': 'other',
'sc.rr.com': 'other',
'live.fr': 'microsoft',
'verizon.net': 'yahoo',
'msn.com': 'microsoft',
'q.com': 'centurylink',
'prodigy.net.mx': 'att',
'frontier.com': 'yahoo',
'anonymous.com': 'other',
'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo',
'outlook.com': 'microsoft',
'mail.com': 'other',
'bellsouth.net': 'other',
'embarqmail.com': 'centurylink',
'cableone.net': 'other',
'hotmail.es': 'microsoft',
'mac.com': 'apple',
'yahoo.co.uk': 'yahoo',
'netzero.com': 'other',
'yahoo.com': 'yahoo',
'live.com.mx': 'microsoft',
'ptd.net': 'other',
'cox.net': 'other',
'aol.com': 'aol',
'juno.com': 'other',
'icloud.com': 'apple',
-999: 'undefined'
}
us_emails = ['gmail', 'net', 'edu']
def transform_email(df, column):
out = pd.DataFrame(index=df.index)
 out[column + '_vendor'] = df[column].map(email_vendors)
 out[column + '_suffix'] = df[column].map(lambda x: str(x ).split('.')[-1])
 out[column + '_us'] = out[column + '_suffix'].map(lambda x: 'us' if str(x)in us_emails else 'other')
 return out<merge>
Ridge= RidgeClassifierCV()
Ridge.fit(X_train, Y_train)
 Y_pred = Ridge.predict(X_test)
 acc_Ridge= round(Ridge.score(X_train, Y_train)* 100, 2)
 acc_Ridge
<merge>
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression', 'Random Forest', 'Naive Bayes', 'Perceptron', 'Grad boost','Decision Tree'],
'Score': [acc_svc, acc_knn, acc_log, acc_random_forest, acc_gaussian, acc_perceptron, acc_grad_boost, acc_decision_tree]})
 models.sort_values(by='Score', ascending=False )
<drop_column>
X_dev, X_eval, y_dev, y_eval = train_test_split(X_train,
Y_train,
test_size=0.2,
random_state=42)
 X_test.head(10 )
train = train.sort_values('TransactionDT' ).drop('TransactionDT', axis=1)
 test = test.sort_values('TransactionDT' ).drop('TransactionDT', axis=1 )<categorify>
dict_clf = {}
paramgrid = {
'n_estimators': [100, 150, 200, 250, 300, 400, 500],
'criterion': ['gini', 'entropy'],
'max_features': ['auto', 'log2'],
'min_samples_leaf': list(range(2, 8))
}
GS = GridSearchCV(RandomForestClassifier(random_state=77),
paramgrid,
cv=4)
 t0 = time.time()
GS.fit(X_dev, y_dev)
 t = time.time() - t0
best_clf = GS.best_estimator_
best_params = GS.best_params_
best_score = GS.best_score_
name = 'RF'
def encode_categorial_features_fit(df, columns_to_encode):
encoders = {}
for c in columns_to_encode:
if c in df.columns:
encoder = LabelEncoder()
encoder.fit(df[c].astype(str ).values)
 encoders[c] = encoder
return encoders
def encode_categorial_features_transform(df, encoders):
out = pd.DataFrame(index=df.index)
 for c in encoders.keys() :
if c in df.columns:
out[c] = encoders[c].transform(df[c].astype(str ).values)
 return out
categorial_features_columns = [
'id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21',
'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29', 'id_30', 'id_31',
'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38',
'DeviceType', 'DeviceInfo', 'ProductCD', 'P_emaildomain', 'R_emaildomain',
'card1', 'card2', 'card3', 'card4', 'card5', 'card6',
'addr1', 'addr2',
'M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9',
'P_emaildomain_vendor', 'P_emaildomain_suffix', 'P_emaildomain_us',
'R_emaildomain_vendor', 'R_emaildomain_suffix', 'R_emaildomain_us'
] + features_interactions
categorial_features_encoders = encode_categorial_features_fit(
 pd.concat([train, test], join='outer', sort=False),
categorial_features_columns )<merge>
best_clf.fit(X_dev, y_dev)
 acc_eval = accuracy_score(y_eval, best_clf.predict(X_eval))
dict_clf[name] = {
'best_par': best_params,
'best_clf': best_clf,
'best_score': best_score,
'score_eval': acc_eval,
'fit_time': t,
}
acc_eval
temp = encode_categorial_features_transform(train, categorial_features_encoders)
 columns_to_drop = list(set(categorial_features_columns)& set(train.columns))
train = train.drop(columns_to_drop, axis=1 ).merge(temp, how='left', left_index=True, right_index=True)
 del temp<merge>
paramgrid = {
'n_estimators': [100, 150, 200, 250, 300, 400, 500],
'max_features': ['auto', 'log2'],
'min_samples_leaf': list(range(2, 7)) ,
'loss' : ['deviance', 'exponential'],
'learning_rate': [0.025, 0.05, 0.075, 0.1],
}
GS = GridSearchCV(GradientBoostingClassifier(random_state=77),
paramgrid,
cv=4)
 t0 = time.time()
GS.fit(X_dev, y_dev)
 t = time.time() - t0
best_clf = GS.best_estimator_
best_params = GS.best_params_
best_score = GS.best_score_
name = 'GB'
best_clf.fit(X_dev, y_dev)
 acc_eval = accuracy_score(y_eval, best_clf.predict(X_eval))
dict_clf[name] = {
'best_par': best_params,
'best_clf': best_clf,
'best_score': best_score,
'score_eval': acc_eval,
'fit_time': t,
}
temp = encode_categorial_features_transform(test, categorial_features_encoders)
 columns_to_drop = list(set(categorial_features_columns)& set(test.columns))
test = test.drop(columns_to_drop, axis=1 ).merge(temp, how='left', left_index=True, right_index=True)
 del temp<drop_column>
for clf in dict_clf.keys() :
print("{0} classifier:
\t- Best score = {1:.2%}".format(clf, dict_clf[clf]['best_score']))
print("\t- Score on evaluation set = {0:.2%}".format(dict_clf[clf]['score_eval']))
print("\t- Fitting time = {0:.1f} min".format(round(dict_clf[clf]['fit_time']/60, 1)))
 print("\t- Best parameters:")
 for par in sorted(dict_clf[clf]['best_par'].keys()):
print("\t\t* {0}: {1}".format(par, dict_clf[clf]['best_par'][par]))
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<prepare_x_and_y>
estimators = [('RF', dict_clf['RF']['best_clf']), 
('GB', dict_clf['GB']['best_clf']), 
('KNN', knn),('svc', svc),('trees', decision_tree)]
voter = VotingClassifier(estimators=estimators, voting='hard')
 voter.fit(X_train, Y_train)
 Y_pred = voter.predict(X_test ).astype(int )
<import_modules><EOS> 
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submission.csv', index=False )