Kaggle competition: Titanic - Machine Learning from Disaster
train = train[train.item_price > 0].reset_index(drop=True)
 train[train.item_cnt_day <= 0].item_cnt_day.unique()
train.loc[train.item_cnt_day < 1, 'item_cnt_day'] = 0<feature_engineering>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 11, 'shop_id'] = 10
test.loc[test.shop_id == 11, 'shop_id'] = 10
train.loc[train.shop_id == 40, 'shop_id'] = 39
test.loc[test.shop_id == 40, 'shop_id'] = 39<feature_engineering>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops['category'] = shops['shop_name'].str.split(' ' ).map(lambda x:x[1] ).astype(str)
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
category = ['Орджоникидзе,', 'ТЦ', 'ТРК', 'ТРЦ','ул.', 'Магазин', 'ТК', 'склад']
shops.category = shops.category.apply(lambda x: x if(x in category)else 'etc')
 shops.category.unique()<groupby>
pd.pivot_table(train, index = 'Pclass', values = ['Survived'] )
category = ['ТЦ', 'ТРК', 'ТРЦ', 'ТК']
shops.category = shops.category.apply(lambda x: x if(x in category)else 'etc')
 print('Category Distribution', shops.groupby(['category'] ).sum() )<categorify>
train['Title'] = train.Name.apply(lambda x: x.split(',')[1].split('.')[0])
 train['Family'] = train.Name.apply(lambda x: x.split(',')[0])
 print('Title values')
 print(train.Title.value_counts())
 print('Family values')
 print(train.Family.value_counts() )
shops['shop_city'] = shops.city
shops['shop_category'] = shops.category
shops['shop_city'] = LabelEncoder().fit_transform(shops['shop_city'])
 shops['shop_category'] = LabelEncoder().fit_transform(shops['shop_category'])
 shops = shops[['shop_id','shop_city', 'shop_category']]
shops.head()<count_unique_values>
test['Title'] = test.Name.apply(lambda x: x.split(',')[1].split('.')[0])
 test['Title'] = test.Title.apply(lambda x: x.replace('Mlle','Miss' ).replace('Mme','Mrs' ).replace('Col','Others' ).replace('Major','Others' ).replace('Don','Others' ).replace('Lady','Others' ).replace('Sir','Others' ).replace('Capt','Others' ).replace('the Countess','Others' ).replace('Ms','Others' ).replace('Jonkheer','Others'))
test.head()
print(len(cats.item_category_name.unique()))
cats.item_category_name.unique()<feature_engineering>
train = train.drop('Name',axis = 1)
 test = test.drop('Name',axis = 1 )
cats['type_code'] = cats.item_category_name.apply(lambda x: x.split(' ')[0] ).astype(str)
 cats.loc[(cats.type_code == 'Игровые')|(cats.type_code == 'Аксессуары'), 'category'] = 'Игры'
cats.loc[cats.type_code == 'PC', 'category'] = 'Музыка'
category = ['Игры', 'Карты', 'Кино', 'Книги','Музыка', 'Подарки', 'Программы', 'Служебные', 'Чистые']
cats['type_code'] = cats.type_code.apply(lambda x: x if(x in category)else 'etc')
 print(cats.groupby(['type_code'] ).sum())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type_code'])
 cats['split'] = cats.item_category_name.apply(lambda x: x.split('-'))
cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]<feature_engineering>
ageMean = train.groupby('Title')['Age'].median()
meanDr = ageMean[0]
meanMaster = ageMean[1]
meanMiss = ageMean[2]
meanMr = ageMean[3]
meanMrs = ageMean[4]
meanOthers = ageMean[5]
meanRev = ageMean[6]
ageMean
items = pd.read_csv('.. /input/' + PATH + '/items.csv')
 items['name_1'], items['name_2'] = items['item_name'].str.split('[', 1 ).str
items['name_1'], items['name_3'] = items['item_name'].str.split('(', 1 ).str
items['name_2'] = items['name_2'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items['name_3'] = items['name_3'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items = items.fillna('0')
 result_1 = Counter(' '.join(items['name_2'].values.tolist() ).split(' ')).items()
result_1 = sorted(result_1, key=itemgetter(1))
result_1 = pd.DataFrame(result_1, columns=['feature', 'count'])
 result_1 = result_1[(result_1['feature'].str.len() > 1)&(result_1['count'] > 200)]
result_2 = Counter(' '.join(items['name_3'].values.tolist() ).split(" ")).items()
result_2 = sorted(result_2, key=itemgetter(1))
result_2 = pd.DataFrame(result_2, columns=['feature', 'count'])
 result_2 = result_2[(result_2['feature'].str.len() > 1)&(result_2['count'] > 200)]
result = pd.concat([result_1, result_2])
 result = result.drop_duplicates(subset=['feature'] ).reset_index(drop=True)
 print('Most common aditional features:', result)
 def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
items['item_name'] = items['item_name'].apply(lambda x: name_correction(x))
items.name_2 = items.name_2.apply(lambda x: x[:-1] if x != '0' else '0' )<feature_engineering>
for x in range(len(train["Age"])) :
if pd.isna(train["Age"][x]):
title = train["Title"][x]
title = title.strip()
if(title == 'Dr'):
train["Age"][x] = meanDr
elif(title == "Master"):
train["Age"][x] = meanMaster
elif(title == "Miss"):
train["Age"][x] = meanMiss
elif(title =="Mr"):
train["Age"][x] = meanMr
elif(title == "Mrs"):
train["Age"][x] = meanMrs
elif(title == "Others"):
train["Age"][x] = meanOthers
elif(title == "Rev"):
train["Age"][x] = meanRev
for x in range(len(test["Age"])) :
if pd.isna(test["Age"][x]):
title = test["Title"][x]
title = title.strip()
if(title == 'Dr'):
test["Age"][x] = meanDr
elif(title == "Master"):
test["Age"][x] = meanMaster
elif(title == "Miss"):
test["Age"][x] = meanMiss
elif(title =="Mr"):
test["Age"][x] = meanMr
elif(title == "Mrs"):
test["Age"][x] = meanMrs
elif(title == "Others"):
test["Age"][x] = meanOthers
elif(title == "Rev"):
test["Age"][x] = meanRev
items['type'] = items.name_2.apply(lambda x: x[0:8] if x.split(' ')[0] == 'xbox' else x.split(' ')[0])
 items.loc[(items.type == 'x360')|(items.type == 'xbox360'), 'type'] = 'xbox 360'
items.loc[items.type == '', 'type'] = 'mac'
items.type = items.type.apply(lambda x: x.replace(' ',''))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == 'рс'), 'type'] = 'pc'
items.loc[(items.type == 'рs3'), 'type'] = 'ps3'
group_sum = items.groupby('type' ).sum()
group_sum.loc[group_sum.item_category_id < 200]<drop_column>
test['AgeGroup'] = test.Age.apply(lambda x: 'Baby' if x <= 5 else('Child' if x <= 12 else('Teenager' if x <= 18 else('Young')if x <= 24 else('Young Adult' if x <= 35 else('Adult' if x <= 60 else 'Senior')))))
drop_list = ['5c5', '5c7', '5f4', '6dv', '6jv', '6l6', 'android', 'hm3', 'j72', 'kf6', 'kf7','kg4',
'ps2', 's3v', 's4v'	,'англ', 'русская', 'только', 'цифро']
items.name_2 = items.type.apply(lambda x: 'etc' if x in drop_list else x)
 items = items.drop(['type'], axis=1)
 items.groupby('name_2' ).sum()<categorify>
train.Ticket
items['name_2'] = LabelEncoder().fit_transform(items['name_2'])
 items['name_3'] = LabelEncoder().fit_transform(items['name_3'])
 items.drop(['item_name', 'name_1'], axis=1, inplace=True)
 items.head()<data_type_conversions>
train['NumericTicket'] = train.Ticket.apply(lambda x: 1 if x.isnumeric() else 0)
 pd.pivot_table(train, index = 'NumericTicket', values = ['Survived'] )
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
train = train.drop('NumericTicket',axis=1)
 train = train.drop('Ticket',axis=1)
 test = test.drop('Ticket',axis=1 )
train['revenue'] = train['item_price'] * train['item_cnt_day']<data_type_conversions>
train['has_cabin'] = train.Cabin.apply(lambda x: 0 if pd.isna(x)else 1)
 train['CabinLetter'] = train.Cabin.apply(lambda x: str(x)[0])
 train.head()
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
test['has_cabin'] = test.Cabin.apply(lambda x: 0 if pd.isna(x)else 1)
 test['CabinLetter'] = test.Cabin.apply(lambda x: str(x)[0])
 train = train.drop('Cabin',axis=1)
 test = test.drop('Cabin',axis=1 )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
train = train.fillna({"Embarked": "S"} )
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
test = test.fillna({"Fare": train.Fare.mean() })
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['shop_city'] = matrix['shop_city'].astype(np.int8)
 matrix['shop_category'] = matrix['shop_category'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
train = train.drop("Age",axis=1)
 test = test.drop("Age",axis=1 )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
to_cat = ['Pclass','SibSp','Parch','has_cabin','Sex','Embarked','Title','AgeGroup','CabinLetter']
for x in to_cat:
train[x] = train[x].astype('category')
 test[x] = test[x].astype('category')
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
sh_train = train.shape
c1 = pd.concat(( train, test), sort=False ).reset_index(drop=True)
 c = pd.get_dummies(c1)
 train = c[:sh_train[0]]
test = c[sh_train[0]:]
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
scale = StandardScaler()
train[['Fare']]= scale.fit_transform(train[['Fare']])
 test[['Fare']]= scale.fit_transform(test[['Fare']] )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
pred = train.drop(['Survived', 'PassengerId'], axis=1)
 target = train["Survived"]
X_train, X_valid, y_train, y_valid = train_test_split(pred, target, test_size = 0.20, random_state = 0)
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
 y_pred = logreg.predict(X_valid)
 acc_logreg = round(accuracy_score(y_pred, y_valid)* 100, 2)
 print(acc_logreg )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
randomforest = RandomForestClassifier()
randomforest.fit(X_train, y_train)
 y_pred = randomforest.predict(X_valid)
 acc_randomforest = round(accuracy_score(y_pred, y_valid)* 100, 2)
 print(acc_randomforest )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'itme_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_item_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'itme_id'], how='left')
 matrix['date_shop_item_avg_item_cnt'] = matrix['date_shop_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_item_avg_item_cnt')
 matrix.drop(['date_shop_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
input_shape = [X_valid.shape[1]]
model = keras.Sequential([
layers.BatchNormalization(input_shape=input_shape),
layers.Dense(25,kernel_initializer='he_uniform', activation='relu'),
layers.BatchNormalization() ,
layers.Dropout(0.5),
layers.Dense(12,kernel_initializer='he_uniform', activation='relu'),
layers.BatchNormalization() ,
layers.Dropout(0.5),
layers.Dense(1, kernel_initializer='glorot_uniform',activation='sigmoid'),
])
 model.compile(
 optimizer='adam',
loss='binary_crossentropy',
metrics=['binary_accuracy'], 
)
 early_stopping = keras.callbacks.EarlyStopping(
 patience=5,
min_delta=0.001,
restore_best_weights=True, 
)
 history = model.fit(
 X_train, y_train,
validation_data=(X_valid, y_valid),
batch_size=512,
epochs=200,
callbacks=[early_stopping], 
)
 history_df = pd.DataFrame(history.history)
 history_df.loc[:, ['loss', 'val_loss']].plot(title="Cross-entropy")
 history_df.loc[:, ['binary_accuracy', 'val_binary_accuracy']].plot(title="Accuracy" )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
ynew = model.predict_classes(test.drop(['PassengerId','Survived'], axis=1))
<merge><EOS> 
ids = test['PassengerId']
predictions = ynew.flatten()
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )