Kaggle competition: Titanic - Machine Learning from Disaster
train[train.item_price == 307980]<filter>
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 df = pd.DataFrame(np.random.randn(50, 4), columns=list('ABCD'))
items[items.item_id == 6066]<filter>
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv" )
train[train.item_id == 6066]<filter>
passenger_id = test_data["PassengerId"]
train = train[train.item_price < 300000]<sort_values>
train_data[['SibSp','Survived']].groupby(['SibSp'],as_index=False ).count().sort_values(by='Survived',ascending=False)
train['item_price'].sort_values() [:5]<filter>
train_data[['Parch','Survived']].groupby(['Parch'],as_index=False ).count().sort_values(by='Survived',ascending=False)
train[train.item_price == -1]<feature_engineering>
train_data[['Pclass','Survived']].groupby(['Pclass'],as_index=False ).count().sort_values(by='Survived',ascending=False)
price_correction = train[(train.shop_id == 32)&(train.item_id == 2973)&(train.date_block_num ==4)&(train.item_price > 0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = price_correction<count_unique_values>
train_data[['Embarked','Survived']].groupby(['Embarked'],as_index=False ).count().sort_values(by='Survived',ascending=False)
shops_train = train.shop_id.nunique()
shops_test = test.shop_id.nunique()
print("Shops in training set = ", shops_train)
 print("Shops in test set = ", shops_test )<feature_engineering>
train_data[['Age','Survived']].groupby(['Age'],as_index=False ).count().sort_values(by='Survived',ascending=False ).head()
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0] )<categorify>
target_variable = train_data["Survived"]
train_data.drop(["Survived"], axis = 1, inplace=True )
LE = preprocessing.LabelEncoder()
LE.fit_transform(shops['city'] )<categorify>
all_data = pd.concat([train_data, test_data], axis = 0 )
shops['city_label'] = LE.fit_transform(shops['city'])
 shops.drop(['shop_name', 'city'], axis = 1, inplace = True)
 shops.head()<count_values>
all_data.drop(["PassengerId", "Ticket" ], axis=1, inplace = True )
len(set(items_test_list)-set(items_train_list))<categorify>
total = all_data.isnull().sum().sort_values(ascending=False)
 percent =(all_data.isnull().sum() /all_data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data[total > 0]
LE = preprocessing.LabelEncoder()
category_split = categories['item_category_name'].str.split('-')
 categories['main_categories_id'] = category_split.map(lambda row: row[0].strip())
 categories['main_categories_id'] = LE.fit_transform(categories['main_categories_id'])
 categories['sub_category_id'] = category_split.map(lambda row: row[1].strip() if len(row)> 1 else row[0].strip())
 categories['sub_category_id'] = LE.fit_transform(categories['sub_category_id'])
 categories.head()<data_type_conversions>
all_data["Cabin"] = all_data["Cabin"].fillna("None" )
train['date'] = pd.to_datetime(train['date'], format = '%d.%m.%Y')
 train.info()<concatenate>
all_data["Age"] = all_data["Age"].fillna(all_data["Age"].mean())
 all_data["Fare"] = all_data["Fare"].fillna(all_data["Fare"].mean() )
cartesian_test = []
cartesian_test.append(np.array(jan))
cartesian_test.append(np.array(feb))<prepare_output>
embarked_mode = all_data["Embarked"].mode() [0]
all_data["Embarked"] = all_data["Embarked"].fillna(embarked_mode )
cartesian_test = np.vstack(cartesian_test)
 cartesian_test_df = pd.DataFrame(cartesian_test, columns = ['shop_id', 'item_id', 'date_block_num'])
 cartesian_test_df.head()<data_type_conversions>
simplify_ages(all_data )
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df<groupby>
simplify_fares(all_data )
x = train.groupby(['shop_id', 'item_id', 'date_block_num'])['item_cnt_day'].sum().rename('item_cnt_month' ).reset_index()
x.head()<merge>
def format_name(df):
df['Lname'] = df.Name.apply(lambda x: x.split(' ')[0])
 df['NamePrefix'] = df.Name.apply(lambda x: x.split(' ')[1])
 return df
new_train = pd.merge(cartesian_df, x, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0)
 new_train['item_cnt_month'] = np.clip(new_train['item_cnt_month'], 0, 20 )<drop_column>
format_name(all_data)
 all_data.drop(['Name'], axis=1, inplace=True )
del x
del cartesian_df
del cartesian
del cartesian_test
del cartesian_test_df
del feb
del jan
del items_test_list
del items_train_list
del train<sort_values>
all_data["family_members"] = all_data["SibSp"] + all_data["Parch"]
all_data.drop(["SibSp", "Parch" ], axis=1 )
new_train.sort_values(['date_block_num','shop_id','item_id'], inplace = True)
 new_train.head()<feature_engineering>
all_data.head()
test.insert(loc=3, column='date_block_num', value=34)
 test['item_cnt_month'] = 0
test.head()<concatenate>
def encode_features(df):
features = ['Sex', 'Age', 'Fare', 'Embarked', 'Lname', 'NamePrefix', 'Cabin']
for feature in features:
le = preprocessing.LabelEncoder()
le = le.fit(df[feature])
 df[feature] = le.transform(df[feature])
 return df
all_data = encode_features(all_data)
 all_data.head()
new_train = new_train.append(test.drop('ID', axis = 1))<merge>
train_data = all_data[:train_data.shape[0]]
test_data = all_data[train_data.shape[0]:]
y = target_variable
new_train = pd.merge(new_train, shops, on=['shop_id'], how='left')
 new_train.head()<merge>
X_train, X_test, y_train, y_test = train_test_split(train_data, y, test_size=0.2, random_state=42 )
new_train = pd.merge(new_train, items.drop('item_name', axis = 1), on=['item_id'], how='left')
 new_train.head()<merge>
clf = RandomForestClassifier()
parameters = {'n_estimators': [4, 6, 9],
'max_features': ['log2', 'sqrt','auto'],
'criterion': ['entropy', 'gini'],
'max_depth': [2, 3, 5, 10],
'min_samples_split': [2, 3, 5],
'min_samples_leaf': [1,5,8]
}
acc_scorer = make_scorer(accuracy_score)
 grid_obj = GridSearchCV(clf, parameters, scoring=acc_scorer)
 grid_obj = grid_obj.fit(X_train, y_train)
 clf = grid_obj.best_estimator_
clf.fit(X_train, y_train )
new_train = pd.merge(new_train, categories.drop('item_category_name', axis = 1), on=['item_category_id'], how='left')
 new_train.head()<merge>
predictions = clf.predict(X_test)
 print("Accuracy of Random forest classifier" , accuracy_score(y_test, predictions))
def generate_lag(train, months, lag_column):
for month in months:
train_shift = train[['date_block_num', 'shop_id', 'item_id', lag_column]].copy()
train_shift.columns = ['date_block_num', 'shop_id', 'item_id', lag_column+'_lag_'+ str(month)]
train_shift['date_block_num'] += month
train = pd.merge(train, train_shift, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 return train<drop_column>
lreg = LogisticRegression()
lreg.fit(X_train, y_train)
 predictions = lreg.predict(X_test)
 print("Accuracy of Logistic Regression" , accuracy_score(y_test, predictions))
del items
del categories
del shops
del test<categorify>
predict_survival = lreg.predict(test_data)
 my_submission = pd.DataFrame({'PassengerId': passenger_id, 'Survived': predict_survival})
 my_submission.to_csv('submission.csv', index=False )