Kaggle competition: Titanic - Machine Learning from Disaster
train_mask, valid_mask = list(LabelKFold(train['people_id'], n_folds=10)) [0]
x_test = test.drop(['people_id','activity_id'],axis=1)
 y = act_train['outcome']
train = train.drop(['people_id', 'activity_id'], axis=1)
 kklo=x_train = np.array(train)[train_mask]
y_train = np.array(y)[train_mask]
x_valid = np.array(train)[valid_mask]
y_valid = np.array(y)[valid_mask]
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'auc'
params['eta'] = 0.1
params['max_depth'] = 10
params['subsample'] = 0.9
params['colsample_bytree'] = 0.9
d_train = xgb.DMatrix(x_train, label=y_train, missing=np.nan)
 d_valid = xgb.DMatrix(x_valid, label=y_valid, missing=np.nan)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 600, watchlist, early_stopping_rounds=40)
 clf.save_model('mymodel.model' )<train_model>
train_data = pd.read_csv(".. /input/titanic/train.csv")
 test_data = pd.read_csv(".. /input/titanic/test.csv")
 train_data.columns
clf = xgb.train(params, d_train, 800, watchlist, xgb_model ='mymodel.model', early_stopping_rounds=40 )<save_to_csv>
def outlier_detect(feature, data):
outlier_index = []
for each in feature:
Q1 = np.percentile(data[each], 25)
 Q3 = np.percentile(data[each], 75)
 IQR = Q3 - Q1
min_quartile = Q1 - 1.5*IQR
max_quartile = Q3 + 1.5*IQR
outlier_list = data[(data[each] < min_quartile)|(data[each] > max_quartile)].index
outlier_index.extend(outlier_list)
 outlier_index = Counter(outlier_index)
 outlier_data = list(i for i, n in outlier_index.items() if n > 3)
 return outlier_data
p_test = clf.predict(xgb.DMatrix(np.array(x_test)))
 sub = pd.DataFrame()
sub['activity_id'] = act_test['activity_id']
sub['outcome'] = p_test
sub.to_csv('submission.csv', index=False )<import_modules>
outlier_data = outlier_detect(["Age","SibSp","Parch","Fare"], train_data)
 train_data.loc[outlier_data]
import numpy as np
import pandas as pd
import xgboost as xgb 
<data_type_conversions>
train_data = train_data.drop(outlier_data, axis=0 ).reset_index(drop=True )
def cleanPeople(people):
people = people.drop(['date'],axis=1)
 people['people_id'] = people['people_id'].apply(lambda x : x.split('_')[1])
 people['people_id'] = pd.to_numeric(people['people_id'] ).astype(int)
 fields = list(people.columns)
 cat_data = fields[1:11]
bool_data = fields[11:]
for data in cat_data:
people[data] = people[data].fillna('type 0')
 people[data] = people[data].apply(lambda x: x.split(' ')[1])
 people[data] = pd.to_numeric(people[data] ).astype(int)
 for data in bool_data:
people[data] = pd.to_numeric(people[data] ).astype(int)
 return people
def cleanAct(data, train=False):
data = data.drop(['date'],axis = 1)
 if train:
data = data.drop(['outcome'],axis=1)
 data['people_id'] = data['people_id'].apply(lambda x : x.split('_')[1])
 data['people_id'] = pd.to_numeric(data['people_id'] ).astype(int)
 data['activity_id'] = data['activity_id'].apply(lambda x: x.split('_')[1])
 data['activity_id'] = pd.to_numeric(data['activity_id'] ).astype(int)
 fields = list(data.columns)
 cat_data = fields[2:13]
for column in cat_data:
data[column] = data[column].fillna('type 0')
 data[column] = data[column].apply(lambda x : x.split(' ')[1])
 data[column] = pd.to_numeric(data[column] ).astype(int)
 return data
people = pd.read_csv(".. /input/people.csv")
 people = cleanPeople(people)
 act_train = pd.read_csv(".. /input/act_train.csv",parse_dates=['date'])
 act_train_cleaned = cleanAct(act_train,train=True)
 act_test = pd.read_csv(".. /input/act_test.csv",parse_dates=['date'])
 act_test_cleaned = cleanAct(act_test)
 train = act_train_cleaned.merge(people,on='people_id', how='left')
 test = act_test_cleaned.merge(people, on='people_id', how='left')
 train = train.drop(['people_id'],axis=1)
 train = train.drop(['activity_id'],axis=1)
 test = test.drop(['people_id','activity_id'],axis=1)
 output = act_train['outcome']
X_train, X_test, y_train, y_test = train_test_split(train,output, test_size=0.2, random_state =7 )<choose_model_class>
data = pd.concat([train_data, test_data], axis=0 ).reset_index(drop=True )
model = xgb.XGBClassifier(max_depth=8,n_estimators=500,learning_rate=0.1,objective='binary:logistic',seed =7,reg_lambda=1 )<train_model>
data[["Sex", "Survived"]].groupby(["Sex"], as_index = False ).mean()
model.fit(X_train,y_train )<compute_train_metric>
data.columns[data.isnull().any() ]
results = model.predict_proba(X_test)
 s = results[:,1]
score = roc_auc_score(y_test,s)
 print(score )<save_to_csv>
data.isnull().sum()
results = model.predict_proba(test)
 s = results[:,1]
activity = act_test['activity_id']
result = pd.DataFrame({'activity_id': activity, 'outcome': s})
 result.to_csv("Result.csv",index=False )<load_from_csv>
data.isnull().sum()
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
data[data["Fare"].isnull() ]
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
data["Fare"] = data["Fare"].fillna(np.mean(data[(( data["Pclass"]==3)&(data["Embarked"]==0)) ]["Fare"]))
data[data["Fare"].isnull() ]
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
data[data["Embarked"].isnull() ]
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
data["Embarked"] = data["Embarked"].fillna(1)
 data[data["Embarked"].isnull() ]
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
data[data["Age"].isnull() ]
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
data_age_nan_index = data[data["Age"].isnull() ].index
for i in data_age_nan_index:
mean_age = data["Age"][(data["Pclass"]==data.iloc[i]["Pclass"])].median()
data["Age"].iloc[i] = mean_age
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
data["Alone"] = [1 if i == 0 else 0 for i in data["Family"]]
data["Family"].replace([0,1,2,3,4,5,6,7,10], [0,1,1,1,0,2,0,2,2], inplace=True)
 data.head()
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
data['Title']=data.Name.str.extract('([A-Za-z]+)\.' )
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
data['Title'].replace(['Mme','Ms','Mlle','Lady','Countess','Dona','Dr','Major','Sir','Capt','Don','Rev','Col', 'Jonkheer'],['Miss','Miss','Miss','Mrs','Mrs','Mrs','Mr','Mr','Mr','Mr','Mr','Other','Other','Other'], inplace=True )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
data['Age_Limit'] = LabelEncoder().fit_transform(data['Age_Limit'])
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
data['Fare_Limit'] = LabelEncoder().fit_transform(data['Fare_Limit'])
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
data['Age']=data['Age'].astype(int)
 data.drop(labels=["SibSp","Parch","Cabin","Fare","Age", "Ticket", "Name", "PassengerId"], axis=1, inplace = True)
 data.head()
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
data = pd.get_dummies(data,columns=["Pclass"])
 data = pd.get_dummies(data,columns=["Embarked"])
 data = pd.get_dummies(data,columns=["Family"])
 data = pd.get_dummies(data,columns=["Age_Limit"])
 data = pd.get_dummies(data,columns=["Fare_Limit"])
 data = pd.get_dummies(data,columns=["Title"])
 data.head()
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
if len(data)==(len(train_data)+ len(test_data)) :
print("success" )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
test = data[len(train_data):]
test.drop(labels="Survived", axis=1, inplace=True )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
train = data[:len(train_data)]
X_train = train.drop(labels = "Survived", axis=1)
 y_train = train["Survived"]
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.3, random_state=42)
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
log_reg = LogisticRegression(random_state=42)
 log_reg.fit(X_train, y_train)
 print("Accuracy: ", log_reg.score(X_test,y_test))
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
rf_reg = RandomForestClassifier(random_state=42)
 rf_reg.fit(X_train, y_train)
 print("Accuracy: ", rf_reg.score(X_test,y_test))
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
svm_clsf = SVC()
svm_clsf.fit(X_train, y_train)
 print("Accuracy: ", svm_clsf.score(X_test,y_test))
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
best_knn = []
for n in range(1,12):
knn = KNeighborsClassifier(n_neighbors=n)
 knn.fit(X_train, y_train)
 best_knn.insert(n, knn.score(X_test,y_test))
best_knn
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
knn_clsf = KNeighborsClassifier(n_neighbors=8)
 knn_clsf.fit(X_train, y_train)
 print("Accuracy: ", knn_clsf.score(X_test,y_test))
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
voting_classfication = VotingClassifier(estimators = [('knn', knn_clsf),('lg', log_reg),('rfg', rf_reg),('svc', svm_clsf)], voting="hard", n_jobs=-1)
 voting_classfication.fit(X_train, y_train)
 print("Accuracy: ", voting_classfication.score(X_test,y_test))
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
test_result = pd.Series(voting_classfication.predict(test), name = "Survived" ).astype(int)
 results = pd.concat([test_data["PassengerId"], test_result],axis = 1)
 results.to_csv("titanic_submission2.csv", index = False )
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts<merge>
from sklearn.model_selection import cross_val_score
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn import tree
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts<feature_engineering>
gnb = GaussianNB()
cv = cross_val_score(gnb,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
matrix['month'] = matrix['date_block_num'] % 12<categorify>
lr = LogisticRegression(max_iter = 2000)
 cv = cross_val_score(lr,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<data_type_conversions>
lr = LogisticRegression(max_iter = 2000)
 cv = cross_val_score(lr,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<data_type_conversions>
dt = tree.DecisionTreeClassifier(random_state = 1)
 cv = cross_val_score(dt,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<feature_engineering>
dt = tree.DecisionTreeClassifier(random_state = 1)
 cv = cross_val_score(dt,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts<filter>
knn = KNeighborsClassifier()
cv = cross_val_score(knn,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts<correct_missing_values>
knn = KNeighborsClassifier()
cv = cross_val_score(knn,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts<drop_column>
rf = RandomForestClassifier(random_state = 1)
 cv = cross_val_score(rf,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<drop_column>
rf = RandomForestClassifier(random_state = 1)
 cv = cross_val_score(rf,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
matrix = matrix[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
svc = SVC(probability = True)
 cv = cross_val_score(svc,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
X_train = matrix[matrix.date_block_num < 34].drop(['item_cnt_month'], axis=1)
 y_train = matrix[matrix.date_block_num < 34]['item_cnt_month']
X_test = matrix[matrix.date_block_num == 34].drop(['item_cnt_month'], axis=1)
<import_modules>
svc_poly = SVC(probability = True, kernel='poly', degree=2, gamma='auto', coef0=1, C=5)
 cv = cross_val_score(svc,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
import lightgbm as lgb
from sklearn.linear_model import LinearRegression<choose_model_class>
svc_rbf = SVC(probability = True, kernel='rbf', gamma=0.5, C=0.1)
 cv = cross_val_score(svc,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
lr = LinearRegression()
lr.fit(X_train.values, y_train)
 pred_lr = lr.predict(X_test.values )<init_hyperparams>
xgb = XGBClassifier(random_state =1)
 cv = cross_val_score(xgb,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
lgb_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread':1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.03,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 2**7,
'bagging_freq':1,
'verbose':0
}
model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)
 pred_lgb = model.predict(X_test )<compute_test_metric>
voting_clf = VotingClassifier(estimators = [('lr',lr),('knn',knn),('rf',rf),('gnb',gnb),('svc',svc),('xgb',xgb), 
('svc_poly', svc_poly),('svc_rbf', svc_rbf)], voting = 'soft' )
X_test_level2 = np.c_[pred_lr, pred_lgb]<define_variables>
cv = cross_val_score(voting_clf,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )
dates = matrix['date_block_num']
dates_train = dates[dates < 34]
dates_test = dates[dates == 34]<prepare_x_and_y>
svc_poly.fit(X_train,y_train)
 test_result = pd.Series(svc_poly.predict(test), name = "Survived" ).astype(int)
 results = pd.concat([test_data["PassengerId"], test_result],axis = 1)
 results.to_csv("titanic_submission.csv", index = False )
dates_train_level2 = dates_train[dates_train.isin([27, 28, 29, 30, 31, 32, 33])]
y_train_level2 = y_train[dates_train.isin([27, 28, 29, 30, 31, 32, 33])]<categorify>
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
X_train_level2 = np.zeros([y_train_level2.shape[0], 2])
 for cur_block_num in [27, 28, 29, 30, 31, 32, 33]:
print(cur_block_num)

X_train = matrix.loc[dates < cur_block_num].drop(['item_cnt_month'], axis=1)
 X_test = matrix.loc[dates == cur_block_num].drop(['item_cnt_month'], axis=1)
 y_train = matrix.loc[dates < cur_block_num, 'item_cnt_month'].values
lr.fit(X_train.values, y_train)
 pred_lr_2 = lr.predict(X_test.values)
 model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)
 pred_lgb_2 = model.predict(X_test)
 X_train_level2[dates_train_level2==cur_block_num] = np.c_[pred_lr_2,pred_lgb_2]<compute_test_metric>
def clf_performance(classifier, model_name):
print(model_name)
 print('Best Score: ' + str(classifier.best_score_))
print('Best Parameters: ' + str(classifier.best_params_))
alphas_to_try = np.linspace(0, 1, 1001)
 best_alpha = 0
r2_train_simple_mix = 0
for alpha in alphas_to_try:
mix = alpha*X_train_level2[:,0] +(1-alpha)*X_train_level2[:,1]
r2 = r2_score(y_train_level2, mix)
 if r2 > r2_train_simple_mix:
best_alpha = alpha
r2_train_simple_mix = r2<prepare_output>
lr = LogisticRegression()
param_grid = {'max_iter' : [2000],
'penalty' : ['l1', 'l2'],
'C' : np.logspace(-4, 4, 20),
'solver' : ['liblinear']}
clf_lr = GridSearchCV(lr, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_lr = clf_lr.fit(X_train,y_train)
 clf_performance(best_clf_lr,'Logistic Regression' )
test_preds = best_alpha*X_test_level2[:,0] +(1-best_alpha)*X_test_level2[:,1]
test_preds = test_preds.clip(0,20 )<create_dataframe>
knn = KNeighborsClassifier()
param_grid = {'n_neighbors' : [3,5,7,9],
'weights' : ['uniform', 'distance'],
'algorithm' : ['auto', 'ball_tree','kd_tree'],
'p' : [1,2]}
clf_knn = GridSearchCV(knn, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_knn = clf_knn.fit(X_train,y_train)
 clf_performance(best_clf_knn,'KNN' )
submissions = pd.DataFrame({
"ID": test.index,
"item_cnt_month": test_preds
})
 submissions.item_cnt_month = submissions.item_cnt_month.fillna(0.0 )<save_to_csv>
svc = SVC(probability = True)
 param_grid = tuned_parameters = [{'kernel': ['rbf'], 'gamma': [.1,.5,1,2,5,10],
'C': [.1, 1, 10, 100, 1000]},
{'kernel': ['linear'], 'C': [.1, 1, 10, 100, 1000]},
{'kernel': ['poly'], 'degree' : [2,3,4,5], 'C': [.1, 1, 10, 100, 1000]}]
clf_svc = GridSearchCV(svc, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_svc = clf_svc.fit(X_train,y_train)
 clf_performance(best_clf_svc,'SVC' )
submissions.to_csv('submission.csv',index=False )<install_modules>
rf = RandomForestClassifier(random_state = 1)
 param_grid = {'n_estimators': [400,450,500,550],
'criterion':['gini','entropy'],
'bootstrap': [True],
'max_depth': [15, 20, 25],
'max_features': ['auto','sqrt', 10],
'min_samples_leaf': [2,3],
'min_samples_split': [2,3]}
clf_rf = GridSearchCV(rf, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_rf = clf_rf.fit(X_train,y_train)
 clf_performance(best_clf_rf,'Random Forest' )
!pip install efficientnet -U<set_options>
param_grid = {
'n_estimators': [450,500,550],
'colsample_bytree': [0.75,0.8,0.85],
'max_depth': [10],
'reg_alpha': [1],
'reg_lambda': [2, 5, 10],
'subsample': [0.55, 0.6,.65],
'learning_rate':[0.5],
'gamma':[.5,1,2],
'min_child_weight':[0.01],
'sampling_method': ['uniform']
}
clf_xgb = GridSearchCV(xgb, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_xgb = clf_xgb.fit(X_train,y_train)
 clf_performance(best_clf_xgb,'XGB' )
<load_from_csv><EOS> 
test_result = pd.Series(best_clf_xgb.predict(test), name = "Survived" ).astype(int)
 results = pd.concat([test_data["PassengerId"], test_result],axis = 1)
 results.to_csv("titanic_submission1.csv", index = False)