Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
logging.getLogger('tensorflow' ).setLevel(logging.ERROR)
 tf.compat.v1.set_random_seed(0 )
finalPred_series = pd.Series(final_pred)
 submission = pd.concat([test['Id'],finalPred_series],axis=1)
 columns=['Id','winPlacePerc']
submission.columns = columns
print(submission)
 submission.to_csv('submission.csv', index=False )<import_modules>
train = pd.read_csv(r'.. /input/titanic/train.csv')
 test = pd.read_csv(r'.. /input/titanic/test.csv' )
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff
from plotly.subplots import make_subplots
import matplotlib.pyplot as plt
from pandas_profiling import ProfileReport
import seaborn as sns
from sklearn import metrics
from scipy import stats
from copy import deepcopy
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
import optuna
from optuna import Trial, visualization
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score,confusion_matrix
from sklearn.metrics import accuracy_score, mean_squared_error<load_from_csv>
def calculate_error_types(prediction_array, true_array):
type1_errors = pd.Series(( true_array==0)&(true_array!=prediction_array))
type2_errors = pd.Series(( true_array==1)&(true_array!=prediction_array))
num_type1 = len(type1_errors.loc[type1_errors==True])
 num_type2 = len(type2_errors.loc[type2_errors==True])
 pct_type1 = num_type1 / len(type1_errors)
 pct_type2 = num_type2 / len(type2_errors)
 pct_errors =(num_type1 + num_type2)/ len(type1_errors)
 return num_type1, pct_type1, num_type2, pct_type2, pct_errors
train_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 test_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 sub_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv')
 train_df.head()<prepare_x_and_y>
def write_model_results(name_str, accuracy, crossvalscores, predictions, y_test, accumulate=False):
num_type1, pct_type1, num_type2, pct_type2, pct_errors = calculate_error_types(predictions, y_test)
 tot_errors = num_type1+num_type2
crossvalscore = crossvalscores.mean()
crossval_std = crossvalscores.std()
global model_results
if 'model_results' in globals() :
if(( name_str in model_results['Model'].unique())&(not accumulate)) :
this_model = model_results.Model==name_str
column_list = ['Crossval','CV_std', 'TestAccuracy','Errors','ErrorPct','Type1','Type1Pct','Type2','Type2Pct']
model_results.loc[this_model, column_list] = [crossvalscore, crossval_std, accuracy, tot_errors,
pct_errors, num_type1, pct_type1, num_type2, pct_type2]
else:
model_results = model_results.append(
 {'Model': name_str, 'Crossval': crossvalscore, 'CV_std': crossval_std, 'TestAccuracy': accuracy,
'Errors': tot_errors,'ErrorPct': pct_errors,
'Type1': num_type1, 'Type1Pct': pct_type1, 'Type2': num_type2, 'Type2Pct': pct_type2},
ignore_index=True)
 else:
model_results = pd.DataFrame(
 data=[[name_str, crossvalscore, crossval_std, accuracy, tot_errors,
pct_errors, num_type1, pct_type1, num_type2, pct_type2]],
columns=['Model', 'Crossval', 'CV_std', 'TestAccuracy', 'Errors', 'ErrorPct',
'Type1', 'Type1Pct', 'Type2', 'Type2Pct'] )
feature_cols = train_df.drop(['id', 'target'], axis=1 ).columns
x = train_df[feature_cols]
y = train_df['target']
print(x.shape, y.shape )<concatenate>
def create_voted_predictions(model_predictions, survived_tilt=0):
voted_predictions = sum(model_predictions)
 num_models = len(model_predictions)
 cutoff = int(np.ceil(( num_models+1)/2)) - survived_tilt
voted_predictions[voted_predictions<cutoff] = 0
voted_predictions[voted_predictions>=cutoff] = 1
return voted_predictions
train_indexs = train_df.index
test_indexs = test_df.index
df = pd.concat(objs=[train_df, test_df], axis=0 ).reset_index(drop=True)
 df = df.drop('id', axis=1)
 len(train_indexs), len(test_indexs )<sort_values>
def get_ticket_survival_arrays(test_indexes=np.array(['none'])) :
if test_indexes[0]=='none':
train_df = train_copy.loc[:, ['Ticket','Survived','Name']]
test_df = test.loc[:, ['Ticket','Name']]
else:
train_df = train_copy.loc[~test_indexes, ['Ticket','Survived','Name']]
test_df = train_copy.loc[test_indexes, ['Ticket','Name']]
grouped = train_df.groupby(['Ticket'])
 for ticket, group in grouped:
for idx, row in group.iterrows() :
group_others = group.drop(idx)
 tot = group_others['Name'].count()
lived = group_others.loc[group_others.Survived==1,'Name'].count()
pct = lived/tot if tot>0 else -1
train_df.loc[idx, 'PctLived'] = pct
tot = group['Name'].count()
lived = group.loc[group.Survived==1,'Name'].count()
pct = lived/tot if tot>0 else -1
test_df.loc[test_df.Ticket==ticket, 'PctLived'] = pct
mean, std = train_df.loc[train_df.PctLived>-1 ,'PctLived'].agg([np.mean,np.std])
 train_df.loc[train_df.PctLived==-1,'PctLived'] = mean
test_df.loc[test_df.PctLived==-1,'PctLived'] = mean
test_df.loc[:,'PctLived'] = test_df.loc[:,'PctLived'].fillna(mean)
 train_df.loc[:,'PctLived'] = train_df.loc[:,'PctLived'].transform(lambda x:(x-mean)/std)
 test_df.loc[:,'PctLived'] = test_df.loc[:,'PctLived'].transform(lambda x:(x-mean)/std)
 return train_df.PctLived.values, test_df.PctLived.values
def fix_skew(features):

 numerical_columns = features.select_dtypes(include=['int64','float64'] ).columns
skewed_features = features[numerical_columns].apply(lambda x: stats.skew(x)).sort_values(ascending=False)
 high_skew = skewed_features[abs(skewed_features)> 0.5]
skewed_features = high_skew.index
for column in skewed_features:
features[column] = boxcox1p(features[column], boxcox_normmax(features[column] + 1))
return features<train_model>
def drop_column_feature_importances(X_train, y_train, random_state = 0):
feature_importances = pd.DataFrame(index=train.loc[:,train.columns!='Survived'].columns)
 for model in [adaboost, logitmodel, randomforest]:
model_clone = clone(model)
 model_clone.random_state = random_state
train_with_dummies=pd.get_dummies(X_train, drop_first=False)
 X, y = np.array(train_with_dummies), np.array(y_train)
 full_scaler = preprocessing.StandardScaler().fit(X)
 X_standardized = full_scaler.transform(X)
 model_clone.fit(X_standardized, y)
 benchmark_score = model_clone.score(X_standardized, y)
 importances = []
for col in X_train.columns:
model_clone = clone(model)
 model_clone.random_state = random_state
train_with_dummies=pd.get_dummies(X_train.drop(col, axis = 1),drop_first=False)
 X = np.array(train_with_dummies)
 full_scaler = preprocessing.StandardScaler().fit(X)
 X_standardized = full_scaler.transform(X)
 model_clone.fit(X_standardized, y)
 drop_col_score = model_clone.score(X_standardized, y)
 importances.append(benchmark_score - drop_col_score)
 modelname = model.__str__()
modelname = modelname[0: modelname.find('(')]
feature_importances[modelname] = importances
feature_importances = feature_importances.transpose()
col_order = feature_importances.mean().sort_values(ascending=False ).index
chart = sns.boxplot(data=feature_importances, color='lightgrey', width=.5, order=col_order,)
 _=plt.gcf().set_size_inches(16,4)
 _=plt.gca().set_title('Feature effect on score, including variance among 3 scikit-learn models.')
 _=plt.gca().axhline(0, linestyle='--', linewidth=.7, color='indianred')
 return feature_importances
param_grid = {
'n_estimators': [5, 10, 15, 20],
'max_depth': [2, 5, 7, 9]
}
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=42)
 clf = XGBRegressor(random_state = 42)
 clf.fit(x_train, y_train )<compute_test_metric>
def cross_validate_entire_process(k=10, ticket_survival_feature=False):
rows = len(X_full)
 cv_indexes = np.linspace(rows//k, rows-(rows//k), k-1 ).astype('int')
 cv_indexes = np.append(cv_indexes, rows)
 start_idx = 0
end_idx = 0
for index in cv_indexes:
end_idx = index
test_indexes = np.isin(np.arange(len(X_full)) , np.r_[start_idx:end_idx])
 X_test = X_full[test_indexes]
X_train = X_full[~test_indexes]
y_test = y_full[test_indexes]
y_train = y_full[~test_indexes]
scaler = preprocessing.StandardScaler().fit(X_train)
 X_train_standardized = scaler.transform(X_train)
 X_test_standardized = scaler.transform(X_test)
 if ticket_survival_feature:
train_survival, test_survival = get_ticket_survival_arrays(test_indexes)
 train_survival = train_survival.reshape(-1,1)
 X_train_standardized = np.append(X_train_standardized,train_survival,axis=1)
 test_survival = test_survival.reshape(-1,1)
 X_test_standardized = np.append(X_test_standardized,test_survival,axis=1)
 _=logitmodel.fit(X_train_standardized,y_train)
 logit_accuracy=logitmodel.score(X_test_standardized,y_test)
 logit_predictions = logitmodel.predict(X_test_standardized)
 _=adaboost.fit(X_train_standardized,y_train)
 adaboost_accuracy=adaboost.score(X_test_standardized,y_test)
 adaboost_predictions = adaboost.predict(X_test_standardized)
 _=randomforest.fit(X_train_standardized,y_train)
 randomforest_accuracy=randomforest.score(X_test_standardized,y_test)
 randomforest_predictions = randomforest.predict(X_test_standardized)
 deep_model_temp = make_deep_model(num_features=X_train_standardized.shape[1])
 _=deep_model_temp.fit(X_train_standardized,y_train,epochs=40,verbose=0)
 deep_accuracy = deep_model_temp.evaluate(x=X_test_standardized, y=y_test, verbose=0)[1]
deep_predictions = deep_model_temp.predict(X_test_standardized)[:, 1]
deep_predictions[deep_predictions<.5] = 0
deep_predictions[deep_predictions>=.5] = 1
deep_predictions=deep_predictions.astype('int')
 all_model_predictions = [logit_predictions, adaboost_predictions, randomforest_predictions, deep_predictions]
ada_rf_deep_predictions = [adaboost_predictions, randomforest_predictions, deep_predictions]
all_no_deep_predictions = [adaboost_predictions, logit_predictions, randomforest_predictions]
voted_allmodels_predictions = create_voted_predictions(all_model_predictions)
 voted_ada_rf_deep_predictions = create_voted_predictions(ada_rf_deep_predictions)
 voted_all_no_deep_predictions = create_voted_predictions(all_no_deep_predictions)
 voted_allmodels_tilted_predictions = create_voted_predictions(all_model_predictions, survived_tilt=1)
 voted_ada_rf_deep_tilted_predictions = create_voted_predictions(ada_rf_deep_predictions, survived_tilt=1)
 voted_all_no_deep_tilted_predictions = create_voted_predictions(all_no_deep_predictions, survived_tilt=1)
 voted_allmodels_accuracy = len(np.where(voted_allmodels_predictions==y_test)[0])/y_test.size
voted_ada_rf_deep_accuracy = len(np.where(voted_ada_rf_deep_predictions==y_test)[0])/y_test.size
voted_all_no_deep_accuracy = len(np.where(voted_all_no_deep_predictions==y_test)[0])/y_test.size
voted_allmodels_tilted_accuracy = len(np.where(voted_allmodels_tilted_predictions==y_test)[0])/y_test.size
voted_ada_rf_deep_tilted_accuracy = len(np.where(voted_ada_rf_deep_tilted_predictions==y_test)[0])/y_test.size
voted_all_no_deep_tilted_accuracy = len(np.where(voted_all_no_deep_tilted_predictions==y_test)[0])/y_test.size
for model_prediction in all_model_predictions:
success = np.array(pd.Series(model_prediction==y_test ).replace({True:1,False:0}))
model_successes[start_idx:end_idx] = model_successes[start_idx:end_idx] + success
write_model_results('LogisticRegression', logit_accuracy, np.array(0), logit_predictions, y_test,accumulate=True)
 write_model_results('AdaBoost', adaboost_accuracy, np.array(0), adaboost_predictions, y_test,accumulate=True)
 write_model_results('RandomForest', randomforest_accuracy, np.array(0), randomforest_predictions, y_test,accumulate=True)
 write_model_results('DeepModel', deep_accuracy, np.array(0), deep_predictions, y_test,accumulate=True)
 write_model_results('Voted_AllModels', voted_allmodels_accuracy, np.array(0), voted_allmodels_predictions, y_test,accumulate=True)
 write_model_results('Voted_Ada_Rf_Deep', voted_ada_rf_deep_accuracy, np.array(0), voted_ada_rf_deep_predictions, y_test,accumulate=True)
 write_model_results('Voted_All_No_Deep', voted_all_no_deep_accuracy, np.array(0), voted_all_no_deep_predictions, y_test,accumulate=True)
 write_model_results('Voted_AllModels_Tilted', voted_allmodels_tilted_accuracy, np.array(0), voted_allmodels_tilted_predictions, y_test,accumulate=True)
 write_model_results('Voted_Ada_Rf_Deep_Tilted', voted_ada_rf_deep_tilted_accuracy, np.array(0), voted_ada_rf_deep_tilted_predictions, y_test,accumulate=True)
 write_model_results('Voted_All_No_Deep_Tilted', voted_all_no_deep_tilted_accuracy, np.array(0), voted_all_no_deep_tilted_predictions, y_test,accumulate=True)
 start_idx = end_idx
aggregated_model_results = model_results.groupby('Model')\ 
.agg({'TestAccuracy':[np.mean,np.std],'Errors':np.mean,'Type1':np.mean,'Type2':np.mean})\ 
.sort_values(( 'TestAccuracy','mean'),ascending=False)\ 
.style.format({('TestAccuracy','mean'):'{:.1%}',('TestAccuracy','std'):'{:.1%}', 
('Errors','mean'):'{:.1f}',('Type1','mean'):'{:.1f}',('Type2','mean'):'{:.1f}'})\ 
.apply(lambda x: ['background: lightgreen' if x.name in ['Voted_AllModels'] else '' for i in x], axis=1)
 print('Cross-validation of models and voted models complete.Updated 'model_results' and 'model_successes'.')
 return aggregated_model_results
predictions = clf.predict(x_test)
 errors = abs(predictions - y_test)
 print('Mean Absolute Error:', round(np.mean(errors), 2), 'degrees.')
<split>
train.head(3)
 summary = [[train[column].dtype, train[column].unique().size, train[column].isna().sum() ] for column in list(train)]
summary_df = pd.DataFrame(summary, index=list(train), columns=['data_type','unique_values','nan_values'])
 summary_df['memory_usage'] = train.memory_usage(deep=True)
 def highlight(df):
fontstyle='color:white; font-weight:bold;'
stylemap = pd.DataFrame().reindex_like(df)
 stylemap.loc[:,:] = 'background-color: '
stylemap.loc[['Age','Cabin','Embarked'],'nan_values']= 'background-color: red; '+ fontstyle
stylemap.loc[['PassengerId','Name','Ticket','Cabin'],'unique_values'] = 'background-color: orange; '+ fontstyle
stylemap.loc[['Age','Fare'],['data_type','unique_values']] = 'background-color: lightseagreen; '+ fontstyle
return stylemap
summary_df.style.apply(highlight, axis=None )
def objective(trial,data=x,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda': trial.suggest_loguniform(
 'lambda', 1e-3, 10.0 
),
'alpha': trial.suggest_loguniform(
 'alpha', 1e-3, 10.0 
),
'colsample_bytree': trial.suggest_categorical(
 'colsample_bytree', [0.5,0.6,0.7,0.8,0.9,1.0] 
),
'subsample': trial.suggest_categorical(
 'subsample', [0.6,0.7,0.8,1.0] 
),
'learning_rate': trial.suggest_categorical(
 'learning_rate', [0.008,0.009,0.01,0.012,0.014,0.016,0.018, 0.02] 
),
'n_estimators': trial.suggest_categorical(
 "n_estimators", [150, 200, 300, 3000] 
),
'max_depth': trial.suggest_categorical(
 'max_depth', [4,5,7,9,11,13,15,17] 
),
'random_state': 42,
'min_child_weight': trial.suggest_int(
 'min_child_weight', 1, 300 
),
}
model = XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<train_model>
train['HadCabin'] = train['Cabin'].notna().replace({False:0,True:1})
 train['HadAge'] = train['Age'].notna().replace({False:0,True:1})
 train['HadEmbarked'] = train['Embarked'].notna().replace({False:0,True:1} )
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=25)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )<find_best_params>
train['CabinLetter'] = train['CabinLetter_Fillna']
study.best_params<train_model>
train_ages = train.loc[:,['Title','Age']]
best_params = study.best_params
best_params['tree_method'] = 'gpu_hist'
best_params['random_state'] = 42
clf = XGBRegressor(**(best_params))
clf.fit(x, y )<predict_on_test>
preds = pd.Series(clf.predict(test_df.drop('id', axis=1)) , name='target')
 preds = pd.concat([test_df['id'], preds], axis=1 )<save_to_csv>
train['Age_Fillna'] = train['Age']
for title in train['Title'].unique() :
nans = train.loc[(train['Title']==title)&(train['Age'].isna())]
non_nan_sample = train.loc[train['Title']==title,'Age'].dropna().sample(n=len(nans), \
replace=False, \
random_state=0 ).values
train.loc[nans.index,'Age_Fillna'] = non_nan_sample
preds.to_csv("submission.csv", index=False )<set_options>
train['Age']=train['Age_Fillna']
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 100)
 %matplotlib inline
sys.version_info<load_from_csv>
train['TicketStub']=train['Ticket'].str.extract(r'([A-Za-z///.]+)',expand=False)
 train['TicketStub']=train['TicketStub'].fillna('numeric:'+ train['Ticket'].str.len().astype('str'))
train['TicketStub']=train['TicketStub'].str.replace('.','' ).str.upper()
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID' )<count_unique_values>
train['FamilySize'] = train['SibSp'].add(train['Parch'])+1
train['FamilySize_Alone'] = train.loc[train['FamilySize']==1,'FamilySize']
train['FamilySize_SmallFamily'] = train.loc[(train['FamilySize']>1)&(train['FamilySize']<=4),'FamilySize']
train['FamilySize_LgFamily'] = train.loc[(train['FamilySize']>4),'FamilySize']
plot_survival_by_category(['FamilySize_Alone', 'FamilySize_SmallFamily', 'FamilySize_LgFamily'],3,sharey=False )
print('train size, item in train, shop in train', train.shape[0], train.item_id.nunique() , train.shop_id.nunique())
 print('train size, item in train, shop in train', test.shape[0], test.item_id.nunique() ,test.shop_id.nunique())
 print('new items:', len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test))<count_missing_values>
train_copy = train.copy()
train.isnull().sum()<count_missing_values>
train_survival, test_survival = get_ticket_survival_arrays()
train.loc[:,'PctLived'] = train_survival
train.isnull().sum()<feature_engineering>
train_copy = train.copy()
use_columns = ['Pclass','Sex','Title','CabinLetter','Age','TicketStub','FamilySize_Code','Fare','Embarked','Survived','PctLived']
drop_columns = set(train.columns)- set(use_columns)
 train.drop(drop_columns, axis=1,inplace=True)
 train.columns
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
non_convertible_columns = \
train.columns[(train.dtypes == 'float64')|(train.dtypes == 'category')|(train.columns=='Survived')]
convertible_columns = set(train.columns)- set(non_convertible_columns)
 for column in convertible_columns:
train[column] = pd.Categorical(train[column] )
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
train_with_dummies=pd.get_dummies(train,drop_first=True)
 train_with_dummies.head(1 )
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<data_type_conversions>
X_full = np.array(train_with_dummies.loc[:,(train_with_dummies.columns !='Survived')])
 y_full = np.array(train_with_dummies.loc[:,(train_with_dummies.columns=='Survived')].iloc[:,0] )
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(X_full, y_full, test_size=.25, random_state=0 )
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
split_scaler = preprocessing.StandardScaler().fit(X_train)
 X_train_standardized = split_scaler.transform(X_train)
 X_test_standardized = split_scaler.transform(X_test )
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float32)) 
<data_type_conversions>
if 'model_results' in globals() :
del(model_results )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
def make_logit_model(X_train, y_train):
logitmodel = LogisticRegression(random_state=0,max_iter=500)
 gridsearch = GridSearchCV(logitmodel,param_grid={'C':np.logspace(-3,2,6), 'solver':['lbfgs','sag'],\
'tol':np.logspace(-3,1,5)}, cv=10,return_train_score=True,iid=True)\ 
.fit(X_train, y_train)
 logitmodel = gridsearch.best_estimator_
return logitmodel, gridsearch
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
<data_type_conversions>
logitmodel, gridsearch = make_logit_model(X_train_standardized, y_train)
 pd.DataFrame(gridsearch.cv_results_)\ 
.loc[:,['params','mean_test_score','mean_train_score','rank_test_score']]\ 
.sort_values(by='rank_test_score' ).head(3)
 print('Setting logitmodel to {}'.format(gridsearch.best_params_))
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
<merge>
logit_predictions = logitmodel.predict(X_test_standardized)
 probas_index_class1 = np.where(logitmodel.classes_==1)[0][0]
logit_probas = logitmodel.predict_proba(X_test_standardized)[:,probas_index_class1]
logit_accuracy = logitmodel.score(X_test_standardized, y_test)
 logit_crossvalscores = cross_val_score(logitmodel, X_train_standardized, y_train,cv=10)
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<statistical_test>
write_model_results('LogisticRegression', logit_accuracy, logit_crossvalscores, logit_predictions, y_test )
matrix = lag_feature(matrix, [1,2,3,6,12], 'item_cnt_month' )<merge>
def make_adaboost_model(X_train, y_train):
adaboost = AdaBoostClassifier(n_estimators=50,learning_rate=1,random_state=0)
 gridsearch = GridSearchCV(adaboost,param_grid={'n_estimators':[10,60,200], 'learning_rate':np.linspace (.01,1,5), \
'random_state':[0]}, cv=10,return_train_score=True,iid=True)\ 
.fit(X_train, y_train)
 adaboost = gridsearch.best_estimator_
return adaboost, gridsearch
def add_group_stats(matrix_, groupby_feats, target, enc_feat, last_periods):
if not 'date_block_num' in groupby_feats:
print('date_block_num must in groupby_feats')
 return matrix_
group = matrix_.groupby(groupby_feats)[target].sum().reset_index()
max_lags = np.max(last_periods)
 for i in range(1,max_lags+1):
shifted = group[groupby_feats+[target]].copy(deep=True)
 shifted['date_block_num'] += i
shifted.rename({target:target+'_lag_'+str(i)},axis=1,inplace=True)
 group = group.merge(shifted, on=groupby_feats, how='left')
 group.fillna(0,inplace=True)
 for period in last_periods:
lag_feats = [target+'_lag_'+str(lag)for lag in np.arange(1,period+1)]
mean = group[lag_feats].sum(axis=1)/float(period)
 mean2 =(group[lag_feats]**2 ).sum(axis=1)/float(period)
 group[enc_feat+'_avg_sale_last_'+str(period)] = mean
group[enc_feat+'_std_sale_last_'+str(period)] =(mean2 - mean**2 ).apply(np.sqrt)
 group[enc_feat+'_std_sale_last_'+str(period)].replace(np.inf,0,inplace=True)
 group[enc_feat+'_avg_sale_last_'+str(period)] /= group[enc_feat+'_avg_sale_last_'+str(period)].mean()
group[enc_feat+'_std_sale_last_'+str(period)] /= group[enc_feat+'_std_sale_last_'+str(period)].mean()
cols = groupby_feats + [f_ for f_ in group.columns.values if f_.find('_sale_last_')>=0]
matrix = matrix_.merge(group[cols], on=groupby_feats, how='left')
 return matrix<drop_column>
adaboost, gridsearch = make_adaboost_model(X_train_standardized, y_train)
 pd.DataFrame(gridsearch.cv_results_)\ 
.loc[:,['params','mean_test_score','mean_train_score','rank_test_score']]\ 
.sort_values(by='rank_test_score' ).head(3)
 print('Setting adaboost to {}'.format(gridsearch.best_params_))
ts = time.time()
matrix = add_group_stats(matrix, ['date_block_num', 'item_id'], 'item_cnt_month', 'item', [6,12])
 matrix = add_group_stats(matrix, ['date_block_num', 'shop_id'], 'item_cnt_month', 'shop', [6,12])
 matrix = add_group_stats(matrix, ['date_block_num', 'item_category_id'], 'item_cnt_month', 'category', [12])
 matrix = add_group_stats(matrix, ['date_block_num', 'city_code'], 'item_cnt_month', 'city', [12])
 matrix = add_group_stats(matrix, ['date_block_num', 'type_code'], 'item_cnt_month', 'type', [12])
 matrix = add_group_stats(matrix, ['date_block_num', 'subtype_code'], 'item_cnt_month', 'subtype', [12])
 time.time() - ts<categorify>
adaboost_index_class1 = np.where(adaboost.classes_==1)[0][0]
adaboost_predictions = adaboost.predict(X_test_standardized)
 adaboost_probas = adaboost.predict_proba(X_test_standardized)[:,probas_index_class1]
adaboost_accuracy = adaboost.score(X_test_standardized,y_test)
 adaboost_crossvalscores = cross_val_score(adaboost, X_train_standardized, y_train,cv=10)
def target_encoding(matrix_, groupby_feats, target, enc_feat, lags):
print('target encoding for',groupby_feats)
 group = matrix_.groupby(groupby_feats ).agg({target:'mean'})
 group.columns = [enc_feat]
group.reset_index(inplace=True)
 matrix = matrix_.merge(group, on=groupby_feats, how='left')
 matrix[enc_feat] = matrix[enc_feat].astype(np.float16)
 matrix = lag_feature(matrix, lags, enc_feat)
 matrix.drop(enc_feat, axis=1, inplace=True)
 return matrix<categorify>
write_model_results('AdaBoost', adaboost_accuracy, adaboost_crossvalscores, adaboost_predictions, y_test )
ts = time.time()
matrix = target_encoding(matrix, ['date_block_num'], 'item_cnt_month', 'date_avg_item_cnt', [1])
 matrix = target_encoding(matrix, ['date_block_num', 'item_id'], 'item_cnt_month', 'date_item_avg_item_cnt', [1,2,3,6,12])
 matrix = target_encoding(matrix, ['date_block_num', 'shop_id'], 'item_cnt_month', 'date_shop_avg_item_cnt', [1,2,3,6,12])
 matrix = target_encoding(matrix, ['date_block_num', 'item_category_id'], 'item_cnt_month', 'date_cat_avg_item_cnt', [1])
 matrix = target_encoding(matrix, ['date_block_num', 'shop_id', 'item_category_id'], 'item_cnt_month', 'date_shop_cat_avg_item_cnt', [1])
 matrix = target_encoding(matrix, ['date_block_num', 'city_code'], 'item_cnt_month', 'date_city_avg_item_cnt', [1])
 matrix = target_encoding(matrix, ['date_block_num', 'item_id', 'city_code'], 'item_cnt_month', 'date_item_city_avg_item_cnt', [1])
 time.time() - ts<data_type_conversions>
def make_random_forest(X_train, y_train):
randomforest = RandomForestClassifier(n_estimators=100,random_state=0)
 gridsearch = GridSearchCV(randomforest,param_grid={'n_estimators':[60], 'max_depth':[2,3,7], \
'max_leaf_nodes':[100,300,500],'random_state':[0]}, cv=10,return_train_score=True, iid=True)\ 
.fit(X_train, y_train)
 randomforest = gridsearch.best_estimator_
return randomforest, gridsearch
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts<merge>
randomforest, gridsearch = make_random_forest(X_train_standardized, y_train)
 pd.DataFrame(gridsearch.cv_results_)\ 
.loc[:,['params','mean_test_score','mean_train_score','rank_test_score']]\ 
.sort_values(by='rank_test_score' ).head(3)
 print('Setting randomforest to {}'.format(gridsearch.best_params_))
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
randomforest_index_class1 = np.where(randomforest.classes_==1)[0][0]
randomforest_predictions = randomforest.predict(X_test_standardized)
 randomforest_probas = randomforest.predict_proba(X_test_standardized)[:,probas_index_class1]
randomforest_accuracy = randomforest.score(X_test_standardized,y_test)
 randomforest_crossvalscores = cross_val_score(randomforest, X_train_standardized, y_train,cv=10 )
matrix['month'] = matrix['date_block_num'] % 12
matrix['year'] =(matrix['date_block_num'] / 12 ).astype(np.int8 )<create_dataframe>
write_model_results('RandomForest', randomforest_accuracy, randomforest_crossvalscores, randomforest_predictions,
y_test )
ts = time.time()
last_sale = pd.DataFrame()
for month in range(1,35):
last_month = matrix.loc[(matrix['date_block_num']<month)&(matrix['item_cnt_month']>0)].groupby(['item_id','shop_id'])['date_block_num'].max()
df = pd.DataFrame({'date_block_num':np.ones([last_month.shape[0],])*month,
'item_id': last_month.index.get_level_values(0 ).values,
'shop_id': last_month.index.get_level_values(1 ).values,
'item_shop_last_sale': last_month.values})
 last_sale = last_sale.append(df)
 last_sale['date_block_num'] = last_sale['date_block_num'].astype(np.int8)
 matrix = matrix.merge(last_sale, on=['date_block_num','item_id','shop_id'], how='left')
 time.time() - ts<create_dataframe>
def make_deep_model(num_features):
num_input_features = num_features
num_hidden_neurons = 13
deep_model = tf.keras.models.Sequential([
tf.keras.layers.Dense(num_input_features, activation='relu'),
tf.keras.layers.Dense(num_hidden_neurons, activation='sigmoid'),
tf.keras.layers.Dropout(rate=0.2, seed=0),
tf.keras.layers.Dense(2, activation='softmax')
 ])
 deep_model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 return deep_model
ts = time.time()
last_sale = pd.DataFrame()
for month in range(1,35):
last_month = matrix.loc[(matrix['date_block_num']<month)&(matrix['item_cnt_month']>0)].groupby('item_id')['date_block_num'].max()
df = pd.DataFrame({'date_block_num':np.ones([last_month.shape[0],])*month,
'item_id': last_month.index.values,
'item_last_sale': last_month.values})
 last_sale = last_sale.append(df)
 last_sale['date_block_num'] = last_sale['date_block_num'].astype(np.int8)
 matrix = matrix.merge(last_sale, on=['date_block_num','item_id'], how='left')
 time.time() - ts<categorify>
deep_model = make_deep_model(num_features=X_train_standardized.shape[1] )
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts<load_pretrained>
deep_model_history = deep_model.fit(x=X_train_standardized, y=y_train, epochs=40, verbose=0, validation_split=.1 )
matrix.to_pickle('data.pkl')
 del matrix
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
deep_probas = deep_model.predict(X_test_standardized)[:, 1]
deep_predictions = deep_probas.copy()
deep_predictions[deep_predictions<.5] = 0
deep_predictions[deep_predictions>=.5] = 1
deep_predictions=deep_predictions.astype('int')
 deep_accuracy = deep_model.evaluate(x=X_test_standardized, y=y_test, verbose=0)[1]
deep_crossvalscores = deep_val_acc[-10]
data = pd.read_pickle('./data.pkl')
 data.head()<create_dataframe>
write_model_results('DeepModel', deep_accuracy, deep_crossvalscores, deep_predictions, y_test )
data = data[[
'date_block_num',
'shop_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code','subtype_code',
'item_cnt_month_lag_1','item_cnt_month_lag_2','item_cnt_month_lag_3','item_cnt_month_lag_6','item_cnt_month_lag_12',
'item_avg_sale_last_6', 'item_std_sale_last_6',
'item_avg_sale_last_12', 'item_std_sale_last_12',
'shop_avg_sale_last_6', 'shop_std_sale_last_6',
'shop_avg_sale_last_12', 'shop_std_sale_last_12',
'category_avg_sale_last_12', 'category_std_sale_last_12',
'city_avg_sale_last_12', 'city_std_sale_last_12',
'type_avg_sale_last_12', 'type_std_sale_last_12',
'subtype_avg_sale_last_12', 'subtype_std_sale_last_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1','date_item_avg_item_cnt_lag_2','date_item_avg_item_cnt_lag_3','date_item_avg_item_cnt_lag_6','date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1','date_shop_avg_item_cnt_lag_2','date_shop_avg_item_cnt_lag_3','date_shop_avg_item_cnt_lag_6','date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month','year',
'item_shop_last_sale','item_last_sale',
'item_shop_first_sale','item_first_sale',
]]
cat_feats = ['shop_id','city_code','item_category_id','type_code','subtype_code']<prepare_x_and_y>
train['PctLived'] = train_copy.PctLived
X_train, y_train = train.loc[:, train.columns!='Survived'], train.loc[:,'Survived']
feature_importances = drop_column_feature_importances(X_train, y_train )
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del data
gc.collect() ; 
<import_modules>
allmodels_predictions = [adaboost_predictions, logit_predictions, randomforest_predictions, deep_predictions]
ada_rf_deep_predictions = [adaboost_predictions, randomforest_predictions, deep_predictions]
all_no_deep_predictions = [adaboost_predictions, logit_predictions, randomforest_predictions]
voted_allmodels_predictions = create_voted_predictions(allmodels_predictions)
 voted_ada_rf_deep_predictions = create_voted_predictions(ada_rf_deep_predictions)
 voted_all_no_deep_predictions = create_voted_predictions(all_no_deep_predictions )
sys.version_info<train_model>
voted_allmodels_accuracy = len(np.where(voted_allmodels_predictions==y_test)[0])/y_test.size
voted_ada_rf_deep_accuracy = len(np.where(voted_ada_rf_deep_predictions==y_test)[0])/y_test.size
voted_all_no_deep_accuracy = len(np.where(voted_all_no_deep_predictions==y_test)[0])/y_test.size
ts = time.time()
model = LGBMRegressor(
 max_depth = 8,
n_estimators = 500,
colsample_bytree=0.7,
min_child_weight = 300,
reg_alpha = 0.1,
reg_lambda = 1,
random_state = 42, 
)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=10,
early_stopping_rounds = 40,
categorical_feature = cat_feats)
 time.time() - ts<save_to_csv>
write_model_results('Voted_AllModels', voted_allmodels_accuracy, np.array(0), voted_allmodels_predictions, y_test)
 write_model_results('Voted_Ada_Rf_Deep', voted_ada_rf_deep_accuracy, np.array(0), voted_ada_rf_deep_predictions, y_test)
 write_model_results('Voted_All_No_Deep', voted_all_no_deep_accuracy, np.array(0), voted_all_no_deep_predictions, y_test)
 model_results.sort_values('TestAccuracy', ascending=False ).style.format({
'Crossval':'{:.1%}', 'CV_std':'{:.1%}', 'TestAccuracy':'{:.1%}',
'ErrorPct':'{:.1%}', 'Type1Pct':'{:.1%}', 'Type2Pct':'{:.1%}'} )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 X_train_level2 = pd.DataFrame({
"ID": np.arange(Y_pred.shape[0]),
"item_cnt_month": Y_pred
})
 X_train_level2.to_csv('lgb_valid.csv', index=False)
 submission = pd.DataFrame({
"ID": np.arange(Y_test.shape[0]),
"item_cnt_month": Y_test
})
 submission.to_csv('lgb_submission.csv', index=False )<import_modules>
del model_results
model_successes = np.zeros(len(X_full))
model_results_initial_features_no_pct_lived = cross_validate_entire_process(k=10)
 model_results_initial_features_no_pct_lived
np.random.seed(233333 )<load_pretrained>
del model_results
model_successes = np.zeros(len(X_full))
model_results_initial_features_with_pct_lived = cross_validate_entire_process(k=10,ticket_survival_feature=True)
 model_results_initial_features_with_pct_lived
data = pd.read_pickle('data.pkl')
 data = data[[
'date_block_num',
'item_cnt_month',
'item_cnt_month_lag_1','item_cnt_month_lag_2','item_cnt_month_lag_3','item_cnt_month_lag_6','item_cnt_month_lag_12',
'item_avg_sale_last_6', 'item_std_sale_last_6',
'item_avg_sale_last_12', 'item_std_sale_last_12',
'shop_avg_sale_last_6', 'shop_std_sale_last_6',
'shop_avg_sale_last_12', 'shop_std_sale_last_12',
'category_avg_sale_last_12', 'category_std_sale_last_12',
'city_avg_sale_last_12', 'city_std_sale_last_12',
'type_avg_sale_last_12', 'type_std_sale_last_12',
'subtype_avg_sale_last_12', 'subtype_std_sale_last_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1','date_item_avg_item_cnt_lag_2','date_item_avg_item_cnt_lag_3','date_item_avg_item_cnt_lag_6','date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1','date_shop_avg_item_cnt_lag_2','date_shop_avg_item_cnt_lag_3','date_shop_avg_item_cnt_lag_6','date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month','year',
'item_shop_last_sale','item_last_sale',
'item_shop_first_sale','item_first_sale',
]]<choose_model_class>
train_survived, test_survived = get_ticket_survival_arrays()
train_copy['PctLived'] = train_survived
train_copy['model_successes'] = model_successes
train_copy.loc[:, 'model_successes'] = train_copy.loc[:, 'model_successes'].astype('int' )
def Sales_prediction_model(input_shape):
in_layer = Input(input_shape)
 x = Dense(16,kernel_initializer='RandomUniform', kernel_regularizer=l2(0.02), activation = "relu" )(in_layer)
 x = Dense(8, kernel_initializer='RandomUniform', kernel_regularizer=l2(0.02), activation = "relu" )(x)
 x = Dense(1, kernel_initializer='RandomUniform', kernel_regularizer=l2(0.02), activation = "relu" )(x)
 model = Model(inputs = in_layer, outputs = x, name='Sales_prediction_model')
 return model
X_train.fillna(0,inplace=True)
 X_valid.fillna(0,inplace=True)
 X_test.fillna(0,inplace=True)
 input_shape = [X_train.shape[1]]
model = Sales_prediction_model(input_shape)
 model.compile(optimizer = Adam(lr=0.0005), loss = ["mse"], metrics=['mse'])
 model.fit(X_train, Y_train, validation_data =(X_valid, Y_valid), batch_size = 10000, epochs=5)
<save_to_csv>
test['CabinLetter'] = test['Cabin'].fillna('X' ).apply(lambda x:x[0] )
Y_pred = model.predict(X_valid ).clip(0, 20)[:,0]
Y_test = model.predict(X_test ).clip(0, 20)[:,0]
X_train_level2 = pd.DataFrame({
"ID": np.arange(Y_pred.shape[0]),
"item_cnt_month": Y_pred
})
 X_train_level2.to_csv('nn_valid.csv', index=False)
 submission = pd.DataFrame({
"ID": np.arange(Y_test.shape[0]),
"item_cnt_month": Y_test
})
 submission.to_csv('nn_submission.csv', index=False )<import_modules>
test['Title']=test['Name'].str.extract(r'^.+,\s (.{0,12}\.) ',expand=False)
 test.loc[test.Title.isin(['Mlle.','Ms.']),'Title'] = 'Miss.'
test.loc[test.Title.isin(['Capt.','Col.','Jonkheer.','Major.']),'Title'] = 'Officer.'
test.loc[test.Title.isin(['Lady.','Sir.','the Countess.']),'Title'] = 'Aristocrat.'
test.loc[test.Title.isin(['Mme.','Dona.']),'Title'] = 'Mrs.'
test.loc[test.Title.isin(['Don.']),'Title'] = 'Mr.'
sys.version_info<train_model>
for title in test['Title'].unique() :
nans = test.loc[(test['Title']==title)&(test['Age'].isna())]
non_nan_sample = train_ages.loc[train_ages['Title']==title,'Age'].dropna().sample(n=len(nans), \
replace=False, \
random_state=0 ).values
test.loc[nans.index,'Age'] = non_nan_sample
ts = time.time()
model = XGBRegressor(
 max_depth=7,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
gamma = 0.005,
eta=0.1,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=10,
early_stopping_rounds = 40, 
)
 time.time() - ts<save_to_csv>
test.loc[:, 'AgeGroup'] = test.loc[:, 'Age'].transform(code_age_group )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 X_train_level2 = pd.DataFrame({
"ID": np.arange(Y_pred.shape[0]),
"item_cnt_month": Y_pred
})
 X_train_level2.to_csv('xgb_valid.csv', index=False)
 submission = pd.DataFrame({
"ID": np.arange(Y_test.shape[0]),
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False )<import_modules>
test['Fare']=test['Fare'].fillna(0 )
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import Ridge, LinearRegression
import gc<prepare_x_and_y>
test['Embarked']=test['Embarked'].fillna('S' )
data = pd.read_pickle('data.pkl')
 Y_train_level2 = data[data.date_block_num == 33]['item_cnt_month']
del data
gc.collect()<load_from_csv>
test['TicketStub']=test['Ticket'].str.extract(r'([A-Za-z///.]+)',expand=False)
 test['TicketStub']=test['TicketStub'].fillna('numeric:'+ test['Ticket'].str.len().astype('str'))
test['TicketStub']=test['TicketStub'].str.replace('.','' ).str.upper()
test.loc[test.groupby('TicketStub')['PassengerId'].transform(len)<=5,'TicketStub']='OTHER'
X_train_level2 = pd.DataFrame()
df = pd.read_csv('./lgb_valid.csv')
 X_train_level2['lgb'] = df['item_cnt_month']
df = pd.read_csv('./xgb_valid.csv')
 X_train_level2['xgb'] = df['item_cnt_month']
df = pd.read_csv('./nn_valid.csv')
 X_train_level2['nn'] = df['item_cnt_month']
X_test_level2 = pd.DataFrame()
df = pd.read_csv('./lgb_submission.csv')
 X_test_level2['lgb'] = df['item_cnt_month']
df = pd.read_csv('./xgb_submission.csv')
 X_test_level2['xgb'] = df['item_cnt_month']
df = pd.read_csv('./nn_submission.csv')
 X_test_level2['nn'] = df['item_cnt_month'] 
<compute_test_metric>
test['FamilySize'] = test['SibSp'].add(test['Parch'])+1
test.loc[:,'TicketSize'] = test.loc[:,'Ticket'].map(passengers_per_ticket)
 test.loc[:,'FamilySize'] = test.loc[:,['FamilySize','TicketSize']].max(axis=1)
 test['FamilySize_Code'] = test['FamilySize'].apply(encode_family_size )
best_alpha = 1;
best_rmse = 100;
for alpha in np.arange(0,1,0.02):
Y_pred_level2 = alpha*X_train_level2['lgb'] +(1-alpha)*X_train_level2['xgb']
rmse = np.sqrt(mean_squared_error(Y_train_level2, Y_pred_level2))
if(rmse<best_rmse):
best_rmse = rmse
best_alpha = alpha
Y_test_level2 = best_alpha*X_test_level2['lgb'] +(1-best_alpha)*X_test_level2['xgb']
print('best alpha:', best_alpha)
 print('weighted average of lgb and xgb validation rmse: ',best_rmse)
 submission = pd.DataFrame({
"ID": np.arange(Y_test_level2.shape[0]),
"item_cnt_month": Y_test_level2
})
 submission.to_csv('./blended_submission1.csv', index=False)
 model = LinearRegression()
model.fit(X_train_level2, Y_train_level2)
 Y_pred_level2 = model.predict(X_train_level2)
 Y_test_level2 = model.predict(X_test_level2)
 rmse = np.sqrt(mean_squared_error(Y_train_level2, Y_pred_level2))
print('Linear regression validation rmse: ',rmse)
 submission = pd.DataFrame({
"ID": np.arange(Y_test_level2.shape[0]),
"item_cnt_month": Y_test_level2
})
 submission.to_csv('./blended_submission2.csv', index=False )<import_modules>
train_survival, test_survival = get_ticket_survival_arrays()
test.loc[:,'PctLived'] = test_survival
train_with_dummies.loc[:,'PctLived'] = train_survival
import numpy as np
import pandas as pd
import scipy
import sklearn
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
from itertools import product
from sklearn.cluster import KMeans
import lightgbm as lgb<define_variables>
test_passenger_ids = test.PassengerId
dpath = '.. /input/competitive-data-science-predict-future-sales/'
adpath ='.. /input/predict-future-sales/'<load_from_csv>
use_columns = ['Pclass','Sex','Title','CabinLetter','Age','TicketStub','FamilySize_Code','Fare','Embarked','PctLived','Survived']
drop_columns = set(test.columns)- set(use_columns)
 test.drop(drop_columns, axis=1,inplace=True)
 test.columns
df_train = pd.read_csv(dpath + 'sales_train.csv')
 df_test = pd.read_csv(dpath + 'test.csv', index_col='ID')
 df_shops = pd.read_csv(dpath + 'shops.csv', index_col='shop_id')
 df_items = pd.read_csv(dpath + 'items.csv', index_col='item_id')
 df_itemcat = pd.read_csv(dpath + 'item_categories.csv', index_col='item_category_id')
 sample_submission = pd.read_csv(dpath + 'sample_submission.csv', index_col='ID' )<load_from_csv>
non_convertible_columns = \
test.columns[(test.dtypes == 'float64')|(test.dtypes == 'category')|(test.columns=='Survived')]
convertible_columns = set(test.columns)- set(non_convertible_columns)
 for column in convertible_columns:
test[column] = pd.Categorical(test[column] )
calendar = pd.read_csv(adpath + 'calendar.csv', dtype='int16' )<categorify>
test_with_dummies=pd.get_dummies(test,drop_first=True )
def shop_name2city(sn):
sn = sn.split() [0]
if sn == 'Цифровой' or sn == 'Интернет-магазин': sn = 'Internet'
if sn[0] == '!': sn = sn[1:]
return sn
df_shops['city'] = df_shops['shop_name'].apply(shop_name2city)
 df_shops['city_enc'] = LabelEncoder().fit_transform(df_shops['city'] ).astype('int8')
 city_info = pd.read_pickle(adpath + 'city_info.pkl')
 df_shops['city_size'] = df_shops['city'].map(city_info['city_size'] )<categorify>
missing_cols = set(train_with_dummies.columns)- set(test_with_dummies.columns)
 for c in missing_cols:
test_with_dummies[c] = 0
test_with_dummies = test_with_dummies[train_with_dummies.columns]
class Items() :
def __init__(self, df_items, df_itemcat):
self.df_items = df_items
self.df_itemcat = df_itemcat
self.set_hl_cat()
self.make_items_ext()
self.item_features = ['item_category_id', 'hl_cat_id']
def set_hl_cat(self):
self.df_itemcat['hl_cat_id'] = self.df_itemcat['item_category_name'].str.split(n=1, expand=True)[0]
self.df_itemcat['hl_cat_id'] = LabelEncoder().fit_transform(self.df_itemcat['hl_cat_id'])
 def make_items_ext(self):
self.df_items = df_items.merge(self.df_itemcat, how = 'left',
left_on = 'item_category_id', right_index = True)
 def get_items_df(self):
return self.df_items[self.item_features].astype('int32' )<concatenate>
X_full = np.array(train_with_dummies.loc[:,(train_with_dummies.columns !='Survived')])
 test_array = np.array(test_with_dummies.loc[:,(test_with_dummies.columns !='Survived')] )
items = Items(df_items, df_itemcat )<prepare_output>
full_scaler = preprocessing.StandardScaler().fit(X_full)
 X_full_standardized = full_scaler.transform(X_full)
 test_array = full_scaler.transform(test_array )
class TT_Extended() :
def __init__(self, df_train, df_test, items, df_shops, calendar, cmode, verbose=True):
self.info = verbose
self.df_train = df_train.copy()
self.df_test = df_test.copy()
self.df_shops = df_shops.copy()
self.calendar = self.set_calender(calendar.copy())
 self.idx_columns = ['date_block_num', 'shop_id', 'item_id']
self.df_test['date_block_num'] = 34
self.df_test['item_cnt_month'] = 0.
 self.df_test['item_cnt_month'] = self.df_test['item_cnt_month'].astype('float32')
 self.df_train[self.idx_columns] = self.df_train[self.idx_columns].astype('int32')
 self.df_test[self.idx_columns] = self.df_test[self.idx_columns].astype('int32')
 self.df_train_cleaning(cmode)
 self.item_mean_features = []
self.shop_mean_features = []
self.df_items = items.get_items_df()
self.item_ext_features = list(self.df_items.columns)
 self.df_items_ext = self.items_ext()
self.df_bb = self.build_bb()
self.df_train_ext = self.df_train_agg(cmin = 0, cmax = 1000, drop = None)
 self.add_train_df()
self.df_train_ext = self.df_train_extention()
def df_train_cleaning(self, mode):
assert mode in ['keep', 'drop', 'block', 'total', 't+b']
if self.info: print('Cleaning train dataframe...( Mode -', mode, ')')
 shop_idx = self.df_train[(self.df_train['shop_id'] == 9)| 
(self.df_train['shop_id'] == 20)].index
self.df_train.drop(shop_idx, inplace=True)
 self.df_train = self.df_train[(self.df_train['item_price'] > 0)& 
(self.df_train['item_price'] < 51000)]
self.df_train = self.df_train[self.df_train['item_cnt_day'] <= 1000]
shop_repl_dict = {0 : 57, 1 : 58, 11 : 10, 40 : 39}
self.df_train['shop_id'] = self.df_train['shop_id'].apply(
 lambda s: shop_repl_dict.get(s)if s in shop_repl_dict else s)
 if mode == 'drop':
self.df_train = self.df_train[self.df_train['item_cnt_day'] > 0]
elif mode == 'block':
item_block_cnt = self.df_train.groupby(['date_block_num', 'shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_block_cnt[item_block_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['date_block_num', 'shop_id', 'item_id'] ).index.isin(items_to_drop)]
elif mode == 'total':
item_total_cnt = self.df_train.groupby(['shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_total_cnt[item_total_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['shop_id', 'item_id'] ).index.isin(items_to_drop)]
elif mode == 't+b':
item_total_cnt = self.df_train.groupby(['shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_total_cnt[item_total_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['shop_id', 'item_id'] ).index.isin(items_to_drop)]
item_block_cnt = self.df_train.groupby(['date_block_num', 'shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_block_cnt[item_block_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['date_block_num', 'shop_id', 'item_id'] ).index.isin(items_to_drop)]
return
def set_calender(self, calendar):
calendar['date_block_num'] =(calendar['year'] - 2013)*12 +(calendar['month'] - 1)
 calendar['hdays'] = calendar['mdays'] - calendar['wdays']
calendar.set_index('date_block_num', inplace=True)
 return calendar
def items_ext(self):
dfi = self.df_items.copy()
dfi['fsb'] = self.df_train.groupby('item_id')['date_block_num'].min()
dfi['fsb'].fillna(34, inplace=True)
 dfi['fsb'] = dfi['fsb'].astype('int8')
 self.item_ext_features += ['fsb']
return dfi
def build_bb(self):
if self.info: print('Building index dataframe...')
 df_work = []
for block_num in self.df_train['date_block_num'].unique() :
cur_shops = self.df_train.loc[self.df_train['date_block_num'] == block_num, 'shop_id'].unique()
cur_items = self.df_train.loc[self.df_train['date_block_num'] == block_num, 'item_id'].unique()
df_work.append(np.array(list(product(*[[block_num], cur_shops, cur_items])) , dtype='int32'))
df_work = pd.DataFrame(np.vstack(df_work), columns = self.idx_columns)
 return df_work
def df_train_agg(self, cmin = 0, cmax = 20, drop = None):
if self.info: print('Aggregation...')
 df_work = self.df_train.groupby(self.idx_columns ).agg({'item_price' : np.mean,
'item_cnt_day': np.sum})
 df_work.reset_index(inplace=True)
 df_work = df_work.rename(columns={'item_cnt_day': 'item_cnt_month'})
 df_work = pd.merge(self.df_bb, df_work, on=self.idx_columns, how='left')
 df_work['item_cnt_month'] = df_work['item_cnt_month'].astype('float32' ).fillna(0.).clip(cmin, cmax)
 df_work['item_price'] = df_work['item_price'].astype('float32' ).fillna(0.)
df_tmp = self.df_train[self.df_train['item_cnt_day'] > 0].groupby(self.idx_columns ).agg({'item_cnt_day': 'count'})
 df_tmp.reset_index(inplace=True)
 df_tmp = df_tmp.rename(columns={'item_cnt_day': 'item_rate_month'})
 df_work = pd.merge(df_work, df_tmp, on=self.idx_columns, how='left')
 df_work['item_rate_month'] = df_work['item_rate_month'].astype('float32' ).fillna(0.)
del df_tmp
if drop: df_work.drop(drop, axis=1, inplace=True)
 return df_work
def add_train_df(self):
self.df_train_ext = pd.concat([self.df_train_ext, self.df_test], ignore_index=True,
sort=False, keys=self.idx_columns)
 def add_item_means(self, df, feature = None):
if feature == None :
group_items = ['date_block_num','item_id']
feature = 'item_cnt'
else:
group_items = ['date_block_num','item_id'] + [feature]
feature_mean_name = feature + '_mean'
if self.info: print('Adding item means for', feature, '...')
 df_tmp = df.groupby(group_items)['item_cnt_month'].mean()
df_tmp = df_tmp.reset_index().rename(columns = {'item_cnt_month': feature_mean_name})
 df = pd.merge(df, df_tmp, on=group_items, how='left')
 self.item_mean_features.append(feature_mean_name)
 del df_tmp
return df
def add_shop_means(self, df, feature):
group_items = ['date_block_num', 'shop_id'] + [feature]
feature_mean_name = feature + '_mean'
if self.info: print('Adding shop means for', feature, '...')
 df_tmp = df.groupby(group_items)['item_cnt_month'].mean()
df_tmp = df_tmp.reset_index().rename(columns = {'item_cnt_month': feature_mean_name})
 df = pd.merge(df, df_tmp, on=group_items, how='left')
 self.shop_mean_features.append(feature)
 del df_tmp
return df
def df_train_extention(self, test_cat_only = False):
df_work = self.df_train_ext.merge(df_shops[['city_enc', 'city_size']], how = 'left', on = 'shop_id')
 df_work = df_work.merge(self.calendar[['mdays', 'wdays', 'hdays']],
how = 'left', left_on = 'date_block_num', right_index = True)
 df_work = df_work.merge(self.df_items_ext[self.item_ext_features],
how = 'left', left_on = 'item_id', right_index = True)
 ssbn = self.df_train.groupby('shop_id')['date_block_num'].min().astype('int8')
 ssbn.name = 'ssbn'
df_work = df_work.merge(ssbn, how = 'left', on = 'shop_id')
 if test_cat_only:
test_cat = df_work[df_work['date_block_num'] == 34]['item_category_id'].unique()
df_work = df_work[df_work['item_category_id'].isin(test_cat)]
return df_work
def make_base_df(self, keepnans = False):
df_work = self.df_train_ext.copy()
if keepnans:
fill_value = None
else:
fill_value = 0
df_work = pd.pivot_table(df_work, values='item_cnt_month', index=['shop_id', 'item_id'],
columns = 'date_block_num', aggfunc=np.sum, fill_value = fill_value)
 df_work.columns.name = ''
return df_work
def add_total_cnt(self, df):
if self.info: print('Adding total count...')
 df_base = self.make_base_df()
for i in range(1, 33):
df_base[i + 1] += df_base[i]
df_base = df_base.shift(1, axis=1 ).loc[:, 1:].astype('int32')
 df_base = df_base.melt(var_name='date_block_num', value_name='total_cnt', ignore_index=False)
 df = df.merge(df_base, how='left', on=self.idx_columns)
 del df_base
return df
def add_item_lags(self, df, feature_name, nlags=3, keepnans = False):
if self.info: print('Adding item lags for', feature_name, '...')
 df_tmp = df[['date_block_num', 'shop_id', 'item_id', feature_name]]
for i in range(nlags, 0, -1):
lag_feature_name = feature_name +'_lag-' + str(i)
 df_shifted = df_tmp.copy()
df_shifted.columns = ['date_block_num', 'shop_id', 'item_id', lag_feature_name]
df_shifted['date_block_num'] += i
df = pd.merge(df, df_shifted, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 if keepnans:
df[lag_feature_name] = df[lag_feature_name].astype('float32')
 else:
df[lag_feature_name] = df[lag_feature_name].fillna(0 ).astype('float32')
 del df_tmp
return df
def add_shop_lags(self, df, feature_name, nlags=3):
mean_feature_name = feature_name + '_mean'
if self.info: print('Adding lags for', mean_feature_name, '...')
 df_tmp = df[['date_block_num', 'shop_id', feature_name, mean_feature_name]]
for i in range(nlags, 0, -1):
lag_feature_name = mean_feature_name + '_lag-' + str(i)
 df_shifted = df_tmp.copy()
df_shifted.columns = ['date_block_num', 'shop_id', feature_name, lag_feature_name]
df_shifted['date_block_num'] += i
df = pd.merge(df, df_shifted.drop_duplicates() , on=['date_block_num', 'shop_id', feature_name], how='left')
 df[lag_feature_name] = df[lag_feature_name].fillna(0 ).astype('float32')
 del df_tmp
del df_shifted
return df
def shop_clustering(self, df_work):
print('No shop clusters provided')
 return df_work
def build_work_db(self, hd, item_mean_features = [], shop_mean_features = [], add_total_cnt = False):
if self.info: print('Building work dataframe...')
 df_work = self.df_train_ext.copy()
df_work = self.add_item_means(df_work)
 for mf in item_mean_features:
df_work = self.add_item_means(df_work, mf)
 for mf in shop_mean_features:
df_work = self.add_shop_means(df_work, mf)
 df_work = self.add_item_lags(df_work, 'item_cnt_month', hd, keepnans = False)
 df_work = self.add_item_lags(df_work, 'item_rate_month', hd)
 df_work = self.add_item_lags(df_work, 'item_price', hd)
 for mf in self.item_mean_features:
df_work = self.add_item_lags(df_work, mf, hd)
 for mf in self.shop_mean_features:
df_work = self.add_shop_lags(df_work, mf, hd)
 df_work.drop(df_work[df_work['date_block_num'] < hd].index, inplace=True)
 df_work.drop(self.item_mean_features, axis=1, inplace=True)
 df_work.drop(['item_category_id_mean', 'item_price', 'item_rate_month'], axis=1, inplace=True)
 self.item_mean_features = []
self.shop_mean_features = []
df_work['qmean'] = df_work[['item_cnt_month_lag-1',
'item_cnt_month_lag-2',
'item_cnt_month_lag-3']].mean(skipna=True, axis=1)
 df_work['new'] = df_work['fsb'] == df_work['date_block_num']
df_work['fsb'] = df_work['date_block_num'] - df_work['fsb']
df_work['ssbn'] = df_work['date_block_num'] - df_work['ssbn']
df_work['month'] =(df_work['date_block_num']%12 ).astype('int8')
 idx = df_work[(df_work['item_price_lag-1'] == 0)&(df_work['item_price_lag-2'] != 0)].index
df_work.loc[idx, 'item_price_lag-1'] = df_work.loc[idx, 'item_price_lag-2']
idx = df_work[(df_work['item_price_lag-2'] == 0)&(df_work['item_price_lag-3'] != 0)].index
df_work.loc[idx, 'item_price_lag-2'] = df_work.loc[idx, 'item_price_lag-3']
df_work['grad-1'] = df_work['item_cnt_month_lag-1']/df_work['item_cnt_month_lag-2']
df_work['grad-1'] = df_work['grad-1'].replace([np.inf, -np.inf], np.nan ).fillna(0.)
df_work['grad-2'] = df_work['item_cnt_month_lag-2']/df_work['item_cnt_month_lag-3']
df_work['grad-2'] = df_work['grad-2'].replace([np.inf, -np.inf], np.nan ).fillna(0.)
df_work = self.shop_clustering(df_work)
 if add_total_cnt: df_work = self.add_total_cnt(df_work)
 col2clip = ['item_cnt_month',
'item_cnt_month_lag-3', 'item_cnt_month_lag-2', 'item_cnt_month_lag-1',
'item_cnt_mean_lag-3', 'item_cnt_mean_lag-2','item_cnt_mean_lag-1',
'city_enc_mean_lag-3', 'city_enc_mean_lag-2', 'city_enc_mean_lag-1',
'item_category_id_mean_lag-3', 'item_category_id_mean_lag-2', 'item_category_id_mean_lag-1',
'qmean']
df_work[col2clip] = df_work[col2clip].clip(0, 20)
 return df_work
def get_work_db(self, hd = 3, item_mean_features = [],
shop_mean_features = [],
drop_features = None,
add_total_cnt = False):
df_work = self.build_work_db(hd, item_mean_features, shop_mean_features, add_total_cnt)
 if drop_features == None:
return df_work
else:
return df_work.drop(drop_features, axis = 1 )<concatenate>
_=logitmodel.fit(X_full_standardized, y_full)
 logit_predictions = logitmodel.predict(test_array)
 _=adaboost.fit(X_full_standardized, y_full)
 adaboost_predictions = adaboost.predict(test_array)
 _=randomforest.fit(X_full_standardized, y_full)
 randomforest_predictions = randomforest.predict(test_array)
 deep_model = make_deep_model(num_features=X_full_standardized.shape[1])
 _=deep_model.fit(X_full_standardized, y_full, epochs=40, verbose=0)
 deep_predictions = deep_model.predict(test_array)[:, 1]
deep_predictions[deep_predictions<.5] = 0
deep_predictions[deep_predictions>=.5] = 1
deep_predictions=deep_predictions.astype('int')
 all_model_predictions = [logit_predictions, adaboost_predictions, randomforest_predictions, deep_predictions]
ada_rf_deep_predictions = [adaboost_predictions, randomforest_predictions, deep_predictions]
all_no_deep_predictions = [adaboost_predictions, logit_predictions, randomforest_predictions]
voted_allmodels_predictions = create_voted_predictions(all_model_predictions)
 voted_ada_rf_deep_predictions = create_voted_predictions(ada_rf_deep_predictions)
 voted_all_no_deep_predictions = create_voted_predictions(all_no_deep_predictions)
 voted_allmodels_tilted_predictions = create_voted_predictions(all_model_predictions, survived_tilt=1)
 voted_ada_rf_deep_tilted_predictions = create_voted_predictions(ada_rf_deep_predictions, survived_tilt=1)
 voted_all_no_deep_tilted_predictions = create_voted_predictions(all_no_deep_predictions, survived_tilt=1 )
%%time
pfs = TT_Extended(df_train, df_test, items, df_shops, calendar, cmode='total' )<set_options>
potential_submissions = dict([('LogisticRegression',logit_predictions),\ 
('AdaBoost',adaboost_predictions),\ 
('RandomForest',randomforest_predictions),\ 
('DeepModel',deep_predictions),\ 
('Voted_AllModels',voted_allmodels_predictions),\ 
('Voted_Ada_Rf_Deep',voted_ada_rf_deep_predictions),\ 
('Voted_All_No_Deep',voted_all_no_deep_predictions),\ 
('Voted_AllModels_Tilted',voted_allmodels_tilted_predictions),\ 
('Voted_Ada_Rf_Deep_Tilted',voted_ada_rf_deep_tilted_predictions),\ 
('Voted_All_No_Deep_Tilted',voted_all_no_deep_tilted_predictions)] )
<drop_column><EOS> 
timestamp = datetime.today().strftime('%y%m%d_%H%M')
 submission_df = pd.DataFrame()
submission_df['PassengerId'] = test_passenger_ids
for i in range(5):
model = model_results_initial_features_with_pct_lived.data.index[i]
submission_df['Survived'] = potential_submissions[model]
file_name = model + '_' + timestamp + '.csv'
submission_df.to_csv(file_name,index=False )