Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
pd.set_option('max_columns', 90)
 PALETTE = ['
BACKCOLOR = '
sns.set_palette(PALETTE)
 warnings.filterwarnings('ignore')
 mpl.rcParams['figure.dpi'] = 120
mpl.rcParams['axes.spines.top'] = False
mpl.rcParams['axes.spines.right'] = False
languages = ['ta', 'hi', 'en']
images_train = read_all(".. /input/level_4a_train/"+LEVEL+"/"+"background", key_prefix='bgr_')
 for language in languages:
images_train.update(read_all(".. /input/level_4a_train/"+LEVEL+"/"+language, key_prefix=language+"_"))
print(len(images_train))
images_test = read_all(".. /input/level_4a_test/kaggle_"+LEVEL, key_prefix='')
 print(len(images_test))<normalization>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 all_data = pd.concat(( train, test)).reset_index(drop=True )
scaler = StandardScaler()
X_scaled_train = scaler.fit_transform(X_train)
 X_scaled_test = scaler.transform(X_test )<choose_model_class>
def multi_table(table_list):
return HTML(
 f"<table><tr> {''.join(['<td>' + table._repr_html_() + '</td>' for table in table_list])} </tr></table>" )
kfold = KFold(5, True, 10)
<find_best_model_class>
multi_table([pd.DataFrame(all_data[i].value_counts())for i in all_data.columns] )
sn_ce = SigmoidNeuron()
for train, test in kfold.split(X_scaled_train,Y_train):
sn_ce.fit(X_scaled_train[train], Y_train[train], epochs=500, learning_rate=0.00002, loss_fn="ce", display_loss=True,initialise=False)
 Y_pred_train = sn_ce.predict(X_scaled_train[test])
 Y_pred_binarised_train =(Y_pred_train >= 0.5 ).astype("int" ).ravel()
accuracy_train = accuracy_score(Y_pred_binarised_train, Y_train[test])
 print("Train Accuracy : ", accuracy_train)
 print("-"*50)
<predict_on_test>
numerical_vars = ['Age', 'SibSp', 'Parch', 'Fare']
ordinal_vars = ['Pclass']
nominal_vars = ['Survived', 'Name', 'Sex', 'Ticket', 'Cabin', 'Embarked']
def print_accuracy(sn):
Y_pred_train = sn.predict(X_scaled_train)
 Y_pred_binarised_train =(Y_pred_train >= 0.5 ).astype("int" ).ravel()
accuracy_train = accuracy_score(Y_pred_binarised_train, Y_train)
 print("Train Accuracy : ", accuracy_train)
 print("-"*50 )<compute_test_metric>
train0 = train[train.Survived == 0]
train1 = train[train.Survived == 1]
cnt = 0
detail_desc = []
for c in train.columns:
if c == 'PassengerId':
continue
if train[c].dtypes != 'object':
desc = pd.DataFrame(columns=['feature', 'data', 'type', 'count', 'mean', 'median', 'std', 'min', 'max', 'skew', 'null'])
 desc.loc[0] = [c, 'Train', train[c].dtype.name, train[c].count() , train[c].mean() , train[c].median() , train[c].std() , train[c].min() , train[c].max() , train[c].skew() , train[c].isnull().sum() ]
desc.loc[1] = [c, 'All', train[c].dtype.name, all_data[c].count() , all_data[c].mean() , all_data[c].median() , all_data[c].std() , all_data[c].min() , all_data[c].max() , all_data[c].skew() , all_data[c].isnull().sum() ]
desc.loc[2] = [c, 'Target=0', train0[c].dtype.name, train0[c].count() , train0[c].mean() , train0[c].median() , train0[c].std() , train0[c].min() , train0[c].max() , train0[c].skew() , train0[c].isnull().sum() ]
desc.loc[3] = [c, 'Target=1', train1[c].dtype.name, train1[c].count() , train1[c].mean() , train1[c].median() , train1[c].std() , train1[c].min() , train1[c].max() , train1[c].skew() , train1[c].isnull().sum() ]
desc = desc.set_index(['feature', 'data'],drop=True)
 detail_desc.append(desc.style.background_gradient() )
print_accuracy(sn_ce )<save_to_csv>
train0 = train[train.Survived == 0]
train1 = train[train.Survived == 1]
cnt = 0
detail_desc = []
for c in train.columns:
if c == 'PassengerId':
continue
if train[c].dtypes == 'object':
desc = pd.DataFrame(columns=['feature', 'data', 'type', 'count', 'null', 'mode', 'value_count'])
 desc.loc[0] = [c, 'Train', train[c].dtype.name, train[c].count() , train[c].isnull().sum() , train[c].mode() , train[c].value_counts() ]
desc = desc.set_index(['feature', 'data'],drop=True)
 detail_desc.append(desc.style.background_gradient() )
Y_pred_test = sn_ce.predict(X_scaled_test)
 Y_pred_binarised_test =(Y_pred_test >= 0.5 ).astype("int" ).ravel()
submission = {}
submission['ImageId'] = ID_test
submission['Class'] = Y_pred_binarised_test
submission = pd.DataFrame(submission)
 submission = submission[['ImageId', 'Class']]
submission = submission.sort_values(['ImageId'])
 submission.to_csv("submisision.csv", index=False )<define_variables>
multi_table(detail_desc )
np.random.seed(100)
 LEVEL = 'level_4a' 
<compute_test_metric>
tmp_train = copy.deepcopy(train)
 tmp_train['AgeBin'] = 6
for i in range(6):
tmp_train.loc[(tmp_train.Age >= 10*i)&(tmp_train.Age < 10*(i + 1)) , 'AgeBin'] = i
tmp_train.head(3 )
class SigmoidNeuron:
def __init__(self):
self.w = None
self.b = None
def perceptron(self, x):
return np.dot(x, self.w.T)+ self.b
def sigmoid(self, x):
return 1.0/(1.0 + np.exp(-x))
def grad_w_mse(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
return(y_pred - y)* y_pred *(1 - y_pred)* x
def grad_b_mse(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
return(y_pred - y)* y_pred *(1 - y_pred)
 def grad_w_ce(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
if y == 0:
return y_pred * x
elif y == 1:
return -1 *(1 - y_pred)* x
else:
raise ValueError("y should be 0 or 1")
 def grad_b_ce(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
if y == 0:
return y_pred
elif y == 1:
return -1 *(1 - y_pred)
 else:
raise ValueError("y should be 0 or 1")
 def fit(self, X, Y, epochs=1, learning_rate=1, initialise=True, loss_fn="mse", display_loss=False):
if initialise:
self.w = np.random.randn(1, X.shape[1])
 self.b = 0
if display_loss:
loss = {}
for i in tqdm_notebook(range(epochs), total=epochs, unit="epoch"):
dw = 0
db = 0
for x, y in zip(X, Y):
if loss_fn == "mse":
dw += self.grad_w_mse(x, y)
 db += self.grad_b_mse(x, y)
 elif loss_fn == "ce":
dw += self.grad_w_ce(x, y)
 db += self.grad_b_ce(x, y)
 self.w -= learning_rate * dw
self.b -= learning_rate * db
if display_loss:
Y_pred = self.sigmoid(self.perceptron(X))
if loss_fn == "mse":
loss[i] = mean_squared_error(Y, Y_pred)
 elif loss_fn == "ce":
loss[i] = log_loss(Y, Y_pred)
 if display_loss:
plt.plot(loss.values())
 plt.xlabel('Epochs')
 if loss_fn == "mse":
plt.ylabel('Mean Squared Error')
 elif loss_fn == "ce":
plt.ylabel('Log Loss')
 plt.show()
def predict(self, X):
Y_pred = []
for x in X:
y_pred = self.sigmoid(self.perceptron(x))
Y_pred.append(y_pred)
 return np.array(Y_pred )<load_pretrained>
cat_dist(tmp_train, var='AgeBin', hue='Survived', msg_show=False )
languages = ['ta', 'hi', 'en']
images_train = read_all(".. /input/level_4a_train/"+LEVEL+"/"+"background", key_prefix='bgr_')
 for language in languages:
images_train.update(read_all(".. /input/level_4a_train//"+LEVEL+"/"+language, key_prefix=language+"_"))
print(len(images_train))
images_test = read_all(".. /input/level_4a_test/kaggle_"+LEVEL, key_prefix='')
 print(len(images_test))<define_variables>
tmp_train = copy.deepcopy(train)
 tmp_train['FareBin'] = pd.cut(tmp_train.Fare, 10)
 tmp_train['FareBin'] = LabelEncoder().fit_transform(tmp_train.FareBin)
 tmp_train.head(3 )
X_train = []
Y_train = []
for key, value in images_train.items() :
X_train.append(value)
 if key[:4] == "bgr_":
Y_train.append(0)
 else:
Y_train.append(1)
 ID_test = []
X_test = []
for key, value in images_test.items() :
ID_test.append(int(key))
X_test.append(value)
 X_train = np.array(X_train)
 Y_train = np.array(Y_train)
 X_test = np.array(X_test)
 print(X_train.shape, Y_train.shape)
 print(X_test.shape )<normalization>
cat_dist(tmp_train, var='FareBin', hue='Survived', msg_show=False )
scaler = MinMaxScaler()
X_scaled_train = scaler.fit_transform(X_train)
 X_scaled_test = scaler.transform(X_test)
<train_model>
cat_dist(train, var='Embarked', hue='Survived' )
sn_ce = SigmoidNeuron()
sn_ce.fit(X_scaled_train, Y_train, epochs=200, learning_rate=0.015, loss_fn="ce", display_loss=True )<predict_on_test>
tmp_all_data = copy.deepcopy(all_data)
 t0 = pd.DataFrame(tmp_all_data.Name)
 t1 = pd.DataFrame(tmp_all_data.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip() ).value_counts())
 multi_table([t0, t1] )
def print_accuracy(sn):
Y_pred_train = sn.predict(X_scaled_train)
 Y_pred_binarised_train =(Y_pred_train >= 0.5 ).astype("int" ).ravel()
accuracy_train = accuracy_score(Y_pred_binarised_train, Y_train)
 print("Train Accuracy : ", accuracy_train)
 print("-"*50 )<compute_test_metric>
tmp_all_data['Title'] = tmp_all_data.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip())
print_accuracy(sn_ce )<prepare_output>
tmp_train.Cabin.value_counts()
Y_pred_test = sn_ce.predict(X_scaled_test)
 Y_pred_binarised_test =(Y_pred_test >= 0.5 ).astype("int" ).ravel()
submission = {}
submission['ImageId'] = ID_test
submission['Class'] = Y_pred_binarised_test
submission = pd.DataFrame(submission)
 submission = submission[['ImageId', 'Class']]
submission = submission.sort_values(['ImageId'])
 submission
submission.to_csv("submisision_6.csv", index=False)
<groupby>
tmp_train['CabinCnt'] = tmp_train.Cabin.apply(lambda x: 0 if pd.isna(x)else len(x.split(' ')))
 tmp_train['CabinClass'] = tmp_train.Cabin.apply(lambda x: str(x)[0] )
submission.groupby('Class' ).count()<set_options>
t0 = pd.DataFrame(tmp_train.CabinCnt.value_counts())
 t1 = pd.DataFrame(tmp_train.CabinClass.value_counts())
 multi_table([t0, t1] )
warnings.filterwarnings('ignore')
 %matplotlib inline 
<load_from_csv>
cat_dist(tmp_train, var='CabinCnt', hue='Survived', msg_show=False )
sputnik_data = pd.read_csv(".. /input/sputnik/train.csv",sep=",")
 sputnik_data<prepare_x_and_y>
cat_dist(tmp_train, var='CabinClass', hue='Survived', msg_show=False )
sputnik_train = sputnik_data.dropna(inplace=False)
 sputnik_train['error'] = np.linalg.norm(sputnik_train[['x', 'y', 'z']].values - sputnik_train[['x_sim', 'y_sim', 'z_sim']].values, axis=1)
 sputnik_train.drop('x',axis=1,inplace=True)
 sputnik_train.drop('y',axis=1,inplace=True)
 sputnik_train.drop('z',axis=1,inplace=True)
 sputnik_train.drop('x_sim',axis=1,inplace=True)
 sputnik_train.drop('y_sim',axis=1,inplace=True)
 sputnik_train.drop('z_sim',axis=1,inplace=True)
 sputnik_train['epoch'] = pd.to_datetime(sputnik_train.epoch,format='%Y-%m-%d %H:%M:%S')
 sputnik_train.index = sputnik_train.epoch
sputnik_train.drop('epoch', axis = 1, inplace = True )<predict_on_test>
tmp_train['IsNumericTicket'] = tmp_train.Ticket.apply(lambda x: 1 if x.isnumeric() else 0)
 tmp_train['TicketType'] = tmp_train.Ticket.apply(lambda x: ''.join(x.split(' ')[:-1] ).replace('.','' ).replace('/','' ).lower() if len(x.split(' ')[:-1])> 0 else 0 )
submission = validation_forecast(0 )<concatenate>
cat_dist(tmp_train, var='IsNumericTicket', hue='Survived' )
for i in tqdm(range(1,600)) :
submission = pd.concat([submission,validation_forecast(i)] )<categorify>
all_data['Age'] = all_data.Age.fillna(train.Age.median())
 all_data['Fare'] = all_data.Fare.fillna(train.Fare.median())
 all_data.dropna(subset=['Embarked'], inplace=True)
 cabins = all_data.Cabin
all_data.drop(['Cabin'], axis=1, inplace=True )
frame = submission.to_frame()<rename_columns>
all_data['CabinCnt'] = cabins.apply(lambda x: 0 if pd.isna(x)else len(x.split(' ')))
 all_data['CabinClass'] = cabins.apply(lambda x: str(x)[0])
 all_data['IsNumericTicket'] = all_data.Ticket.apply(lambda x: 1 if x.isnumeric() else 0)
 all_data['TicketType'] = all_data.Ticket.apply(lambda x: ''.join(x.split(' ')[:-1] ).replace('.','' ).replace('/','' ).lower() if len(x.split(' ')[:-1])> 0 else 0)
 all_data['Title'] = all_data.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip())
 all_data['Family'] = all_data.SibSp + all_data.Parch
frame.columns = ['error']<save_to_csv>
numeric_vars = ['Age', 'SibSp', 'Parch', 'Fare', 'CabinCnt', 'Family']
ordinal_vars = ['Pclass']
nominal_vars = ['Name', 'Sex', 'Ticket', 'Embarked', 'CabinClass', 'IsNumericTicket', 'TicketType', 'Title']
all_data[nominal_vars] = all_data[nominal_vars].astype('str')
 for feature in numeric_vars:
all_data[feature] = np.log1p(all_data[feature])
 scaler = StandardScaler()
numeric_vars = all_data.columns[(all_data.dtypes != 'object')&(all_data.columns != 'PassengerId')&(all_data.columns != 'Survived')&(all_data.columns != 'IsTrain')]
all_data[numeric_vars] = scaler.fit_transform(all_data[numeric_vars] )
frame.to_csv('submit.csv' )<set_options>
all_data.drop(['PassengerId', 'Name', 'Ticket'], axis=1, inplace=True)
 data_dummies = pd.get_dummies(all_data)
 X_train = data_dummies[data_dummies.Survived.notnull() ].drop(['Survived'], axis=1)
 y_train = data_dummies[data_dummies.Survived.notnull() ].Survived
X_test = data_dummies[data_dummies.Survived.isnull() ].drop(['Survived'], axis=1 )
warnings.filterwarnings('ignore')
 %matplotlib inline<load_from_csv>
all_data.Title = all_data.Title.apply(lambda x: 'Others' if x in list(all_data.Title.value_counts() [all_data.Title.value_counts() < 8].index)else x)
 all_data.TicketType = all_data.TicketType.apply(lambda x: 'Others' if x in list(all_data.TicketType.value_counts() [all_data.TicketType.value_counts() < 10].index)else x )
df = pd.read_csv('.. /input/sputnik/train.csv')
 df['epoch'] = pd.to_datetime(df.epoch)
 df.index = df.epoch
df.drop('epoch', axis = 1, inplace = True)
 df.head()<filter>
data_dummies = pd.get_dummies(all_data)
 X_train = data_dummies[data_dummies.Survived.notnull() ].drop(['Survived'], axis=1)
 X_test = data_dummies[data_dummies.Survived.isnull() ].drop(['Survived'], axis=1 )
train = df.loc[df['type'] == 'train']<feature_engineering>
allow_tuning = False
train['error'] = np.linalg.norm(train[['x', 'y', 'z']].values - train[['x_sim', 'y_sim', 'z_sim']].values, axis=1 )<filter>
def xgb_gridsearch(params_grid_xgb, features, values, X, y, last=False):
x_train, x_test = train_test_split(X, test_size=.2, random_state=42)
 y_train_tmp, y_test_tmp = train_test_split(y, test_size=.2, random_state=42)
 cv = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 3, random_state = 42)
 model_xgb = XGBClassifier(use_label_encoder = False,
objective = 'binary:logistic')
 for i in range(len(features)) :
params_grid_xgb[features[i]] = values[i]
search_xgb = GridSearchCV(model_xgb, params_grid_xgb, verbose = 0,
scoring = 'neg_log_loss', cv = cv ).fit(x_train, y_train_tmp, early_stopping_rounds = 15,
eval_set = [[x_test, y_test_tmp]],
eval_metric = 'logloss', verbose = False)
 for i in range(len(features)) :
print(f"{features[i]}: {search_xgb.best_params_[features[i]]}")
 if not last:
for k, v in search_xgb.best_params_.items() :
search_xgb.best_params_[k] = [v]
return search_xgb, search_xgb.best_params_
test = df.loc[df['type'] == 'test']<define_variables>
if allow_tuning:
params_knn = {
'n_neighbors' : range(1, 10),
'weights' : ['uniform', 'distance'],
'algorithm' : ['auto', 'ball_tree','kd_tree'],
'p' : [1,2]
}
model_knn = knn()
search_knn = GridSearchCV(model_knn, params_knn, cv=5, scoring='accuracy', n_jobs=-1, verbose=1 ).fit(X_train, y_train)
 print(search_knn.best_params_ )
train_frames = []
for i in range(600):
train_frames.append(train.loc[train['sat_id'] == i] )<filter>
if allow_tuning:
params_logistic = {
'max_iter': [2000],
'penalty': ['l1', 'l2'],
'C': np.logspace(-4, 4, 20),
'solver': ['liblinear']
}
model_logistic = LogisticRegression()
search_logistic = GridSearchCV(model_logistic, params_logistic, cv=5, scoring='accuracy', n_jobs=-1, verbose=1 ).fit(X_train, y_train)
 print(search_logistic.best_params_ )
test_frames = []
for i in range(600):
test_frames.append(test.loc[test['sat_id'] == i] )<choose_model_class>
if allow_tuning:
params_svc = [{'kernel': ['rbf'], 'gamma': [.01,.1,.5, 1, 2, 5, 10], 'C': [.1, 1, 10, 100, 1000], 'probability': [True]},
{'kernel': ['poly'], 'degree' : [2, 3, 4, 5], 'C': [.01,.1, 1, 10, 100, 1000], 'probability': [True]}]
model_svc = SVC()
search_svc = GridSearchCV(model_svc, params_svc, cv=5, scoring='accuracy', n_jobs=-1, verbose=1 ).fit(X_train, y_train)
 print(search_svc.best_params_ )
season=24<prepare_x_and_y>
if allow_tuning:
params_svc = {'kernel': ['rbf'], 'gamma': [i/10000 for i in range(90, 110)], 'C': range(50, 80, 10), 'probability': [True]}
model_svc = SVC()
search_svc = GridSearchCV(model_svc, params_svc, cv=5, scoring='accuracy', n_jobs=-1, verbose=1 ).fit(X_train, y_train)
 print(search_svc.best_params_ )
sample_data=train_frames[0].x<compute_test_metric>
test_stationarity(sample_data )<split>
if allow_tuning:
params_rf = {
'n_estimators': [95, 100, 105],
'criterion':['entropy'],
'bootstrap': [True, False],
'max_depth': [40, 45, 50],
'max_features': [4, 5, 6],
'min_samples_leaf': [1, 2, 3],
'min_samples_split': [9, 10, 11],
'random_state': [734]}
model_rf = RandomForestClassifier()
search_rf = GridSearchCV(model_rf, params_rf, cv=5, scoring='accuracy', n_jobs=-1, verbose=1 ).fit(X_train, y_train)
 search_rf.best_params_['random_state']=242
search_rf.best_estimator_.random_state=242
print(search_rf.best_params_ )
rcParams['figure.figsize'] = 12, 7
sample_data_diff = sample_data - sample_data.shift(1)
 sample_data_diff.dropna(inplace = True)
 test_stationarity(sample_data_diff, window = 48 )<define_variables>
if allow_tuning:
params_xgb = {'n_estimators': [1000],
'learning_rate': [0.1],
'max_depth': [5],
'min_child_weight': [1],
'gamma': [0],
'subsample': [0.8],
'colsample_bytree': [0.8],
'n_jobs': [-1],
'objective': ['binary:logistic'],
'use_label_encoder': [False],
'eval_metric': ['logloss'],
'scale_pos_weight': [1]}
search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['learning_rate'],
[[0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15, 0.2]],
X_train, y_train)
 search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['max_depth', 'min_child_weight'],
[range(3, 10), range(1, 6)],
X_train, y_train)
 search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['gamma'],
[[0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 2]],
X_train, y_train)
 search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['subsample', 'colsample_bytree'],
[[i/100.0 for i in range(75,90,5)], [i/100.0 for i in range(75,90,5)]],
X_train, y_train)
 search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['reg_alpha'],
[[1e-5, 1e-2, 0.1, 1, 100]],
X_train, y_train)
 params_xgb['n_estimators'] = [5000]
search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['learning_rate'],
[[0.001, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15, 0.2]],
X_train, y_train, last=True)
 x_train, x_test = train_test_split(X_train, test_size=.2, random_state=42)
 y_train_tmp, y_test_tmp = train_test_split(y_train, test_size=.2, random_state=42)
 model_xgb = XGBClassifier(**params_xgb)
 model_xgb = model_xgb.fit(x_train, y_train_tmp, eval_set=[(x_test, y_test_tmp)], eval_metric=['logloss'], early_stopping_rounds=15, verbose=0)
 search_xgb.best_estimator_.n_estimators = model_xgb.best_iteration
d=1
D=0<import_modules>
if allow_tuning:
model_knn = search_knn.best_estimator_
model_logistic = search_logistic.best_estimator_
model_svc = search_svc.best_estimator_
model_rf = search_rf.best_estimator_
model_xgb = search_xgb.best_estimator_
else:
model_knn = knn(algorithm='auto',
n_neighbors=9,
p=1,
weights='uniform')
 model_logistic = LogisticRegression(C=0.08858667904100823,
max_iter=2000,
penalty='l2',
solver='liblinear')
 model_svc = SVC(C=70,
gamma=0.0106,
kernel='rbf',
probability=True)
 model_rf = RandomForestClassifier(bootstrap=True,
criterion='entropy',
max_depth=50, max_features=6,
min_samples_leaf=1,
min_samples_split=10,
n_estimators=100,
random_state=734)
 model_xgb = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.8,
enable_categorical=False, eval_metric='logloss', gamma=0.8,gpu_id=-1, importance_type=None, interaction_constraints='',
learning_rate=0.15, max_delta_step=0, max_depth=5,
min_child_weight=1, missing=np.nan, monotone_constraints='() ',
n_estimators=15, n_jobs=-1, num_parallel_tree=1, predictor='auto',
random_state=0, reg_alpha=1e-05, reg_lambda=1, scale_pos_weight=1,
subsample=0.8, tree_method='exact', use_label_encoder=False,
validate_parameters=1, verbosity=0)
 models = {
'knn': model_knn,
'logistic': model_logistic,
'svc': model_svc,
'rf': model_rf,
'xgb': model_xgb
}
from scipy.stats import linregress<compute_train_metric>
def select_models(start, cnt, goal, estimators, voting):
if cnt == goal:
estimators_copy = copy.deepcopy(estimators)
 voting_name = f'{voting}_' + '_'.join([i[0] for i in list(estimators_copy)])
 models[voting_name] = VotingClassifier(estimators=estimators_copy, voting=voting)
 return
for i in range(start, 5):
estimators.append(list(models.items())[i])
 select_models(i + 1, cnt + 1, goal, estimators, voting)
 estimators.pop()
def prediction(train,test,period = 24):
count = test.shape[0]//period
left = test.shape[0]%period
trend = train.error[-47:].rolling(window=24 ).mean()
os1 =(train.iloc[-period:].error.values ).tolist() *count
if left != 0:
os1 += train.iloc[-period:-(period-left)].error.values.tolist()
seas = np.array(os1)
 a0,*_ = linregress(np.arange(24),trend.dropna().values)
 seas+=np.array(range(seas.shape[0])*a0)
 return seas<predict_on_test>
select_models(0, 0, 2, [], 'hard')
 select_models(0, 0, 3, [], 'hard')
 select_models(0, 0, 4, [], 'hard')
 select_models(0, 0, 5, [], 'hard')
 select_models(0, 0, 2, [], 'soft')
 select_models(0, 0, 3, [], 'soft')
 select_models(0, 0, 4, [], 'soft')
 select_models(0, 0, 5, [], 'soft' )
pr = prediction(train_frames[0],test_frames[0] )<prepare_x_and_y>
result_by_model = pd.DataFrame({'model name': models.keys() , 'model': models.values() , 'score': 0} )
y = np.hstack(( np.asarray(train_frames[0].error), pr))<concatenate>
for name, model in models.items() :
result_by_model.loc[result_by_model['model name'] == name, 'score'] = cross_val_score(model, X_train,y_train,cv=5 ).mean()
x = np.hstack(( np.asarray(train_frames[0].index),(np.asarray(test_frames[0].index))))<predict_on_test>
result_by_model.sort_values('score', ascending=False ).reset_index(drop=True )
<define_variables><EOS> 
model_name = 'rf'
models[model_name].fit(X_train, y_train)
 y_pred = models[model_name].predict(X_test ).astype('int')
 submission = pd.DataFrame({'PassengerId': test.PassengerId,
'Survived': y_pred})
 submission.to_csv('submission.csv', index = False )