Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
warnings.filterwarnings('ignore' )
stage = 1
csvlogger = callbacks.CSVLogger(learn=learn, filename='history_stage_'+str(stage)+'_'+model_name, append=True)
 saveModel = callbacks.SaveModelCallback(learn, every='epoch',
monitor='top_3_accuracy', mode='max',
name='stage_'+str(stage))
reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'top_3_accuracy', mode = 'max', patience = 1, factor = 0.5)
<train_model>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )
lr = 3e-3
learn.fit_one_cycle(4, slice(lr))<save_model>
def merge_data(train, test):
return pd.concat([train, test], sort = True ).reset_index(drop=True)
 def divide_data(data):
return data.iloc[:891], data.iloc[891:].drop(['Survived'], axis = 1)
 data = merge_data(train, test )
learn.save('stage-1-SE_Resnext101' )<choose_model_class>
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 data['Title'].unique()
stage = 2
csvlogger = callbacks.CSVLogger(learn=learn, filename='history_stage_'+str(stage)+'_'+model_name, append=True)
 saveModel = callbacks.SaveModelCallback(learn, every='epoch',
monitor='top_3_accuracy', mode='max',
name='stage_'+str(stage))
reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'top_3_accuracy', mode = 'max', patience = 1, factor = 0.5)
<train_model>
data.groupby('Title')['Sex'].count()
data['Title'] = data['Title'].replace(['Capt', 'Col', 'Countess', 'Don', 'Dr',
'Jonkheer', 'Major', 'Sir', 'Rev', 'Dona'], 'Rare')
 data['Title'] = data['Title'].replace(['Lady', 'Mlle', 'Mme', 'Ms'],
['Mrs', 'Miss', 'Miss', 'Mrs'] )
learn.fit_one_cycle(10 , slice(1e-5, 1e-3))<save_model>
def family_survival() :
data['Last_Name'] = data['Name'].apply(lambda x: str.split(x, ",")[0])
 default_survival_rate = 0.5
data['Family_survival'] = default_survival_rate
for grp, grp_df in data[['Survived', 'Name', 'Last_Name',
'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
ID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == ID,
'Family_survival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == ID,
'Family_survival'] = 0
for _, grp_df in data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_survival'] == 0)|(
 row['Family_survival'] == 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
ID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == ID,
'Family_survival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == ID,
'Family_survival'] = 0
return data
learn.save('stage-2-SE_Resnext101' )<define_variables>
test = ImageList.from_folder('.. /input/ifood-2019-fgvc6/test_set')
 len(test )<load_pretrained>
data['Age'] = data.groupby(['Title', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
learn.export('/tmp/export.pkl')
 learn = load_learner('/tmp/', test=test)
 preds, _ = learn.get_preds(ds_type=DatasetType.Test )<define_variables>
def age_category(age):
if age <=2:
return 0
if 2 < age <= 18:
return 1
if 18 < age <= 35:
return 2
if 35 < age <= 65:
return 3
else:
return 4
data['Age'] = data['Age'].apply(age_category )
fnames = [f.name for f in learn.data.test_ds.items]
fnames[:4] 
<create_dataframe>
data['Age*Pclass'] = data['Age']*data['Pclass']
col = ['img_name']
test_df = pd.DataFrame(fnames, columns=col)
 test_df['label'] = ''<feature_engineering>
data[data['Fare'].isnull() ]
for i, pred in T(enumerate(predictions), total=len(predictions)) :
test_df.loc[i, 'label'] = ' '.join(str(int(i)) for i in np.argsort(pred)[::-1][:3] )<save_to_csv>
data.loc[data['Fare'].isnull() , 'Fare'] = data.loc[(data['Embarked'] == 'S')
 &(data['Pclass'] == 3)&(data['SibSp'] == 0)]['Fare'].median()
test_df.to_csv('submission_SE_Resnext101_fastai_mixup_2.csv', index=False )<save_to_csv>
data['Fare'].value_counts()
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html )<load_pretrained>
def fare_category(fare):
if fare <= 7.91:
return 0
if 7.91 < fare <= 14.454:
return 1
if 14.454 < fare <= 31:
return 2
if 31 < fare <= 99:
return 3
if 99 < fare <= 250:
return 4
else:
return 5
data['Fare'] = data['Fare'].apply(fare_category )
create_download_link(test_df, filename='submission_SE_Resnext101_fastai_mixup_2.csv' )<import_modules>
data[data['Embarked'].isnull() ]
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from keras.callbacks import ModelCheckpoint
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.normalization import BatchNormalization
from keras import optimizers
from keras import initializers
import numpy as np
from matplotlib import pyplot as plt<define_variables>
data.loc[(data['Fare'] < 80)&(data['Pclass'] == 1)]['Embarked'].value_counts()
data.loc[data['Embarked'].isnull() , 'Embarked'] = 'S'
classes = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']<load_from_csv>
folder = '/kaggle/input/fashion-mnist-itba-lab-2020/'
x = np.load(folder+'train_images.npy')
 y = np.loadtxt(folder+'train_labels.csv', delimiter=',', skiprows=1)
 x_test = np.load(folder+'test_images.npy' )<split>
x_train, x_valid, y_train, y_valid = train_test_split(x, y, test_size = 0.1 )<categorify>
def ticket_category(Ticket_length):
if Ticket_length <= 6:
return 0
if 6 < Ticket_length <= 10:
return 1
else:
return 2
y_train_categorical = to_categorical(y_train)
 y_val_categorical = to_categorical(y_valid )<choose_model_class>
data['Family_size'] = data['Parch'] + data['SibSp'] + 1
output_size = 10
model_single_layer = Sequential()
model_single_layer.add(Flatten(input_shape=x_train.shape[1:]))
model_single_layer.add(Dense(output_size, name='Salida'))
model_single_layer.add(Activation('softmax'))
model_single_layer.summary()<choose_model_class>
data[['Family_size', 'Survived']].groupby('Family_size' ).mean()
lr = 0.000001
SGD = optimizers.sgd(lr=lr)
 model_single_layer.compile(loss = 'categorical_crossentropy', optimizer=SGD, metrics=['accuracy'] )<train_model>
data['Single'] = data['Family_size'].map(lambda x: 1 if x == 1 else 0)
batch_size = 512
model_single_layer.fit(x_train,
y_train_categorical,
epochs=20, batch_size=batch_size,
verbose=1,
validation_data =(x_valid, y_val_categorical)
 )<compute_test_metric>
data['Is_Married'] = 0
data['Is_Married'] = data['Title'].map(lambda x: 1 if x == 'Mrs' else 0 )
loss, acc = model_single_layer.evaluate(x_valid, y_val_categorical, verbose=0)
 print(acc, loss )<predict_on_test>
data = data.drop(['Name', 'PassengerId', 'Ticket', 'Cabin', 'Last_Name'], axis = 1 )
test_prediction = model_single_layer.predict(x_test)
 print(test_prediction.shape )<prepare_output>
def object_to_int(df):
if df.dtype=='object':
df = LabelEncoder().fit_transform(df)
 return df
data = data.apply(lambda x: object_to_int(x))
test_labels = np.argmax(test_prediction, axis = 1)
 print(test_labels )<save_to_csv>
data= pd.get_dummies(data, columns = ['Age', 'Title', 'Embarked'], drop_first = True )
df = pandas.DataFrame(data={"Category": test_labels} ).astype(int)
 df.to_csv("./submission.csv", sep=',',index=True, index_label='Id' )<import_modules>
def add_polynomial_features(frame, poly_degree=2, interaction=False):
poly = PolynomialFeatures(degree = poly_degree, interaction_only = interaction, include_bias = False)
 poly_features = poly.fit_transform(frame[['Age', 'Name_length', 'Fare']])
 df_poly = pd.DataFrame(poly_features, columns = poly.get_feature_names())
 return pd.concat([frame, df_poly.drop(['x0'], axis=1)], axis=1)
import json
from sklearn.model_selection import train_test_split
from sklearn.base import BaseEstimator
from sklearn.metrics import f1_score
import scipy<load_from_csv>
train, test = divide_data(data )
train = pd.read_csv('/kaggle/input/finec-1941-hw6/train.csv', encoding='utf-8')
 test = pd.read_csv('/kaggle/input/finec-1941-hw6/test.csv', encoding='utf-8')
 for col in(['production_companies', 'production_countries', 'spoken_languages', 'cast']):
train[col] = train[col].apply(json.loads)
 test[col] = test[col].apply(json.loads)
 train.head().T<split>
X = train.drop(['Survived'], axis = 1)
 y = train['Survived']
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size = 0.25, random_state = 12345 )
x_train = train.drop(['index', 'genre_id'], axis=1)
 y_train = train['genre_id']
x_test = test.drop('index', axis=1)
 np.random.seed(514229)
 x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.33 )<train_model>
class ClusteringClassifier(BaseEstimator):
def __init__(self, clustering):

 self.clustering = clustering
def fit(self, x, y):
self.clustering.fit(x)
 clustered = self.clustering.predict(x)
 self.cluster_labels = {}
for c in np.unique(clustered):
ys = y[clustered == c]
most_popular_label = scipy.stats.mode(ys ).mode[0]
self.cluster_labels[c] = most_popular_label
return self
def predict(self, x):
clustered = self.clustering.predict(x)
 prediction = [self.cluster_labels[c] for c in clustered]
return np.array(prediction)
<compute_train_metric>
model = ClusteringClassifier(clustering=ConstClustering())
 model.fit(x_train, y_train)
 pred_train = model.predict(x_train)
 pred_val = model.predict(x_val)
 print('f1 train:', f1_score(y_train, pred_train, average='macro'))
print('f1 val:', f1_score(y_val, pred_val, average='macro'))<save_to_csv>
lr = LogisticRegression(random_state = 12345)
 parameters_lr = {'C': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 3, 5, 7, 10, 15, 20, 25, 30, 50],
'penalty':['l1', 'l2', 'elasticnet', 'none'],
'solver':['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'class_weight': [1, 3, 10],
'max_iter': [200, 500, 800, 1000, 2000]}
search_lr = RandomizedSearchCV(lr, parameters_lr, cv=5, scoring = 'accuracy', n_jobs = -1, random_state = 12345)
 search_lr.fit(X_train, y_train)
 best_lr = search_lr.best_estimator_
predict_lr = best_lr.predict(X_valid)
 auc_lr = cross_val_score(best_lr, X_valid, y_valid, scoring = 'roc_auc', cv = 10, n_jobs = -1)
 acc_lr = cross_val_score(best_lr, X_valid, y_valid, scoring = 'accuracy', cv = 10, n_jobs = -1)
 print('AUC-ROC for Logistic Regression on validation dataset:', sum(auc_lr)/len(auc_lr))
print('Accuracy for Logistic Regression on validation dataset:', sum(acc_lr)/len(acc_lr))
pred = model.predict(x_test)
 pd.Series(pred, index=test['index'], name='genre_id' ).to_frame().to_csv(f"default_const_submit.csv" )<import_modules>
dt = DecisionTreeClassifier(random_state = 12345)
 parameters_dt = {'criterion': ['gini', 'entropy'],
'max_depth':range(1, 100, 1),
'min_samples_leaf': range(1, 20),
'max_features':range(1, X_train.shape[1]+1)}
search_dt = RandomizedSearchCV(dt, parameters_dt, cv=5, scoring = 'accuracy', n_jobs = -1, random_state = 12345)
 search_dt.fit(X_train, y_train)
 best_dt = search_dt.best_estimator_
acc_dt = cross_val_score(best_dt, X_valid, y_valid, scoring = 'accuracy', cv = 10, n_jobs = -1)
 auc_dt = cross_val_score(best_dt, X_valid, y_valid, scoring = 'roc_auc', cv = 10, n_jobs = -1)
 print('AUC-ROC for Decision Tree on validation dataset:', sum(auc_dt)/len(auc_dt))
print('Accuracy for Decision Tree on validation dataset:', sum(acc_dt)/len(acc_dt))
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression as LR
from sklearn.metrics import mean_squared_error
import lightgbm as lgb
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV<set_options>
confusion_m(best_dt, 'Confusion matrix for Logistic Regression')
 feature_importance(best_dt.feature_importances_, 'Feature importance for Decision Tree' )
warnings.filterwarnings("ignore" )<compute_test_metric>
rf = RandomForestClassifier(random_state = 12345)
 parameters_rf = {'n_estimators': range(1, 1800, 25),
'criterion': ['gini', 'entropy'],
'max_depth':range(1, 100),
'min_samples_split': range(1, 12),
'min_samples_leaf': range(1, 12),
'max_features':['auto', 'log2', 'sqrt', 'None']}
search_rf = RandomizedSearchCV(rf, parameters_rf, cv=5, scoring = 'accuracy', n_jobs = -1, random_state = 12345)
 search_rf.fit(X_train, y_train)
 best_rf = search_rf.best_estimator_
predict_rf = best_rf.predict(X_valid)
 auc_rf = cross_val_score(best_rf, X_valid, y_valid, scoring = 'roc_auc', cv = 10, n_jobs = -1)
 acc_rf = cross_val_score(best_rf, X_valid, y_valid, scoring = 'accuracy', cv = 10, n_jobs = -1)
 print('AUC-ROC for Random Forest on validation dataset:', sum(auc_rf)/len(auc_rf))
print('Accuracy for Random Forest on validation dataset:', sum(acc_rf)/len(acc_rf))
def rsi(values):
up = values[values>0].mean()
down = -1*values[values<0].mean()
return 100 * up /(up + down)
<feature_engineering>
confusion_m(best_rf, 'Confusion matrix for Random Forest')
 feature_importance(best_rf.feature_importances_, 'Feature importance for Random Forest' )
def abands(df):
df['AB_Middle_Band'] = df['close'].rolling(window = 20, center=False ).mean()
df['aupband'] = df['high'] *(1 + 4 *(df['high']-df['low'])/(df['high']+df['low']))
df['AB_Upper_Band'] = df['aupband'].rolling(window=20, center=False ).mean()
df['adownband'] = df['low'] *(1 - 4 *(df['high']-df['low'])/(df['high']+df['low']))
df['AB_Lower_Band'] = df['adownband'].rolling(window=20, center=False ).mean()<feature_engineering>
xgb = XGBClassifier(random_state = 12345, eval_metric='auc')
 parameters_xgb = {'eta': [0.01, 0.05, 0.1, 0.001, 0.005, 0.04, 0.2, 0.0001],
'min_child_weight':range(1, 5),
'max_depth':range(1, 6),
'learning_rate': [0.01, 0.05, 0.1, 0.001, 0.005, 0.04, 0.2],
'n_estimators':range(0, 2001, 50)}
search_xgb = RandomizedSearchCV(xgb, parameters_xgb, cv = 5, scoring = 'accuracy', n_jobs = -1, random_state = 12345)
 search_xgb.fit(X_train, y_train)
 best_xgb = search_xgb.best_estimator_
predict_xgb = best_xgb.predict(X_valid)
 auc_xgb = cross_val_score(best_xgb, X_valid, y_valid, scoring = 'roc_auc', cv = 10, n_jobs = -1)
 acc_xgb = cross_val_score(best_xgb, X_valid, y_valid, scoring = 'accuracy', cv = 10, n_jobs = -1)
 print('AUC-ROC for XGBoost on validation dataset:', sum(auc_xgb)/len(auc_xgb))
print('Accuracy for XGBoost on validation dataset:', sum(acc_xgb)/len(acc_xgb))
def STOK(df, n):
df['STOK'] =(( df['close'] - df['low'].rolling(window=n, center=False ).mean())/(df['high'].rolling(window=n, center=False ).max() - df['low'].rolling(window=n, center=False ).min())) * 100
df['STOD'] = df['STOK'].rolling(window = 3, center=False ).mean()<categorify>
confusion_m(best_xgb, 'Confusion matrix for XGBoost')
 feature_importance(best_xgb.feature_importances_, 'Feature importance for XGBoost' )
def psar(df, iaf = 0.02, maxaf = 0.2):
length = len(df)
 high =(df['high'])
 low =(df['low'])
 close =(df['close'])
 psar = df['close'][0:len(df['close'])]
psarbull = [None] * length
psarbear = [None] * length
bull = True
af = iaf
ep = df['low'][0]
hp = df['high'][0]
lp = df['low'][0]
for i in range(2,length):
if bull:
psar[i] = psar[i - 1] + af *(hp - psar[i - 1])
 else:
psar[i] = psar[i - 1] + af *(lp - psar[i - 1])
 reverse = False
if bull:
if df['low'][i] < psar[i]:
bull = False
reverse = True
psar[i] = hp
lp = df['low'][i]
af = iaf
else:
if df['high'][i] > psar[i]:
bull = True
reverse = True
psar[i] = lp
hp = df['high'][i]
af = iaf
if not reverse:
if bull:
if df['high'][i] > hp:
hp = df['high'][i]
af = min(af + iaf, maxaf)
 if df['low'][i - 1] < psar[i]:
psar[i] = df['low'][i - 1]
if df['low'][i - 2] < psar[i]:
psar[i] = df['low'][i - 2]
else:
if df['low'][i] < lp:
lp = df['low'][i]
af = min(af + iaf, maxaf)
 if df['high'][i - 1] > psar[i]:
psar[i] = df['high'][i - 1]
if df['high'][i - 2] > psar[i]:
psar[i] = df['high'][i - 2]
if bull:
psarbull[i] = psar[i]
else:
psarbear[i] = psar[i]
df['psar'] = psar 
<feature_engineering>
cb = CatBoostClassifier(random_state = 12345, iterations = 300, eval_metric='Accuracy', verbose = 100)
 parameters_cb = {'depth': range(6, 11),
'learning_rate': [0.001, 0.01, 0.05, 0.1, 0.0001]}
search_cb = RandomizedSearchCV(cb, parameters_cb, cv = 5, scoring = 'accuracy', n_jobs = -1, random_state = 12345)
 search_cb.fit(X_train, y_train, verbose = 100)
 best_cb = search_cb.best_estimator_
predict_cb = best_cb.predict(X_valid)
 auc_cb = cross_val_score(best_cb, X_valid, y_valid, scoring = 'roc_auc', cv = 10, n_jobs = -1)
 acc_cb = cross_val_score(best_cb, X_valid, y_valid, scoring = 'accuracy', cv = 10, n_jobs = -1)
 print('AUC-ROC for CatBoost on validation dataset:', sum(auc_cb)/len(auc_cb))
print('Accuracy for CatBoost on validation dataset:', sum(acc_cb)/len(acc_cb))
window = 12
def preprocess(data_in):
starting_ids = []
data = data_in.copy()
for k in range(5):
starting_ids.append(data[data.asset == k]['id'].min())
 data['diff'] = data['high'] - data['low']
data['diff*volume'] = data['diff'] * data['volume']
data ['vol/trades'] = data.volume / data.trades
data.drop('id',axis = 1, inplace = True)
 data.drop('asset',axis = 1, inplace = True)
 data['Momentum_1D'] =(data['close']-data['close'].shift(1)).fillna(0)
 data['RSI_14D'] = data['Momentum_1D'].rolling(center=False, window=14 ).apply(rsi ).fillna(0)
 data['RSI_1D'] = data['Momentum_1D'].rolling(center=False, window=1 ).apply(rsi ).fillna(0)
 data['RSI_2D'] = data['Momentum_1D'].rolling(center=False, window=2 ).apply(rsi ).fillna(0)
 data['RSI_4D'] = data['Momentum_1D'].rolling(center=False, window=4 ).apply(rsi ).fillna(0)
 data['RSI_8D'] = data['Momentum_1D'].rolling(center=False, window=8 ).apply(rsi ).fillna(0)
 data['BB_Middle_Band'], data['BB_Upper_Band'], data['BB_Lower_Band'] = bbands(data['close'], length=20, numsd=1)
 data['BB_Middle_Band'] = data['BB_Middle_Band'].fillna(0)
 data['BB_Upper_Band'] = data['BB_Upper_Band'].fillna(0)
 data['BB_Lower_Band'] = data['BB_Lower_Band'].fillna(0)
 listofzeros = [0] * 25
up, down = aroon(data)
 aroon_list = [x - y for x, y in zip(up,down)]
if len(aroon_list)==0:
aroon_list = [0] * data.shape[0]
data['Aroon_Oscillator'] = aroon_list
else:
data['Aroon_Oscillator'] = listofzeros+aroon_list
data["PVT"] =(data['Momentum_1D']/ data['close'].shift(1)) *data['volume']
data["PVT"] = data["PVT"] - data["PVT"].shift(1)
 data["PVT"] = data["PVT"].fillna(0)
 abands(data)
 data.fillna(0)
 STOK(data, 4)
 data.fillna(0)
 data['ROC'] =(( data['close'] - data['close'].shift(12)) /(data['close'].shift(12)))*100
data.fillna(0)
 data['VWAP'] = np.cumsum(data['volume'] *(data['high'] + data['low'])/2)/ np.cumsum(data['volume'])
 data.fillna(0)
 scaler = StandardScaler(with_std=False)
 data = scaler.fit_transform(data)
 return data<load_from_csv>
confusion_m(best_cb, 'Confusion matrix for CatBoost')
 feature_importance(best_cb.feature_importances_, 'Feature importance for CatBoost' )
data = pd.read_csv('.. /input/train.csv')
 target = data.pop('y')
 data_test = pd.read_csv('.. /input/test.csv' )<define_variables>
et = ExtraTreesClassifier(random_state = 12345)
 parameters_et = {'n_estimators': range(50, 501, 25),
'criterion': ['gini', 'entropy'],
'max_depth':range(1, 100),
'min_samples_split': range(1, 12),
'min_samples_leaf': range(1, 12),
'max_features':['auto', 'log2', 'sqrt', 'None']}
search_et = RandomizedSearchCV(et, parameters_et, cv=5, scoring = 'accuracy', n_jobs = -1, random_state = 12345)
 search_et.fit(X_train, y_train)
 best_et = search_et.best_estimator_
predict_et = best_et.predict(X_valid)
 auc_et = cross_val_score(best_et, X_valid, y_valid, scoring = 'roc_auc', cv = 10, n_jobs = -1)
 acc_et = cross_val_score(best_et, X_valid, y_valid, scoring = 'accuracy', cv = 10, n_jobs = -1)
 print('AUC-ROC for Extra Trees on validation dataset:', sum(auc_et)/len(auc_et))
print('Accuracy for Extra Trees on validation dataset:', sum(acc_et)/len(acc_et))
target_by_asset = []
test_by_asset = []
train_by_asset = []
PredID = []
preds_df = []
scores = []
best_params = []
grid_scores = []
grid_preds_df = []
for k in range(5):
train_by_asset.append(data[data['asset']==k])
 target_by_asset.append(target[data['asset']==k])
 PredID.append(data_test[data_test['asset']==k]['id'])
 train_by_asset[k] = preprocess(train_by_asset[k])
 test_by_asset.append(data_test[data_test['asset']==k])
 test_by_asset[k] = preprocess(test_by_asset[k])
 X_train, X_test, y_train, y_test = train_test_split(train_by_asset[k], target_by_asset[k], test_size=0.33, random_state=777, shuffle=False)
 lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
 params = {
'boosting_type': 'gbdt',
'colsample_bytree': 0.65,
'learning_rate': 0.005,
'n_estimators': 40,
'num_leaves': 16,
'objective': 'regression',
'random_state': 501,
'reg_alpha': 1,
'reg_lambda': 0,
'subsample': 0.7,
'max_depth' : -1,
'max_bin': 512,
'subsample_for_bin': 200,
'subsample_freq': 1,
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 5,
'scale_pos_weight': 1,
}
gbm = lgb.train(params,
lgb_train,
num_boost_round=20,
valid_sets=lgb_eval,
early_stopping_rounds=5)
 mdl = lgb.LGBMRegressor(boosting_type= 'gbdt',
objective = 'regrssion',
n_jobs = -1,
silent = True,
max_depth = params['max_depth'],
max_bin = params['max_bin'],
subsample_for_bin = params['subsample_for_bin'],
subsample = params['subsample'],
subsample_freq = params['subsample_freq'],
min_split_gain = params['min_split_gain'],
min_child_weight = params['min_child_weight'],
min_child_samples = params['min_child_samples'],
scale_pos_weight = params['scale_pos_weight'])
 gridParams = {
'learning_rate': [0.005],
'n_estimators': [40],
'num_leaves': [6,8,12,16],
'boosting_type' : ['gbdt'],
'objective': ['regression'],
'random_state' : [501],
'colsample_bytree' : [0.65, 0.66],
'subsample' : [0.7,0.75],
'reg_alpha' : [0,1,1.2],
'reg_lambda' : [0,1,1.2,1.4],
}
grid = RandomizedSearchCV(mdl, gridParams,
verbose=0,
cv=4,
n_jobs=-1)
 grid.fit(train_by_asset[k], target_by_asset[k])
 print(grid.best_params_)
 print(grid.best_score_)
 best_params.append(grid.best_params_)
 grid_scores.append(grid.best_score_)
 grid_pred = grid.predict(test_by_asset[k])
 grid_pred[:12] = np.zeros(12)
 out = pd.DataFrame(PredID[k])
 out['expected'] = grid_pred
grid_preds_df.append(out)
 y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)
 print('The rmse of prediction is:', mean_squared_error(y_test, y_pred)** 0.5)
 scores.append(mean_squared_error(y_test, y_pred)** 0.5)
 preds = gbm.predict(test_by_asset[k], num_iteration=gbm.best_iteration)
 preds[:12] = np.zeros(12)
 out = pd.DataFrame(PredID[k])
 out['expected'] = preds
preds_df.append(out )<save_to_csv>
confusion_m(best_et, 'Confusion matrix for ExtraTreesClassifier')
 feature_importance(best_et.feature_importances_, 'Feature importance for ExtraTreesClassifier' )
out_df = pd.DataFrame(np.concatenate(preds_df), columns=['id','expected'])
 out_df.id = out_df.id.astype('int')
 out_df.to_csv('by_asset_lgbm.csv',index = False )<save_to_csv>
vc = VotingClassifier(estimators=[('lr', best_lr),('xgb', best_xgb),('rf', best_rf),('et', best_et),('cb', best_cb)], voting='soft')
 vc.fit(X_train, y_train)
 predict_vc = vc.predict(X_valid)
 auc_vc = cross_val_score(vc, X_valid, y_valid, scoring = 'roc_auc', cv = 5, n_jobs = -1)
 acc_vc = cross_val_score(vc, X_valid, y_valid, scoring = 'accuracy', cv = 5, n_jobs = -1)
 print('AUC-ROC for ensemble models on validation dataset:', sum(auc_vc)/len(auc_vc))
print('Accuracy for ensemble models on validation dataset:', sum(acc_vc)/len(acc_vc))
out_df = pd.DataFrame(np.concatenate(grid_preds_df), columns=['id','expected'])
 out_df.id = out_df.id.astype('int')
 out_df.to_csv('grid_by_asset.csv',index = False )<define_variables>
confusion_m(vc, 'Confusion matrix for VotingClassifier' )
regr = ElasticNetCV(cv=5, random_state=0, l1_ratio = [.1,.5,.7,.9,.95,.99, 1])
 scores = []
target_by_asset = []
test_by_asset = []
train_by_asset = []
PredID = []
preds_df = []
scores = []
best_params = []
grid_scores = []
grid_preds_df = []
for k in range(5):
train_by_asset.append(data[data['asset']==k])
 target_by_asset.append(target[data['asset']==k])
 PredID.append(data_test[data_test['asset']==k]['id'])
 train_by_asset[k] = preprocess(train_by_asset[k])
 test_by_asset.append(data_test[data_test['asset']==k])
 test_by_asset[k] = preprocess(test_by_asset[k])
 X_train, X_test, y_train, y_test = train_test_split(np.nan_to_num(train_by_asset[k],0), target_by_asset[k], test_size=0.33, random_state=777, shuffle=False)
 regr.fit(X_train,y_train)
 y_pred = regr.predict(X_test)
 print('The rmse of prediction is:', mean_squared_error(y_test, y_pred)** 0.5)
 scores.append(mean_squared_error(y_test, y_pred)** 0.5)
 preds = regr.predict(np.nan_to_num(test_by_asset[k],0))
out = pd.DataFrame(PredID[k])
 out['expected'] = preds
preds_df.append(out )<save_to_csv>
models = ['logistic_regression', 'decision_tree', 'random_forest',
'xgboost', 'catboost',
'extra_trees', 'voting']
dict_values = {'accuracy': [acc_lr.mean() , acc_dt.mean() , acc_rf.mean() ,
acc_xgb.mean() , acc_cb.mean() ,
acc_et.mean() , acc_vc.mean() ],
'auc_roc': [auc_lr.mean() , auc_dt.mean() , auc_rf.mean() ,
auc_xgb.mean() , auc_cb.mean() ,
auc_et.mean() , auc_vc.mean() ]}
df_score = pd.DataFrame(dict_values, index = models, columns = ['accuracy', 'auc_roc'])
 df_score
out_df = pd.DataFrame(np.concatenate(preds_df), columns=['id','expected'])
 out_df.id = out_df.id.astype('int')
 out_df.to_csv('Elastic_net.csv',index = False )<import_modules>
X_train = X_train.values
test = test.values
ntrain = X_train.shape[0]
ntest = test.shape[0]
n_folds = 5
kf = KFold(n_splits = n_folds, random_state = 12345)
 def get_oof(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( n_folds, ntest))
for i,(train_index, test_index)in enumerate(kf.split(x_train)) :
x_tr = x_train[train_index]
y_tr = y_train.iloc[train_index]
x_te = x_train[test_index]
clf.fit(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )
import pandas as pd
from pathlib import Path
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression<load_from_csv>
et_oof_train, et_oof_test = get_oof(best_et, X_train, y_train, test)
 rf_oof_train, rf_oof_test = get_oof(best_rf,X_train, y_train, test)
 cb_oof_train, cb_oof_test = get_oof(best_cb, X_train, y_train, test)
 xgb_oof_train, xgb_oof_test = get_oof(best_xgb,X_train, y_train, test)
 lr_oof_train, lr_oof_test = get_oof(best_lr,X_train, y_train, test )
data_dir = Path('.. /input')
 train_df = pd.read_csv(data_dir / 'train.csv')
 test_df = pd.read_csv(data_dir / 'test.csv')
 sample_submission = pd.read_csv(data_dir / 'sampleSubmission.csv' )<define_variables>
X_train = np.concatenate(( et_oof_train, rf_oof_train, cb_oof_train, xgb_oof_train, lr_oof_train), axis=1)
 X_test = np.concatenate(( et_oof_test, rf_oof_test, cb_oof_test, xgb_oof_test, lr_oof_test), axis=1 )
features = ['di_minus', 'rsi', 'cci']<prepare_x_and_y>
best_model = XGBClassifier(n_estimators= 2000, max_depth= 4, min_child_weight= 2, gamma=0.9,
subsample=0.8, colsample_bytree=0.8, objective= 'binary:logistic',
nthread= -1,scale_pos_weight=1 ).fit(X_train, y_train )
X_train = train_df[features]
y_train = train_df['y']
X_test = test_df[features]<normalization>
y_test = best_model.predict(X_test )
scaler = StandardScaler(with_std=False)
 X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<train_model>
submission = pd.DataFrame({'PassengerId': list(range(892, 1310)) , 'Survived': y_test})
 submission['Survived'] = submission['Survived'].astype(int)
 submission.to_csv('submission.csv', index=False)
 print(submission )