Kaggle competition: Titanic - Machine Learning from Disaster
train_data['cont13_cont4_mul'] = train_data['cont13']*train_data['cont4']
train_data['cont13_cont11_mul'] = train_data['cont13']*train_data['cont11']
train_data['cont13_cont7_mul'] = train_data['cont13']*train_data['cont7']
train_data['cont13_cont2_mul'] = train_data['cont13']*train_data['cont2']
train_data['cont13_cont10_mul'] = train_data['cont13']*train_data['cont10']
test_data['cont13_cont4_mul'] = test_data['cont13']*test_data['cont4']
test_data['cont13_cont11_mul'] = test_data['cont13']*test_data['cont11']
test_data['cont13_cont7_mul'] = test_data['cont13']*test_data['cont7']
test_data['cont13_cont2_mul'] = test_data['cont13']*test_data['cont2']
test_data['cont13_cont10_mul'] = test_data['cont13']*test_data['cont10']<categorify>
df_train = pd.read_csv('/kaggle/input/titanic/train.csv')
 df_train.head()
num_bins = int(1 + np.log2(len(train_data)))
 train_data.loc[:,'bins'] = pd.cut(train_data['target'].to_numpy() ,bins=num_bins,labels=False)
 features = [f'cont{x}' for x in range(1,15)]
features += [
'cont13_cont4_mul',
'cont13_cont11_mul',
'cont13_cont7_mul',
'cont13_cont2_mul',
'cont13_cont10_mul',
]
target_feature = 'target'
train_data = train_data.query('target >=5')
 bins = train_data['bins'].to_numpy()
target = train_data[target_feature].to_numpy()
train_data = train_data[features].to_numpy()
test_data = test_data[features].to_numpy()
scaler = PowerTransformer()
train_data = scaler.fit_transform(train_data)
 test_data = scaler.transform(test_data )<compute_test_metric>
df_train.isnull().sum()
def rmse_score(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))<init_hyperparams>
df_train['Survived'].value_counts()
nfolds = 5
seed = 42
lgb_params={'objective':'regression',
'metrics':'rmse',
'boosting':'gbdt',
'min_data_per_group': 5,
'num_leaves': 256,
'max_depth': -1,
'learning_rate': 0.005,
'subsample_for_bin': 200000,
'lambda_l1': 1.074622455507616e-05,
'lambda_l2': 2.0521330798729704e-06,
'n_jobs': -1,
'cat_smooth': 1.0,
'verbose':-1,
'feature_pre_filter': False,
'bagging_fraction': 0.8206341150202605,
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 0.001,
'bagging_freq': 6,
'feature_fraction': 0.5,
'min_gain_to_split': 0.0,
'min_child_samples': 20}
xgb_params = {'lambda': 0.0030282073258141168,
'alpha': 0.01563845128469084,
'colsample_bytree': 0.55,
'subsample': 0.7,
'learning_rate': 0.01,
'max_depth': 15,
'random_state': 2020,
'min_child_weight': 257,
}
cat_params = {'iterations': 4000,
'depth': 7,
'verbose':200,
'learning_rate': 0.03618554870098452,
'random_strength': 29,
'bagging_temperature': 0.5528282676740153,
'od_type': 'IncToDec',
'early_stopping_rounds':100,
'random_state': seed
}
lgb_final_params={'objective':'regression',
'metrics':'rmse',
'num_leaves' : 287,
'max_bin' : 817,
'learning_rate' : 0.017005497730644668,
'max_depth' : 5,
'min_child_weight' : 5,
'feature_fraction' : 0.40159350277696726,
'bagging_fraction' : 0.780876382367899,
'bagging_freq' : 8,
'min_child_samples' : 56,
'lambda_l1' : 0.011468211453769755,
'lambda_l2' : 0.00026850241464538264,
'boosting':'gbdt',
'verbose':-1}<train_model>
df_train['Survived'].value_counts() /len(df_train)*100
final_preds = np.zeros(test_data.shape[0])
 kfold = StratifiedKFold(n_splits=nfolds,random_state=seed)
 for f,(train_idx, valid_idx)in enumerate(kfold.split(X=train_data,y=bins)) :
print(f"Fold: {f}")
 X_train, X_valid, y_train, y_valid = train_data[train_idx],train_data[valid_idx],target[train_idx],target[valid_idx]
print(f"{r_}LGBM TRAIN{sr_}")
 lgb_train = lgb.Dataset(X_train,y_train)
 lgb_valid = lgb.Dataset(X_valid,y_valid,reference=lgb_train)
 lgb_model = lgb.train(lgb_params,
lgb_train,
valid_sets=[lgb_train,lgb_valid],
num_boost_round=10000,
verbose_eval=200,
early_stopping_rounds=100, 
)
 print(f"{g_}XGB TRAIN{sr_}")
 xgb_train = xgb.DMatrix(X_train,label=y_train)
 xgb_valid = xgb.DMatrix(X_valid,label=y_valid)
 xgb_model = xgb.train(xgb_params,
xgb_train,
10000,
verbose_eval=200,
evals=[(xgb_train,'train'),(xgb_valid,'valid')],
early_stopping_rounds=100)
 print(f"{y_}CAT TRAIN{sr_}")
 cat_train = Pool(X_train,y_train)
 cat_valid = Pool(X_valid,y_valid)
 cat_model = CatBoost(cat_params)
 cat_model.fit(cat_train,eval_set=cat_valid)
 print("STACKING")
 final_train = np.column_stack([lgb_model.predict(X_train),xgb_model.predict(xgb.DMatrix(X_train)) ,cat_model.predict(X_train)])
 final_valid = np.column_stack([lgb_model.predict(X_valid),xgb_model.predict(xgb.DMatrix(X_valid)) ,cat_model.predict(X_valid)])
 final_test = np.column_stack([lgb_model.predict(test_data),xgb_model.predict(xgb.DMatrix(test_data)) ,cat_model.predict(test_data)])
 print(f"{b_}FINAL TRAINING{sr_}")
 lgb_train = lgb.Dataset(final_train,y_train)
 lgb_valid = lgb.Dataset(final_valid,y_valid)
 lgb_final = lgb.train(lgb_final_params, lgb_train, valid_sets=[lgb_train,lgb_valid],early_stopping_rounds=100,
verbose_eval=200, num_boost_round=1000)
 final_preds += lgb_final.predict(final_test)/nfolds<save_to_csv>
df_train = df_train.drop(['PassengerId','Ticket','Name','Cabin'],axis=1 )
sample.target = final_preds.ravel()
sample.to_csv("submission.csv",index=False)
 sample.head()<install_modules>
df_train['Sex'].value_counts() /len(df_train)*100
!pip install --upgrade xgboost
xgb.__version__<load_from_csv>
df_train[df_train['Survived']==0]['Sex'].value_counts() /len(df_train[df_train['Survived']==0])*100
sub = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv")
 data = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 final_test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv" )<train_model>
df_train[df_train['Survived']==1]['Sex'].value_counts() /len(df_train[df_train['Survived']==1])*100
print('Training Data')
 print(data.isnull().sum())
 print()
print()
print('Testing Data')
 print(final_test.isnull().sum() )<prepare_x_and_y>
df_train[df_train['Survived']==1]['Pclass'].value_counts() /len(df_train[df_train['Survived']==1])*100
columns = final_test.columns[1:]
train = data[columns]
target = data['target']<split>
df_train[df_train['Survived']==0]['Pclass'].value_counts() /len(df_train[df_train['Survived']==0])*100
x_train, x_test, y_train, y_test =train_test_split(
 train, target, random_state= 2021, test_size = 0.20)
 xgb_initial = xgb.XGBRegressor()
xgb_initial.fit(x_train, y_train)
 initial_preds = xgb_initial.predict(x_test )<compute_test_metric>
df_train['Family']=df_train['SibSp']+df_train['Parch']
mean_squared_error(y_test, initial_preds, squared=False)
<split>
df_train['Embarked'].value_counts() /len(df_train )
def objective(trial, X_data = train, Y_data = target):
x_train, x_test, y_train, y_test = train_test_split(
 X_data, Y_data, random_state= 2021, test_size = 0.20)
 param = {
'tree_method':'gpu_hist',
'predictor': 'gpu_predictor',
'learning_rate': trial.suggest_discrete_uniform('learning_rate',0.01,0.50,0.05),
'colsample_bylevel': trial.suggest_discrete_uniform('colsample_bylevel',0.01,0.91, 0.1),
'colsample_bynode': trial.suggest_discrete_uniform('colsample_bytree',0.01,0.91, 0.1),
'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree',0.01,0.91, 0.1),
'max_depth': trial.suggest_int('max_depth', 1,10),
'subsample': trial.suggest_discrete_uniform('subsample', 0.20,1, 0.05),
'n_estimators': trial.suggest_categorical('n_estimators',[4000,5000,6000,7000]),
'min_child_weight': trial.suggest_int('min_child_weight', 1,401,step=2),
'lambda': trial.suggest_loguniform('lambda', 1e-8, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-8, 10.0),
'random_state': trial.suggest_categorical('random_state',[2000,3000,4000]),
'gamma': trial.suggest_discrete_uniform('gamma',0.01,2.01, 0.1)
 }
model = xgb.XGBRegressor(**param)
 model.fit(x_train,y_train,eval_set=[(x_train,y_train)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(x_test)
 rmse = mean_squared_error(y_test, preds,squared=False)
 return rmse 
<find_best_params>
df_train[df_train['Survived']==1]['Embarked'].value_counts() /len(df_train[df_train['Survived']==1])*100
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials= 100 )<train_model>
df_train[df_train['Survived']==0]['Embarked'].value_counts() /len(df_train[df_train['Survived']==0])*100
print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params)
 print('Best objective value:', study.best_value)
<find_best_params>
df_train['Sex'] = pd.get_dummies(df_train['Sex'])
 df_train['Age'] = df_train['Age'].fillna(df_train['Age'].median())
 df_train['Embarked'] = df_train['Embarked'].map({'C':0,'Q':1,'S':2} )
best_trial = study.best_trial.params
best_trial['tree_method'] = 'gpu_hist'
best_trial['predictor'] = 'gpu_predictor'<init_hyperparams>
df_train.isnull().sum()
best_trial= {'learning_rate': 0.01,
'colsample_bylevel': 0.6100000000000001,
'colsample_bytree': 0.91,
'max_depth': 10,
'subsample': 0.8,
'min_child_weight': 67,
'lambda': 0.012157425362490908,
'alpha': 7.278941365308569e-08,
'random_state': 3000,
'gamma': 1,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor'} 
<prepare_x_and_y>
df_train = df_train.dropna()
final_test = xgb.DMatrix(final_test[columns] )<define_variables>
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split, cross_validate, GridSearchCV
from sklearn.dummy import DummyClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, roc_curve, accuracy_score, precision_score, recall_score, precision_recall_curve
import time
from sklearn import __version__ as sklearn_version
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape<train_model>
X = df_train.drop(['Survived'],axis=1)
 y = df_train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2,stratify=y, random_state=42 )
NUM_FOLDS=10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 fold_rmse =[]
for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
train_df = xgb.DMatrix(train_df, label=train_target)
 val_df = xgb.DMatrix(val_df, label=val_target)
 model = xgb.train(best_trial, train_df, 2000)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(final_test)
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
rmse_iter = mean_squared_error(temp_oof, val_target, squared=False)
 print(rmse_iter)
 fold_rmse.append(rmse_iter )<save_to_csv>
columns = ['Model Name', 'accuracy','precision','recall','ROC AUC score','run time']
results = pd.DataFrame(columns=columns )
sub['target'] = test_preds
sub.to_csv('submission2_post_competition.csv', index=False )<split>
def metrics(model_name,y_test,y_pred):
accuracy = accuracy_score(y_test,y_pred)
 roc_auc =roc_auc_score(y_test, y_pred)
 precision = precision_score(y_pred=y_pred, y_true=y_test,zero_division=1)
 recall = recall_score(y_pred=y_pred, y_true=y_test,zero_division=1)
 print(classification_report(y_test, y_pred,zero_division=1))
cf_matrix = confusion_matrix(y_test, y_pred)
 sns.heatmap(cf_matrix, annot=True,fmt='3', cmap='Blues')
 plt.xlabel('Predicted user status',fontsize=12)
 plt.ylabel('True user status',fontsize=12)
 plt.title('%s Confusion Matrix' % model_name,fontsize=20)
 plt.show()
fpr, tpr, threshold = roc_curve(y_test, y_pred)
 plt.plot([0,1], [0,1], 'k--')
 plt.plot(fpr, tpr, label=model_name)
 plt.xlabel('False Positive Rate',fontsize=12)
 plt.ylabel('True Positive Rate',fontsize=12)
 plt.title('%s ROC Curve'% model_name,fontsize=20)
 plt.legend(fontsize=12)
 plt.show()
return pd.DataFrame([[model_name,accuracy, precision, recall,roc_auc,t1]],columns=columns )
def objective_2(trial, X_data = train, Y_data = target):
x_train, x_test, y_train, y_test = train_test_split(
 X_data, Y_data, random_state= 2021, test_size = 0.20)
 param = {
'tree_method':'gpu_hist',
'predictor': 'gpu_predictor',
'learning_rate': 0.01,
'colsample_bylevel': trial.suggest_discrete_uniform('colsample_bylevel',0.01,0.91, 0.1),
'colsample_bynode': trial.suggest_discrete_uniform('colsample_bytree',0.01,0.91, 0.1),
'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree',0.01,0.91, 0.1),
'max_depth': trial.suggest_int('max_depth', 1,10),
'subsample': trial.suggest_discrete_uniform('subsample', 0.20,1, 0.05),
'n_estimators': trial.suggest_categorical('n_estimators',[4000,5000,6000,7000]),
'min_child_weight': trial.suggest_int('min_child_weight', 1,401,step=2),
'lambda': trial.suggest_loguniform('lambda', 1e-8, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-8, 10.0),
'random_state': trial.suggest_categorical('random_state',[2000,3000,4000]),
'gamma': trial.suggest_discrete_uniform('gamma',0.01,2.01, 0.1)
 }
model = xgb.XGBRegressor(**param)
 model.fit(x_train,y_train,eval_set=[(x_train,y_train)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(x_test)
 rmse = mean_squared_error(y_test, preds,squared=False)
 return rmse 
<find_best_params>
model_name = 'Dummy'
model = DummyClassifier(strategy='most_frequent')
 pipe_dummy = make_pipeline(
 SimpleImputer(strategy='median'),
StandardScaler() ,
model)
 t0 = time.time()
pipe_dummy.fit(X_train,y_train)
 t1 = time.time() - t0
y_pred = pipe_dummy.predict(X_test)
 print('time to run in seconds: ',format(t1))
results = results.append(metrics(model_name,y_test,y_pred))
study_2 = optuna.create_study(direction='minimize')
 study_2.optimize(objective_2, n_trials= 100 )<train_model>
model_name = 'Naive Bayes'
NB = GaussianNB()
params_NB = {'var_smoothing': np.logspace(0,-9, num=100)}
grid = GridSearchCV(estimator=NB, param_grid=params_NB, cv=5)
 grid = grid.fit(X_train,y_train)
 model = grid.best_estimator_
pipe_NB = make_pipeline(
 SimpleImputer(strategy='median'),
StandardScaler() ,
model)
 t0 = time.time()
pipe_NB.fit(X_train,y_train)
 t1 = time.time() - t0
y_pred = pipe_NB.predict(X_test)
 print('time to run in seconds: ',format(t1))
results = results.append(metrics(model_name,y_test,y_pred))
print('Number of finished trials:', len(study_2.trials))
print('Best trial:', study_2.best_trial.params)
 print('Best objective value:', study_2.best_value)
<find_best_params>
model_name = 'Logistic Regression'
param_grid = [{'penalty' : ['l1', 'l2'],
'C' : np.logspace(0, 4, 10),
'solver' : ['liblinear']}]
LR = LogisticRegression()
grid = GridSearchCV(estimator=LR, param_grid=param_grid, cv=5)
 grid = grid.fit(X_train,y_train)
 model = grid.best_estimator_
pipe_LR = make_pipeline(
 SimpleImputer(strategy='median'),
StandardScaler() ,
model)
 t0 = time.time()
pipe_LR.fit(X_train,y_train)
 t1 = time.time() - t0
y_pred = pipe_LR.predict(X_test)
 print('time to run in seconds: ',format(t1))
results = results.append(metrics(model_name,y_test,y_pred))
best_trial_2 = study_2.best_trial.params
best_trial_2['tree_method'] = 'gpu_hist'
best_trial_2['predictor'] = 'gpu_predictor'
best_trial_2['learning_rate'] = 0.01<define_variables>
model_name = 'kNN'
knn = KNeighborsClassifier()
param_grid = {'n_neighbors': [3, 5, 7, 9, 11],
'weights': ['uniform', 'distance']
}
grid = GridSearchCV(estimator=knn, param_grid=param_grid, cv=5)
 grid = grid.fit(X_train,y_train)
 model = grid.best_estimator_
pipe_kNN = make_pipeline(
 SimpleImputer(strategy='median'),
StandardScaler() ,
model)
 t0 = time.time()
pipe_kNN.fit(X_train,y_train)
 t1 = time.time() - t0
y_pred = pipe_kNN.predict(X_test)
 print('time to run in seconds: ',format(t1))
results = results.append(metrics(model_name,y_test,y_pred))
train_oof = np.zeros(( 300000,))
test_preds_2 = 0
train_oof.shape<init_hyperparams>
model_name = 'Random Forest'
rfc=RandomForestClassifier(random_state=1234)
 param_grid = {'n_estimators': [100,200],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [5,10,15,20,25,50],
'criterion' :['gini', 'entropy']}
grid = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5)
 grid = grid.fit(X_train,y_train)
 model = grid.best_estimator_
pipe_rfc = make_pipeline(
 SimpleImputer(strategy='mean'),
StandardScaler() ,
model)
 t0 = time.time()
pipe_rfc.fit(X_train,y_train)
 t1 = time.time() - t0
y_pred = pipe_rfc.predict(X_test)
 print('time to run in seconds: ',format(t1))
results = results.append(metrics(model_name,y_test,y_pred))
best_trial_2 = {'colsample_bylevel': 0.91,
'colsample_bytree': 0.6100000000000001,
'max_depth': 10,
'subsample': 0.5,
'min_child_weight': 21,
'lambda': 2.4118345076896113e-05,
'alpha': 3.234942680594196e-08,
'random_state': 3000,
'gamma': 1.51,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor',
'learning_rate': 0.01}<split>
df_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 df_test.head()
NUM_FOLDS=10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 fold_rmse_2 =[]
for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
train_df = xgb.DMatrix(train_df, label=train_target)
 val_df = xgb.DMatrix(val_df, label=val_target)
 model = xgb.train(best_trial_2, train_df, 2000)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(final_test)
 train_oof[val_ind] = temp_oof
test_preds_2 += temp_test/NUM_FOLDS
rmse_iter = mean_squared_error(temp_oof, val_target, squared=False)
 print(rmse_iter)
 fold_rmse_2.append(rmse_iter )<save_to_csv>
df_test.isnull().sum()
sub['target'] = test_preds_2
sub.to_csv('submission3_post_competition.csv', index=False )<install_modules>
passengerId = df_test['PassengerId']
df_test['Family'] = df_test['SibSp'] + df_test['Parch']
df_test = df_test.drop(['Cabin','Name','Ticket','PassengerId'],axis=1 )
!pip install tensorflow_addons==0.9.1<set_options>
df_test['Age'] = df_test['Age'].fillna(df_test['Age'].median())
 df_test['Fare'] = df_test['Fare'].fillna(df_test['Fare'].median() )
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500 )<define_variables>
df_test['Embarked'] = df_test['Embarked'].map({'C':0,'Q':1,'S':2})
 df_test['Sex'] = pd.get_dummies(df_test['Sex'] )
EPOCHS = 180
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = 0.001
SPLITS = 5
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<load_from_csv>
prediction = []
for i in range(len(df_test)) :
test = pipe_rfc.predict([df_test.loc[i]])
 prediction.append(test )
def read_data() :
train_data = pd.read_csv('.. /input/data-without-drift/train_clean.csv')
 clean_train = pd.read_csv('.. /input/liverpool-noiseremoval/clean_train_signal.csv')
 train_oofs = pd.read_csv('.. /input/liverpool-lgbm-oofs/oofs_train.csv')
 test_data = pd.read_csv('.. /input/data-without-drift/test_clean.csv')
 clean_test = pd.read_csv('.. /input/liverpool-noiseremoval/clean_test_signal.csv')
 test_oofs = pd.read_csv('.. /input/liverpool-lgbm-oofs/oofs_test.csv')
 train = pd.DataFrame()
train['signal']=clean_train['signal'].values
train['open_channels']=train_data['open_channels'].values
train['oofs']=train_oofs['oofs'].values
test = pd.DataFrame()
test['signal'] = clean_test['signal'].values
test['oofs'] = test_oofs['oofs'].values
sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = cbr(inp, 64, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 16, 3, 12)
 x = BatchNormalization()(x)
 x = wave_block(x, 32, 3, 8)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 4)
 x = BatchNormalization()(x)
 x = wave_block(x, 128, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 30:
lr = LR
elif epoch < 40:
lr = LR / 3
elif epoch < 50:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score: {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f')
 def run_everything() :
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...')
 run_everything()<save_to_csv>
prediction = pd.DataFrame(prediction, columns = ['Survived'] )
train_or = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 sub = pd.read_csv('submission_wavenet.csv')
 sub[700000:800000]['open_channels']=sub[700000:800000]['open_channels'].values + train_or[4000000:4100000]['open_channels'].values
sub.to_csv('final_submission_wavenet.csv', index=False, float_format='%.4f' )<import_modules>
test_data_predictions = pd.concat([passengerId,prediction],axis=1 )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<define_search_space>
test_data_predictions.to_csv('Titanic_survivor_prediction.csv',index=False )
seed = 42
test_size = 0.2
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 workspace = "./unet"
backbone = [1, 1, 1, 1]
encoder_channels = np.array([64, 128, 256, 512, 1024])*2
decoder_channels = np.array([512, 256, 128, 64])*2
fold = 0
time_step = 4000
time_step_test = 10000
stride = 2
batch_size = 10
TRAIN = True
PREDICT = True
ENSEMBLE = False
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 print(device )<load_from_csv>
pd.read_csv('Titanic_survivor_prediction.csv' )