Kaggle competition: CareerCon 2019 - Help Navigate Robots
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.60, 0.85]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
X_test = pd.read_csv(".. /input/X_test.csv")
 X_train = pd.read_csv(".. /input/X_train.csv")
 y_train = pd.read_csv(".. /input/y_train.csv" )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
y_train.group_id.nunique()
NFOLD = 5
kf = KFold(n_splits=NFOLD )<predict_on_test>
y_train.groupby(['group_id', 'surface'] ).nunique().head(10 )
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
X_train = pd.merge(y_train[['series_id', 'group_id', 'surface']], X_train, on='series_id')
 X_train.head()
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
train_X = pd.read_csv('.. /input/X_train.csv' ).iloc[:,3:].values.reshape(-1,128,10)
 test_X = pd.read_csv('.. /input/X_test.csv' ).iloc[:,3:].values.reshape(-1,128,10)
 print('train_X shape:', train_X.shape, ', test_X shape:', test_X.shape )
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
train_group = y_train['group_id'].values
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
def sq_dist(a,b):

 return np.sum(( a-b)**2, axis=1)
 def find_run_edges(data, edge):

 if edge == 'left':
border1 = 0
border2 = -1
elif edge == 'right':
border1 = -1
border2 = 0
else:
return False
edge_list = []
linked_list = []
for i in range(len(data)) :
dist_list = sq_dist(data[i, border1, :4], data[:, border2, :4])
 min_dist = np.min(dist_list)
 closest_i = np.argmin(dist_list)
 if closest_i == i:
print('Sample', i, 'linked with itself.Next closest sample used instead.')
 closest_i = np.argsort(dist_list)[1]
dist_list = sq_dist(data[closest_i, border2, :4], data[:, border1, :4])
 rev_dist = np.min(dist_list)
 closest_rev = np.argmin(dist_list)
 if closest_rev == closest_i:
print('Sample', i, '(back-)linked with itself.Next closest sample used instead.')
 closest_rev = np.argsort(dist_list)[1]
if(i != closest_rev):
edge_list.append(i)
 else:
linked_list.append([i, closest_i, min_dist])
 return edge_list, linked_list
def find_runs(data, left_edges, right_edges):

 data_runs = []
for start_point in left_edges:
i = start_point
run_list = [i]
while i not in right_edges:
tmp = np.argmin(sq_dist(data[i, -1, :4], data[:, 0, :4]))
if tmp == i:
tmp = np.argsort(sq_dist(data[i, -1, :4], data[:, 0, :4])) [1]
i = tmp
run_list.append(i)
 data_runs.append(np.array(run_list))
return data_runs
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
train_left_edges, train_left_linked = find_run_edges(train_X, edge='left')
 train_right_edges, train_right_linked = find_run_edges(train_X, edge='right')
 print('Found', len(train_left_edges), 'left edges and', len(train_right_edges), 'right edges.' )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )<prepare_output>
train_runs = find_runs(train_X, train_left_edges, train_right_edges )
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
y_train['run_id'] = 0
y_train['run_pos'] = 0
for run_id in range(len(train_runs)) :
for run_pos in range(len(train_runs[run_id])) :
series_id = train_runs[run_id][run_pos]
y_train.at[ series_id, 'run_id' ] = run_id
y_train.at[ series_id, 'run_pos' ] = run_pos
y_train.to_csv('y_train_with_runs.csv', index=False)
 y_train.tail()
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
flat_list = [series_id for run in train_runs for series_id in run]
print(len(flat_list), len(np.unique(flat_list)) )
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()<save_to_csv>
surface_cnt_train = []
for i in range(len(train_runs)) :
surface_cnt_train.append(len(np.unique(y_train['surface'].iloc[train_runs[i]])) )
df.to_csv('submission.csv', index=False )<import_modules>
print('Id of runs with different surfaces:')
 for i in range(len(surface_cnt_train)) :
if surface_cnt_train[i] > 1:
print(str(i)+ ', number of surfaces: ' + str(surface_cnt_train[i]))
from pathlib import Path
import numpy as np
import pandas as pd
import lightgbm as lgb
import os
from sklearn.model_selection import KFold
from tqdm.notebook import tqdm
import seaborn as sbn
from datetime import datetime, timedelta
from sklearn import datasets, linear_model
import matplotlib.pyplot as plt
import statsmodels.api as sm
import random
from scipy.stats import norm<set_options>
X_train = pd.merge(y_train[['series_id', 'run_id']], X_train, on='series_id')
 X_train.head()
pd.options.mode.chained_assignment = None<define_variables>
test_X = pd.read_csv('.. /input/X_test.csv' ).iloc[:,3:].values.reshape(-1,128,10 )
path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression')
 assert path.exists()<load_from_csv>
test_left_edges, test_left_linked = find_run_edges(test_X, edge='left')
 test_right_edges, test_right_linked = find_run_edges(test_X, edge='right')
 print('Found', len(test_left_edges), 'left edges and', len(test_right_edges), 'right edges.' )
TRAIN_TYPES={"Patient": "category",
"Weeks": "int16", "FVC": "int32", 'Percent': 'float32', "Age": "uint8",
"Sex": "category", "SmokingStatus": "category" }
SUBMISSION_TYPES={"Patient_Week": "category", "FVC": "int32", "Confidence": "int16"}
def read_data(path):
train_df = pd.read_csv(path/'train.csv', dtype = TRAIN_TYPES)
 test_df = pd.read_csv(path/'test.csv', dtype = TRAIN_TYPES)
 submission_df = pd.read_csv(path/'sample_submission.csv', dtype = SUBMISSION_TYPES)
 train_df.drop_duplicates(keep='first', inplace=True, subset=['Patient','Weeks'])
 return train_df, test_df, submission_df<load_from_disk>
test_runs = find_runs(test_X, test_left_edges, test_right_edges )
train_df, test_df, submission_df = read_data(path )<merge>
flat_list = [series_id for run in test_runs for series_id in run]
print(len(flat_list), len(np.unique(flat_list)) )
def prepare_submission(df, test_df):
df['Patient'] = df['Patient_Week'].apply(lambda x:x.split('_')[0])
 df['Weeks'] = df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
df = df[['Patient','Weeks','Confidence','Patient_Week']]
df = df.merge(test_df.drop('Weeks', axis=1 ).copy() , on=['Patient'])
 return df<categorify>
lost_samples = np.array([ i for i in range(len(test_X)) if i not in np.concatenate(test_runs)])
 print(lost_samples)
 print(len(lost_samples))
submission_df = prepare_submission(submission_df, test_df )<feature_engineering>
find_run_edges(test_X[lost_samples], edge='left')[1][0]
def adapt_percent_in_submission() :
previous_match = None
for i, r in submission_df.iterrows() :
in_training = train_df[(train_df['Patient'] == r['Patient'])&(train_df['Weeks'] == r['Weeks'])]
if(len(in_training)> 0):
previous_match = in_training['Percent'].item()
submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match
elif previous_match is not None:
submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match<concatenate>
lost_run = np.array(lost_samples[find_runs(test_X[lost_samples], [0], [5])[0]])
 test_runs.append(lost_run )
train_df['WHERE'] = 'train'
test_df['WHERE'] = 'val'
submission_df['WHERE'] = 'test'
data = train_df.append([test_df, submission_df] )<feature_engineering>
X_test = pd.read_csv(".. /input/X_test.csv" )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<filter>
y_test = pd.DataFrame(columns = ['series_id', 'run_id', 'run_pos'])
 y_test['series_id'] = X_test.series_id.unique()
base = data.loc[data.Weeks == data.min_week]<categorify>
y_test['run_id'] = 0
y_test['run_pos'] = 0
for run_id in range(len(test_runs)) :
for run_pos in range(len(test_runs[run_id])) :
series_id = test_runs[run_id][run_pos]
y_test.at[ series_id, 'run_id' ] = run_id
y_test.at[ series_id, 'run_pos' ] = run_pos
y_test.head()
base = base[['Patient','FVC', 'Percent']].copy()
base.columns = ['Patient','min_FVC', 'min_Percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base<merge>
X_test = pd.merge(y_test[['series_id', 'run_id']], X_test, on='series_id')
 X_test.head()
data = data.merge(base, on='Patient', how='left' )<feature_engineering>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.degrees(math.atan2(t0, t1))
t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.degrees(math.asin(t2))
t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.degrees(math.atan2(t3, t4))
return X, Y, Z
data['base_week'] = data['Weeks'] - data['min_week']
data['base_week'] = data['base_week']
del base<filter>
def fe_step0(data):
data['norm_quat'] = data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2 + data['orientation_W']**2
data['mod_quat'] =(data['norm_quat'])**0.5
data['norm_X'] = data['orientation_X'] / data['mod_quat']
data['norm_Y'] = data['orientation_Y'] / data['mod_quat']
data['norm_Z'] = data['orientation_Z'] / data['mod_quat']
data['norm_W'] = data['orientation_W'] / data['mod_quat']
return data
data[data['Patient'] == 'ID00421637202311550012437']<data_type_conversions>
X_train = fe_step0(X_train)
 X_test = fe_step0(X_test)
 print(X_train.shape)
 X_train.head(5 )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<rename_columns>
def fe_step1(data):

 x, y, z, w = data['norm_X'].tolist() , data['norm_Y'].tolist() , data['norm_Z'].tolist() , data['norm_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 data['euler_x'] = nx
data['euler_y'] = ny
data['euler_z'] = nz
return data
data = data.rename(columns={"Age": "age", "min_FVC": "BASE", "base_week": "week", "Percent": "percent"})
 FE += ['age','week','BASE', 'percent']
FE<filter>
X_train = fe_step1(X_train)
 X_test = fe_step1(X_test)
 print(X_train.shape)
 X_train.head(5 )
train_df = data.loc[data.WHERE=='train']
test_df = data.loc[data.WHERE=='val']
submission_df = data.loc[data.WHERE=='test']
del data<sort_values>
def feat_eng(data, test_data=False):

 df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
if test_data:
col_for_group_by = ['run_id']
else:
col_for_group_by = ['run_id', 'surface']
for col in data.columns:
if col in ['row_id','series_id','measurement_number', 'group_id', 'run_id', 'run_pos', 'surface']:
continue
df[col + '_mean'] = data.groupby(col_for_group_by)[col].mean()
df[col + '_median'] = data.groupby(col_for_group_by)[col].median()
df[col + '_max'] = data.groupby(col_for_group_by)[col].max()
df[col + '_min'] = data.groupby(col_for_group_by)[col].min()
df[col + '_std'] = data.groupby(col_for_group_by)[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(col_for_group_by)[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mean_change_of_abs_change'] = data.groupby(col_for_group_by)[col].apply(mean_change_of_abs_change)
 df[col + '_abs_max'] = data.groupby(col_for_group_by)[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(col_for_group_by)[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df
train_df.sort_values(['Patient', 'Weeks'], inplace=True )<prepare_x_and_y>
X_train = feat_eng(X_train)
 X_test = feat_eng(X_test, test_data=True)
 print("New features: ",X_train.shape )
y = train_df['FVC']
y<define_variables>
feature_list.extend(['totl_anglr_vel', 'totl_linr_acc', 'acc_vs_vel'] )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )<define_variables>
X_train = pd.merge(y_train[['series_id', 'run_id', 'surface']], X_train, on=['run_id', 'surface'])
 X_train.drop(['run_id', 'surface'], axis=1, inplace=True)
 X_train.head()
seed_everything(42 )<compute_test_metric>
X_test = pd.merge(y_test[['series_id', 'run_id']], X_test, on='run_id')
 X_test.drop(['run_id'], axis=1, inplace=True)
 X_test.head()
C1_val = 70
C2_val = 1000
C1, C2 = C1_val, C2_val
q = np.array([0.2, 0.50, 0.8])
 def score(y_true, y_pred):
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = np.max(sigma)
 delta = np.abs(y_true[:, 0] - fvc_pred)
 delta = np.min(delta)
 sq2 = np.sqrt(2.)
metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 def qloss(y_true, y_pred):
print('y_true.shape', y_true.shape)
 print('y_pred.shape', y_pred.shape)
 e = y_true - y_pred
v = np.max(q*e,(q-1)*e)
 return np.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
y_true = y_true.reshape(-1, 1)
 return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss<init_hyperparams>
y_gr = y_train[['group_id','surface','series_id']].groupby(['group_id','surface'] ).count().reset_index()
y_gr.rename(columns={'series_id':'cnt'}, inplace=True)
 y_gr.head()
n_estimators = 30000
if not 'sub_row' in locals() :
sub_row = 0.75
if not 'bagging_freq' in locals() :
bagging_freq = 1
if not 'learning_rate' in locals() :
learning_rate = 0.4
leave_size = 4
lgb_params = {
"objective": 'quantile',
'n_jobs': 1,
'max_depth': leave_size + 1,
'num_leaves': 2**leave_size-1,
"min_data_in_leaf": 2**(leave_size + 1)-1,
"n_estimators": n_estimators,
'learning_rate': 8e-3,
'colsample_bytree': 0.9,
'boosting_type': 'gbdt',
"early_stopping_rounds": 100,
'verbosity': 1000,
"metric": ["rmse", "mse"]
}<define_variables>
y_gr_temp = y_gr[y_gr.surface!='hard_tiles']
y_gr_train, y_gr_val = train_test_split(y_gr_temp, test_size=0.15, random_state=11, stratify=y_gr_temp.surface)
 y_gr_train = y_gr_train.append(y_gr[y_gr.surface=='hard_tiles'] )
cat_feats = ['Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']<define_variables>
y_gr_val[['group_id', 'surface']].groupby('surface' ).count()
NFOLD = 5
kf = KFold(n_splits=NFOLD, shuffle=False)
 pred = {a: np.zeros(( train_df.shape[0])) for a in q.tolist() }<define_search_space>
y_gr_train[['group_id', 'surface']].groupby('surface' ).count()
ensemble_weights = [6./7, 1./7]
assert np.sum(ensemble_weights)== 1.0<predict_on_test>
y_train_data = y_train.merge(y_gr_train.group_id, on='group_id' )
def ols_quantile(m, X, q):
mean_pred = m.predict(X)
 se = np.sqrt(m.scale)
 return mean_pred + norm.ppf(q)* se<train_model>
y_val_data = y_train.merge(y_gr_val.group_id, on='group_id' )
models = []
linear_models = []
for cnt,(tr_idx, val_idx)in tqdm(enumerate(kf.split(X)) , total=NFOLD):
X_train, y_train = X.loc[tr_idx], y.loc[tr_idx]
X_valid, y_valid = X.loc[val_idx], y.loc[val_idx]
print(f"FOLD {cnt}", X_train.shape, y_train.shape, X_valid.shape, y_valid.shape)
 for qi, quantile_alpha in enumerate(q.tolist()):
lgb_params['alpha'] = quantile_alpha
m_lgb_regressor = lgb.LGBMRegressor(**lgb_params)
 m_lgb_regressor.fit(X=X_train, y=y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)],
eval_names=['train mloss', 'valid mloss'],
eval_metric=lgb_params['metric'],
verbose=lgb_params['verbosity'],
early_stopping_rounds=lgb_params["early_stopping_rounds"],
categorical_feature=cat_feats)
 lin_model = sm.OLS(y_train, X_train ).fit()
lin_predict = ols_quantile(lin_model, X_valid, quantile_alpha)
 lgb_predict = m_lgb_regressor.predict(X_valid)
 pred[quantile_alpha][val_idx] = np.average([lin_predict, lgb_predict], axis = 0, weights=ensemble_weights)
 models.append(m_lgb_regressor)
 linear_models.append(lin_model )<compute_test_metric>
y_train_data[['series_id', 'surface']].groupby('surface' ).count()
full_preds = np.vstack([pred[a] for a in q] ).T
score(np.array(y ).reshape(-1, 1), full_preds )<predict_on_test>
y_val_data[['group_id', 'surface']].groupby('surface' ).count()
pred = []
for i in range(NFOLD):
cur_pred = []
for j, quantile in enumerate(q):
model_idx = i * 3 + j
model = models[model_idx]
lin_predict = ols_quantile(linear_models[model_idx], submission_df[FE], quantile)
 dbmc_predict = model.predict(submission_df[FE])
 cur_pred.append(np.average([lin_predict, dbmc_predict], axis = 0, weights=ensemble_weights))
pred.append(np.array(cur_pred ).T )<prepare_output>
X = y_train_data.merge(X_train, on='series_id')
 X = X.drop(['group_id', 'run_id', 'run_pos', 'series_id', 'surface'], axis=1)
 X = np.array(X)
 X_val = y_val_data.merge(X_train, on='series_id')
 X_val = X_val.drop(['group_id', 'run_id', 'run_pos', 'series_id', 'surface'], axis=1)
 X_val = np.array(X_val )
submission_df['FVC1'] = final_preds[:,1]
submission_df['Confidence1'] = final_preds[:, 2] - final_preds[:, 0]<feature_engineering>
X_test = X_test.sort_values('series_id')
 X_t = X_test.drop(['series_id'], axis=1)
 X_t = np.array(X_t )
submission_df.loc[~submission_df.FVC1.isnull() ,'FVC'] = submission_df.loc[~submission_df.FVC1.isnull() ,'FVC1']
submission_df.loc[~submission_df.FVC1.isnull() ,'Confidence'] = submission_df.loc[~submission_df.FVC1.isnull() ,'Confidence1']<save_to_csv>
n_featurs = X.shape[1]
print(n_featurs )
submission_df[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<load_from_csv>
rf_clf = RandomForestClassifier(n_estimators=100, max_depth=11, verbose=1,
class_weight='balanced',
random_state=11, n_jobs=10 )
submission_final_df = pd.read_csv("submission.csv" )<set_options>
rf_clf.fit(X, y_train_data['surface'] )
warnings.simplefilter('ignore')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 quantiles = [0.2, 0.5, 0.8]<categorify>
print('Accuracy on train data: ', rf_clf.score(X, y_train_data['surface']))
print('Accuracy on validation data: ', rf_clf.score(X_val, y_val_data['surface']))
def make_X(dt, dense_cols, cat_feats):
X = {"dense": dt[dense_cols].to_numpy() }
for i, v in enumerate(cat_feats):
X[v] = dt[[v]].to_numpy()
return X
class Loader:
def __init__(self, X, y, shuffle=True, batch_size=64, cat_cols=[]):
self.X_cont = X["dense"]
self.X_cat = np.concatenate([X[k] for k in cat_cols], axis=1)
 self.y = y
self.shuffle = shuffle
self.batch_size = batch_size
self.n_conts = self.X_cont.shape[1]
self.len = self.X_cont.shape[0]
n_batches, remainder = divmod(self.len, self.batch_size)
 if remainder > 0:
n_batches += 1
self.n_batches = n_batches
self.remainder = remainder
self.idxes = np.array([i for i in range(self.len)])
 def __iter__(self):
self.i = 0
if self.shuffle:
ridxes = self.idxes
np.random.shuffle(ridxes)
 self.X_cat = self.X_cat[[ridxes]]
self.X_cont = self.X_cont[[ridxes]]
if self.y is not None:
self.y = self.y[[ridxes]]
return self
def __next__(self):
if self.i >= self.len:
raise StopIteration
if self.y is not None:
y = torch.FloatTensor(self.y[self.i:self.i + self.batch_size].astype(np.float32))
else:
y = None
xcont = torch.FloatTensor(self.X_cont[self.i:self.i + self.batch_size])
 xcat = torch.LongTensor(self.X_cat[self.i:self.i + self.batch_size])
 batch =(xcont, xcat, y)
 self.i += self.batch_size
return batch
def __len__(self):
return self.n_batches 
<choose_model_class>
pred_train = rf_clf.predict(X)
 pred_val = rf_clf.predict(X_val )
class model_nn(nn.Module):
def __init__(self, hidden_dim, output_dim, emb_dims, n_cont):
super().__init__()
self.emb_layers = nn.ModuleList([nn.Embedding(x, y)for x, y in emb_dims])
 n_embs = sum([y for x, y in emb_dims])
 self.n_embs = n_embs
self.n_cont = n_cont
inp_dim = n_embs + n_cont
self.inp_dim = inp_dim
self.fc0 = nn.Linear(inp_dim, hidden_dim)
 self.relu0 = nn.ELU()
self.fc1 = nn.Linear(hidden_dim, hidden_dim)
 self.relu1 = nn.ELU()
self.fc2 = nn.Linear(hidden_dim, output_dim)
 self.fc3 = nn.Linear(hidden_dim, output_dim)
 self.fc4 = nn.Linear(hidden_dim, output_dim)
 def encode_and_combine_data(self, cat_data):
xcat = [el(cat_data[:, k])for k, el in enumerate(self.emb_layers)]
xcat = torch.cat(xcat, 1)
 return xcat
def forward(self, cont_data, cat_data):
cont_data = cont_data.to(device)
 cat_data = cat_data.to(device)
 cat_data = self.encode_and_combine_data(cat_data)
 x = torch.cat([cont_data, cat_data], dim=1)
 hz = self.fc0(x)
 hz = self.relu0(hz)
 hz = self.fc1(hz)
 hz = self.relu1(hz)
 out1 = self.fc2(hz)
 out2 = self.fc3(hz)
 out3 = self.fc4(hz)
 return torch.cat([out1, out2, out3], dim=1 )<compute_test_metric>
X = y_train.merge(X_train, on='series_id')
 X = X.drop(['group_id', 'run_id', 'run_pos', 'series_id', 'surface'], axis=1)
 X = np.array(X )
def quantile_loss(preds, target, quantiles):
assert not target.requires_grad
assert preds.size(0)== target.size(0)
 losses = []
for i, q in enumerate(quantiles):
errors = target - preds[:, i]
losses.append(torch.max(( q - 1)* errors, q * errors ).unsqueeze(1))
loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))
return loss
def metric(outputs, target):
confidence = np.abs(outputs[:, 2] - outputs[:, 0])
 clip = np.where(confidence > 70, confidence, 70)
 delta = np.abs(outputs[:, 1] - target)
 delta = np.where(delta > 1000, 1000, delta)
 metrics =(delta*np.sqrt(2)/clip)+ np.log(clip*np.sqrt(2))
return np.mean(metrics)
<define_variables>
rf_clf = RandomForestClassifier(n_estimators=100, max_depth=11, verbose=1,
class_weight='balanced',
random_state=11, n_jobs=10 )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
Model_Root = 'models'<load_from_csv>
rf_clf.fit(X, y_train['surface'] )
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient', 'FVC', 'Percent']].copy()
base.columns = ['Patient','min_FVC', 'min_percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age_week'] = data['Age'].values + data['base_week'].values/53
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['min_percent_norm'] =(data['min_percent'] - data['min_percent'].min())/(data['min_percent'].max() - data['min_percent'].min())
 data['age_week_norm'] =(data['age_week'] - data['age_week'].min())/(data['age_week'].max() - data['age_week'].min())
 cat_feat = ['Sex', 'SmokingStatus']
uniques = []
for i, v in enumerate(cat_feat):
data[v] = OrdinalEncoder(dtype="int" ).fit_transform(data[[v]])
 uniques.append(len(data[v].unique()))
tr = data.loc[data.WHERE == 'train']
chunk = data.loc[data.WHERE == 'val']
sub = data.loc[data.WHERE == 'test']
FE += ['age', 'week', 'BASE', 'min_percent_norm', 'age_week_norm']
print(list(tr))
print(list(sub))
print(FE)
 print(cat_feat)
 print(uniques)
 dims = [1, 2]
emb_dims = [(x, y)for x, y in zip(uniques, dims)]
n_cont = len(FE )<train_model>
subm_result = rf_clf.predict(X_t )
hidden_dim = 256
out_dim = 1
kfold = 6
groups = np.asarray(tr['Patient'].values)
 skf = GroupKFold(n_splits=kfold)
 avg_preds = np.zeros(( len(sub), len(quantiles)))
 models = []
for i,(train_index, test_index)in enumerate(skf.split(tr, tr['FVC'].values, groups=groups)) :
print('[Fold %d/%d]' %(i + 1, kfold))
model_path = f"{Model_Root}/nn_model_%s.pth" % i
if os.path.isfile(model_path):
final_model = torch.load(model_path)
 X_test = make_X(sub, FE, cat_feat)
 test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)
 preds = []
for i,(X_cont, X_cat, y)in enumerate(tqdm(test_loader)) :
print(X_cont.shape, X_cat.shape)
 out = final_model(X_cont, X_cat)
 preds.append(out)
 preds = torch.cat(preds, dim=0 ).detach().cpu().numpy()
avg_preds += preds
else:
X_train, X_valid = tr.iloc[train_index], tr.iloc[test_index]
y_train, y_valid = tr.iloc[train_index]['FVC'].values, tr.iloc[test_index]['FVC'].values
X_train = make_X(X_train.reset_index() , FE, cat_feat)
 X_valid = make_X(X_valid.reset_index() , FE, cat_feat)
 train_loader = Loader(X_train, y_train, cat_cols=cat_feat, batch_size=16, shuffle=True)
 val_loader = Loader(X_valid, y_valid, cat_cols=cat_feat, batch_size=64, shuffle=True)
 model = model_nn(hidden_dim, out_dim, emb_dims, n_cont ).to(device)
 final_model = model_training(model, train_loader, val_loader, epochs=1000,
batch_size=64, lr=0.01, patience=20,
model_path='nn_model_%s.pth' % i)
 models.append(final_model)
 for model in models:
X_test = make_X(sub, FE, cat_feat)
 test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)
 preds = []
with torch.no_grad() :
for i,(X_cont, X_cat, y)in enumerate(tqdm(test_loader)) :
out = model(X_cont, X_cat)
 preds.append(out)
 preds = torch.cat(preds, dim=0 ).detach().cpu().numpy()
avg_preds += preds
print(avg_preds)
 avg_preds = avg_preds/kfold
print(avg_preds)
 sub['Patient_Week'] = sub['Patient_Week'].values
sub['FVC'] = avg_preds[:, 1]
sub['Confidence'] = avg_preds[:, 2]-avg_preds[:, 0]
sub[['Patient_Week', 'FVC', 'Confidence']].to_csv("submission.csv", index=False)
<install_modules>
submission = pd.DataFrame(columns = ['series_id', 'surface'])
 submission['series_id'] = X_test.series_id.unique()
submission['surface'] = subm_result
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index<import_modules>
submission.to_csv('submission.csv', index=False )
<define_variables><EOS> 
submission.to_csv('submission.csv', index=False )