Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<split>
warnings.filterwarnings("ignore")
 print(os.listdir(".. /input"))
tr_p, vl_p = train_test_split(P,shuffle=True, train_size= 0.8 )<compute_test_metric>
train_df = pd.read_csv(".. /input/X_train.csv", squeeze = True)
 test_df = pd.read_csv(".. /input/X_test.csv" )
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )<define_variables>
print("Dimension of train data", train_df.shape[0]//128)
 train_df.head()
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
y_train = pd.read_csv(".. /input/y_train.csv", skipinitialspace = True, squeeze = True)
 y_train.head()
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
y_train["surface_label"] = y_train["surface"].map({'fine_concrete':0, 'concrete':1, 'soft_tiles':2, 'tiled':3, 'soft_pvc':4,
'hard_tiles_large_space':5, 'carpet':6, 'hard_tiles':7, 'wood':8} )
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
train_df = pd.merge(train_df, y_train[["surface_label", "series_id"]], on = "series_id")
 train_df.head()
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
print("Test data dimension",test_df.shape[0]//128)
 test_df.head()
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
y_train["surface"].value_counts()
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
print("Number of duplicates row in train data {}".format(sum(train_df.duplicated())))
 print("Number of duplicates row in test data {}".format(sum(test_df.duplicated())) )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
print("Total number of null/nan values in train data
{}".format(train_df.isnull().sum()))
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
print("Total number of null/nan values in test data
{}".format(test_df.isnull().sum()))
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
train_data = train_df.drop(["surface_label"], axis = 1)
 train_data.columns[3:]
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
def sma(x, y, z):
sum = 0
for i in range(len(x)) :
sum +=(abs(x[i])+ abs(y[i])+ abs(z[i]))
return sum/len(x )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
train_data['sma'] = sma(train_data['angular_velocity_X'], train_data['angular_velocity_Y'], train_data['angular_velocity_Z'])
 test_df['sma'] = sma(test_df['angular_velocity_X'], test_df['angular_velocity_Y'], test_df['angular_velocity_Z'] )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
def feat_eng(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 +
data['angular_velocity_Z'])** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 +
data['linear_acceleration_Z'])**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 +
data['orientation_Z'])**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_q25'] = data.groupby(['series_id'])[col].quantile(0.25)
 df[col + '_q75'] = data.groupby(['series_id'])[col].quantile(0.75)
 df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
X_train = feat_eng(train_data)
 test_data = feat_eng(test_df)
 print(X_train.shape)
 X_train.head()
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
tsne_data = X_train
x_tsne = MinMaxScaler().fit_transform(tsne_data)
 y_tsne = y_train["surface"]
x_tsne = np.nan_to_num(x_tsne )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
def perform_model(test_data, model, X_train, y_train, X_test, y_test, class_labels, cm_normalize=True, \
print_cm=True, cm_cmap=plt.cm.Greens):
results = dict()
train_start_time = datetime.now()
print('training the model.. ')
 model.fit(X_train, y_train)
 print('Done
')
 train_end_time = datetime.now()
results['training_time'] = train_end_time - train_start_time
print('training_time(HH:MM:SS.ms)- {}
'.format(results['training_time']))
print('Predicting test data')
 test_start_time = datetime.now()
y_pred = model.predict(X_test)
 prediction = model.predict(test_data)
 test_end_time = datetime.now()
print('Done
')
 results['testing_time'] = test_end_time - test_start_time
print('testing time(HH:MM:SS:ms)- {}
'.format(results['testing_time']))
results['predicted'] = y_pred
accuracy = metrics.accuracy_score(y_true=y_test, y_pred=y_pred)
 results['accuracy'] = accuracy
print('---------------------')
 print('| Accuracy |')
 print('---------------------')
 print('
{}
'.format(accuracy))
cm = metrics.confusion_matrix(y_test, y_pred)
 results['confusion_matrix'] = cm
if print_cm:
print('--------------------')
 print('| Confusion Matrix |')
 print('--------------------')
 print('
{}'.format(cm))
plt.figure(figsize=(8,8))
plt.grid(b=False)
 plot_confusion_matrix(cm, classes=class_labels, normalize=True, title='Normalized confusion matrix', cmap = cm_cmap)
 plt.show()
print('-------------------------')
 print('| Classifiction Report |')
 print('-------------------------')
 classification_report = metrics.classification_report(y_test, y_pred)
 results['classification_report'] = classification_report
print(classification_report)
 results['model'] = model
return prediction, results
NFOLD = 5
kf = KFold(n_splits=NFOLD )<predict_on_test>
def print_grid_search_attributes(model):
print('--------------------------')
 print('| Best Estimator |')
 print('--------------------------')
 print('
\t{}
'.format(model.best_estimator_))
print('--------------------------')
 print('| Best parameters |')
 print('--------------------------')
 print('\tParameters of best estimator :
\t{}
'.format(model.best_params_))
print('---------------------------------')
 print('| No of CrossValidation sets |')
 print('--------------------------------')
 print('
\tTotal numbre of cross validation sets: {}
'.format(model.n_splits_))
print('--------------------------')
 print('| Best Score |')
 print('--------------------------')
 print('
\tAverage Cross Validate scores of best estimator :
\t{}
'.format(model.best_score_))
%%time
cnt = 0
EPOCHS = 1500
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
X_train = np.nan_to_num(X_train)
 test_data = np.nan_to_num(test_data )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train["surface_label"], test_size = 0.3, random_state = 4, stratify = y_train["surface_label"], shuffle = True)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
labels = ['fine_concrete', 'concrete', 'soft_tiles', 'tiled', 'soft_pvc', 'hard_tiles_large_space', 'carpet', 'hard_tiles', 'wood']
parameters = {'C':[0.01, 0.1, 1, 10, 20, 30], 'penalty':['l2','l1']}
log_reg = linear_model.LogisticRegression()
log_reg_grid = GridSearchCV(log_reg, param_grid = parameters, cv = 10, verbose = 1, n_jobs = -1)
 predict_lr, log_reg_grid_results = perform_model(test_data, log_reg_grid, X_train, y_train, X_test, y_test, class_labels = labels )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
param_grid = {'max_depth': np.arange(5,8,1), \
'n_estimators':np.arange(130,170,10)}
gbdt = GradientBoostingClassifier()
gbdt_grid = GridSearchCV(gbdt, param_grid=param_grid, n_jobs=-1)
 predict_gbdt, gbdt_grid_results = perform_model(test_data, gbdt_grid, X_train, y_train, X_test, y_test, class_labels=labels)
 print_grid_search_attributes(gbdt_grid_results['model'] )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
params = {'n_estimators': np.arange(10,201,20), 'max_depth':np.arange(3,15,2)}
rfc = RandomForestClassifier()
rfc_grid = GridSearchCV(rfc, param_grid=params, cv = 10, n_jobs=-1)
 labels = ['fine_concrete', 'concrete', 'soft_tiles', 'tiled', 'soft_pvc', 'hard_tiles_large_space', 'carpet', 'hard_tiles', 'wood']
predict_rf, rfc_grid_results = perform_model(test_data, rfc_grid, X_train, y_train, X_test, y_test, class_labels = labels)
 print_grid_search_attributes(rfc_grid_results['model'] )
<prepare_output><EOS> 
submission = pd.read_csv(".. /input/sample_submission.csv")
 submission["surface"] = predict_rf
submission["surface"] = submission["surface"].map({0:'fine_concrete', 1:'concrete', 2:'soft_tiles', 3:'tiled', 4:'soft_pvc',
5:'hard_tiles_large_space', 6:'carpet', 7:'hard_tiles', 8:'wood'})
 submission.to_csv("sample_submission.csv", index = False )