Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<compute_test_metric>
import pandas as pd
import numpy as np
import zipfile
import os
from matplotlib import pyplot as plt
from sklearn.linear_model import LogisticRegression,LinearRegression
from sklearn.model_selection import train_test_split,StratifiedKFold
from sklearn.metrics import accuracy_score
from scipy import fftpack
import xgboost
import warnings
from sklearn.model_selection import cross_val_predict, cross_validate
import seaborn as sns; sns.set()
from collections import Counter
from sklearn.neighbors import KNeighborsClassifier
from keras.layers import LSTM,Input,Dense,Flatten,SpatialDropout1D,Dropout,CuDNNLSTM,Reshape,Concatenate
from keras.layers import Lambda,concatenate,BatchNormalization
from keras.models import Model
from keras.utils import to_categorical
from keras.optimizers import Adam
from keras import backend as K
from sklearn.preprocessing import LabelEncoder
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )<define_variables>
directory = ".. /input/"
X_test_path = os.path.join(directory,"X_test.csv")
 X_train_path = os.path.join(directory,"X_train.csv")
 X_test_data = pd.read_csv(X_test_path)
 X_train_data = pd.read_csv(X_train_path)
 y_train_data = pd.read_csv(os.path.join(directory,"y_train.csv"))
sample_submission = pd.read_csv(os.path.join(directory,"sample_submission.csv"))
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
le = LabelEncoder()
le.fit(list(y_train_data["surface"]))
y_train_dataset_for_nn = to_categorical(le.transform(list(y_train_data["surface"])) )
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
def dataset_for_nn(X_dataset):
num_samples = X_dataset.shape[0]//128
X_dataset_for_nn = np.zeros(( num_samples,128,10))
for i in range(num_samples):
subset = np.array(X_dataset.iloc[i*128:(i+1)*128,3:])
 X_dataset_for_nn[i,:,:] = subset
return X_dataset_for_nn
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
X_train_for_nn = dataset_for_nn(X_train_data)
 X_test_for_nn = dataset_for_nn(X_test_data )
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
def freqs(dataset,width):
X = np.abs(fftpack.fft(dataset))
squeezed_dataset = []
for i in range(64//width):
squeezed_dataset.append(np.mean(X[i*width:(i+1)*width]))
return squeezed_dataset
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
def X_features(X_dataset,width=3):
num_samples = len(list(set(X_dataset["series_id"])))
 num_cols = 64//width
features = np.zeros(( num_samples,40+10*num_cols))
for i in range(num_samples):
X_train_subset = np.array(X_dataset.iloc[i*128:(i+1)*128,3:])
 features[i,:10] = np.mean(X_train_subset,axis=0)
 features[i,10:20] = np.std(X_train_subset,axis=0)
 features[i,20:30] = np.max(X_train_subset,axis=0)-np.min(X_train_subset,axis=0)
 features[i,30:40] = X_train_subset[-1,:]-X_train_subset[0,:]
for j in range(X_train_subset.shape[1]):
features[i,40+j*num_cols:40+(j+1)*num_cols] = freqs(X_train_subset[:,j],width)
 return features
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
X_train_features = X_features(X_train_data)
 X_test_features = X_features(X_test_data )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
def LSTM_NN(drop):
inp = Input(shape=(128,10))
x = SpatialDropout1D(0.1 )(inp)
 inp_2 = Input(shape=(250,))
x_2 = Dense(250, input_shape=(250,), activation="sigmoid" )(inp_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = Dense(120, activation="sigmoid" )(x_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = Dense(60, activation="sigmoid" )(x_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = BatchNormalization()(x_2)
 x = CuDNNLSTM(units=200, return_sequences=True, return_state=False, go_backwards=False )(x)
 x = Dropout(drop )(x)
 x = CuDNNLSTM(units=100, return_sequences=False, return_state=False, go_backwards=False )(x)
 x = concatenate([x,x_2])
 x = Dropout(drop )(x)
 outp = Dense(9, activation="sigmoid" )(x)
 model = Model(inputs=[inp,inp_2], outputs=outp)
 model.compile(loss='categorical_crossentropy', optimizer=Adam())
 return model
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
%%time
n_splits=5
kfold = StratifiedKFold(n_splits=n_splits, random_state=10, shuffle=True)
 y_test = np.zeros(( X_test_for_nn.shape[0],9*n_splits))
train_preds = np.zeros(( X_train_for_nn.shape[0],9))
X = X_train_for_nn
X_test = X_test_for_nn
Y = np.array(list(y_train_data["surface"]))
for i,(train_index, valid_index)in enumerate(kfold.split(X, Y)) :
X_train, X_val = X[list(train_index),:,:],X[list(valid_index),:,:]
X_train_feat, X_val_feat = X_train_features[list(train_index),:],X_train_features[list(valid_index),:]
Y_train, Y_val = Y[list(train_index)], Y[list(valid_index)]
Y_train = to_categorical(le.transform(Y_train))
Y_val = to_categorical(le.transform(Y_val))
model = LSTM_NN(0.15)
 model.fit([X_train,X_train_feat], Y_train, epochs=120, validation_data=([X_val,X_val_feat], Y_val), verbose=2)
 y_pred = model.predict([X_val,X_val_feat], verbose=2)
 y_test[:,i*9:(i+1)*9] = model.predict([X_test,X_test_features])
 train_preds[list(valid_index),:] = np.squeeze(y_pred )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
res_train = np.argmax(train_preds,axis=1)
 ans_train = np.argmax(to_categorical(le.transform(Y)) ,axis=1)
 print("CV score",round(accuracy_score(res_train,ans_train),4))
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
def most_frequent(List):
occurence_count = Counter(List)
 return occurence_count.most_common(1)[0][0]
res_test_inter = np.zeros(( y_test.shape[0],n_splits))
res_test=[]
for i in range(n_splits):
inter_arr = y_test[:,i*9:(i+1)*9]
res_test_inter[:,i] = np.argmax(inter_arr,axis=1)
 for j in range(y_test.shape[0]):
res_test.append(int(most_frequent(res_test_inter[j,:])) )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
test_for_sub=le.inverse_transform(res_test)
 print(test_for_sub[:5])
 test_size = len(list(set(X_test_data["series_id"])))
 Y_test_pred_array = np.zeros(( test_size,2))
Y_test_for_submission = pd.DataFrame(Y_test_pred_array,columns = ["series_id","surface"])
 Y_test_for_submission.iloc[:,0] = list(range(test_size))
Y_test_for_submission.iloc[:,1] = test_for_sub
Y_test_for_submission.to_csv("submission.csv",index=None )
<filter><EOS> 
FileLink('submission.csv' )