Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
shortrecipes = train_data[train_data['ingredients'].str.len() <= 2]
print("It seems that {} recipes consist of less than or equal to 2 ingredients!".format(len(shortrecipes)) )<sort_values>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
print("Explore the ingredients in the shortest recipes in our training set:" + "
")
 print(list(train_data[train_data['ingredients'].str.len() == 1].ingredients.values))
print("And there corresponding labels" + "
")
 print(list(train_data[train_data['ingredients'].str.len() == 1].cuisine.values))<define_variables>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
allingredients = []
for item in train_data['ingredients']:
for ingr in item:
allingredients.append(ingr )<count_values>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE = 128
countingr = Counter()
for ingr in allingredients:
countingr[ingr] += 1<count_unique_values>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
print("The most commonly used ingredients(with counts)are:")
 print("
")
 print(countingr.most_common(20))
print("
")
 print("The number of unique ingredients in our training sample is {}.".format(len(countingr)) )<define_variables>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
features = []
for item in train_data['ingredients']:
features.append(item )<define_variables>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
ingredients = []
for item in train_data['ingredients']:
for ingr in item:
ingredients.append(ingr )<feature_engineering>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
tfidf = TfidfVectorizer(vocabulary= list(set([str(i ).lower() for i in ingredients])) , max_df=0.99, norm='l2', ngram_range=(1, 4))
X_tr = tfidf.fit_transform([str(i)for i in features])
 feature_names = tfidf.get_feature_names()<categorify>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
def top_feats_by_class(trainsample,target,featurenames, min_tfidf=0.1, top_n=10):

 dfs = []
labels = np.unique(target)
 for label in labels:
ids = np.where(target==label)
 D = trainsample[ids].toarray()
D[D < min_tfidf] = 0
tfidf_means = np.nanmean(D, axis=0)
 topn_ids = np.argsort(tfidf_means)[::-1][:top_n]
top_feats = [(featurenames[i], tfidf_means[i])for i in topn_ids]
df = pd.DataFrame(top_feats)
 df.columns = ['feature', 'tfidf']
df['cuisine'] = label
dfs.append(df)
 return dfs<prepare_x_and_y>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
target = train_data['cuisine']<concatenate>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
result_tfidf = top_feats_by_class(X_tr, target, feature_names, min_tfidf=0.1, top_n=5 )<train_model>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
print("How training data looks like at this stage(example of one recipe):")
 print(str(features[0])+ '
')
 print("Number of instances: "+ str(len(features)) + '
')
 print("And the target variable:")
 print(target[0] )<define_variables>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
features_test = []
for item in test_data['ingredients']:
features_test.append(item )<define_variables>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(300, activation="relu", name="d1" )(z)
 x = L.Dropout(0.2 )(x)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 x = L.Dropout(0.2 )(x)
 p1 = L.Dense(3, activation="sigmoid", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.005, amsgrad=False), metrics=[score])
 return model
print("How test data looks like at this stage(example of one recipe):")
 print(str(features_test[0])+ '
')
 print("Number of instances: "+ str(len(features_test)) )<define_variables>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
features_processed= []
for item in features:
newitem = []
for ingr in item:
ingr.lower()
ingr = re.sub("[^a-zA-Z]"," ",ingr)
 ingr = re.sub(( r'\b(oz|ounc|ounce|pound|lb|inch|inches|kg|to)\b'), ' ', ingr)
 newitem.append(ingr)
 features_processed.append(newitem)
 features_test_processed= []
for item in features_test:
newitem = []
for ingr in item:
ingr.lower()
ingr = re.sub("[^a-zA-Z]"," ",ingr)
 ingr = re.sub(( r'\b(oz|ounc|ounce|pound|lb|inch|inches|kg|to)\b'), ' ', ingr)
 newitem.append(ingr)
 features_test_processed.append(newitem )<train_model>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
count_m = []
for recipe in features_processed:
if not recipe:
count_m.append([recipe])
 else: pass
print("Empty instances in the preprocessed training sample: " + str(len(count_m)) )<define_variables>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=200,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
count_m = []
for recipe in features_test_processed:
if not recipe:
count_m.append([recipe])
 else: pass
print("Empty instances in the preprocessed test sample: " + str(len(count_m)) )<choose_model_class>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
vectorizer = CountVectorizer(analyzer = "word",
ngram_range =(1,1),
binary = True,
tokenizer = None,
preprocessor = None,
stop_words = None,
max_df = 0.99 )<feature_engineering>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
train_X = vectorizer.fit_transform([str(i)for i in features_processed])
 test_X = vectorizer.transform([str(i)for i in features_test_processed] )<categorify>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
lb = LabelEncoder()
train_Y = lb.fit_transform(target )<train_model>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
vclf=VotingClassifier(estimators=[('clf1',LogisticRegression(random_state = 42)) , 
('clf2',SVC(kernel='linear',random_state = 42,probability=True)) , 
('clf3',RandomForestClassifier(n_estimators = 600,random_state = 42)) ],
voting='soft', weights = [1,1,1])
 vclf.fit(train_X, train_Y )<compute_train_metric>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
kfold = model_selection.KFold(n_splits=10, random_state=42)
 valscores = model_selection.cross_val_score(vclf, train_X, train_Y, cv=kfold)
 print('Mean accuracy on 10-fold cross validation: ' + str(np.mean(valscores)) )<save_to_csv>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )