Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<predict_on_test>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
y_j_sub_conv=model_loss.predict(x_j_sub_conv,verbose=1)
 y_j_sub_conv2=model_acc.predict(x_j_sub_conv,verbose=1 )<categorify>
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image
y_j_sub=y_lbr.inverse_transform(y_j_sub_conv)
 y_j_sub2=y_lbr.inverse_transform(y_j_sub_conv2 )<save_to_csv>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )
df_submission[y_field]= y_j_sub
df_submission.head()
print("Generate Submission File...")
 sub=df_submission.drop(x_field,axis=1)
 sub.head(50)
 sub.to_csv('svm_output_loss.csv', index=True)
 sub.to_csv('submission.csv', index=True)
 with open('submission.csv', mode='r', encoding='utf-8')as f:
csvtext=f.read()
print(csvtext )<save_to_csv>
seed_everything(42 )
df_submission[y_field]= y_j_sub2
df_submission.head()
print("Generate Submission File...")
 sub=df_submission.drop(x_field,axis=1)
 sub.head(50)
 sub.to_csv('svm_output_acc.csv', index=True)
 sub.to_csv('submission.csv', index=True)
 with open('submission.csv', mode='r', encoding='utf-8')as f:
csvtext=f.read()
print(csvtext )<load_from_disk>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
data = pd.read_json('.. /input/train.json')
 data.head()<split>
Dropout_model = 0.38
FVC_weight = 0.2
Confidence_weight = 0.2
GaussianNoise_stddev = 0.2
n_data = len(data.id)
 r_train = 1
r_eval = 0
data = data.sample(frac=1 ).reset_index(drop=True)
 i_train = int(n_data*r_train)
 data_train = data[:i_train]
i_eval = int(n_data*(1-r_eval))
data_eval = data[i_eval:]
print('n_train, n_eval = {}, {}'.format(len(data_train), len(data_eval)) )<prepare_x_and_y>
commits_df = pd.DataFrame(columns = ['commit_num', 'FVC_weight', 'Dropout_model', 'LB_score', 'seed', 'GaussianNoise_stddev'] )
x_train_name = data_train.ingredients.values
y_train_name = data_train.cuisine.values
x_eval_name = data_eval.ingredients.values
y_eval_name = data_eval.cuisine.values
tx = Tokenizer(filters='', split=None)
 tx.fit_on_texts(x_train_name)
 x_train = tx.texts_to_matrix(x_train_name)
 x_eval = tx.texts_to_matrix(x_eval_name)
 input_dim = x_train.shape[1]
ty = Tokenizer(filters='', split=None)
 ty.fit_on_texts(y_train_name)
 y_train = ty.texts_to_matrix(y_train_name)
 y_eval = ty.texts_to_matrix(y_eval_name)
 output_dim = y_train.shape[1]
y_inv = dict(map(reversed, ty.word_index.items()))
print('Shape x_train, x_eval =', x_train.shape, x_eval.shape)
 print('Shape y_train, y_eval =', y_train.shape, y_eval.shape)
 i = 0
print('x_train_name[{}] = {}'.format(i, x_train_name[i]))
print('x_train[{}]: N_total = {}, N_0 = {}, N_1 = {}'.format(
 i, len(x_train[i]), len(x_train[i][x_train[i]==0]), len(x_train[i][x_train[i]==1])))
 print('y_train_name[{}] = {}'.format(i, y_train_name[i]))
print('y_train[{}]: N_total = {}, N_0 = {}, N_1 = {}'.format(
 i, len(y_train[i]), len(y_train[i][y_train[i]==0]), len(y_train[i][y_train[i]==1])))
 print('y_inv =', y_inv )<train_model>
n=0
commits_df.loc[n,'commit_num'] = 1
commits_df.loc[n,'Dropout_model'] = 0.4
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8111
model = Sequential()
model.add(Dense(256, input_dim=input_dim, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(output_dim, activation='softmax'))
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
 model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
 model.fit(x_train, y_train, epochs=40, batch_size=128)
 if r_eval > 0:
score = model.evaluate(x_eval, y_eval, batch_size=128)
 print(model.metrics_names, '=', score)
<load_from_disk>
n=1
commits_df.loc[n,'commit_num'] = 4
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8105
test = pd.read_json('.. /input/test.json')
 test.head()<save_to_csv>
n=2
commits_df.loc[n,'commit_num'] = 6
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.35
commits_df.loc[n,'LB_score'] = -6.8158
x_test = tx.texts_to_matrix(test.ingredients)
 y_test = model.predict(x_test)
 y_test_name = [y_inv[k] for k in np.argmax(y_test, axis=1)]
out = pd.DataFrame({
'id': test.id,
'cuisine': y_test_name })
 out.to_csv('submission.csv', index=False )<feature_engineering>
n=3
commits_df.loc[n,'commit_num'] = 8
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8107
def load_data(filename, flag):
with open(filename,'r')as fp:
json_data = json.load(fp)
 id_list = []
ingredients_list = []
cuisine_list = []
for row in json_data:
id_list.append(row['id'])
 ingredients_list.append(row['ingredients'])
 if(flag == 'train'):
cuisine_list.append(row['cuisine'])
 if(flag == 'train'):
df = pd.DataFrame(columns = ['id','ingredients','cuisine'])
 df['id'] = pd.Series(id_list)
 df['ingredients'] = pd.Series(ingredients_list)
 df['cuisine'] = pd.Series(cuisine_list)
 else:
df = pd.DataFrame(columns = ['id','ingredients'])
 df['id'] = pd.Series(id_list)
 df['ingredients'] = pd.Series(ingredients_list)
 return df<load_from_disk>
n=4
commits_df.loc[n,'commit_num'] = 9
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.3
commits_df.loc[n,'LB_score'] = -6.8125
train = load_data('.. /input/train.json','train')
 test = load_data('.. /input/test.json','test')
 len(train), len(test)
<feature_engineering>
n=5
commits_df.loc[n,'commit_num'] = 10
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8089
def removespaces(ingredients):
ingreds = []
for ingred in ingredients:
ingred_mod = ingred.replace(" ","_")
 ingreds.append(ingred_mod)
 return ingreds
def removebracs(ingredients):
ingreds = ""
for ingred in ingredients:
ingred_mod = ingred.replace(" ","_")
 ingreds = ingreds + ingred_mod + ","
return ingreds[:-1]
train['ingredients_tfidf'] = train['ingredients'].apply(removebracs)
 test['ingredients_tfidf'] = test['ingredients'].apply(removebracs)
 train.head()<feature_engineering>
n=6
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8100
vectorizer = TfidfVectorizer()
X_train = vectorizer.fit_transform(train['ingredients_tfidf'])
 X_test = vectorizer.transform(test['ingredients_tfidf'] )<train_on_grid>
n=7
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.175
commits_df.loc[n,'LB_score'] = -6.8096
lb = LabelEncoder()
y = lb.fit_transform(train['cuisine'])
 C = 1.0
svc = svm.SVC(kernel='linear', C=C)
 ovr = OneVsRestClassifier(svc)
 model = ovr.fit(X_train, y)
 y_preds=model.predict(X_test)
 preds = lb.inverse_transform(y_preds)
<save_to_csv>
n=8
commits_df.loc[n,'commit_num'] = 14
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.225
commits_df.loc[n,'LB_score'] = -6.8100
sub = pd.DataFrame({'id': test['id'], 'cuisine': preds}, columns=['id', 'cuisine'])
 sub.to_csv('svm_output.csv', index=False)
<import_modules>
n=9
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'Dropout_model'] = 0.32
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
import json
import numpy as np<load_from_disk>
n=10
commits_df.loc[n,'commit_num'] = 16
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8093
with open('.. /input/train.json')as file:
training_data = json.load(file)
 with open('.. /input/test.json')as file:
testing_data = json.load(file )<categorify>
n=11
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.5
commits_df.loc[n,'LB_score'] = -6.8283
def cleanup_ingredient(ingredient):
x = re.sub('('s)|(\’s)','',ingredient)
 x = re.sub('[0-9%\(\)\/\.™\,®'\&]|(lb\.) |(oz\.) ','',x)
 x = re.sub('\-',' ', x)
 x = re.sub(' +',' ', x)
 x = re.sub('(low fat)|(full fat)|(reduced fat)|(fat free)|(skimmed)|(fatfree)|(lowfat)|(nonfat)|(non fat)|(low sodium)|(reduced sodium)|(less sodium)|(no salt added)|(homemade)|(gluten free)|(salt free)|(reduc sodium)|(cholesterol free)|(s real)|(light)|(free range)|(skim)','',x)
 x = re.sub('(small)|(medium)|(large)','', x)
 x = re.sub('(heinz)|(hellmannâ€)|(hellmanns)|(hellmann)|(kikkoman)|(kraft)|(taco bell)|(mccormick)|(mcintosh)|(knorr)|(or best food)|(best food)', '', x)
 x = re.sub('[éèê]', 'e', x)
 x = re.sub('[íî]', 'i', x)
 x = re.sub('[â]', 'a', x)
 x = re.sub(' +',' ', x)
 x = x.strip()
return x<import_modules>
n=12
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8087
from sklearn.preprocessing import LabelEncoder
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.svm import LinearSVC, SVC
from sklearn.multiclass import OneVsRestClassifier
from sklearn.feature_extraction.text import TfidfVectorizer<define_variables>
n=13
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'Dropout_model'] = 0.39
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090
training_recipes = [" ".join([cleanup_ingredient(ingredient)for ingredient in recipe['ingredients']])for recipe in training_data]
testing_recipes = [" ".join([cleanup_ingredient(ingredient)for ingredient in recipe['ingredients']])for recipe in testing_data] 
<categorify>
n=14
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'Dropout_model'] = 0.37
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
ingredient_encoder = TfidfVectorizer(binary=True )<categorify>
n=15
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.19
commits_df.loc[n,'LB_score'] = -6.8093
X_train = ingredient_encoder.fit_transform(training_recipes )<categorify>
n=16
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8402
X_test = ingredient_encoder.transform(testing_recipes )<categorify>
n=17
commits_df.loc[n,'commit_num'] = 26
commits_df.loc[n,'Dropout_model'] = 0.385
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
cuisine_encoder = LabelEncoder()
y_train = cuisine_encoder.fit_transform([recipe['cuisine'] for recipe in training_data] )<split>
n=18
commits_df.loc[n,'commit_num'] = 27
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.21
commits_df.loc[n,'LB_score'] = -6.8091
X_train, X_devtest, y_train, y_devtest = train_test_split(X_train, y_train, test_size=0.4)
 X_devtest, X_testest, y_devtest, y_testest = train_test_split(X_devtest, y_devtest, test_size=0.5)
 X_train = X_train.astype('float16')
 X_devtest = X_devtest.astype('float16')
 X_testest = X_testest.astype('float16' )<train_on_grid>
n=19
commits_df.loc[n,'commit_num'] = 28
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090
%matplotlib inline
degree = 3
gamma = 1.4
C = 13.0
best_C = C
best_gamma = gamma
best_degree = degree
best_score = -1
best_model = None
hist = {'C':[], 'acc': [], 'gamma':[], 'degree':[]}
svm = SVC(C=C, kernel='rbf', gamma=gamma, degree=degree, max_iter=-1)
 svm = OneVsRestClassifier(svm, n_jobs=4)
 svm.fit(X_train, y_train)
 best_model = svm
best_score = svm.score(X_devtest, y_devtest)
<compute_test_metric>
n=20
commits_df.loc[n,'commit_num'] = 29
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'GaussianNoise_stddev'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8092
print("{} {} {}".format(best_C, best_gamma, best_score))
print("Train accuracy {}".format(best_model.score(X_train, y_train)))
 print("Cross validation accuracy {}".format(best_model.score(X_devtest, y_devtest)))
 print("Holdout test accuracy {}".format(best_model.score(X_testest, y_testest)) )<predict_on_test>
n=21
commits_df.loc[n,'commit_num'] = 31
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'GaussianNoise_stddev'] = 0.21
commits_df.loc[n,'LB_score'] = -6.8093
y_test = best_model.predict(X_test )<categorify>
n=22
commits_df.loc[n,'commit_num'] = 32
commits_df.loc[n,'Dropout_model'] = 0.24
commits_df.loc[n,'FVC_weight'] = 0.14
commits_df.loc[n,'GaussianNoise_stddev'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8106
y_pred = cuisine_encoder.inverse_transform(y_test )<define_variables>
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1
ids = [recipe['id'] for recipe in testing_data]<save_to_csv>
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0
with open('submission.csv','w')as file:
file.write('id,cuisine
')
 for id_,cuisine in zip(ids, y_pred):
file.write("{},{}
".format(id_, cuisine))<load_from_disk>
commits_df['GaussianNoise_stddev'] = 0.2
commits_df.loc[commits_df['commit_num'] == 28, 'GaussianNoise_stddev'] = 0.25
train_df = pd.read_json(".. /input/train.json" ).set_index("id")
 train_df.head()<split>
commits_df.sort_values(by=['LB_score'], ascending = False )
le = LabelEncoder()
y = le.fit_transform(train_df["cuisine"].values)
 X = train_df["ingredients"].values
x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size = 0.33 )<feature_engineering>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
tfidf_vectorizer = TfidfVectorizer(analyzer = "word", preprocessor = lambda x: x, tokenizer = lambda x: x, token_pattern = None)
 tfidf_train = tfidf_vectorizer.fit_transform(x_train)
 tfidf_valid = tfidf_vectorizer.transform(x_valid)
 tfidf_train<find_best_params>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
LogisticRegression().get_params().keys()<predict_on_test>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
logreg = LogisticRegression(penalty = "L2", dual = False, tol = 1e-5, C = 1.0, fit_intercept = True, intercept_scaling = 1,
class_weight = None, random_state = None, solver = "saga", max_iter = 250, multi_class = "ovr",
verbose = 0, warm_start = False, n_jobs = 1)
 logreg.fit(tfidf_train, y_train)
 train_pred = logreg.predict(tfidf_train)
 valid_pred = logreg.predict(tfidf_valid )<compute_test_metric>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
print("Train Accuracy: "+str(accuracy_score(y_train, train_pred)))
 print("Valid Accuracy: "+str(accuracy_score(y_valid, valid_pred)))
 print("Train F1 Score: "+str(f1_score(y_train, train_pred, average = "weighted")))
 print("Valid F1 Score: "+str(f1_score(y_valid, valid_pred, average = "weighted")))
 print("Train Precision: "+str(precision_score(y_train, train_pred, average = "weighted")))
 print("Valid Precision: "+str(precision_score(y_valid, valid_pred, average = "weighted")))
 print("Train Recall: "+str(recall_score(y_train, train_pred, average = "weighted")))
 print("Valid Recall: "+str(recall_score(y_valid, valid_pred, average = "weighted")) )<define_search_space>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(GaussianNoise_stddev )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
Cs = np.geomspace(1e-8, 1e8, 30)
 penalties = ["L1", "L2"]
accuracy_train = {}
accuracy_valid = {}
accuracy_diff = {}
f1_train = {}
f1_valid = {}
f1_diff = {}
precision_train = {}
precision_valid = {}
precision_diff = {}
recall_train = {}
recall_valid = {}
recall_diff = {}
for i, penalty in enumerate(penalties):
accuracy_train[penalty] = np.zeros(len(Cs))
f1_train[penalty] = np.zeros(len(Cs))
precision_train[penalty] = np.zeros(len(Cs))
recall_train[penalty] = np.zeros(len(Cs))
accuracy_valid[penalty] = np.zeros(len(Cs))
f1_valid[penalty] = np.zeros(len(Cs))
precision_valid[penalty] = np.zeros(len(Cs))
recall_valid[penalty] = np.zeros(len(Cs))
for j, C in enumerate(Cs):
start_time = time.time()
print("("+str(( i*30)+(j+1)) +")Starting penalty="+penalty+", C="+str(C))
logreg = LogisticRegression(penalty = penalty, dual = False, tol = 1e-5, C = C, fit_intercept = True, intercept_scaling = 1,
class_weight = None, random_state = None, solver = "saga", max_iter = 250, multi_class = "ovr",
verbose = 0, warm_start = False, n_jobs = 1)
 logreg.fit(tfidf_train, y_train)
 train_pred = logreg.predict(tfidf_train)
 valid_pred = logreg.predict(tfidf_valid)
 accuracy_train[penalty][j] = accuracy_score(y_train, train_pred)
 accuracy_valid[penalty][j] = accuracy_score(y_valid, valid_pred)
 f1_train[penalty][j] = f1_score(y_train, train_pred, average = "weighted")
 f1_valid[penalty][j] = f1_score(y_valid, valid_pred, average = "weighted")
 precision_train[penalty][j] = precision_score(y_train, train_pred, average = "weighted")
 precision_valid[penalty][j] = precision_score(y_valid, valid_pred, average = "weighted")
 recall_train[penalty][j] = recall_score(y_train, train_pred, average = "weighted")
 recall_valid[penalty][j] = recall_score(y_valid, valid_pred, average = "weighted")
 end_time = time.time()
print("("+str(( i*30)+(j+1)) +")Finished penalty="+penalty+", C="+str(C)+" in "+str(end_time - start_time)+"s")
 accuracy_diff[penalty] = np.absolute(accuracy_train[penalty] - accuracy_valid[penalty])
 f1_diff[penalty] = np.absolute(f1_train[penalty] - f1_valid[penalty])
 precision_diff[penalty] = np.absolute(precision_train[penalty] - precision_valid[penalty])
 recall_diff[penalty] = np.absolute(recall_train[penalty] - recall_valid[penalty] )<compute_test_metric>
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )
xx = np.argmax(accuracy_valid["L1"])
 yx = np.argmax(accuracy_valid["L2"])
 print("Max L1 validation accuracy is at C="+str(Cs[xx]))
print("Train Accuracy : "+str(accuracy_train["L1"][xx]))
print("Valid Accuracy : "+str(accuracy_valid["L1"][xx]))
print("Difference Accuracy : "+str(accuracy_diff["L1"][xx]))
print("Max L2 validation accuracy is at C="+str(Cs[yx]))
print("Train Accuracy : "+str(accuracy_train["L2"][yx]))
print("Valid Accuracy : "+str(accuracy_valid["L2"][yx]))
print("Difference Accuracy : "+str(accuracy_diff["L2"][yx]))<train_on_grid>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
logreg = LogisticRegression(dual = False, tol = 1e-5, fit_intercept = True, intercept_scaling = 1, class_weight = None,
random_state = None, solver = "saga", max_iter = 1000, multi_class = "ovr",
verbose = 0, warm_start = False, n_jobs = 1)
 params = {
"penalty": ["L1", "L2"],
"C": stats.lognorm(0.75, scale = Cs[xx])
 }
clf = RandomizedSearchCV(logreg, params, scoring = "accuracy", cv = 5)
 clf.fit(tfidf_train, y_train)
 print(clf.best_params_ )<find_best_params>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
print(clf.best_params_ )<categorify>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
C_best = clf.best_params_["C"]
penalty_best = clf.best_params_["penalty"]
tfidf_vectorizer = TfidfVectorizer(analyzer = "word", preprocessor = lambda x: x, tokenizer = lambda x: x, token_pattern = None)
 tfidf = tfidf_vectorizer.fit_transform(train_df["ingredients"])
 le = LabelEncoder()
y = le.fit_transform(train_df["cuisine"])
 logreg = LogisticRegression(dual = False, tol = 1e-5, fit_intercept = True, intercept_scaling = 1, class_weight = None,
random_state = None, solver = "saga", max_iter = 100000, multi_class = "ovr",
verbose = 0, warm_start = False, n_jobs = 1, C = C_best, penalty = penalty_best)
 logreg.fit(tfidf, y)
 pred = logreg.predict(tfidf)
 print("Accuracy: "+str(accuracy_score(y, pred)) )<save_to_csv>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
test_df = pd.read_json(".. /input/test.json" ).set_index("id")
 test_tfidf = tfidf_vectorizer.transform(test_df["ingredients"])
 pred_test = logreg.predict(test_tfidf)
 test_df["cuisine"] = le.inverse_transform(pred_test)
 test_df.drop(columns = ["ingredients"], inplace = True)
 test_df.to_csv("tuned_logistic_regression_submission.csv" )<load_pretrained>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
print("Read Dataset...")
 def read_dataset(path):
	return json.load(open(path))
train = read_dataset('.. /input/train.json')
 test = read_dataset('.. /input/test.json')
 shuffle(train)
 shuffle(train)
 print("Prepare text data of Train and Test...")
 def generate_text(data):
	text_data = [" ".join(doc['ingredients'] ).lower() for doc in data]
	return text_data
train_text = generate_text(train)
 test_text = generate_text(test)
 target = [doc['cuisine'] for doc in train]
print("TF-IDF on text data...")
 tfidf = TfidfVectorizer(binary=True)
 def tfidf_features(txt, flag):
if flag == "train":
	x = tfidf.fit_transform(txt)
 else:
	 x = tfidf.transform(txt)
 x = x.astype('float16')
 return x
X = tfidf_features(train_text, flag="train")
 X_test = tfidf_features(test_text, flag="test")
 print("Label Encode the Target Variable...")
 lb = LabelEncoder()
y = lb.fit_transform(target )<categorify>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
print("Label Encode the Target Variable...")
 lb = LabelEncoder()
y = lb.fit_transform(target)
 print("TF-IDF on text data...")
 tfidf = TfidfVectorizer(binary=True)
 def tfidf_features(txt, flag):
if flag == "train":
	x = tfidf.fit_transform(txt)
 else:
	 x = tfidf.transform(txt)
 return x
X = tfidf_features(train_text, flag="train")
 X_test = tfidf_features(test_text, flag="test")
<data_type_conversions>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
X=X.toarray().tolist()
X_test = X_test.toarray().tolist()
print(y[1:3] )<feature_engineering>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
Y=y
class_weight_ = class_weight.compute_class_weight('balanced',
np.unique(y),
y)
 class_weights={}
for i in range(len(class_weight_)) :
class_weights[i]=100/class_weight_[i]
X=np.array(X)
 X_test=np.array(X_test )<choose_model_class>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
model = Sequential()
model.add(Dense(500, activation='relu',input_shape=(3010,)))
 model.add(Dropout(0.25))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(len(Y[0]), activation='softmax'))
sgd = optimizers.SGD(lr=.1, decay=1e-6, momentum=0.9, nesterov=True)
 model.compile(optimizer='nadam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 model.summary()<train_model>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
<import_modules>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
from sklearn.cross_validation import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from sklearn import tree<train_model>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
model = DecisionTreeClassifier(criterion = "gini", random_state = 1000,
max_depth=1000, min_samples_leaf=1)
 model.fit(X, Y )<compute_test_metric>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
score = model.score(X[33000:],Y[33000:])
 print('Test accuracy:', score )<predict_on_test>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
Ans= model.predict(X_test)
 print(Ans[0])
 Ans= lb.inverse_transform(Ans )<save_to_csv>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
print("Generate Submission File...")
 test_id = [doc['id'] for doc in test]
sub = pd.DataFrame({'id': test_id, 'cuisine': Ans}, columns=['id', 'cuisine'])
 sub.to_csv('svm_output.csv', index=False )<load_pretrained>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
def initi(m):
if isinstance(m,nn.Linear):
nn.init.xavier_uniform(m.weight.data)
 with open('.. /input/train.json')as f:
train = json.load(f)
 with open('.. /input/test.json')as f:
test = json.load(f)
 print('imported' )<prepare_x_and_y>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
shuffle(train)
 xx=[d['ingredients'] for d in train]
yy=[d['cuisine'] for d in train] 
<define_variables>
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
word_list=set()
for x in xx:
word_list|=set(tuple(x))
word_list=list(word_list)
 cat_list=list(set(yy))

<define_variables>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
newx=[]
for num,rec in enumerate(xx):
newx.append(np.zeros(len(word_list)))
 for ing in rec:
if ing in word_list:
ing_location=word_list.index(ing)
 newx[num][ing_location]=1
newx=np.array(newx)
 newy2=[]
for num,rec in enumerate(yy):
newy2.append(cat_list.index(rec))
newy2=np.array(newy2)
 print('First 3 recepies of one hot vectors')
 print(newx[0:5])
 print('First 5 cuisines in numerical')
 print(newy2[0:5] )<compute_test_metric>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
def stats(prevloss):
train_loss_history.append(loss.data[0])
 valpred=model(txval)
 valloss=loss_fn(valpred,tyval)
 val_loss_history.append(valloss.data[0])
 actualy=tyval.cpu().data.numpy()
predy=valpred.cpu().data.numpy()
val_accuracy=(np.argmax(predy,axis=1)==actualy ).sum() /len(predy)
 val_accuracy_history.append(val_accuracy)
 if t%10==0:
print('Iteration
print('Current valdiation loss')
 print(round(valloss.data[0],2))
print('Previous validation loss')
 print(round(prevloss,2))
print('Validation accuracy: {} %'.format(round(100*val_accuracy,3)))

prevloss=valloss.data[0]
return(prevloss)
<categorify>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
dtype = torch.cuda.FloatTensor
SPLIT = 0.8
tv=int(np.ceil(len(newx)*SPLIT))
xtrain=newx[0:tv]
ytrain=newy2[0:tv]
xval=newx[tv:]
yval=newy2[tv:]

 txval=Variable(torch.from_numpy(xval.astype(float)).float() ).type(dtype)
 tyval=Variable(torch.from_numpy(yval.astype(float)).long() ).cuda()
tx=Variable(torch.from_numpy(xtrain.astype(float)).float() ).type(dtype)
 ty=Variable(torch.from_numpy(ytrain.astype(float)).long() ).cuda()
print(tx.shape)
<choose_model_class>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
batch_size=50
model= nn.Sequential(
 nn.Linear(6714,20),
nn.LeakyReLU() , 
)
 model.cuda()
model.zero_grad()
model.apply(initi)
 loss_fn=nn.CrossEntropyLoss()
print('built new model')
<choose_model_class>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
lr=1e-4
optimizer = torch.optim.Adam(model.parameters() , lr=lr,betas=(0.9,0.99))
optimizer.zero_grad()
print('made new optimizer')
<define_variables>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
epochs=170
itt=0
train_loss_history=[]
val_loss_history=[]
val_accuracy_history=[]
prevloss=90
steps_back=0<find_best_params>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
for t in range(epochs):
itt=0
for curr in range(batch_size,len(tx),batch_size):
x=tx[itt:curr]
y=ty[itt:curr]
pred=model(x)
 loss=loss_fn(pred,y)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
itt+=batch_size
prevloss=stats(prevloss)
 print('Trained!')
<feature_engineering>
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()
<data_type_conversions><EOS> 
df.to_csv('submission.csv', index=False )