Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<choose_model_class>
%matplotlib inline
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
batch_size = 5000
hidden_units = [150, 150, 150]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
epochs = 200
clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 if inference:
clf = keras.models.load_model('keras_nn')
 else:
clf.fit(X, y, epochs=epochs, batch_size=5000)
 clf.save('keras_nn')
 models = []
models.append(clf)
 th = 0.5000
f = np.median
models = models[-3:] 
<split>
IMG_ROWS = 28
IMG_COLS = 28
NUM_CLASSES = 10
TEST_SIZE = 0.1
RANDOM_STATE = 2018
NO_EPOCHS = 150
PATIENCE = 20
VERBOSE = 1
BATCH_SIZE = 128
IS_LOCAL = False
if(IS_LOCAL):
PATH=".. /input/digit-recognizer/"
else:
PATH=".. /input/"
print(os.listdir(PATH))
if cv:
oof_probas = np.zeros(y.shape)
 val_idx_all = []
N_SPLITS = 5
gkf = GroupKFold(n_splits=N_SPLITS)
 for fold,(train_idx, val_idx)in enumerate(gkf.split(train.action.values, groups=train.date.values)) :
X_train, X_val = X.iloc[train_idx], X.iloc[val_idx].values
y_train, y_val = y[train_idx], y[val_idx]
clf.fit(X_train, y_train, epochs=epochs, batch_size=5000)
 oof_probas[val_idx] += clf(X_val, training=False ).numpy()
score = roc_auc_score(y_val, oof_probas[val_idx])
 print(f'FOLD {fold} ROC AUC:\t {score}')
 del X_train, X_val, y_train, y_val
gc.collect()
val_idx_all.append(val_idx)
 val_idx = np.concatenate(val_idx_all)
<compute_test_metric>
train_file = PATH+"train.csv"
test_file = PATH+"test.csv"
train_df = pd.read_csv(train_file)
 test_df = pd.read_csv(test_file )
if cv:
auc_oof = roc_auc_score(y[val_idx], oof_probas[val_idx])
 print(auc_oof )<compute_test_metric>
print("MNIST train - rows:",train_df.shape[0]," columns:", train_df.shape[1])
 print("MNIST test - rows:",test_df.shape[0]," columns:", test_df.shape[1] )
def determine_action(df, thresh):

 action =(df.weight * df.resp > thresh ).astype(int)
 return action
def date_weighted_resp(df):

 cols = ['weight', 'resp', 'action']
weighted_resp = np.prod(df[cols], axis=1)
 return weighted_resp.sum()
def calculate_t(dates_p):

 e_1 = dates_p.sum() / np.sqrt(( dates_p**2 ).sum())
 e_2 = np.sqrt(250/np.abs(len(dates_p)))
 return e_1 * e_2
def calculate_u(df, thresh):

 df = df.copy()
dates_p = df.groupby('date' ).apply(date_weighted_resp)
 t = calculate_t(dates_p)
 return t, min(max(t, 0), 6)* dates_p.sum()
def plot_roc_curve(fpr, tpr, label=None):
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, linewidth=2, label=label)
 plt.plot([0, 1], [0, 1], 'k--', label='Random')
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.legend(loc='lower right')
 plt.grid()
def plot_precision_recall_curve(precisions, recalls, thresholds):
plt.figure(figsize=(8, 6))
plt.plot(thresholds, precisions[:-1], 'b--', label='Precision')
 plt.plot(thresholds, recalls[:-1], 'g-', label='Recall')
 plt.xlabel('Thresholds')
 plt.legend(loc='lower left')
 plt.grid()
def plot_thresh_u_t(df, oof):
threshs = np.linspace(0, 1, 1000)
 ts = []
us = []
for thresh in threshs:
df['action'] = np.where(oof >= thresh, 1, 0)
 t, u = calculate_u(df, thresh)
 ts.append(t)
 us.append(u)
 ts = np.array(ts)
 us = np.array(us)
 ts = np.where(np.isnan(ts), 0.0, ts)
 us = np.where(np.isnan(us), 0.0, us)
 tmax = np.argmax(ts)
 umax = np.argmax(us)
 print(f'Max Utility Score: {us[umax]}')
 fig, axes = plt.subplots(1, 2, figsize=(14, 4))
axes[0].plot(threshs, ts)
 axes[0].set_title('Different t scores by threshold')
 axes[0].set_xlabel('Threshold')
 axes[0].axvline(threshs[tmax])
 axes[1].plot(threshs, us)
 axes[1].set_title('Different u scores by threshold')
 axes[1].set_xlabel('Threshold')
 axes[1].axvline(threshs[umax], color='r', linestyle='--', linewidth=1.2)
 print(f'Optimal Threshold: {threshs[umax]}')
 return threshs[umax] 
<feature_engineering>
def get_classes_distribution(data):
label_counts = data["label"].value_counts()
total_samples = len(data)
 for i in range(len(label_counts)) :
label = label_counts.index[i]
count = label_counts.values[i]
percent =(count / total_samples)* 100
print("{}: {} or {}%".format(label, count, percent))
get_classes_distribution(train_df )
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
x_tt = np.append(x_tt, [[1]], axis=1)
 x_tt = neutralize_array(x_tt, neutralizer)
 pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= opt_thresh, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<install_modules>
def sample_images_data(data, hasLabel=True):
sample_images = []
sample_labels = []
if(hasLabel):
for k in range(0,10):
samples = data[data["label"] == k].head(4)
 for j, s in enumerate(samples.values):
img = np.array(samples.iloc[j, 1:] ).reshape(IMG_ROWS,IMG_COLS)
 sample_images.append(img)
 sample_labels.append(samples.iloc[j, 0])
 else:
samples = data.iloc[random.sample(range(1, 10000), 40),]
for j, s in enumerate(samples.values):
img = np.array(samples.iloc[j, 0:] ).reshape(IMG_ROWS,IMG_COLS)
 sample_images.append(img)
 sample_labels.append(-1)
 print("Total number of sample images to plot: ", len(sample_images))
return sample_images, sample_labels
train_sample_images, train_sample_labels = sample_images_data(train_df )
def install(package):
subprocess.check_call([sys.executable, "-m", "pip","install",package])
 install(".. /input/fastremap/fastremap-1.10.2-cp37-cp37m-manylinux1_x86_64.whl")
 install(".. /input/fillvoids/fill_voids-2.0.0-cp37-cp37m-manylinux1_x86_64.whl")
 install(".. /input/finalmask")
 install("pydicom" )<set_options>
def data_preprocessing(raw, hasLabel=True):
start_pixel = 0
if(hasLabel):
start_pixel = 1
if(hasLabel):
out_y = keras.utils.to_categorical(raw.label, NUM_CLASSES)
 else:
out_y = None
num_images = raw.shape[0]
x_as_array = raw.values[:,start_pixel:]
x_shaped_array = x_as_array.reshape(num_images, IMG_ROWS, IMG_COLS, 1)
 out_x = x_shaped_array / 255
return out_x, out_y
sns.set(style="whitegrid")
 sns.set_context("paper")
<define_variables>
X, y = data_preprocessing(train_df)
 X_test, y_test = data_preprocessing(test_df,hasLabel=False )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<load_from_csv>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=TEST_SIZE, random_state=RANDOM_STATE )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
train=pd.read_csv(f"{ROOT}/train.csv")
 train.head()<load_from_csv>
print("MNIST train - rows:",X_train.shape[0]," columns:", X_train.shape[1:4])
 print("MNIST valid - rows:",X_val.shape[0]," columns:", X_val.shape[1:4])
 print("MNIST test - rows:",X_test.shape[0]," columns:", X_test.shape[1:4] )
sample_submission=pd.read_csv(f"{ROOT}/sample_submission.csv")
 test=pd.read_csv(f"{ROOT}/test.csv")
 test.head()<merge>
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),activation='relu', padding="same",
kernel_initializer='he_normal',input_shape=(IMG_ROWS, IMG_COLS, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=(3, 3), activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size=(3, 3), strides=2,padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3, 3), strides=2,padding='same', activation='relu'))
model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(NUM_CLASSES, activation='softmax'))
train['Patient_Week']=train['Patient']+'_'+train['Weeks'].astype(str)
 lists=train['Patient_Week'][train.duplicated(['Patient_Week'], keep=False)].unique().tolist()
for patient_week in lists:
new_row=train.loc[train['Patient_Week']==patient_week].groupby(['Patient','Weeks','Age','Sex','SmokingStatus','Patient_Week'] ).mean().copy()
train=train[train['Patient_Week']!=patient_week]
train=train.append(new_row.reset_index())
 add=train.copy()
add.rename(columns={'Weeks':'base_weeks','FVC':'base_fvc'},inplace=True)
 final=train.merge(add,on='Patient')
 final.drop(['Patient_Week_x','Age_y','Sex_y','SmokingStatus_y','Percent_y'],axis=1,inplace=True)
 final.rename(columns={'Weeks':'base_week','FVC':'base_fvc','base_fvc':'FVC','Percent_x':'base_percent','Patient_Week_y':'Patient_Week','Age_x':'Age','Sex_x':'sex','SmokingStatus_x':'smokingstatus','base_weeks':'predict_week'},inplace=True)
 final['weeks_passed']=final['predict_week']-final['base_week']
cols=['Patient','Patient_Week', 'base_week', 'base_fvc', 'base_percent', 'Age', 'sex','smokingstatus','predict_week','weeks_passed', 'FVC']
final=final[cols]
final=final.loc[final['weeks_passed']!=0]
final.reset_index(drop=True,inplace=True)
 final.head() 
<feature_engineering>
model.compile(loss = "categorical_crossentropy", optimizer="adam", metrics=["accuracy"] )
test.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'},inplace=True)
 Week=sample_submission['Patient_Week'].apply(lambda x : x.split('_')[1] ).unique()
Week=np.tile(Week, len(test['Patient']))
test=test.loc[test.index.repeat(146)].reset_index(drop=True)
 test['predict_week']=Week
test['Patient_Week']=test['Patient']+'_'+test['predict_week']
test['weeks_passed']=test['predict_week'].astype(int)-test['base_Week'].astype(int)
 test.rename(columns={'base_Week':'base_week','base_FVC':'base_fvc','base_Percent':'base_percent','base_Age':'Age','Sex':'sex','SmokingStatus':'smokingstatus'},inplace=True)
 cols=['Patient','Patient_Week','base_week','base_fvc','base_percent','Age','sex','smokingstatus','predict_week','weeks_passed']
test=test[cols]
print(test.shape)
 test.head()<load_pretrained>
plot_model(model, to_file='model.png')
 SVG(model_to_dot(model ).create(prog='dot', format='svg'))
file_path= '.. /input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/10.dcm'
dataset = pydicom.dcmread(file_path)
<compute_test_metric>
NO_EPOCHS = 10
def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = abs(y_pred[:,2] - y_pred[:,0])
 fvc_pred = y_pred[:,1]
sigma_clip = tf.maximum(sigma, 70)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, 1000)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return metric
def qloss(y_true, y_pred):
qs = [0.8,0.5,0.2]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return kb.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss<load_pretrained>
earlystopper = EarlyStopping(monitor='loss', patience=PATIENCE, verbose=VERBOSE)
 checkpointer = ModelCheckpoint('best_model.h5',
monitor='val_acc',
verbose=VERBOSE,
save_best_only=True,
save_weights_only=True)
 history = model.fit(X_train, y_train,
batch_size=BATCH_SIZE,
epochs=NO_EPOCHS,
verbose=1,
validation_data=(X_val, y_val),
callbacks=[earlystopper, checkpointer] )
input_image = sitk.ReadImage('.. /input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/12.dcm')
 segmentation = mask.apply(input_image)
 plt.figure(figsize=(10,10))
plt.imshow(segmentation[0] )<load_from_csv>
print("run model - predict validation set")
 score = model.evaluate(X_val, y_val, verbose=0)
 print(f'Last validation loss: {score[0]}, accuracy: {score[1]}')
 model_optimal = model
model_optimal.load_weights('best_model.h5')
 score = model_optimal.evaluate(X_val, y_val, verbose=0)
 print(f'Best validation loss: {score[0]}, accuracy: {score[1]}' )
atten=pd.read_csv('.. /input/attent-1/atten(1 ).csv')
 patients=os.listdir(f"{ROOT}/{how}")
 avg_atten_test=[]
for patient in patients:
try:
mid=mid_image_test(patient,True,True)
 postives=mid>mid.min()
mid[postives].mean()
avg_atten_test.append(mid[postives].mean())
 except:
avg_atten_test.append(np.nan)
 continue<prepare_x_and_y>
def predict_show_classes(model, X_val, y_val):
predicted_classes = model.predict_classes(X_val)
 y_true = np.argmax(y_val,axis=1)
 correct = np.nonzero(predicted_classes==y_true)[0]
incorrect = np.nonzero(predicted_classes!=y_true)[0]
print("Correct predicted classes:",correct.shape[0])
 print("Incorrect predicted classes:",incorrect.shape[0])
 target_names = ["Class {}:".format(i)for i in range(NUM_CLASSES)]
print(classification_report(y_true, predicted_classes, target_names=target_names))
return correct, incorrect
final1=final.copy()
final1=final1.merge(atten,on='Patient')
 X1=final1[['base_fvc','base_percent','Age','sex','smokingstatus','weeks_passed','avg_atten','base_week']].copy()
y1=final1.FVC.copy()<categorify>
correct, incorrect = predict_show_classes(model, X_val, y_val )
enc = OneHotEncoder(handle_unknown='ignore')
 enc.fit(X1[['sex','smokingstatus']])
 encoded=pd.DataFrame(enc.transform(X1[['sex','smokingstatus']] ).toarray())
 X1=X1.join(encoded)
 X1.drop(['smokingstatus','sex'],axis=1,inplace=True)
 scaler=preprocessing.MinMaxScaler().fit(X1)
 X1=pd.DataFrame(scaler.transform(X1))
X1.head()<categorify>
correct, incorrect = predict_show_classes(model_optimal, X_val, y_val )
atten_test=pd.DataFrame({'Patient':patients,'avg_atten':avg_atten_test})
 atten_test['avg_atten']=atten_test['avg_atten'].fillna(atten["avg_atten"].mean())
 X_test=test.merge(atten_test,on='Patient')
 X_test=X_test[['base_fvc','base_percent','Age','sex','smokingstatus','weeks_passed','avg_atten','base_week']].copy()
encoded=pd.DataFrame(enc.transform(X_test[['sex','smokingstatus']] ).toarray())
 X_test=X_test.join(encoded)
 X_test.drop(['smokingstatus','sex'],axis=1,inplace=True)
 X_test=pd.DataFrame(scaler.transform(X_test))
X1=X1.astype(np.float32)
 y1=y1.astype(np.float32 )<choose_model_class>
y_cat = model.predict(X_test, batch_size=64 )
inputs= keras.Input(shape=[11])
 dense = layers.Dense(100, activation="relu")
 x = dense(inputs)
 x = layers.Dense(100, activation="relu" )(x)
 output1 = layers.Dense(3,activation='linear' )(x)
 model = keras.Model(inputs=inputs, outputs=output1)
 model.summary()<train_model>
y_pred = np.argmax(y_cat,axis=1 )
model.compile(loss=mloss(0.8),optimizer='adam',metrics=score)
 model.fit(X1, y1,batch_size=512,epochs=130 )<predict_on_test>
output_file = "submission.csv"
with open(output_file, 'w')as f :
f.write('ImageId,Label
')
 for i in range(len(y_pred)) :
f.write("".join([str(i+1),',',str(y_pred[i]),'
']))
preds_high=model.predict(X_test)[:,0]
preds_low=model.predict(X_test)[:,2]
preds=model.predict(X_test)[:,1]<prepare_output>
y_cat = model_optimal.predict(X_test, batch_size=64)
 y_pred = np.argmax(y_cat,axis=1)
 output_file = "submission_optimal.csv"
with open(output_file, 'w')as f :
f.write('ImageId,Label
')
 for i in range(len(y_pred)) :
f.write("".join([str(i+1),',',str(y_pred[i]),'
']))