Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<define_variables>
from IPython.display import clear_output
categorical_features = [
'item_id', 'dept_id', 'store_id', 'state_id',
'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'
]<filter>
!pip install -U efficientnet
clear_output()
drop = ["id", "date", "sales", "day", "wm_yr_wk", "weekday", "cat_id"]
train_columns = train.columns[~train.columns.isin(drop)]
print(train_columns )<drop_column>
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
train_sets = [train[train["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del train; gc.collect()<drop_column>
import efficientnet.tfkeras as efn
from matplotlib import pyplot as plt
from tensorflow.keras.applications import InceptionResNetV2
val_sets = [validation[validation["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del validation; gc.collect()<create_dataframe>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
train_pools = [lgb.Dataset(
 data=train_sets[category][train_columns],
label=train_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del train_sets; gc.collect()<create_dataframe>
img = plt.imread('.. /input/plant-pathology-2020-fgvc7/images/Train_10.jpg')
 print(img.shape)
 plt.imshow(img )
val_pools = [lgb.Dataset(
 data=val_sets[category][train_columns],
label=val_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del val_sets; del evaluation; gc.collect()<init_hyperparams>
train_df = pd.read_csv("/kaggle/input/plant-pathology-2020-fgvc7/train.csv")
 test_df = pd.read_csv("/kaggle/input/plant-pathology-2020-fgvc7/test.csv")
 sub_df = pd.read_csv("/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv" )
if TEST_RUN:
ITERATIONS=1
else:
ITERATIONS = 1200
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : ITERATIONS,
'num_leaves': 128,
"min_data_in_leaf": 100,
"early_stopping_round": 50
}<train_model>
img_size = 896
EPOCHS = 40
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
n_classes = 4
%%time
models = [lgb.train(params,
train_pools[category],
valid_sets = val_pools[category],
verbose_eval=20)
 for category in range(3)]
for cat,model in enumerate(models):
model.save_model(f"model_cat{cat}_{MODEL_VERSION}.lgb" )<set_options>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
del train_pools, val_pools; gc.collect()<data_type_conversions>
train_tpu_paths = train_df.image_id.apply(lambda x: GCS_DS_PATH+"/images/"+x+".jpg" ).values
train_labels = train_df.iloc[:, 1:].values
test_tpu_paths = test_df.image_id.apply(lambda x: GCS_DS_PATH+"/images/"+x+".jpg" ).values
df = load_data(train=False)
 make_date_features(df)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8' )<feature_engineering>
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if tf.random.uniform([1])>0.4:
image = tf.image.adjust_brightness(image, 0.2)
 if label is None:
return image
else:
return image, label
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'id'])
 dt.loc[dt['date'] == str(day),f'rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print("Lag features done")
 print(type(dt)) 
<define_variables>
train_set =(tf.data.Dataset 
.from_tensor_slices(( train_tpu_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO))
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_tpu_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
%%time

 END_DATE = EVAL_SPLIT
ZERO_THRESHOLD = 0.01
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 for i in range(3):
prediction_data_cat = prediction_data.loc[(prediction_data['date'] == str(pred_date)) &(prediction_data["cat_id"] == i), train_columns]
prediction = models[i].predict(prediction_data_cat)
 df.loc[(df['date'] == str(pred_date)) &(df.index.isin(prediction_data_cat.index)) , 'sales'] = prediction 
<set_options>
def get_model_eff() :
base_model = efn.EfficientNetB7(weights='noisy-student', include_top=False, pooling='avg', input_shape=(None, None, 3))
x = base_model.output
predictions = tf.keras.layers.Dense(n_classes, activation="softmax" )(x)
 return tf.keras.models.Model(inputs=base_model.input, outputs=predictions )
del prediction_data
gc.collect()<feature_engineering>
with strategy.scope() :
model1 = get_model_eff()
model1.compile(optimizer='adam', loss='categorical_crossentropy',metrics=['accuracy'] )
submission_val = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_val.loc[submission_val['sales'] < 0, 'sales'] = 0
submission_val.sort_values('id', inplace=True)
 submission_val['day'] = submission_val['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_val.columns )<prepare_output>
hist1 = model1.fit(train_set,
steps_per_epoch=train_labels.shape[0]//BATCH_SIZE,
epochs = EPOCHS,
callbacks=[lr_callback],
)
submission_eval = submission_val.copy()
submission_eval['id'] = submission_eval['id'].str.replace('validation', 'evaluation')
 submission = pd.concat([submission_val, submission_eval], axis=0, sort=False)
 print(submission.columns)
 print(submission.head(1))<save_to_csv>
%%time
preds1 = model1.predict(test_dataset )
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )<import_modules>
with strategy.scope() :
gc.collect()
import pandas as pd, numpy as np
import math, json, gc, random, os, sys
from matplotlib import pyplot as plt
from tqdm import tqdm
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold<load_from_csv>
def get_model_inc() :
base_model = InceptionResNetV2(weights='imagenet', include_top=False, pooling='avg', input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = tf.keras.layers.Dense(n_classes, activation="softmax" )(x)
 return tf.keras.models.Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model2 = get_model_inc()
model2.compile(optimizer='adam', loss='categorical_crossentropy',metrics=['accuracy'])
 model2.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['accuracy'] 
)
train = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines=True)
 sample_sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv" )<define_variables>
hist2 = model2.fit(train_set,
steps_per_epoch=train_labels.shape[0]//BATCH_SIZE,
epochs = EPOCHS,
callbacks=[lr_callback],
)
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']<categorify>
%%time
preds2 = model2.predict(test_dataset )
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def get_pair_index_structure(structure):
structure = np.array([struc for struc in structure], dtype="<U4")
 open_index = np.where(structure == "(")[0]
closed_index = np.where(structure == ")")[0]
structure[open_index] = range(0, len(open_index))
structure[closed_index] = range(len(open_index)-1, -1, -1)
 structure[structure == "."] = -1
structure = structure.astype(int)
 pair_structure = np.array([-1]*len(structure))
for i in range(len(open_index)) :
start, end = np.where(structure == i)[0]
pair_structure[start] = end
pair_structure[end] = start
return pair_structure<categorify>
preds =(preds1+preds2)/2
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
return np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 train_inputs_all = preprocess_inputs(train)
 train_labels_all = np.array(train[target_cols].values.tolist() ).transpose(( 0, 2, 1))<compute_test_metric>
sub_df.iloc[:, 1:]=preds
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1)
 def gru_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer = 'orthogonal'))
def build_model(model_type=1,seq_len=107, pred_len=68, dropout=0.4,
embed_dim=100, hidden_dim=128):
inputs = tf.keras.layers.Input(shape=(seq_len, 3))
embed = tf.keras.layers.Embedding(input_dim=len(token2int), output_dim=embed_dim )(inputs)
 reshaped = tf.reshape(
 embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = tf.keras.layers.SpatialDropout1D (.2 )(reshaped)
 if model_type == 0:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif model_type == 1:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif model_type == 2:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif model_type == 3:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif model_type == 4:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = tf.keras.layers.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
model.compile(optimizer=adam, loss=MCRMSE)
 return model<choose_model_class>
sub_df.to_csv("submission.csv", index=False )
<prepare_output><EOS> 
su=pd.read_csv("submission.csv" )