Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<groupby>
import numpy as np
import pandas as pd
import os
%%time
num_cols = [item for item in df_train.columns if "totals" in item]
sum_cols = num_cols + ["fullVisitorId"]
df_train_x_agg_sum = df_train_x_encoded[sum_cols].groupby("fullVisitorId",as_index=False ).sum().sort_values("fullVisitorId" ).reset_index(drop=True ).copy()
df_test_x_agg_sum = df_test_x_encoded[sum_cols].groupby("fullVisitorId",as_index=False ).sum().sort_values("fullVisitorId" ).reset_index(drop=True ).copy()
df_train_y_agg_sum = df_train_y_encoded[sum_cols].groupby("fullVisitorId",as_index=False ).sum().sort_values("fullVisitorId" ).reset_index(drop=True ).copy()
df_train_x_agg_sum['totals_transactionRevenue'] = np.log1p(df_train_x_agg_sum['totals_transactionRevenue'])
 df_test_x_agg_sum['totals_transactionRevenue'] = np.log1p(df_test_x_agg_sum['totals_transactionRevenue'])
 df_train_y_agg_sum['totals_transactionRevenue'] = np.log1p(df_train_y_agg_sum['totals_transactionRevenue'])
 df_train_x_agg = pd.merge(df_train_x_agg_sum,df_train_x_agg_last, how='left',on="fullVisitorId" ).sort_values("fullVisitorId" ).reset_index(drop=True ).copy()
df_train_y_agg = pd.merge(df_train_y_agg_sum,df_train_y_agg_last, how='left',on="fullVisitorId" ).sort_values("fullVisitorId" ).reset_index(drop=True ).copy()
df_test_x_agg = pd.merge(df_test_x_agg_sum,df_test_x_agg_last, how='left',on="fullVisitorId" ).sort_values("fullVisitorId" ).reset_index(drop=True ).copy()
df_train_y_agg=pd.merge(id_train.to_frame() , df_train_y_agg, how='left',on="fullVisitorId" ).sort_values("fullVisitorId" ).reset_index(drop=True ).copy()
df_train_y_agg["totals_transactionRevenue"]=df_train_y_agg["totals_transactionRevenue"].fillna(0 )<filter>
sys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')
 ! pip install -e /kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle
df_train_x_agg[df_train_x_agg.totals_transactionRevenue>0]<train_on_grid>
print("Tensorflow version " + tf.__version__)
train_x = df_train_x_agg.drop(["fullVisitorId"],axis=1 ).values
train_y = df_train_y_agg.totals_transactionRevenue.values
test_x_id = df_test_x_agg.fullVisitorId.values
test_x = df_test_x_agg.drop(["fullVisitorId"],axis=1 ).values
grid_params={'max_depth': [4,5,6],
'subsample': [0.95],
'colsample_bytree': [1.0]
}
xgb_model = xgb.XGBClassifier(early_stopping_rounds=50,eval_metric="rmse",verbose=3,n_jobs=-1)
 grid_search = GridSearchCV(xgb_model,
grid_params,
cv=3,
scoring="neg_mean_squared_error",
n_jobs=1,
verbose=3)
 grid_search.fit(train_x,train_y)
 predict = grid_search.predict(test_x )<save_to_csv>
cfg = {"smoothing":0.00,
"arch_fn":efn,
"name": [f"EfficientNetB{i}" for i in [4,5,6]],
"num_class":5,
"epochs":20,
"kfold":[5,5,5],
"seed":42,
"verbose":1,
"resize":512,
"crop_size":480,
"rotation":0.0,
"shear":0.0,
"h-zoom":5.0,
"w-zoom":5.0,
"h-shift":5.0,
"w-shift":5.0,
"path_models":".. /input/cassavatpumodelsbaseline/"
}
AUTOTUNE = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
FILENAMES = tf.io.gfile.glob(".. /input/cassava-leaf-disease-classification" + '/test_tfrecords/ld_test*.tfrec')
 BATCH_SIZE = 128 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]
predict_df=pd.DataFrame()
predict_df["fullVisitorId"]=test_x_id
predict_df["PredictedLogRevenue"]=predict
predict_df.to_csv("xgboost_cv.csv", index=False )<set_options>
os.environ['PYTHONHASHSEED']=str(cfg["seed"])
 os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
random.seed(cfg["seed"])
 np.random.seed(cfg["seed"])
 tf.random.set_seed(cfg["seed"] )
%matplotlib inline
plt.style.use('ggplot')
<load_from_csv>
def read_tfrecord(example, labeled, return_image_name=False):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"image_name": tf.io.FixedLenFeature([], tf.string)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 if return_image_name:
return image, tf.reshape(tf.one_hot([label], depth=cfg["num_class"], axis=-1), [-1]), example["image_name"]
return image, tf.reshape(tf.one_hot([label], depth=cfg["num_class"], axis=-1), [-1])
 idnum = example['image_name']
return image, idnum
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def load_df(csv_path='.. /input/train.csv', JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource'], load_all=False):
if not load_all:
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'}, nrows=250000)
 else:
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'})
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 return df<load_from_csv>
def load_dataset(filenames, labeled=True, ordered=False, return_image_name=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled, return_image_name=return_image_name), num_parallel_calls=AUTOTUNE)
 return dataset
%%time
train = load_df(".. /input/train_v2.csv" )<load_from_csv>
def get_test_dataset(filenames, ordered=True, tta=False):
dataset = load_dataset(filenames, labeled=False, ordered=ordered)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
%%time
test = load_df(".. /input/test_v2.csv", load_all=True )<data_type_conversions>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["resize"]
ROT_ = cfg["rotation"]
SHR_ = cfg["shear"]
HZOOM_ = cfg["h-zoom"]
WZOOM_ = cfg["w-zoom"]
HSHIFT_ = cfg["h-shift"]
WSHIFT_ = cfg["w-shift"]
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
train['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)
 test['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)
 train['trafficSource.isTrueDirect'].fillna(False, inplace=True)
 test['trafficSource.isTrueDirect'].fillna(False, inplace=True)
 train['date'] = pd.to_datetime(train['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))
test['date'] = pd.to_datetime(test['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))<drop_column>
def get_model(cfg, name):
model_input = tf.keras.Input(shape=(cfg['resize'], cfg['resize'], 3), name='inputs')
 constructor = getattr(cfg["arch_fn"], name)
 x = constructor(include_top=False, weights=None,
input_shape=(cfg['resize'], cfg['resize'], 3),
pooling=None )(model_input)
 x = tf.keras.layers.GlobalAveragePooling2D(name='avg_pool' )(x)
 x = tf.keras.layers.Dropout(0.25 )(x)
 outputs = tf.keras.layers.Dense(cfg["num_class"], activation='softmax', name="outputs" )(x)
 model = tf.keras.Model(model_input, outputs, name=name+'_{0}'.format(cfg["resize"]))
return model
def compile_new_model(cfg, name):
with strategy.scope() :
model = get_model(cfg, name)
 losses = tf.keras.losses.CategoricalCrossentropy(label_smoothing = cfg['smoothing'])
 model.compile(
 optimizer = tf.keras.optimizers.Adam(lr=1e-3),
loss = losses,
metrics = tf.keras.metrics.CategoricalAccuracy() 
)
 return model
cols_to_drop = [col for col in train.columns if train[col].nunique(dropna=False)== 1]
train.drop(cols_to_drop, axis=1, inplace=True)
 test.drop([col for col in cols_to_drop if col in test.columns], axis=1, inplace=True)
 print(f'Dropped {len(cols_to_drop)} columns.' )<data_type_conversions>
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 tr = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv' )
for col in ['visitNumber', 'totals.hits', 'totals.pageviews', 'totals.transactionRevenue']:
train[col] = train[col].astype(float )<drop_column>
preds_all = []
for num_model in range(len(cfg["name"])) :
preds_model = []
for fold in range(cfg["kfold"][num_model]):
print(f"
ds_test = generator(JPEG_PATH,submission.image_id.values)
 K.clear_session()
with strategy.scope() :
model = compile_new_model(cfg, cfg['name'][num_model])
 print('Loading best model...')
 model.load_weights(cfg["path_models"]+ cfg['name'][num_model] + "/" + cfg['name'][num_model]+'-fold-%i.h5'%fold)
 preds = model.predict(ds_test, verbose=True)
 preds_model.append(preds)
 preds_model = np.stack(preds_model ).mean(0)
 preds_all.append(preds_model)
 preds_all = np.stack(preds_all )
train.drop(['customDimensions', 'hits', 'trafficSource.referralPath', 'trafficSource.source', 'totals.totalTransactionRevenue'], axis=1, inplace=True)
 test.drop(['customDimensions', 'hits', 'trafficSource.referralPath', 'trafficSource.source', 'totals.totalTransactionRevenue'], axis=1, inplace=True )<count_missing_values>
<count_values><EOS> 
submission["label"] = preds_all.mean(0 ).argmax(1)
 submission.to_csv("submission.csv", index=False )