Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<load_from_csv>
warnings.simplefilter(action = 'ignore', category = FutureWarning)
 print("Tensorflow version " + tf.__version__ )
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv' )<load_from_csv>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
SEED = 414
seed_everything(SEED )
submit = pd.read_csv(".. /input/ieee-gb-2-make-amount-useful-again/submission.csv" )<data_type_conversions>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
REPLICAS = strategy.num_replicas_in_sync
print("REPLICAS: ", REPLICAS )
def make_day(df):
def fillna(x):
if "nan" in x:
return np.nan
else:
return x
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 df['Date'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds=x)))
 df['Day'] =(df["Date"].dt.year - 2017)* 365 + df["Date"].dt.dayofyear
df["ID1"] = df["card1"].astype(str)+df["card2"].astype(str)+df["card3"].astype(str)+df["card4"].astype(str)+\
df["card5"].astype(str)+df["card6"].astype(str)
 df["ID_D1"] = fillna(df["ID1"].astype(str)+(df["Day"] - df["D1"] ).astype(str))
df["ID_D1D10"] = fillna(df["ID_D1"].astype(str)+(df["Day"] - df["D10"] ).astype(str))
df["ID_D1D12"] = fillna(df["ID_D1"].astype(str)+(df["Day"] - df["D12"] ).astype(str))
return df<categorify>
GCS_DS_PATH = '.. /input/cassava-leaf-disease-classification'
print(GCS_DS_PATH )
train_transaction = make_day(train_transaction)
 test_transaction = make_day(test_transaction )<groupby>
BATCH_SIZE = 16 * REPLICAS
WARMUP_EPOCHS = 3
WARMUP_LEARNING_RATE = 1e-4 * REPLICAS
EPOCHS = 20
LEARNING_RATE = 5e-5 * REPLICAS
ES_PATIENCE = 5
CHANNELS = 3
N_CLASSES = 5
DIM = 512
HEIGHT = 512
WIDTH = 512
CLASSES = ['0', '1', '2', '3', '4']
AUTO = tf.data.experimental.AUTOTUNE
q_id1 = train_transaction[["ID_D1D10", "isFraud"]].groupby("ID_D1D10" ).agg({"isFraud": ["count", "mean"]} ).reset_index()
q_id1.columns = ["ID_D1D10", "isFraud_countD1D10", "isFraud_meanD1D10"]
q_id2 = train_transaction[["ID_D1D10", "isFraud"]].groupby("ID_D1D10" ).agg({"isFraud": ["count", "mean"]} ).reset_index()
q_id2.columns = ["ID_D1D12", "isFraud_countD1D12", "isFraud_meanD1D12"]<merge>
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
test_transaction = pd.merge(test_transaction, q_id1, how="left", on="ID_D1D10")
 test_transaction = pd.merge(test_transaction, q_id2, how="left", on="ID_D1D12" )<merge>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
submit = pd.merge(submit, test_transaction, how="left", on="TransactionID" )<define_variables>
def transform(image, DIM=512):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
q = "isFraud_countD1D10 > 1 and isFraud_meanD1D10 > 0.7 and ProductCD != 'C'"<feature_engineering>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=512):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
submit["isFraud"][submit.query(q ).index] = 1<define_variables>
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=BATCH_SIZE, dim=512):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
q = "isFraud_countD1D12 > 1 and isFraud_meanD1D12 == 1 and ProductCD == 'C'"<define_variables>
TEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + '/test_tfrecords/*.tfrec')
q = "isFraud_countD1D10 > 5 and isFraud_meanD1D10 == 0"<feature_engineering>
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print('Dataset: {} unlabeled test images'.format(NUM_TEST_IMAGES))
submit["isFraud"][submit.query(q ).index] = 0<save_to_csv>
sys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')
 ! pip install /kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle
submit[["TransactionID", "isFraud"]].to_csv("postprocessed.csv", index=False )<set_options>
import efficientnet.keras as efn
pd.options.display.max_rows = 500
pd.options.display.max_columns = 100<load_from_csv>
def create_model_efnB6() :
base_model = efn.EfficientNetB6(weights=None,
include_top=False,
input_shape=[HEIGHT, WIDTH, 3])
 model = tf.keras.Sequential([
base_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 return model
train = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv')
 train_ind = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv')
 test = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv')
 test_ind = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv' )<load_from_csv>
with strategy.scope() :
model_efnB6 = create_model_efnB6()
train_len = len(train)
<merge>
TTA = 1
print('Predicting Test with TTA...')
 test_ds = get_dataset(TEST_FILENAMES,labeled=False,return_image_names=False,augment=False,
repeat=False,shuffle=False)
 test_ct = count_data_items(TEST_FILENAMES);
STEPS = TTA * test_ct/BATCH_SIZE/REPLICAS
if STEPS < 1:
STEPS = 1
test_df = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 probabilities = np.zeros(( test_df.shape[0],5))
for f in range(5):
model_efnB6.load_weights(".. /input/cassava-leaf-disease-classification-kfold/model_efnB6_train_fold_%i.h5"%f, by_name=True)
 prob = model_efnB6.predict(test_ds, verbose=2)
 probabilities += prob/5
predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print(probabilities )
<concatenate><EOS> 
print('Generating submission.csv file...')
 ds = get_dataset(TEST_FILENAMES,labeled=False,return_image_names=True,augment=False,
repeat=False,shuffle=False)
 test_ids = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())])
 np.savetxt(
 'submission.csv',
np.rec.fromarrays([test_ids, predictions]),
fmt=['%s', '%d'],
delimiter=',',
header='image_id,label',
comments='', 
)
 !head submission.csv