Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<define_variables>
%%capture
sys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')
 ! pip install -e /kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle
train_index_list=list(range(len(x_train)))
 epoch_list=[]<define_variables>
print("Tensorflow version " + tf.__version__ )
for i in range(1000):
shuffle(train_index_list)
 epoch_list+=train_index_list<define_variables>
DIM = 512
IMAGE_SIZE = [DIM, DIM]
EFFNET = 6
PHASE = 'inference'
PRETRAINED_WEIGHTS = 'imagenet'
GCS_PATH1 = KaggleDatasets().get_gcs_path(f'cassava-tfrecords-{DIM}x{DIM}')if PHASE=='train' else f"/kaggle/input/cassava-tfrecords-{DIM}x{DIM}"
GCS_PATH2 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')if PHASE=='train' else "/kaggle/input/cassava-leaf-disease-classification"
AUTOTUNE = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
CLASSES = ['0', '1', '2', '3', '4']
EPOCHS = 12
FOLDS = 5
print(GCS_PATH1)
 print(GCS_PATH2 )
def gen_flow(X,y):
batch_size=32
i=0
imgs=[]
masks=[]
while True:
for j in range(32):
imgs.append(x_train[i*32+j])
 masks.append(y_train[i*32+j])
 yield imgs,masks
i+=1<categorify>
test_df = pd.read_csv(GCS_PATH2 + '/sample_submission.csv')
 train_df = pd.read_csv(GCS_PATH2 + '/train.csv' )
def gen_flow_for_two_inputs(X, y):
genX1 = gen.flow(X,y, batch_size=batch_size)
 while True:
X=genX1.next()
img_mask=np.concatenate([X[0],X[1]],axis=3)
 seq_det=seq.to_deterministic()
img_mask_aug=seq_det.augment_images(img_mask)
 img_mask_aug=crop_batch(img_mask_aug,0.1)
 img_aug=img_mask_aug[:,:,:,0]
mask_aug=np.round(img_mask_aug[:,:,:,1])
 yield np.expand_dims(img_aug,axis=3),np.expand_dims(mask_aug,axis=3 )<choose_model_class>
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
sgd = SGD(lr=0.01, decay=1e-4, momentum=0.9, nesterov=True)
 model.compile(loss=bce_dice_loss, optimizer=sgd, metrics=["accuracy",iouMetric] )<train_model>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform_mat(image, DIM=IMAGE_SIZE[0]):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d, [DIM, DIM,3] )
early_stopping = EarlyStopping(patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.1, patience=20, min_lr=0.000001, verbose=1)
 epochs = 200
batch_size = 16
gen = ImageDataGenerator()
gen_flow = gen_flow_for_two_inputs(x_train, y_train)
 history = model.fit_generator(gen_flow,
validation_data=[x_valid, y_valid],
epochs=epochs,
steps_per_epoch=len(x_train)/ batch_size,
callbacks=[model_checkpoint, reduce_lr] )<load_pretrained>
def to_float32(image, label):
return tf.cast(image, tf.float32), label
model.load_weights("./keras.model",{"bce_dice_loss":bce_dice_loss,'iouMetric':iouMetric})
<predict_on_test>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid] )<compute_train_metric>
def read_tfrecord(example, labeled):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
return image, idnum
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )<find_best_params>
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)
 return dataset
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]<categorify>
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH1 + '/ld_train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH2 + '/test_tfrecords/ld_test*.tfrec')
 print('Train Files:',len(TRAINING_FILENAMES))
print('Test Files:',len(TEST_FILENAMES))
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs<predict_on_test>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
preds_test = model.predict(x_test )<compute_test_metric>
TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(
 tf.io.gfile.glob(GCS_PATH1 + '/ld_train*.tfrec'),
test_size=0.35, random_state=5 
)
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH2 + '/test_tfrecords/ld_test*.tfrec' )
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }<save_to_csv>
def dropout(image, DIM=DIM, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image
def data_augment(image, label, dim=DIM, droprate=0.65, dropsize=0.15, dropct=8):
image = tf.image.random_flip_left_right(image)
 if(droprate!=0)&(dropct!=0)&(dropsize!=0):
image = dropout(image, DIM=dim, PROBABILITY=droprate, CT=dropct, SZ=dropsize)
 return image, label
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )<set_options>
def get_training_dataset(training_fikenames=TRAINING_FILENAMES):
dataset = load_dataset(training_fikenames, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
%matplotlib inline<categorify>
def get_validation_dataset(valid_filenames=VALID_FILENAMES, ordered=False):
dataset = load_dataset(valid_filenames, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def rle_decode(rle_mask):

 s = rle_mask.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(101*101, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(101,101 )<load_from_csv>
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
df = pd.read_csv('.. /input/submit-802/test_18000-null.csv')
 i = 0
j = 0
plt.figure(figsize=(30,15))
plt.subplots_adjust(bottom=0.2, top=0.8, hspace=0.2)
 while True:
if str(df.loc[i,'rle_mask'])!=str(np.nan):
decoded_mask = rle_decode(df.loc[i,'rle_mask'])
 plt.subplot(1,6,j+1)
 plt.imshow(decoded_mask)
 plt.title('ID: '+df.loc[i,'id'])
 j = j + 1
if j>5:
break
i = i + 1 
<define_variables>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
test_path = '.. /input/tgs-salt-identification-challenge/test/images/'<categorify>
np.set_printoptions(threshold=15, linewidth=80)
 label2name = {"0": "Bacterial Blight",
"1": "Brown Streak Disease",
"2": "Green Mottle",
"3": "Mosaic Disease",
"4": "Healthy"}
def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_plant(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
plt.xticks([])
 plt.yticks([])
 else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
plt.xticks([])
 plt.yticks([])
 for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else label2name[CLASSES[label]] + f'({str(label)})'
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_plant(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.tight_layout()
plt.show()
def rle_encode(im):

 pixels = im.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<save_to_csv>
testing_dataset = get_test_dataset()
testing_dataset = testing_dataset.unbatch().batch(20)
 test_batch = iter(testing_dataset )
df.to_csv('crf_correction.csv',index=False )<import_modules>
display_batch_of_images(next(test_batch))
import pandas as pd
import numpy as np<import_modules>
def onehot(image,label):
CLASSES = 5
return image,tf.one_hot(label,CLASSES )
import pandas as pd
import numpy as np<categorify>
def cutmix(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
CLASSES = 5
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 b = tf.random.uniform([],0,1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[j,ya:yb,0:xa,:]
two = image[k,ya:yb,xa:xb,:]
three = image[j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
def mixup(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
CLASSES = 5
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 a = tf.random.uniform([],0,1)*P
img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
def transform(image,label):
DIM = IMAGE_SIZE[0]
CLASSES = 5
SWITCH = 0.5
CUTMIX_PROB = 0.666
MIXUP_PROB = 0.666
image1 = []
for j in range(AUG_BATCH):
img = transform_mat(image[j,])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 image1.append(img)
 image1 = tf.reshape(tf.stack(image1),(AUG_BATCH,DIM,DIM,3))
image2, label2 = cutmix(image1, label, CUTMIX_PROB)
 image3, label3 = mixup(image1, label, MIXUP_PROB)
 imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image4,label4
le = LabelEncoder()
def get_dictionary(s):
try:
i = eval(s)
 except:
i = {}
return i
def prepare(df):
global json_cols
global train_dict
df[['release_month', 'release_day', 'release_year']] = df['release_date'].str.split('/', expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'].map(lambda x : x if x > 100 else x +(2000 if(x <= 18)else 1900))
releaseDate = pd.to_datetime(df['release_date'])
 df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter
df['rating'] = df.rating.fillna(df['rating'].median())
 df['totalVotes'] = df.totalVotes.fillna(df['rating'].median())
 df['weighteRating'] = np.log(df['rating'] * df['totalVotes'])
 df['originalBudget'] = df['budget']
df['inflationBudget'] = df['budget'] + df['budget'] * 1.8 / 100 *(2018 - df['release_year'])
 df['budget'] = np.log1p(df['budget'])
 df['genders_0_crew'] = df['crew'].map(lambda x : sum([1 for i in get_dictionary(x)if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].map(lambda x : sum([1 for i in get_dictionary(x)if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].map(lambda x : sum([1 for i in get_dictionary(x)if i['gender'] == 2]))
df['_collection_name'] = df['belongs_to_collection'].map(lambda x : [i['name'] for i in get_dictionary(x)])
 df['_collection_name'] = le.fit_transform(list(df['_collection_name'].fillna('' ).astype(str)))
 df['_num_Keywords'] = df['Keywords'].map(lambda x : len(eval(x)) if get_dictionary(x)!= {} else 0)
 df['_num_cast'] = df['cast'].map(lambda x : len(eval(x)) if get_dictionary(x)!= {} else 0)
 df['_popularity_mean_year'] = df['popularity'] / df.groupby('release_year')['popularity'].transform('mean')
 df['_budget_runtime_ratio'] = df.budget / df.runtime
df['_release_year_popularity_ratio'] = df.release_year / df.popularity
df['_release_year_polularity_ratio2'] = df.popularity / df.release_year
df['_popularity_totalVote_ratio'] = df.totalVotes / df.popularity
df['_rating_popularity_ratio'] = df.rating / df.popularity
df['_rating_totalVotes_ratio'] = df.totalVotes / df.rating
df['_totalVotes_releaseYear_ratio'] = df.totalVotes / df.release_year
df['has_homepage'] = 1
df.loc[df['homepage'].isnull() , 'has_homepage'] = 0
df['isbelongs_to_collectionNA'] = 0
df.loc[df['belongs_to_collection'].isnull() , 'isbelongs_to_collectionNA'] = 1
df['isTaglineNA'] = 0
df.loc[df['tagline'].isnull() , 'isTaglineNA'] = 1
df['isOriginalLanguage'] = 0
df.loc[df['original_language'] == 'en', 'isOriginalLanguage'] = 1
df['isTitleDifferent'] = 1
df.loc[df['original_title'] == df['title'], 'isTitleDifferent'] = 0
df['isMovieReleased'] = 1
df.loc[df['status'] == 'Released', 'isMovieReleased'] = 0
df['collection_id'] = df['belongs_to_collection'].map(lambda x : np.nan if len(get_dictionary(x)) == 0 else get_dictionary(x)[0]['id'])
 df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].map(lambda x : len(get_dictionary(x)))
 df['cast_count'] = df['cast'].map(lambda x : len(get_dictionary(x)))
 df['crew_count'] = df['crew'].map(lambda x : len(get_dictionary(x)))
 df['meanruntimeByYear'] = df.groupby('release_year')['runtime'].transform('mean' ).round()
df['meanPopularityByYear'] = df.groupby('release_year')['popularity'].transform('mean' ).round()
df['meanBudgetByYear'] = df.groupby('release_year')['budget'].transform('mean')
 df['meanTotalVotesByYear'] = df.groupby('release_year')['totalVotes'].transform('mean' ).round()
df['meanTotalVotesByRating'] = df.groupby('rating')['totalVotes'].transform('mean' ).round()
df['medianBudgetByYear'] = df.groupby('release_year')['budget'].transform('median')
 for col in ['genres', 'production_countries', 'spoken_languages', 'production_companies']:
df[col] = df[col].map(lambda x : sorted(list(set([n if n in train_dict['genres'] else 'genres' + '_etc' for n in [i['name'] for i in get_dictionary(x)]])))).map(lambda x : ','.join(map(str, x)))
 temp = df[col].str.get_dummies(sep=',')
 df = pd.concat([df, temp], axis=1, sort=False)
 df.drop(['genres_etc'], axis=1, inplace=True)
 df = df.drop(['id', 'revenue', 'belongs_to_collection',
'genres', 'homepage', 'imdb_id', 'overview',
'runtime', 'poster_path', 'production_companies',
'production_countries', 'release_date', 'spoken_languages',
'status', 'title', 'Keywords', 'cast', 'crew',
'original_language', 'original_title', 'tagline',
'collection_id'], axis=1)
 df.fillna(value=0.0, inplace=True)
 return df
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test['revenue'] = np.nan
TrainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 TestAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.merge(train, TrainAdditionalFeatures, how='left', on=['imdb_id'])
 test = pd.merge(test, TestAdditionalFeatures, how='left', on=['imdb_id'])
 additionalTrainData = pd.read_csv('.. /input/tmdb-box-office-prediction-more-training-data/additionalTrainData.csv')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].astype('str')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].str.replace('-', '/')
 train = pd.concat([train, additionalTrainData])
 train['revenue'] = np.log1p(train['revenue'])
 y = train['revenue'].values
json_cols = ['genres', 'production_companies', 'production_countries',
'spoken_languages', 'Keywords', 'cast', 'crew']
def get_json_dict(df):
global json_cols
result = {}
for e_col in json_cols:
d = {}
rows = df[e_col].values
for row in rows:
if row is None: continue
if type(row)is str:
for i in get_dictionary(row):
if i['name'] not in d:
d[i['name']] = 0
d[i['name']] += 1
result[e_col] = d
return result
train_dict = get_json_dict(train)
 test_dict = get_json_dict(test)
 for col in json_cols:
remove = []
train_id = set(list(train_dict[col].keys()))
test_id = set(list(test_dict[col].keys()))
remove += list(train_id - test_id)+ list(test_id - train_id)
 for i in train_id.union(test_id)- set(remove):
if train_dict[col][i] < 10 or i == '' :
remove += [i]
for i in remove :
if i in train_dict[col] :
del train_dict[col][i]
if i in test_dict[col] :
del test_dict[col][i]
all_data = prepare(pd.concat([train, test] ).reset_index(drop = True))
train = all_data.loc[:train.shape[0] - 1,:]
test = all_data.loc[train.shape[0]:,:] 
<import_modules>
def get_training_dataset(dataset=TRAINING_FILENAMES, do_aug=True):
if do_aug:
dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)
 dataset = dataset.repeat()
dataset = dataset.batch(AUG_BATCH)
 if do_aug:
dataset = dataset.map(transform, num_parallel_calls=AUTOTUNE)
 dataset = dataset.unbatch()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
from sklearn.model_selection import KFold<choose_model_class>
def get_validation_dataset(dataset, do_onehot=True):
dataset = dataset.batch(BATCH_SIZE)
 if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTOTUNE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTOTUNE)
 return dataset
random_seed = 2019
k = 10
fold = list(KFold(k, shuffle=True, random_state=random_seed ).split(train))
np.random.seed(random_seed)
<define_variables>
class DataGenerator(tf.keras.utils.Sequence):
def __init__(self, path, list_IDs, labels, batch_size, img_size, img_channel):
self.path = path
self.list_IDs = list_IDs
self.labels = labels
self.batch_size = batch_size
self.img_size = img_size
self.img_channel = img_channel
self.indexes = np.arange(len(self.list_IDs))
def __len__(self):
return int(np.floor(len(self.list_IDs)/self.batch_size))
def __getitem__(self, index):
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indexes]
X, y = self.__data_generation(list_IDs_temp)
 return X, y
def __data_generation(self, list_IDs_temp):
X = np.empty(( self.batch_size, self.img_size, self.img_size, self.img_channel))
y = np.empty(( self.batch_size, 5), dtype=int)
 for i, ID in enumerate(list_IDs_temp):
img = cv2.imread(self.path+ID)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = cv2.resize(img,(self.img_size, self.img_size))
X[i, ] = img
y[i, ] = self.labels[i]
X = X.astype('float32')
 X /= 255.
 return X, y
def preprocess_image(image_path, desired_size=DIM):
img = cv2.imread(image_path)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 im = cv2.resize(img,(desired_size,desired_size), interpolation = cv2.INTER_AREA)
 im = np.array(im)
 return im
test_generator = DataGenerator('.. /input/cassava-leaf-disease-classification/'+'test_images/',
test_df['image_id'],
test_df['label'],
1,
DIM,
3 )
result_dict = {}
val_pred = np.zeros(train.shape[0])
 test_pred = np.zeros(test.shape[0])
 final_err = 0
verbose = False<import_modules>
def get_lr_callback(batch_size=8, show = False):
lr_start = 0.000015000
lr_max = 0.000000250 * strategy.num_replicas_in_sync * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
if show:
plt.figure(figsize =(8, 5))
plt.plot(np.arange(1, 12), [lrfn(x)for x in np.arange(1, 12)], marker = 'o')
 plt.xlabel('epoch')
 plt.ylabel('learaning_rate');
plt.show()
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
import xgboost as xgb<train_model>
model_dict = {0: efn.EfficientNetB0,
1: efn.EfficientNetB1,
2: efn.EfficientNetB2,
3: efn.EfficientNetB3,
4: efn.EfficientNetB4,
5: efn.EfficientNetB5,
6: efn.EfficientNetB6,
7: efn.EfficientNetB7,}
def get_model() :
with strategy.scope() :
inp = tf.keras.layers.Input(shape=(DIM,DIM,3))
base = model_dict[EFFNET](input_shape=(DIM,DIM,3),weights=PRETRAINED_WEIGHTS,include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(64,activation='relu' )(x)
 x = tf.keras.layers.Dense(5,activation='softmax' )(x)
 model = tf.keras.Model(inputs=inp, outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.CategoricalCrossentropy(
 from_logits=False, label_smoothing=0.0001,
name='categorical_crossentropy' 
)
 model.compile(optimizer=opt,loss=loss,metrics=['categorical_accuracy'])
 return model
def xgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective':'reg:linear',
'eta': 0.01,
'max_depth':6,
'subsample':0.6,
'colsample_bytree':0.7,
'eval_metric':'rmse',
'seed':random_seed,
'silent':True,
}
record = dict()
model = xgb.train(params,
xgb.DMatrix(trn_x, trn_y),
100000,
[(xgb.DMatrix(trn_x, trn_y), 'train'), 
(xgb.DMatrix(val_x, val_y), 'valid')],
verbose_eval = verbose,
early_stopping_rounds = 500,
callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
val_pred = model.predict(xgb.DMatrix(val_x), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(test), ntree_limit=model.best_ntree_limit)
 return {'val' : val_pred,
'test' : test_pred,
'error': record['valid']['rmse'][best_idx],
'importance': [i for k, i in model.get_score().items() ]}<load_from_csv>
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH1 + '/ld_train*.tfrec')
 kfold = KFold(FOLDS, shuffle = True, random_state = 42)
 probabilities = []
for f,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print() ; print('='*50)
 print(f' fold: {f+1} | model: EfficientNetB{EFFNET} | image_size: {DIM}')
 print('='*50)
 if PHASE == 'train':
train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)
 sv = tf.keras.callbacks.ModelCheckpoint(
 'effnet-b%i_img-%i_fold-%02d_best.h5'%(EFFNET, DIM, f+1), monitor='val_categorical_accuracy', verbose=0, save_best_only=True,
save_weights_only=True, mode='max', save_freq='epoch')
 model = get_model()
history = model.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [get_lr_callback(BATCH_SIZE), sv],
validation_data = get_validation_dataset(val_dataset), 
)
 plt.figure(figsize=(15,5))
plt.plot(np.arange(len(model.history.history['categorical_accuracy'])) ,model.history.history['categorical_accuracy'],'-o',label='Train categorical_accuracy',color='
plt.plot(np.arange(len(model.history.history['categorical_accuracy'])) ,model.history.history['val_categorical_accuracy'],'-o',label='Val categorical_accuracy',color='
x = np.argmax(model.history.history['val_categorical_accuracy']); y = np.max(model.history.history['val_categorical_accuracy'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('categorical_accuracy',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(len(model.history.history['loss'])) ,model.history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(len(model.history.history['loss'])) ,model.history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(model.history.history['val_loss']); y = np.min(model.history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('Fold: %i | Image Size: %i | model: EfficientNetB%i | Batch_size: %i'%(f+1, DIM ,EFFNET, BATCH_SIZE))
plt.legend(loc=3)
 plt.show()
model.save_weights('effnet-b%i_img-%i_fold-%02d_last.h5'%(EFFNET, DIM, f+1))
del model; z = gc.collect()
elif PHASE == 'inference':
model = get_model()
weight_path = '/kaggle/input/casava-efficientnetb6-cutmixupdropout-models/effnet-b6_img-512_fold-%02d_best.h5'%(f+1)
 print('loading weight:', weight_path.split('/')[-1])
 model.load_weights(weight_path)
 preds = model.predict(test_generator, verbose=1)
 probabilities.append(preds)
 else:
print('PHASE must be train or inference.')
 predictions = np.array(probabilities ).mean(0 ).argmax(-1)
 print(predictions )
<prepare_output><EOS> 
if PHASE == 'inference':
test_df['label'] = predictions
test_df = test_df[["image_id","label"]]
test_df.to_csv('submission.csv',index=False)
 test_df