Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<define_variables>
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )<load_pretrained>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore' )
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 sys.path.insert(0, "/kaggle/input/blazeface-pytorch")
 sys.path.insert(0, "/kaggle/input/deepfakes-inference-demo")
 facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False)
 frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet)
 input_size = 150
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 class Pooling(nn.Module):
def __init__(self):
super(Pooling, self ).__init__()
self.p1 = nn.AdaptiveAvgPool2d(( 1,1))
self.p2 = nn.AdaptiveMaxPool2d(( 1,1))
def forward(self, x):
x1 = self.p1(x)
 x2 = self.p2(x)
 return(x1+x2)* 0.5
model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)))
 class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-kernel-data/model_50epochs_lr0001_patience5_factor01_batchsize32.pth'))
net.append(model )<predict_on_test>
BATCH_SIZE = 16 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 5
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions)
 model.eval()
predictions = predict_on_video_set(test_videos, num_workers=4 )<set_options>
database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'
submission = pd.read_csv(f'{database_base_path}sample_submission.csv')
 display(submission.head())
 TEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/ld_test*.tfrec')
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print(f'GCS: test: {NUM_TEST_IMAGES}' )
!nvidia-smi<save_to_csv>
model_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-tpu-tensorflow-training/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )
submission_df_one = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_one.to_csv("submission_xception.csv", index=False)
 fsub = 0.52*submission_df_resnext['label'] + 0.50*submission_df_one['label']
final = pd.DataFrame({'filename': test_videos, "label": fsub})
 final.to_csv('submission.csv', index=False )<install_modules>
model_path_list_2 = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')
 model_path_list_2.sort()
print('Models to predict:')
 print(*model_path_list_2, sep='
' )
!pip install.. /input/pytorchcv/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet<set_options>
def model_fn(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='inputs')
 base_model = efn.EfficientNetB3(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 model = tf.keras.Sequential([
base_model,
L.Dropout (.25),
L.Dense(N_CLASSES, activation='softmax', name='output')
 ])
 return model
def model_fn_2(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='inputs')
 base_model = efn.EfficientNetB4(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 x = L.Dropout (.5 )(base_model.output)
 output = L.Dense(N_CLASSES, activation='softmax', name='output' )(x)
 model = Model(inputs=inputs, outputs=output)
 return model
with strategy.scope() :
model = model_fn(( None, None, CHANNELS), N_CLASSES)
 model_2 = model_fn_2(( None, None, CHANNELS), N_CLASSES)
 model.summary()
%matplotlib inline
warnings.filterwarnings("ignore" )<define_variables>
files_path = f'{database_base_path}test_images/'
test_preds = np.zeros(( len(os.listdir(files_path)) , N_CLASSES))
print('First model')
 for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True)
 for step in range(TTA_STEPS):
print(f'TTA step {step+1}/{TTA_STEPS}')
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/(TTA_STEPS * len(model_path_list))
else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 print('
Second model')
 for model_path in model_path_list_2:
print(model_path)
 K.clear_session()
model_2.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True)
 for step in range(TTA_STEPS):
print(f'TTA step {step+1}/{TTA_STEPS}')
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model_2.predict(x_test)/(TTA_STEPS * len(model_path_list))
else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model_2.predict(x_test)/ len(model_path_list)
 test_preds = np.argmax(test_preds, axis=-1)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_ds.unbatch())]
<set_options><EOS> 
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )