Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<set_options>
geffnet_path = '.. /input/hongnangeffnet/gen-efficientnet-pytorch-master-hongnan'
timm_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
vit_path = '.. /input/vision-transformer-pytorch/VisionTransformer-Pytorch'
sys_paths = [geffnet_path, timm_path, vit_path]
for paths in sys_paths:
sys.path.append(paths)
plt.rcParams['figure.figsize'] =(12,6 )<categorify>
!pip install -q torch==1.6.0
!pip install -q torchvision==0.7.0
!pip install -q albumentations==0.5.1
def load_data(data='train',n=2):
df = pd.DataFrame()
for i in range(n):
if data=='train':
if i > 8 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))
elif data=='test':
if i > 2 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))
df = pd.concat([df,dfpart])
 del dfpart
return df 
<load_from_csv>
InteractiveShell.ast_node_interactivity = "all"
df_train = load_data(n=9)
 df_test = load_data('test',n=4 )<count_missing_values>
class config:
seed = 1992
num_classes = 5
batch_size = 32
image_size = 512
image_col_name = 'image_id'
class_col_name = 'label'
weights_dir = '.. /input/cassavahongnan/'
hn_test_path = '.. /input/cassava-test-images'
effnet = 'tf_efficientnet_b5_ns'
effnet_paths = ['.. /input/cassavaroots/baseline_cld_fold0_epoch8_tf_efficientnet_b4_ns_512.pth',
'.. /input/cassavaroots/baseline_cld_fold1_epoch9_tf_efficientnet_b4_ns_512.pth',
'.. /input/cassavaroots/baseline_cld_fold2_epoch9_tf_efficientnet_b4_ns_512.pth',
'.. /input/cassavaroots/baseline_cld_fold3_epoch5_tf_efficientnet_b4_ns_512.pth',
'.. /input/cassavaroots/baseline_cld_fold4_epoch11_tf_efficientnet_b4_ns_512.pth']
resnext = 'resnext50_32x4d'
resnext_paths = ['.. /input/cassavaroots/resnext50_32x4d_fold0.pth',
'.. /input/cassavaroots/resnext50_32x4d_fold1.pth',
'.. /input/cassavaroots/resnext50_32x4d_fold2.pth',
'.. /input/cassavaroots/resnext50_32x4d_fold3.pth',
'.. /input/cassavaroots/resnext50_32x4d_fold4.pth']
vit = 'ViT-B_16'
vit_image_size = 384
vit_paths = ['.. /input/cassavadocumentedweights/vit_base_16.pt']
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 paths = {'sample_path': '.. /input/cassava-leaf-disease-classification/sample_submission.csv',
'test_path': '.. /input/cassava-leaf-disease-classification/test_images',
'model_weight_path_folder': '.. /input/efficientnet-weights'}
print(f'<compute_test_metric>
config = config
def rmse(y_true, y_pred):
return round(np.sqrt(mean_squared_error(y_true, y_pred)) , 5)
 def split_data(df=df_train,rate=.8):
df = df.sort_values('date' ).copy()
df.drop(['fullVisitorId','visitId','visitStartTime'],axis=1,inplace=True)
 df['Revenue'] = np.log1p(df['Revenue'])
 global X_train,X_valid,y_train,y_valid
n_train = int(len(df)*rate)
 X_train = df.drop(['Revenue','date'],axis=1 ).iloc[:n_train]
X_valid = df.drop(['Revenue','date'],axis=1 ).iloc[n_train:]
y_train = df['Revenue'].iloc[:n_train]
y_valid = df['Revenue'].iloc[n_train:]
print(X_train.shape,X_valid.shape)
 def encode_data(verbose=False):
global df_train_encoded,df_test_encoded
df_train_encoded = df_train.copy()
df_test_encoded = df_test.copy()
for col in df_train.columns:
if df_train_encoded[col].dtype == 'object' and col not in ['fullVisitorId','visitId','visitStartTime','date']:
if verbose:
print(col)
 lb = LabelEncoder()
lb.fit(list(df_train_encoded[col].unique())+ list(df_test_encoded[col].unique()))
df_train_encoded[col] = lb.transform(df_train_encoded[col])
 df_test_encoded[col] = lb.transform(df_test_encoded[col])
 def run_xgb() :
params = {
'objective':'reg:linear',
'eval_metric':'rmse',
'learning_rate':.01,
'eta': 0.1,
'max_depth': 10,
'subsample': 0.6,
'colsample_bytree': 0.6,
'lambda':1,
'random_state': 42,
'silent':True
}
xgb_train_data = xgb.DMatrix(X_train, y_train)
 xgb_val_data = xgb.DMatrix(X_valid, y_valid)
 model = xgb.train(params, xgb_train_data,
num_boost_round=1000,
evals= [(xgb_train_data, 'train'),(xgb_val_data, 'valid')],
early_stopping_rounds=10,
verbose_eval=20 
)
 return model
def submit() :
test_matrix = xgb.DMatrix(X_test)
 y_pred = clf.predict(test_matrix,ntree_limit=clf.best_ntree_limit)
 df_test['PredictedLogRevenue'] = y_pred
submit = df_test[['PredictedLogRevenue','fullVisitorId']].groupby('fullVisitorId' ).PredictedLogRevenue.sum().reset_index()
submit.to_csv('submit.csv',index=False)
 test(y_pred)
 def test(predict):
y_test = df_test['totals_transactionRevenue']
print(rmse(y_test,predict)) 
<feature_engineering>
def seed_all(seed: int):
if not seed:
seed = 10
print("[ Using Seed : ", seed, " ]")
 os.environ['PYTHONHASHSEED'] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.cuda.manual_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.enabled = False
def seed_worker(worker_id):
worker_seed = torch.initial_seed() % 2**32
np.random.seed(worker_seed)
 random.seed(worker_seed )
def prepare_data(df_train,df_test,
del_col=['fullVisitorId','visitId','visitStartTime','date']):
df_train = df_train.sort_values('date' ).copy()
df_train = df_train.drop(del_col,axis=1 ).copy()
df_test = df_test.drop(del_col,axis=1 ).copy()
df_train['totals_transactionRevenue'] = np.log1p(df_train['totals_transactionRevenue'])
 df_test['totals_transactionRevenue'] = np.log1p(df_test['totals_transactionRevenue'])
 global X_train,X_valid,y_train,y_valid,X_test,y_test
n_train = int(len(df_train)*.8)
 X_train = df_train.drop(['totals_transactionRevenue'],axis=1 ).iloc[:n_train]
X_valid = df_train.drop(['totals_transactionRevenue'],axis=1 ).iloc[n_train:]
y_train = df_train['totals_transactionRevenue'].iloc[:n_train]
y_valid = df_train['totals_transactionRevenue'].iloc[n_train:]
X_test = df_test.drop(['totals_transactionRevenue'],axis=1)
 y_test = df_test['totals_transactionRevenue'] 
<categorify>
seed_all(seed=config.seed )
encode_data(verbose=True)
 prepare_data(df_train_encoded,df_test_encoded,del_col=['fullVisitorId','visitId',
'visitStartTime','date','totals_totalTransactionRevenue','totals_transactions'])
<compute_test_metric>
sys.path.append('.. /input/pytorch-image-models/pytorch-image-models-master')
 image_size = 512
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 OUTPUT_DIR = './'
MODEL_DIR = '.. /input/cassavaroots/'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'
class CFG:
debug=False
num_workers=4
model_name='resnext50_32x4d'
size=512
batch_size=32
seed=1992
num_classes=5
target_col='label'
resnext = 'resnext50_32x4d'
n_fold=5
trn_fold=[0, 1, 2, 3, 4]
inference=True
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test['filepath'] = test.image_id.apply(lambda x: os.path.join('.. /input/cassava-leaf-disease-classification/test_images', f'{x}'))
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['image_id'].values
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform:
augmented = self.transform(image=image)
 image = augmented['image']
return image
def get_transforms(*, data):
if data == 'valid':
return A.Compose([
A.Resize(CFG.size, CFG.size),
A.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
])
 class CustomResNext(nn.Module):
def __init__(self, config, pretrained=False):
super().__init__()
self.model = timm.create_model(model_name=config.resnext, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, config.num_classes)
 def forward(self, input_neurons):
output_predictions = self.model(input_neurons)
 return output_predictions
def load_state(model_path):
model = CustomResNext(CFG, pretrained=False)
 try:
model.load_state_dict(torch.load(model_path)['model'], strict=True)
 state_dict = torch.load(model_path)['model']
except:
state_dict = torch.load(model_path)['model']
state_dict = {k[7:] if k.startswith('module.')else k: state_dict[k] for k in state_dict.keys() }
return state_dict
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state)
 model.eval()
with torch.no_grad() :
y_preds = model(images)
 print(y_preds)
 avg_preds.append(y_preds.softmax(1 ).to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs
model = CustomResNext(CFG, pretrained=False)
 states = [load_state(MODEL_DIR+f'{CFG.model_name}_fold{fold}.pth')for fold in CFG.trn_fold]
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,
num_workers=CFG.num_workers, pin_memory=True)
 predictions = inference(model, states, test_loader, device)
 predictions
<set_options>
class Cassava(torch.utils.data.Dataset):
def __init__(self,
df: pd.DataFrame,
config: type,
transforms: type = None,
test: bool = False,
transform_norm: bool = False, meta_features=None):
self.df = df
self.config = config
self.transforms = transforms
self.test = test
self.transform_norm = transform_norm
self.meta_features = meta_features
if self.transforms is None:
assert self.transform_norm is False
print('Transforms is None and Transform Normalization is not '
'initialized!')
 self.image_extension = get_file_type(
 image_folder_path=config.paths['test_path'], allowed_extensions=None)
 self.df_has_ext = check_df_ext(df=self.df, col_name=config.image_col_name)
 if self.df_has_ext is True:
self.image_extension = ""
def __len__(self):

 return len(self.df)
 def __getitem__(self, idx: int):

 image_id = self.df[self.config.image_col_name].values[idx]
label = None
label = torch.zeros(1)
 if self.test:
image_path = os.path.join(
 self.config.paths['test_path'], "{}{}".format(image_id,
self.image_extension))
else:
label = self.df[self.config.class_col_name].values[idx]
label = torch.as_tensor(data=label, dtype=torch.int64, device=None)
 image_path = os.path.join(
 self.config.paths['train_path'], "{}{}".format(image_id,
self.image_extension))
image = cv2.imread(image_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform_norm is False:
image = image.astype(np.float32)/ 255.0
if self.transforms is not None:
image = self.transforms.augment(image)
 else:
image = torch.as_tensor(data=image,
dtype=torch.float32,
device=None)
 if self.meta_features is not None:
meta = np.array(self.df.iloc[idx][self.meta_features].values, dtype=np.float32)
 return image_id,(image, meta), label
return image_id, image, label
!pip install tensorflow_addons
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<define_variables>
class Augmentation(ABC):
@abstractmethod
def augment(image):

 class AlbumentationsAugmentation(Augmentation):
def __init__(self, transforms: albumentations.core.composition.Compose):
self.transforms = transforms
def augment(self, image):
albu_dict = {"image": image}
transform = self.transforms(**albu_dict)
 return transform["image"]
def get_albu_transforms(config):
transforms_test = albumentations.Compose(
 [
albumentations.Resize(height=config.image_size, width=config.image_size, p=1.0),
albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 transforms_tta = albumentations.Compose(
 [
albumentations.RandomResizedCrop(height=config.image_size, width=config.image_size, p=1.0),
albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 return transforms_test, transforms_tta
def get_albu_transforms_vit(config):
transforms_test = albumentations.Compose(
 [
albumentations.RandomResizedCrop(height=config.vit_image_size, width=config.vit_image_size, p=1.0),
albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 transforms_tta = albumentations.Compose(
 [
albumentations.RandomResizedCrop(height=config.vit_image_size, width=config.vit_image_size, p=1.0),
albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 return transforms_test, transforms_tta
EPOCHS = 110
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = 0.001
SPLITS = 5
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<load_from_csv>
class CustomEfficientNet(nn.Module):
def __init__(self, config: type, pretrained: bool=True):
super().__init__()
self.config = config
self.model = geffnet.create_model(
 model_weight_path_folder=config.paths['model_weight_path_folder'],
model_name=config.effnet,
pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, config.num_classes)
 def forward(self, input_neurons):
output_predictions = self.model(input_neurons)
 return output_predictions
class CustomResNext(nn.Module):
def __init__(self, config, pretrained=False):
super().__init__()
self.model = timm.create_model(model_name=config.resnext, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, config.num_classes)
 def forward(self, input_neurons):
output_predictions = self.model(input_neurons)
 return output_predictions
class enet_v2(nn.Module):
def __init__(self, backbone, out_dim, pretrained=False):
super(enet_v2, self ).__init__()
self.enet = timm.create_model(backbone, pretrained=pretrained)
 in_ch = self.enet.classifier.in_features
self.myfc = nn.Linear(in_ch, out_dim)
 self.enet.classifier = nn.Identity()
def forward(self, x):
x = self.enet(x)
 x = self.myfc(x)
 return x
vit_model = VisionTransformer.from_name('ViT-B_16', num_classes=config.num_classes )
def read_data() :
train = pd.read_csv('/kaggle/input/clean-kalman/train_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/clean-kalman/test_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = wave_block(inp, 16, 3, 12)
 x = wave_block(x, 32, 3, 8)
 x = wave_block(x, 64, 3, 4)
 x = wave_block(x, 128, 3, 1)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 30:
lr = LR
elif epoch < 40:
lr = LR / 3
elif epoch < 50:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f')
 def run_everything() :
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...')
 run_everything()<install_modules>
sample = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv' )
!pip install --no-warn-conflicts -q tensorflow-addons<import_modules>
def inference_by_fold(config, model, states_dicts, test_loader,tta_test_loader=None,use_tta=False):
model.to(config.device)
 model.eval()
with torch.no_grad() :
all_folds_preds = []
all_folds_tta_preds = []
for fold_num, state in enumerate(states_dicts):
if 'model_state_dict' in state:
model.load_state_dict(state['model_state_dict'])
 elif 'model' in state:
state = state['model']
state = {k[7:] if k.startswith('module.')else k: state[k] for k in state.keys() }
model.load_state_dict(state)
 else:
model.load_state_dict(state)
 current_fold_preds = []
current_fold_tta_preds = []
for data_test in tqdm(test_loader):
_, images, _ = data_test
images = images.to(config.device ).float()
logits = model(images)
 softmax_preds = torch.nn.Softmax(dim=1 )(input=logits ).to('cpu' ).numpy()
current_fold_preds.append(softmax_preds)
 current_fold_preds = np.concatenate(current_fold_preds, axis = 0)
 all_folds_preds.append(current_fold_preds)
 if use_tta:
for data_test_tta in tqdm(tta_test_loader):
_, tta_images, _ = data_test_tta
tta_images = tta_images.to(config.device ).float()
tta_logits = model(tta_images)
 tta_softmax_preds = torch.nn.Softmax(dim=1 )(input=tta_logits ).to('cpu' ).numpy()
combined_preds = np.mean([softmax_preds, tta_softmax_preds], axis=0)
 current_fold_tta_preds.append(combined_preds)
 current_fold_tta_preds = np.concatenate(current_fold_tta_preds, axis=0)
 all_folds_tta_preds.append(current_fold_tta_preds)
 avg_preds = np.mean(all_folds_preds, axis=0)if not use_tta else np.mean(all_folds_tta_preds, axis=0)
 return avg_preds
GRU, GRUCell, LSTMCell, SimpleRNNCell, SimpleRNN, TimeDistributed, RNN, RepeatVector)
 warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline<define_variables>
pred_resnext = submission(config, base_model_name=config.resnext)
 pred_resnext
EPOCHS=50
NNBATCHSIZE=16
BATCHSIZE = 50000
SEED = 321
SELECT = True
SPLITS = 5
LR = 0.01
fe_config = [ 
(True, 4000),
]<define_variables>
sample[config.class_col_name] = predictions_ensemble.argmax(1)
 sample[[config.image_col_name, config.class_col_name]].to_csv('submission.csv', index=False)
 display(sample.head() )
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED)
<load_from_csv>
all_preds_list = \
[np.array([[0.01, 0.02, 0.03, 0.9 , 0.04],
[0.02, 0.8 , 0.05, 0.06, 0.07],
[0.01, 0.8 , 0.03, 0.09, 0.07]]),
np.array([[0.03, 0.05 , 0.01 , 0.88 , 0.03],
[0.01 , 0.82 , 0.02 , 0.07 , 0.08],
[0.005 , 0.81 , 0.0205, 0.0555, 0.109]]),
np.array([[0.05, 0.03, 0.08, 0.83, 0.01],
[0.05, 0.89, 0.01, 0.02, 0.03],
[0.03, 0.78, 0.05, 0.02, 0.12]]),
np.array([[0.02, 0.01, 0.03, 0.92, 0.02],
[0.05, 0.85, 0.01, 0.01, 0.08],
[0.02, 0.88, 0.03, 0.05, 0.02]]),
np.array([[0.01, 0.02, 0.02, 0.93, 0.02],
[0.03, 0.76, 0.06, 0.06, 0.09],
[0.02, 0.83, 0.01, 0.07, 0.07]])]
np.mean(all_preds_list, axis=None)
 np.mean(all_preds_list, axis=0)
 np.mean(all_preds_list, axis=1 )
def read_data(base : os.path.abspath)-> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv('.. /input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('.. /input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv(os.path.join(base + '/sample_submission.csv'), dtype={'time': np.float32})
 return train, test, sub 
<create_dataframe>
fold_1_pred_1 = np.array([[0.01, 0.02, 0.03, 0.9, 0.04], [0.02, 0.8, 0.05, 0.06, 0.07]])
 fold_1_pred_2 = np.array([[0.01, 0.8, 0.03, 0.09, 0.07]])
 fold_2_pred_1 = np.array([[0.03, 0.05, 0.01, 0.88, 0.03], [0.01, 0.82, 0.02, 0.07, 0.08]])
 fold_2_pred_2 = np.array([[0.005, 0.81, 0.0205, 0.0555, 0.109]])
 fold_3_pred_1 = np.array([[0.05, 0.03, 0.08, 0.83, 0.01], [0.05, 0.89, 0.01, 0.02, 0.03]])
 fold_3_pred_2 = np.array([[0.03, 0.78, 0.05, 0.02, 0.12]])
 fold_4_pred_1 = np.array([[0.02, 0.01, 0.03, 0.92, 0.02], [0.05, 0.85, 0.01, 0.01, 0.08]])
 fold_4_pred_2 = np.array([[0.02, 0.88, 0.03, 0.05, 0.02]])
 fold_5_pred_1 = np.array([[0.01, 0.02, 0.02, 0.93, 0.02], [0.03, 0.76, 0.06, 0.06, 0.09]])
 fold_5_pred_2 = np.array([[0.02, 0.83, 0.01, 0.07, 0.07]] )
def batching(df : pd.DataFrame,
batch_size : int)-> pd.DataFrame :
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df 
<create_dataframe>
all_folds_preds = []
fold_1_preds = []
fold_1_preds = [fold_1_pred_1, fold_1_pred_2]
fold_1_preds before np.concatenate
',fold_1_preds)
 fold_1_preds = np.concatenate(fold_1_preds, axis=0)
 fold_1_preds_ = np.concatenate([fold_1_pred_1, fold_1_pred_2], axis=0)
 ',fold_1_preds)
 ',fold_1_preds_)
 all_folds_preds.append(fold_1_preds)
 print('All Folds Pred list after Fold 1 is
{}
'.format(all_folds_preds))
fold_2_preds = []
fold_2_preds = [fold_2_pred_1, fold_2_pred_2]
fold_2_preds = np.concatenate(fold_2_preds, axis=0)
 all_folds_preds.append(fold_2_preds)
 print('All Folds Pred list after Fold 1+2 is
{}
'.format(all_folds_preds))
fold_3_preds = []
fold_3_preds = [fold_3_pred_1, fold_3_pred_2]
fold_3_preds = np.concatenate(fold_3_preds, axis=0)
 all_folds_preds.append(fold_3_preds)
 print('All Folds Pred list after fold 1+2+3 is
{}
'.format(all_folds_preds))
fold_4_preds = []
fold_4_preds = [fold_4_pred_1, fold_4_pred_2]
fold_4_preds = np.concatenate(fold_4_preds, axis=0)
 all_folds_preds.append(fold_4_preds)
 print('All Folds Pred list after fold 1+2+3+4 is
{}
'.format(all_folds_preds))
fold_5_preds = []
fold_5_preds = [fold_5_pred_1, fold_5_pred_2]
fold_5_preds
fold_5_preds = np.concatenate(fold_5_preds, axis=0)
 fold_5_preds
all_folds_preds.append(fold_5_preds)
 print('All Folds Pred list after fold 1+2+3+4+5 is
{}
'.format(all_folds_preds))
avg_pred_without_concat = np.mean(all_folds_preds, axis=0)
 print('Average predictions is
{}'.format(avg_pred_without_concat))
def lag_with_pct_change(df : pd.DataFrame,
shift_sizes : Optional[List]=[1, 2],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df.groupby('group')['signal'].shift(shift_size ).fillna(0)
 df['signal_shift_neg_'+str(shift_size)] = df.groupby('group')['signal'].shift(-1*shift_size ).fillna(0)
 if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
df['pct_change_shift_pos_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(shift_size ).fillna(0)
 df['pct_change_shift_neg_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(-1*shift_size ).fillna(0)
 return df 
<create_dataframe>
def inference_by_fold(model, states_dicts, test_loader, device):
model.to(device)
 model.eval()
with torch.no_grad() :
all_folds_preds = []
for fold_num, state in enumerate(states_dicts):
if 'model_state_dict' not in state:
model.load_state_dict(state)
 else:
model.load_state_dict(state['model_state_dict'])
 current_fold_preds = []
for data in tqdm(test_loader, position=0, leave=True):
img_ids, images, labels = data
images = images.to(device)
 logits = model(images)
 softmax_preds = torch.nn.Softmax(dim=1 )(input=logits ).to('cpu' ).numpy()
current_fold_preds.append(softmax_preds)
 current_fold_preds = np.concatenate(current_fold_preds, axis = 0)
 all_folds_preds.append(current_fold_preds)
 avg_preds = np.mean(all_folds_preds, axis=0)
 return avg_preds
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
batch_size : int)-> pd.DataFrame:
df = batching(df, batch_size=batch_size)
 if create_all_data_feats:
df = lag_with_pct_change(df, [1, 2, 3, 4, 5, 6], add_pct_change=False, add_pct_change_lag=False)
 return df 
<feature_engineering>
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame)-> Tuple[pd.DataFrame , pd.DataFrame, List]:
use_cols = [col for col in df.columns if col not in ['index','group', 'open_channels', 'time']]
df = df.replace([np.inf, -np.inf], np.nan)
 df_test = df_test.replace([np.inf, -np.inf], np.nan)
 for col in use_cols:
col_mean = pd.concat([df[col], df_test[col]], axis=0 ).mean()
df[col] = df[col].fillna(col_mean)
 df_test[col] = df_test[col].fillna(col_mean)
 gc.collect()
return df, df_test, use_cols 
<create_dataframe>
class TheModelClass(nn.Module):
def __init__(self):
super(TheModelClass, self ).__init__()
self.conv1 = nn.Conv2d(in_channels=3, out_channels =6, kernel_size =5)
 self.pool = nn.MaxPool2d(2, 2)
 self.conv2 = nn.Conv2d(6, 16, 5)
 self.fc1 = nn.Linear(16 * 5 * 5, 120)
 self.fc2 = nn.Linear(120, 84)
 self.fc3 = nn.Linear(84, 10)
 def forward(self, x):
x = self.pool(F.relu(self.conv1(x)))
 x = self.pool(F.relu(self.conv2(x)))
 x = x.view(-1, 16 * 5 * 5)
 x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = self.fc3(x)
 return x
model = TheModelClass()
optimizer = torch.optim.SGD(model.parameters() , lr=0.001, momentum=0.9)
 print("Model's state_dict:")
 for param_tensor in model.state_dict() :
print(param_tensor, "\t", model.state_dict() [param_tensor].size())
 print("Optimizer's state_dict:")
 for var_name in optimizer.state_dict() :
print(var_name, "\t", optimizer.state_dict() [var_name])
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
batch_col : Text,
feats : List,
sample_submission: pd.DataFrame,
nn_epochs : int,
nn_batch_size : int)-> NoReturn:
oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
gc.collect()
K.clear_session()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_schedule = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, verbose=0, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)
 cb_Early_Stop=EarlyStopping(monitor='val_loss',patience=10)
 cb_clr = CyclicLR(base_lr=1e-7, max_lr = 1e-4, step_size= int(1.0*(test.shape[0])/(nn_batch_size*4)) , mode='exp_range', gamma=1.0, scale_fn=None, scale_mode='cycle')
 cb_prg = tfa.callbacks.TQDMProgressBar(leave_epoch_progress=False,leave_overall_progress=False, show_epoch_progress=False,show_overall_progress=True)
 model.fit(train_x,train_y,
epochs=nn_epochs,
callbacks=[cb_prg, cb_schedule, MacroF1(model, valid_x,valid_y)],
batch_size=nn_batch_size,verbose=0,
validation_data=(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 logger.info(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ =f1_score(np.argmax(train_tr, axis=2 ).reshape(-1), np.argmax(oof_, axis=1), average = 'macro')
 logger.info(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis=1 ).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return 
<choose_model_class>
def count_parameters(model):
table = PrettyTable(["Modules", "Parameters"])
 total_params = 0
for name, parameter in model.named_parameters() :
if not parameter.requires_grad: continue
param = parameter.numel()
table.add_row([name, param])
 total_params+=param
print(table)
 print(f"Total Trainable Params: {total_params}")
 return total_params
count_parameters(model)
class CyclicLR(tf.keras.callbacks.Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1 /(2.**(x - 1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma **(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1 + self.clr_iterations /(2 * self.step_size))
x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(
 self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
K.set_value(self.model.optimizer.lr, self.clr())
<compute_train_metric>
def categorical_focal_loss(gamma=2.0, alpha=0.25):

 def focal_loss(y_true, y_pred):
epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.0-epsilon)
 cross_entropy = -y_true*K.log(y_pred)
 weight = alpha * y_true * K.pow(( 1-y_pred), gamma)
 loss = weight * cross_entropy
loss = K.sum(loss, axis=1)
 return loss
return focal_loss 
<choose_model_class>