Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<save_to_csv>
warnings.simplefilter("ignore")
<save_to_csv>
print('Train images: %d' %len(os.listdir(
 os.path.join(WORK_DIR, "train_images"))))
sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 sample_submission.isFraud = preds
sample_submission.to_csv('sub_dl.csv',index=False)
 plt.hist(sample_submission.isFraud,bins=100)
 plt.ylim(( 0,5000))
plt.title('DL Submission')
 plt.show()<load_from_csv>
with open(os.path.join(WORK_DIR, "label_num_to_disease_map.json")) as file:
print(json.dumps(json.loads(file.read()), indent=4))
X_test['isFraud'] = sample_submission.isFraud.values
X_train['isFraud'] = y_train.values
comb = pd.concat([X_train[['isFraud']],X_test[['isFraud']]],axis=0)
 uids = pd.read_csv('/kaggle/input/ieee-submissions-and-uids/uids_v4_no_multiuid_cleaning.. csv',usecols=['TransactionID','uid'] ).rename({'uid':'uid2'},axis=1)
 comb = comb.merge(uids,on='TransactionID',how='left')
 mp = comb.groupby('uid2' ).isFraud.agg(['mean'])
 comb.loc[comb.uid2>0,'isFraud'] = comb.loc[comb.uid2>0].uid2.map(mp['mean'])
 uids = pd.read_csv('/kaggle/input/ieee-submissions-and-uids/uids_v1_no_multiuid_cleaning.csv',usecols=['TransactionID','uid'] ).rename({'uid':'uid3'},axis=1)
 comb = comb.merge(uids,on='TransactionID',how='left')
 mp = comb.groupby('uid3' ).isFraud.agg(['mean'])
 comb.loc[comb.uid3>0,'isFraud'] = comb.loc[comb.uid3>0].uid3.map(mp['mean'])
 sample_submission.isFraud = comb.iloc[len(X_train):].isFraud.values
sample_submission.to_csv('sub_dl_PP.csv',index=False )<load_from_csv>
train_labels = pd.read_csv(os.path.join(WORK_DIR, "train.csv"))
train_labels.head()
<load_from_csv>
BATCH_SIZE = 8
STEPS_PER_EPOCH = len(train_labels)*0.8 / BATCH_SIZE
VALIDATION_STEPS = len(train_labels)*0.2 / BATCH_SIZE
EPOCHS = 20
TARGET_SIZE = 512
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 train_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv')
 train_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv')
 test_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv')
 test_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv' )<sort_values>
train_labels.label = train_labels.label.astype('str')
 train_datagen = ImageDataGenerator(validation_split = 0.2,
preprocessing_function = None,
rotation_range = 45,
zoom_range = 0.2,
horizontal_flip = True,
vertical_flip = True,
fill_mode = 'nearest',
shear_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1)
 train_generator = train_datagen.flow_from_dataframe(train_labels,
directory = os.path.join(WORK_DIR, "train_images"),
subset = "training",
x_col = "image_id",
y_col = "label",
target_size =(TARGET_SIZE, TARGET_SIZE),
batch_size = BATCH_SIZE,
class_mode = "sparse")
 validation_datagen = ImageDataGenerator(validation_split = 0.2)
 validation_generator = validation_datagen.flow_from_dataframe(train_labels,
directory = os.path.join(WORK_DIR, "train_images"),
subset = "validation",
x_col = "image_id",
y_col = "label",
target_size =(TARGET_SIZE, TARGET_SIZE),
batch_size = BATCH_SIZE,
class_mode = "sparse" )
identity_cols_withNA = 100*train_identity.isnull().sum().sort_values(ascending=False)/train_identity.shape[0]
identity_cols_withNA = identity_cols_withNA[identity_cols_withNA > 0]
print('train_identity has {} columns with null entries.The 10 most empty columns are shown below'.format(identity_cols_withNA.shape[0]))
identity_cols_withNA[:10]<sort_values>
generator = train_datagen.flow_from_dataframe(train_labels.iloc[20:21],
directory = os.path.join(WORK_DIR, "train_images"),
x_col = "image_id",
y_col = "label",
target_size =(TARGET_SIZE, TARGET_SIZE),
batch_size = BATCH_SIZE,
class_mode = "sparse")
 aug_images = [generator[0][0][0]/255 for i in range(10)]
fig, axes = plt.subplots(2, 5, figsize =(20, 10))
axes = axes.flatten()
for img, ax in zip(aug_images, axes):
ax.imshow(img)
 ax.axis('off')
 plt.tight_layout()
plt.show()
transaction_cols_withNA = 100*train_transaction.isnull().sum().sort_values(ascending=False)/train_transaction.shape[0]
transaction_cols_withNA = transaction_cols_withNA[transaction_cols_withNA > 0]
print('train_transaction has {} columns with null entries.The 10 most empty columns are shown below'.format(transaction_cols_withNA.shape[0]))
transaction_cols_withNA[:10]<define_variables>
identity_cat_cols = ['DeviceType','DeviceInfo']
for i in range(12,39):
identity_cat_cols.append('id_'+str(i))
transaction_cat_cols = ['ProductCD','addr1','addr2','P_emaildomain','R_emaildomain']
for i in range(1,7):
transaction_cat_cols.append('card'+str(i))
for i in range(1,10):
transaction_cat_cols.append('M'+str(i))<count_unique_values>
def create_model() :
conv_base = EfficientNetB0(include_top = False, weights = None,
input_shape =(TARGET_SIZE, TARGET_SIZE, 3))
model = conv_base.output
model = layers.GlobalAveragePooling2D()(model)
 model = layers.Dense(5, activation = "softmax" )(model)
 model = models.Model(conv_base.input, model)
 model.compile(optimizer = Adam(lr = 0.001),
loss = "sparse_categorical_crossentropy",
metrics = ["acc"])
 return model
print('Number of unique entries per categorical column(including nulls)in train_identity:
')
 for col in identity_cat_cols:
cat_number = len(train_identity[col].unique())
 print(col+': '+str(cat_number))<count_unique_values>
print('Our EfficientNet CNN has %d layers' %len(model.layers))
print('Number of unique entries per categorical column(including nulls)in train_transaction:
')
 for col in transaction_cat_cols:
cat_number = len(train_transaction[col].unique())
 print(col+': '+str(cat_number))<categorify>
model.load_weights('.. /input/cassava-leaf-disease-models/basic_EfNetB0_imagenet_512.h5' )
def reduce_mem_usage(df,cat_cols):
start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in cat_cols:
df[col] = df[col].astype('category')
 num_cols = list(set(df.columns)- set(cat_cols))
for col in num_cols:
col_type = df[col].dtype
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.uint8 ).min and c_max < np.iinfo(np.uint8 ).max:
df[col] = df[col].astype(np.uint8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.uint16 ).min and c_max < np.iinfo(np.uint16 ).max:
df[col] = df[col].astype(np.uint16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.uint32 ).min and c_max < np.iinfo(np.uint32 ).max:
df[col] = df[col].astype(np.uint32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 elif c_min > np.iinfo(np.uint64 ).min and c_max < np.iinfo(np.uint64 ).max:
df[col] = df[col].astype(np.uint64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<merge>
model_save = ModelCheckpoint('./EffNetB0_512_8_best_weights.h5',
save_best_only = True,
save_weights_only = True,
monitor = 'val_loss',
mode = 'min', verbose = 1)
 early_stop = EarlyStopping(monitor = 'val_loss', min_delta = 0.001,
patience = 5, mode = 'min', verbose = 1,
restore_best_weights = True)
 reduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.3,
patience = 2, min_delta = 0.001,
mode = 'min', verbose = 1)
 history = model.fit(
 train_generator,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
validation_data = validation_generator,
validation_steps = VALIDATION_STEPS,
callbacks = [model_save, early_stop, reduce_lr]
)
print('Reducing memory usage of train_identity
')
 train_identity = reduce_mem_usage(train_identity,identity_cat_cols)
 print('
Reducing memory usage of train_transaction
')
 train_transaction = reduce_mem_usage(train_transaction,transaction_cat_cols)
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 train_mem_usage = train.memory_usage(index = True ).sum() /(1024**2)
 print('
Memory usage of train is {:.2f} MB'.format(train_mem_usage))
train.head()<merge>
model.save('./EffNetB0_512_8.h5' )
cols_to_rename = {}
for i in range(10):
cols_to_rename['id-0'+str(i)] = 'id_0'+str(i)
 for i in range(10,39):
cols_to_rename['id-'+str(i)] = 'id_'+str(i)
 test_identity = test_identity.rename(columns=cols_to_rename)
 print('Reducing memory usage of test_identity
')
 test_identity = reduce_mem_usage(test_identity,identity_cat_cols)
 print('
Reducing memory usage of test_transaction
')
 test_transaction = reduce_mem_usage(test_transaction,transaction_cat_cols)
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left')
 test_mem_usage = test.memory_usage(index = True ).sum() /(1024**2)
 print('
Memory usage of test is {:.2f} MB'.format(test_mem_usage))
del train_transaction
del train_identity
del test_identity
del test_transaction
gc.collect()
test.head()<categorify>
def all_activations_vis(img, layers = 10):
layer_outputs = [layer.output for layer in model.layers[:layers]]
activation_model = models.Model(inputs = model.input, outputs = layer_outputs)
 activations = activation_model.predict(img)
 layer_names = []
for layer in model.layers[:layers]:
layer_names.append(layer.name)
 images_per_row = 3
for layer_name, layer_activation in zip(layer_names, activations):
n_features = layer_activation.shape[-1]
size = layer_activation.shape[1]
n_cols = n_features // images_per_row
display_grid = np.zeros(( size * n_cols, images_per_row * size))
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0, :, :, col * images_per_row + row]
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255 ).astype('uint8')
 display_grid[col * size :(col + 1)* size,
row * size :(row + 1)* size] = channel_image
scale = 1./ size
plt.figure(figsize=(scale * 5 * display_grid.shape[1],
scale * 5 * display_grid.shape[0]))
plt.title(layer_name)
 plt.grid(False)
 plt.axis('off')
 plt.imshow(display_grid, aspect = 'auto', cmap = 'viridis' )
cat_cols = transaction_cat_cols + identity_cat_cols
num_cols = list(set(train.drop(['TransactionID','isFraud','TransactionDT'],axis=1 ).columns)- set(cat_cols))
train_copy = train.copy()
test_copy = test.copy()
for col in cat_cols:
le = LabelEncoder()
le.fit(list(train_copy[col].astype(str ).values)+ list(test_copy[col].astype(str ).values))
train_copy[col] = le.transform(list(train_copy[col].astype(str ).values))
test_copy[col] = le.transform(list(test_copy[col].astype(str ).values))
X = train_copy[cat_cols + num_cols]
y = train_copy['isFraud']
X_test = test_copy[cat_cols + num_cols]
del train_copy
del test_copy
gc.collect()<init_hyperparams>
ss = pd.read_csv(os.path.join(WORK_DIR, "sample_submission.csv"))
ss
params = {'num_leaves': 256,
'min_child_samples': 79,
'objective': 'binary',
'max_depth': 8,
'learning_rate': 0.02,
"boosting_type": "gbdt",
"subsample_freq": 3,
"subsample": 0.9,
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3,
'reg_lambda': 0.3,
'colsample_bytree': 0.9,
}
n_splits = 4
skf = StratifiedKFold(n_splits=n_splits,shuffle=True)
 skf.get_n_splits(X, y)
 scores = []
i = 1
y_pred = np.zeros(len(X_test.iloc[:,0]))
feat_imp = pd.Series(0, index=X.columns)
 for train_index, valid_index in skf.split(X, y):
print('Training the model using fold number {}...'.format(i))
X_train, X_valid = X.iloc[train_index,:], X.iloc[valid_index,:]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
model = lgb.LGBMClassifier(**params, n_estimators=500, n_jobs = -1)
 model.fit(X_train, y_train,eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='auc', early_stopping_rounds=200,verbose=False)
 y_pred_valid = model.predict_proba(X_valid)[:, 1]
y_pred += model.predict_proba(X_test)[:, 1]/n_splits
score = metrics.roc_auc_score(y_valid, y_pred_valid)
 scores.append(score)
 feat_imp += pd.Series(model.feature_importances_, index=X.columns)/n_splits
print('AUC score: {:.3f}'.format(score))
i += 1
print('mean AUC score over the validation sets: {:.3f}
'.format(np.mean(scores)))
 fig,ax = plt.subplots(figsize=(10,10))
_ = feat_imp.nlargest(20 ).sort_values().plot(kind='barh',ax=ax)
 _ = ax.set_xlabel('Average importance(a.u.) ')
 _ = ax.set_ylabel('Features' )<save_to_csv>
preds = []
for image_id in ss.image_id:
image = Image.open(os.path.join(WORK_DIR, "test_images", image_id))
image = image.resize(( TARGET_SIZE, TARGET_SIZE))
image = np.expand_dims(image, axis = 0)
 preds.append(np.argmax(model.predict(image)))
 ss['label'] = preds
ss
<drop_column><EOS> 
ss.to_csv('submission.csv', index = False )