Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<categorify>
print("tensorflow version" + tf.__version__)
_=
all_data['_meanHitsPerDay'] = all_data.groupby(['_day'])['totals.hits'].transform('mean')
 all_data['_meanHitsPerWeekday'] = all_data.groupby(['_weekday'])['totals.hits'].transform('mean')
 all_data['_meanHitsPerMonth'] = all_data.groupby(['_month'])['totals.hits'].transform('mean')
 all_data['_sumHitsPerDay'] = all_data.groupby(['_day'])['totals.hits'].transform('sum')
 all_data['_sumHitsPerWeekday'] = all_data.groupby(['_weekday'])['totals.hits'].transform('sum')
 all_data['_sumHitsPerMonth'] = all_data.groupby(['_month'])['totals.hits'].transform('sum')
 for feature in ['totals.hits', 'totals.pageviews']:
info = all_data.groupby('fullVisitorId')[feature].mean()
all_data['_usermean_' + feature] = all_data.fullVisitorId.map(info)
 for feature in ['visitNumber']:
info = all_data.groupby('fullVisitorId')[feature].max()
all_data['_usermax_' + feature] = all_data.fullVisitorId.map(info)
 del info<feature_engineering>
BASE_DIR = ".. /input/cassava-leaf-disease-classification"
all_data['_source.country'] = all_data['trafficSource.source'] + '_' + all_data['geoNetwork.country']
all_data['_campaign.medium'] = all_data['trafficSource.campaign'] + '_' + all_data['trafficSource.medium']
all_data['_browser.category'] = all_data['device.browser'] + '_' + all_data['device.deviceCategory']
all_data['_browser.os'] = all_data['device.browser'] + '_' + all_data['device.operatingSystem']<sort_values>
with open(os.path.join(BASE_DIR, "label_num_to_disease_map.json")) as file:
map_classes = json.loads(file.read())
 map_classes = {int(k): v for k, v in map_classes.items() }
print(json.dumps(map_classes, indent=4))
null_cnt = all_data.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )<data_type_conversions>
input_files = os.listdir(os.path.join(BASE_DIR, "train_images"))
print(f"Number of train images: {len(input_files)}" )
all_data.drop(['visitId','visitStartTime'],axis=1,inplace=True)
 for i, t in all_data.loc[:, all_data.columns != 'fullVisitorId'].dtypes.iteritems() :
if t == object:
all_data[i].fillna('unknown', inplace=True)
 all_data[i] = pd.factorize(all_data[i])[0] 
<prepare_x_and_y>
img_shapes = {}
for image_name in os.listdir(os.path.join(BASE_DIR, "train_images")) [:300]:
image = cv2.imread(os.path.join(BASE_DIR, "train_images", image_name))
img_shapes[image.shape] = img_shapes.get(image.shape, 0)+ 1
print(img_shapes )
train = all_data[all_data['totals.transactionRevenue'].notnull() ]
test = all_data[all_data['totals.transactionRevenue'].isnull() ].drop(['totals.transactionRevenue'], axis=1 )<prepare_x_and_y>
df_train = pd.read_csv(os.path.join(BASE_DIR , "train.csv"))
df_train["class_name"] = df_train["label"].map(map_classes)
 df_train[:10]
train_id = train['fullVisitorId']
test_id = test['fullVisitorId']
Y_train_reg = train.pop('totals.transactionRevenue')
 X_train = train.drop(['fullVisitorId'], axis=1)
 X_test = test.drop(['fullVisitorId'], axis=1)
 print(X_train.shape, X_test.shape )<create_dataframe>
BATCH_SIZE = 8
STEPS_PER_EPOCH = len(df_train)*0.8 / BATCH_SIZE
VALIDATION_STEPS = len(df_train)*0.2 / BATCH_SIZE
EPOCHS = 20
TARGET_SIZE = 512
del all_data, train, test, train_rev
gc.collect()
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )<init_hyperparams>
df_train.label = df_train.label.astype('str')
 train_datagen = tf.keras.preprocessing.image.ImageDataGenerator(validation_split = 0.2,
rescale = 1./255,
rotation_range = 45,
zoom_range = 0.2,
horizontal_flip = True,
vertical_flip = True,
fill_mode = 'nearest',
shear_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1)
 train_generator = train_datagen.flow_from_dataframe(df_train,
directory = os.path.join(BASE_DIR, "train_images"),
subset = "training",
x_col = "image_id",
y_col = "label",
target_size =(TARGET_SIZE, TARGET_SIZE),
batch_size = BATCH_SIZE,
class_mode = "sparse")
 validation_datagen = tf.keras.preprocessing.image.ImageDataGenerator(validation_split = 0.2)
 validation_generator = validation_datagen.flow_from_dataframe(df_train,
directory = os.path.join(BASE_DIR, "train_images"),
subset = "validation",
x_col = "image_id",
y_col = "label",
target_size =(TARGET_SIZE, TARGET_SIZE),
batch_size = BATCH_SIZE,
class_mode = "sparse" )
params={'learning_rate': 0.01,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
'random_state':42,
'bagging_fraction': 0.6,
'feature_fraction': 0.6
}
folds = GroupKFold(n_splits=5)
 oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds.split(X_train, Y_train_reg, groups=train_id)) :
trn_x, trn_y = X_train.iloc[trn_], Y_train_reg.iloc[trn_]
val_x, val_y = X_train.iloc[val_], Y_train_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=3000)
 reg.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=50, verbose=500)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 sub_preds += reg.predict(X_test, num_iteration=reg.best_iteration_)/ folds.n_splits
pred = sub_preds<save_to_csv>
def create_model() :
conv_base = EfficientNetB0(include_top = False, weights = None,
input_shape =(TARGET_SIZE, TARGET_SIZE, 3))
model = conv_base.output
model = layers.GlobalAveragePooling2D()(model)
 model = layers.Dense(5, activation = "softmax" )(model)
 model = models.Model(conv_base.input, model)
 model.compile(optimizer = Adam(lr = 0.001),
loss = "sparse_categorical_crossentropy",
metrics = ["acc"])
 return model
submission = pd.DataFrame({'fullVisitorId':test_id, 'PredictedLogRevenue':pred})
 submission["PredictedLogRevenue"] = np.expm1(submission["PredictedLogRevenue"])
 submission["PredictedLogRevenue"] = submission["PredictedLogRevenue"].apply(lambda x : 0.0 if x < 0 else x)
 submission["PredictedLogRevenue"] = submission["PredictedLogRevenue"].fillna(0.0)
 submission_sum = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId' ).sum().reset_index()
submission_sum["PredictedLogRevenue"] = np.log1p(submission_sum["PredictedLogRevenue"])
 submission_sum.to_csv("submission.csv", index=False)
 submission_sum.head(20 )<import_modules>
model.load_weights('.. /input/weight1/myweights3.h5' )
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
import xgboost as xgb
from pprint import pprint
import math
from scipy.stats import kurtosis, skew
from IPython import embed
from IPython.terminal.embed import InteractiveShellEmbed
from sklearn.model_selection import KFold
import random<set_options>
early_stop = EarlyStopping(monitor = 'val_loss', min_delta = 0.001,
patience = 5, mode = 'min', verbose = 1,
restore_best_weights = True)
 reduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.3,
patience = 2, min_delta = 0.001,
mode = 'min', verbose = 1)
 history = model.fit(
 train_generator,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
validation_data = validation_generator,
validation_steps = VALIDATION_STEPS,
callbacks = [early_stop, reduce_lr] 
)
 model.save('./myEffNetB0_512_8__3.h5')
 model.save_weights('./myweights4.h5' )
plt.rcParams['figure.figsize'] =(12,6 )<categorify>
ss = pd.read_csv(os.path.join(BASE_DIR, "sample_submission.csv"))
ss
def load_data(data='train',n=2):
df = pd.DataFrame()
for i in range(n):
if data=='train':
if i > 8 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))
elif data=='test':
if i > 2 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))
df = pd.concat([df,dfpart])
 del dfpart
return df 
<load_pretrained>
preds = []
for image_id in ss.image_id:
image = Image.open(os.path.join(BASE_DIR, "test_images", image_id))
image = image.resize(( TARGET_SIZE, TARGET_SIZE))
image = np.expand_dims(image, axis = 0)
 preds.append(np.argmax(model.predict(image)))
 ss['label'] = preds
ss
%%time
df_train = load_data(n=9)
 df_test = load_data('test',n=4 )<drop_column>
ss.to_csv('submission.csv', index = False )