Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<create_dataframe>
warnings.filterwarnings('ignore')
df_sub = pd.DataFrame()
df_sub['id'] = sub['id']
df_sub['revenue'] = reve[0]<save_to_csv>
training_folder = '.. /input/cassava-leaf-disease-classification/train_images/'
df_sub.to_csv('submission_1.csv', index=False )<set_options>
samples_df = pd.read_csv(".. /input/cassava-leaf-disease-classification/train.csv")
 samples_df = shuffle(samples_df, random_state=42)
 samples_df["filepath"] = training_folder+samples_df["image_id"]
samples_df.head()
%matplotlib inline
warnings.filterwarnings(action="ignore")
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 print(os.listdir(".. /input"))<load_from_csv>
training_percentage = 0.8
training_item_count = int(len(samples_df)*training_percentage)
 validation_item_count = len(samples_df)-int(len(samples_df)*training_percentage)
 training_df = samples_df[:training_item_count]
validation_df = samples_df[training_item_count:]
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
batch_size = 8
image_size = 512
input_shape =(image_size, image_size, 3)
 dropout_rate = 0.4
classes_to_predict = sorted(training_df.label.unique() )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
training_data = tf.data.Dataset.from_tensor_slices(( training_df.filepath.values, training_df.label.values))
validation_data = tf.data.Dataset.from_tensor_slices(( validation_df.filepath.values, validation_df.label.values))
X_train = train.drop(['revenue'],axis=1)
 y_train = train['revenue']
print(X_train.shape, y_train.shape )<feature_engineering>
def load_image_and_label_from_path(image_path, label):
img = tf.io.read_file(image_path)
 img = tf.image.decode_jpeg(img, channels=3)
 return img, label
AUTOTUNE = tf.data.experimental.AUTOTUNE
training_data = training_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE)
 validation_data = validation_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE )
dict_columns = ['belongs_to_collection', 'genres', 'production_companies','production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )<concatenate>
training_data_batches = training_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE)
 validation_data_batches = validation_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE )
X = pd.concat([X_train, test], axis=0, ignore_index=True )<feature_engineering>
adapt_data = tf.data.Dataset.from_tensor_slices(training_df.filepath.values)
 def adapt_mode(image_path):
img = tf.io.read_file(image_path)
 img = tf.image.decode_jpeg(img, channels=3)
 img = layers.experimental.preprocessing.Rescaling(1.0 / 255 )(img)
 return img
adapt_data = adapt_data.map(adapt_mode, num_parallel_calls=AUTOTUNE)
 adapt_data_batches = adapt_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE )
X['has_homepage'] = X['homepage'].isnull() == False
X['is_original_english'] = X['original_language'] == 'en'
X['has_collection'] = X['belongs_to_collection'].isnull() == False
X['has_two_titles'] = X['original_title'] != X['title']
X.drop(['status','original_language','poster_path', 'homepage', 'imdb_id','belongs_to_collection', 'id'], axis=1, inplace=True)
 X.head(2 )<feature_engineering>
data_augmentation_layers = tf.keras.Sequential(
 [
layers.experimental.preprocessing.RandomCrop(height=image_size, width=image_size),
layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical"),
layers.experimental.preprocessing.RandomRotation(0.25),
layers.experimental.preprocessing.RandomZoom(( -0.2, 0)) ,
layers.experimental.preprocessing.RandomContrast(( 0.2,0.2))
]
)
X.loc[pd.isnull(X['spoken_languages'])== True,'spoken_languages'] = 0
X['lang'] = list(map(lambda x: [i['iso_639_1'] for i in eval(x)] if x!=0 else [], X['spoken_languages'].values))
X['n_lang'] = X['lang'].apply(lambda x: len(x))
spoken_features = ['' + i for i in ['', 'la', 'it', 'cs', 'ta', 'pt', 'hu', 'zh', 'pl', 'ar', 'en', 'ja', 'de', 'ko', 'cn', 'tr',
'he', 'sv', 'el', 'ru', 'fr', 'es', 'hi', 'th']]
for i in spoken_features:
X[i] = X['lang'].apply(lambda x: i[7:] in x)
 X.drop(['original_title', 'spoken_languages', 'lang'], axis=1, inplace=True )<feature_engineering>
image = Image.open(".. /input/cassava-leaf-disease-classification/train_images/3412658650.jpg")
 plt.imshow(image)
 plt.show()
X.loc[pd.isnull(X['genres'])== True,'genres'] = 0
genres = set(' '.join([' '.join(i)for i in list(map(lambda x: [i['name'] for i in eval(x)] if x!=0 else [], X['genres'].values)) ] ).split())
 X['genres'] = list(map(lambda x: [i['name'] for i in eval(x)] if x!=0 else [], X['genres'].values))
for i in genres:
X['genre_' + i] = X['genres'].apply(lambda x: i in x )<feature_engineering>
image = tf.expand_dims(np.array(image), 0 )
X['n_genres'] = X['genres'].apply(lambda x: len(x))
X['release_month'] = 0
X['release_day'] = 0
X['release_year'] = 0
X = pd.concat([X, X['release_date'].str.split('/', expand=True)], axis=1)
 X.head(2 )<data_type_conversions>
plt.figure(figsize=(10, 10))
for i in range(9):
augmented_image = data_augmentation_layers(image)
 ax = plt.subplot(3, 3, i + 1)
 plt.imshow(augmented_image[0])
 plt.axis("off" )
X.iloc[:,-1] = X.iloc[:,-1].fillna('0' ).astype(int )<feature_engineering>
efficientnet = EfficientNetB3(weights=".. /input/efficientnetb3-notop/efficientnetb3_notop.h5",
include_top=False,
input_shape=input_shape,
drop_connect_rate=dropout_rate)
 inputs = Input(shape=input_shape)
 augmented = data_augmentation_layers(inputs)
 efficientnet = efficientnet(augmented)
 pooling = layers.GlobalAveragePooling2D()(efficientnet)
 dropout = layers.Dropout(dropout_rate )(pooling)
 outputs = Dense(len(classes_to_predict), activation="softmax" )(dropout)
 model = Model(inputs=inputs, outputs=outputs)
 model.summary()
year_mod = []
for i in X.iloc[:,-1].values:
if i in range(0, 19):
year_mod.extend([2000 + i])
 else:
year_mod.extend([1900 + i])
 year_mod
X['release_year'] = year_mod<categorify>
%%time
model.get_layer('efficientnetb3' ).get_layer('normalization' ).adapt(adapt_data_batches )
X = pd.concat([X, pd.get_dummies(X[0], prefix='release_month')], axis=1)
 X.head(2 )<data_type_conversions>
epochs = 8
decay_steps = int(round(len(training_df)/batch_size)) *epochs
cosine_decay = CosineDecay(initial_learning_rate=1e-4, decay_steps=decay_steps, alpha=0.3)
 callbacks = [ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True)]
model.compile(loss="sparse_categorical_crossentropy", optimizer=tf.keras.optimizers.Adam(cosine_decay), metrics=["accuracy"] )
X['release_date'] = pd.to_datetime(X['release_date'])
 X['release_weekday'] = X['release_date'].dt.weekday.fillna(8 ).astype(int )<feature_engineering>
history = model.fit(training_data_batches,
epochs = epochs,
validation_data=validation_data_batches,
callbacks=callbacks )
X.loc[:,'production_companies'] = X.loc[:,'production_companies'].fillna('[]')
 companies = ','.join([','.join(i)for i in list(map(lambda x: [i['name'] for i in eval(x)], X['production_companies'].values)) ] ).split(',')
 unique_companies = set(companies)
 X['production_companies'] = list(map(lambda x: [i['name'] for i in eval(x)], X['production_companies'].values))<feature_engineering>
model.load_weights("best_model.h5" )
prod_count = {i: sum([1 for j in companies if i == j])for i in unique_companies}
most_famous_prod = [k for k,v in prod_count.items() if v > 100 and k]
famous_prod = [k for k,v in prod_count.items() if 30 <= v < 100 and k]
X['n_production_companies'] = X['production_companies'].apply(lambda x: len(x))
X['most_famous_prod'] = X['production_companies'].apply(lambda x: sum([1 for i in x if i in most_famous_prod]))
X['famous_prod'] = X['production_companies'].apply(lambda x: sum([1 for i in x if i in famous_prod]))
X.head(2 )<feature_engineering>
test_time_augmentation_layers = tf.keras.Sequential(
 [
layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical"),
layers.experimental.preprocessing.RandomZoom(( -0.2, 0)) ,
layers.experimental.preprocessing.RandomContrast(( 0.2,0.2))
]
)
X.loc[:,'production_countries'] = X.loc[:,'production_countries'].fillna('[]')
 countries = ','.join([','.join(i)for i in list(map(lambda x: [i['iso_3166_1'] for i in eval(x)], X['production_countries'].values)) ] ).split(',')
 unique_countries = set(countries)
 X['production_countries'] = list(map(lambda x: [i['iso_3166_1'] for i in eval(x)], X['production_countries'].values))<feature_engineering>
def run_predictions_over_image_list(image_list, folder):
predictions = []
with tqdm(total=len(image_list)) as pbar:
for image_filename in image_list:
pbar.update(1)
 predictions.append(predict_and_vote(image_filename, folder))
return predictions
country_count = {i: sum([1 for j in countries if i == j])for i in unique_countries}
most_famous_countries= [k for k,v in country_count.items() if v > 100 and k]
famous_countries = [k for k,v in country_count.items() if 30 <= v < 100 and k]
X['n_production_countries'] = X['production_countries'].apply(lambda x: len(x))
X['most_famous_countries'] = X['production_countries'].apply(lambda x: sum([1 for i in x if i in most_famous_countries]))
X['famous_countries'] = X['production_countries'].apply(lambda x: sum([1 for i in x if i in famous_countries]))<drop_column>
validation_df["results"] = run_predictions_over_image_list(validation_df["image_id"], training_folder )
X['has_tagline'] = X['tagline'].apply(lambda x: pd.isnull(x))
X.drop(['genres', 'overview', 'production_companies', 'production_countries', 'release_date', 'tagline', 'release_month', 'release_day', 0, 2,
'title', 'Keywords', 'cast','crew'], axis=1, inplace=True)
 X.head(2 )<feature_engineering>
true_positives = 0
prediction_distribution_per_class = {"0":{"0": 0, "1": 0, "2":0, "3":0, "4":0},
"1":{"0": 0, "1": 0, "2":0, "3":0, "4":0},
"2":{"0": 0, "1": 0, "2":0, "3":0, "4":0},
"3":{"0": 0, "1": 0, "2":0, "3":0, "4":0},
"4":{"0": 0, "1": 0, "2":0, "3":0, "4":0}}
number_of_images = len(validation_df)
 for idx, pred in validation_df.iterrows() :
if int(pred["label"])== pred.results:
true_positives+=1
prediction_distribution_per_class[str(pred["label"])][str(pred.results)] += 1
print("accuracy: {}%".format(true_positives/number_of_images*100))
X['budget_log'] = np.log1p(X['budget'] )<data_type_conversions>
test_folder = '.. /input/cassava-leaf-disease-classification/test_images/'
submission_df = pd.DataFrame(columns={"image_id","label"})
 submission_df["image_id"] = os.listdir(test_folder)
 submission_df["label"] = 0
X['inflationBudget'] = X['budget'] + X['budget']*1.8/100*(2019-X['release_year'])
 X['runtime'] = X['runtime'].fillna(X['runtime'].mean())
 X[1] = X[1].fillna(1)
 for f in X.dtypes[(X.dtypes == 'bool')|(X.dtypes == 'object')].index:
X[f] = X[f].astype(int )<drop_column>
submission_df["label"] = run_predictions_over_image_list(submission_df["image_id"], test_folder )
data_dropping_names = data.drop(['original_title','overview','tagline','title'], axis=1)
 train = data_dropping_names[data_dropping_names['source'] == 'train'].copy()
test = data_dropping_names[data_dropping_names['source'] == 'test'].copy()
train_labels = train['revenue_log']
train.drop(['id', 'revenue', 'source', 'revenue_log'], axis=1, inplace=True)
 test_final = test.drop(['id', 'source','revenue','revenue_log'], axis=1 )<choose_model_class>
submission_df.to_csv("submission.csv", index=False )