Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<train_model>
!pip install.. /input/pytorch-image-models/timm-0.3.1-py3-none-any.whl
for i in range(count_network):
list_models[i].load_weights(f'bestmodel{i + 1}.hdf5')
 print(f'Model №{i + 1}')
 _, acc = list_models[i].evaluate(X_train_list[i], Y_train_list[i])
 _, acc2 = list_models[i].evaluate(X_valid_list[i], Y_valid_list[i])
 print()<predict_on_test>
import numpy as np
import os
import pandas as pd
from fastai.vision.all import *
import albumentations
def get_predict(models, data, method_voting='soft', count_classes=10):
if method_voting == 'soft':
for_test = np.zeros(( data.shape[0], count_classes))
for i in range(len(models)) :
for_test += models[i].predict(data)
 return np.argmax(for_test, axis=1)
 elif method_voting == 'hard':
for_test = np.zeros(( data.shape[0], len(models)))
 for i in range(len(models)) :
for_test[:, i] += models[i].predict_classes(data)
 return np.int32(mode(for_test, axis=1)[0].T.flatten() )<save_to_csv>
set_seed(999,reproducible=True )
submit = pd.DataFrame(get_predict(list_models, X_test), columns=['Label'], index=pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')['ImageId'])
 submit2 = pd.DataFrame(get_predict(list_models, X_test, method_voting='hard'), columns=['Label'],
index=pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')['ImageId'])
 submit.index.name = 'ImageId'
submit.to_csv('submittion.csv')
 submit2.index.name = 'ImageId'
submit2.to_csv('submittion2.csv' )<merge>
train_df = pd.read_csv(dataset_path/'train.csv' )
comparison = submit.join(submit2, lsuffix='_1', rsuffix='_2')
 comparison.loc[~(comparison['Label_1'] == comparison['Label_2'])]<set_options>
train_df['path'] = train_df['image_id'].map(lambda x:dataset_path/'train_images'/x)
 train_df = train_df.drop(columns=['image_id'])
 train_df = train_df.sample(frac=1 ).reset_index(drop=True)
 train_df.head(10 )
%matplotlib inline
plt.style.use('ggplot')
 %matplotlib notebook
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
<load_from_csv>
im = Image.open(train_df['path'][1])
 width, height = im.size
print(width,height )
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 sam_sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 print("train dataset:", train.shape,"
","test dataset: ",test.shape,"
","sample_submission dataset:", sam_sub.shape )<sort_values>
class AlbumentationsTransform(RandTransform):
"A transform handler for multiple `Albumentation` transforms"
split_idx,order=None,2
def __init__(self, train_aug, valid_aug): store_attr()
def before_call(self, b, split_idx):
self.idx = split_idx
def encodes(self, img: PILImage):
if self.idx == 0:
aug_img = self.train_aug(image=np.array(img)) ['image']
else:
aug_img = self.valid_aug(image=np.array(img)) ['image']
return PILImage.create(aug_img )
train.sort_values(by='revenue', ascending=False ).head(20)[['title','revenue','release_date']]<feature_engineering>
def get_train_aug(sz): return albumentations.Compose([
albumentations.RandomResizedCrop(sz,sz),
albumentations.Transpose(p=0.5),
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.ShiftScaleRotate(p=0.5),
albumentations.HueSaturationValue(
 hue_shift_limit=0.2,
sat_shift_limit=0.2,
val_shift_limit=0.2,
p=0.5 
),
albumentations.RandomBrightnessContrast(
 brightness_limit=(-0.1,0.1),
contrast_limit=(-0.1, 0.1),
p=0.5 
),
albumentations.CoarseDropout(p=0.5),
albumentations.Cutout(p=0.5)
 ])
 def get_valid_aug(sz): return albumentations.Compose([
albumentations.CenterCrop(sz,sz, p=1.) ,
albumentations.Resize(sz,sz)
 ], p=1.)
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1542,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'revenue'] = 6800000
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3197,'budget'] = 8000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000<feature_engineering>
def get_dls(sz,bs):
item_tfms = AlbumentationsTransform(get_train_aug(sz), get_valid_aug(sz))
batch_tfms = [Normalize.from_stats(*imagenet_stats)]
dls = ImageDataLoaders.from_df(train_df,
valid_pct=0.2,
seed=999,
label_col=0,
fn_col=1,
bs=bs,
item_tfms=item_tfms,
batch_tfms=batch_tfms)
 return dls
def date_features(df):
df['release_date'] = pd.to_datetime(df['release_date'])
 df['release_year'] = df['release_date'].dt.year
df['release_month'] = df['release_date'].dt.month
df['release_day'] = df['release_date'].dt.day
df['release_quarter'] = df['release_date'].dt.quarter
df.drop(columns=['release_date'], inplace=True)
 return df
train=date_features(train)
 test=date_features(test)
 train['release_year'].head(10 )<filter>
dls = get_dls(456,16 )
train['release_year'].iloc[np.where(train['release_year']> 2019)][:10]<feature_engineering>
dls.show_batch()
train['release_year']=np.where(train['release_year']> 2019, train['release_year']-100, train['release_year'])
 test['release_year']=np.where(test['release_year']> 2019, test['release_year']-100, test['release_year'] )<categorify>
def create_timm_body(arch:str, pretrained=True, cut=None, n_in=3):
"Creates a body from any model in the `timm` library."
model = create_model(arch, pretrained=pretrained, num_classes=0, global_pool='')
 _update_first_layer(model, n_in, pretrained)
 if cut is None:
ll = list(enumerate(model.children()))
cut = next(i for i,o in reversed(ll)if has_pool_type(o))
if isinstance(cut, int): return nn.Sequential(*list(model.children())[:cut])
 elif callable(cut): return cut(model)
 else: raise NamedError("cut must be either integer or function")
 def create_timm_model(arch:str, n_out, cut=None, pretrained=True, n_in=3, init=nn.init.kaiming_normal_, custom_head=None,
concat_pool=True, **kwargs):
"Create custom architecture using `arch`, `n_in` and `n_out` from the `timm` library"
body = create_timm_body(arch, pretrained, None, n_in)
 if custom_head is None:
nf = num_features_model(nn.Sequential(*body.children())) *(2 if concat_pool else 1)
 head = create_head(nf, n_out, concat_pool=concat_pool, **kwargs)
 else: head = custom_head
model = nn.Sequential(body, head)
 if init is not None: apply_init(model[1], init)
 return model
fillna_column = {'release_year':'mode','release_month':'mode',
'release_day':'mode'}
for k,v in fillna_column.items() :
if v == 'mode':
fill = train[k].mode() [0]
else:
fill = v
print(k, ': ', fill)
 train[k].fillna(value = fill, inplace = True)
 test[k].fillna(value = fill, inplace = True )<data_type_conversions>
def timm_learner(dls, arch:str, loss_func=None, pretrained=True, cut=None, splitter=None,
y_range=None, config=None, n_out=None, normalize=True, **kwargs):
"Build a convnet style learner from `dls` and `arch` using the `timm` library"
if config is None: config = {}
if n_out is None: n_out = get_c(dls)
 assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
 model = create_timm_model(arch, n_out, default_split, pretrained, y_range=y_range, **config)
 learn = Learner(dls, model, loss_func=loss_func, splitter=default_split, **kwargs)
 if pretrained: learn.freeze()
return learn
def year_month_together(df):
year = df["release_year"].astype(int ).copy().astype(str)
 month=df['release_month'].astype(int ).copy().astype(str)
 day=df['release_day'].astype(int ).copy().astype(str)
 df["release_date"]= month.str.cat(day.str.cat(year,sep="/"), sep ="/")
 df['release_date']=pd.to_datetime(df['release_date'],format="%m/%d/%Y")
 df['release_dow'] = df['release_date'].dt.dayofweek
return df
train=year_month_together(train)
 test=year_month_together(test)
 train['release_date'].head(10 )<categorify>
learn = timm_learner(dls,
'tf_efficientnet_b5_ns',
opt_func=ranger,
loss_func=LabelSmoothingCrossEntropy() ,
cbs=[GradientAccumulation(n_acc=32)],
metrics = [accuracy] ).to_native_fp16()
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )<count_values>
learn.lr_find()
train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()<feature_engineering>
learn.freeze()
learn.fit_flat_cos(1,1e-1, wd=0.1, cbs=[MixUp() ] )
train['collection_name'] = train['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 train['has_collection'] = train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 test['collection_name'] = test['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 test['has_collection'] = test['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 train = train.drop(['belongs_to_collection'], axis=1)
 test = test.drop(['belongs_to_collection'], axis=1 )<count_values>
learn.save('stage-1' )
train['collection_name'].value_counts() [1:10]<feature_engineering>
learn = learn.load('stage-1' )
train['num_genres'] = train['genres'].apply(lambda x: len(x)if x != {} else 0)
 train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_genres = [m[0] for m in Counter([i for j in list_of_genres for i in j] ).most_common(15)]
for g in top_genres:
train['genre_' + g] = train['all_genres'].apply(lambda x: 1 if g in x else 0)
 test['num_genres'] = test['genres'].apply(lambda x: len(x)if x != {} else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_genres:
test['genre_' + g] = test['all_genres'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['genres'], axis=1)
 test = test.drop(['genres'], axis=1 )<string_transform>
learn.unfreeze()
learn.lr_find()
drama=train.loc[train['genre_Drama']==1,]
comedy=train.loc[train['genre_Comedy']==1,]
action=train.loc[train['genre_Action']==1,]
thriller=train.loc[train['genre_Thriller']==1,]
text_drama = " ".join(review for review in drama.title)
 text_comedy = " ".join(review for review in comedy.title)
 text_action = " ".join(review for review in action.title)
 text_thriller = " ".join(review for review in thriller.title)
 wordcloud1 = WordCloud(stopwords=stopwords, background_color="white",colormap="Reds" ).generate(text_drama)
 wordcloud2 = WordCloud(stopwords=stopwords, background_color="white",colormap="Blues" ).generate(text_comedy)
 wordcloud3 = WordCloud(stopwords=stopwords, background_color="white",colormap="Greens" ).generate(text_action)
 wordcloud4 = WordCloud(stopwords=stopwords, background_color="white",colormap="Greys" ).generate(text_thriller)
 fig = plt.figure(figsize=(20, 10))
plt.subplot(221)
 plt.imshow(wordcloud1, interpolation='bilinear')
 plt.title('drama movie title')
 plt.axis("off")
 plt.subplot(222)
 plt.imshow(wordcloud2, interpolation='bilinear')
 plt.title('comedy movie title')
 plt.axis("off")
 plt.show()
fig = plt.figure(figsize=(20, 10))
plt.subplot(223)
 plt.imshow(wordcloud3, interpolation='bilinear')
 plt.title('action movie title')
 plt.axis("off")
 plt.subplot(224)
 plt.imshow(wordcloud4, interpolation='bilinear')
 plt.title('thriller movie title')
 plt.axis("off")
 plt.show() 
<groupby>
learn.unfreeze()
learn.fit_flat_cos(10, 1e-3,cbs=[MixUp() ,SaveModelCallback() ] )
drama_revenue=drama.groupby(['release_year'] ).mean() ['revenue']
comedy_revenue=comedy.groupby(['release_year'] ).mean() ['revenue']
action_revenue=action_revenue=action.groupby(['release_year'] ).mean() ['revenue']
thriller_revenue=thriller.groupby(['release_year'] ).mean() ['revenue']
revenue_concat = pd.concat([drama_revenue,comedy_revenue,action_revenue,thriller_revenue], axis=1)
 revenue_concat.columns=['drama','comedy','action','thriller']
revenue_concat.index=train.groupby(['release_year'] ).mean().index<feature_engineering>
learn = learn.to_native_fp32()
train['num_companies'] = train['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 train['all_production_companies'] = train['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_companies = [m[0] for m in Counter([i for j in list_of_companies for i in j] ).most_common(30)]
for g in top_companies:
train['production_company_' + g] = train['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 test['num_companies'] = test['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 test['all_production_companies'] = test['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_companies:
test['production_company_' + g] = test['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_companies', 'all_production_companies'], axis=1)
 test = test.drop(['production_companies', 'all_production_companies'], axis=1 )<filter>
learn.save('stage-2' )
Warner_Bros=train.loc[train['production_company_Warner Bros.']==1,]
Universal_Pictures=train.loc[train['production_company_Universal Pictures']==1,]
Twentieth_Century_Fox_Film=train.loc[train['production_company_Twentieth Century Fox Film Corporation']==1,]
Columbia_Pictures=train.loc[train['production_company_Columbia Pictures']==1,]
MGM=train.loc[train['production_company_Metro-Goldwyn-Mayer(MGM)']==1,]
New_Line_Cinema=train.loc[train['production_company_New Line Cinema']==1,]
Touchstone_Pictures=train.loc[train['production_company_Touchstone Pictures']==1,]
Walt_Disney=train.loc[train['production_company_Walt Disney Pictures']==1,]
Warner_Bros_revenue=Warner_Bros.groupby(['release_year'] ).mean() ['revenue']
Universal_Pictures_revenue=Universal_Pictures.groupby(['release_year'] ).mean() ['revenue']
Twentieth_Century_Fox_Film_revenue=Twentieth_Century_Fox_Film.groupby(['release_year'] ).mean() ['revenue']
Columbia_Pictures_revenue=Columbia_Pictures.groupby(['release_year'] ).mean() ['revenue']
MGM_revenue=MGM.groupby(['release_year'] ).mean() ['revenue']
New_Line_Cinema_revenue=New_Line_Cinema.groupby(['release_year'] ).mean() ['revenue']
Touchstone_Pictures_revenue=Touchstone_Pictures.groupby(['release_year'] ).mean() ['revenue']
Walt_Disney_revenue=Walt_Disney.groupby(['release_year'] ).mean() ['revenue']
prod_revenue_concat = pd.concat([Warner_Bros_revenue,Universal_Pictures_revenue,Twentieth_Century_Fox_Film_revenue,Columbia_Pictures_revenue,
MGM_revenue,New_Line_Cinema_revenue,Touchstone_Pictures_revenue,Walt_Disney_revenue], axis=1)
 prod_revenue_concat.columns=['Warner_Bros','Universal_Pictures','Twentieth_Century_Fox_Film','Columbia_Pictures','MGM','New_Line_Cinema','Touchstone_Pictures','Walt_Disney']
fig = plt.figure(figsize=(13, 7))
prod_revenue_concat.agg("mean",axis='rows' ).sort_values(ascending=True ).plot(kind='barh',x='Production Companies',y='Revenue',title='Mean Revenue(100 million dollars)of Top 8 Most Common Production Companies')
 plt.xlabel('Revenue(100 million dollars)' )<feature_engineering>
sample_df = pd.read_csv(dataset_path/'sample_submission.csv')
 sample_df.head()
train['num_countries'] = train['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 train['all_countries'] = train['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_countries = [m[0] for m in Counter([i for j in list_of_countries for i in j] ).most_common(25)]
for g in top_countries:
train['production_country_' + g] = train['all_countries'].apply(lambda x: 1 if g in x else 0)
 test['num_countries'] = test['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 test['all_countries'] = test['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_countries:
test['production_country_' + g] = test['all_countries'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_countries', 'all_countries'], axis=1)
 test = test.drop(['production_countries', 'all_countries'], axis=1 )<feature_engineering>
_sample_df = sample_df.copy()
_sample_df['path'] = _sample_df['image_id'].map(lambda x:dataset_path/'test_images'/x)
 _sample_df = _sample_df.drop(columns=['image_id'])
 test_dl = dls.test_dl(_sample_df )
train['num_languages'] = train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 train['all_languages'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['iso_639_1'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in list_of_languages for i in j] ).most_common(30)]
for g in top_languages:
train['language_' + g] = train['all_languages'].apply(lambda x: 1 if g in x else 0)
 test['num_languages'] = test['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 test['all_languages'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['iso_639_1'] for i in x])) if x != {} else '')
 for g in top_languages:
test['language_' + g] = test['all_languages'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['spoken_languages', 'all_languages'], axis=1)
 test = test.drop(['spoken_languages', 'all_languages'], axis=1 )<string_transform>
test_dl.show_batch()
text_drama = " ".join(review for review in drama['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else ''))
text_comedy = " ".join(review for review in comedy['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else ''))
text_action = " ".join(review for review in action['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else ''))
text_thriller = " ".join(review for review in thriller['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else ''))
wordcloud1 = WordCloud(stopwords=stopwords, background_color="white",colormap="Reds" ).generate(text_drama)
 wordcloud2 = WordCloud(stopwords=stopwords, background_color="white",colormap="Blues" ).generate(text_comedy)
 wordcloud3 = WordCloud(stopwords=stopwords, background_color="white",colormap="Greens" ).generate(text_action)
 wordcloud4 = WordCloud(stopwords=stopwords, background_color="white",colormap="Greys" ).generate(text_thriller)
 fig = plt.figure(figsize=(25, 20))
plt.subplot(221)
 plt.imshow(wordcloud1, interpolation='bilinear')
 plt.title('Drama Keywords')
 plt.axis("off")
 plt.subplot(222)
 plt.imshow(wordcloud2, interpolation='bilinear')
 plt.title('Comedy Keywords')
 plt.axis("off")
 plt.show()
fig = plt.figure(figsize=(25, 20))
plt.subplot(223)
 plt.imshow(wordcloud3, interpolation='bilinear')
 plt.title('Action Keywords')
 plt.axis("off")
 plt.subplot(224)
 plt.imshow(wordcloud4, interpolation='bilinear')
 plt.title('Thriller Keywords')
 plt.axis("off")
 plt.show()<feature_engineering>
preds, _ = learn.tta(dl=test_dl, n=15, beta=0 )
train['num_Keywords'] = train['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_' + g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_keywords:
test['keyword_' + g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['Keywords', 'all_Keywords'], axis=1)
 test = test.drop(['Keywords', 'all_Keywords'], axis=1 )<feature_engineering>
sample_df['label'] = preds.argmax(dim=-1 ).numpy()
<filter><EOS> 
sample_df.to_csv('submission.csv',index=False )