Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<feature_engineering>
from fastai.vision.all import *
list_of_cast_genders = list(train['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 list_of_cast_characters = list(train['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 train['genders_0'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['cast_character_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['cast_character_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )<feature_engineering>
path = Path('/kaggle/input/cassava-leaf-disease-classification/')
 path.ls()
list_of_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 train['num_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0)
 train['all_crew'] = train['crew'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(30)]
for g in top_crew_names:
train['crew_name_' + g] = train['all_crew'].apply(lambda x: 1 if g in x else 0)
 test['num_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 test['all_crew'] = test['crew'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_crew_names:
test['crew_name_' + g] = test['all_crew'].apply(lambda x: 1 if g in x else 0 )<filter>
train_df = pd.read_csv(path/'train.csv')
 train_df.head()
with open(path/'label_num_to_disease_map.json')as f:
label_dict = json.load(f)
 label_dict = {int(k):v for k,v in label_dict.items() }
df = train_df.set_index('image_id')
 labels = df.to_dict() ['label']
def get_label(labels, x):
x = Path(x)
 return labels[x.name]
def get_data(labels, bs=64, presize=500, resize=384):
tfms = [Rotate(90), Warp(magnitude=0.4, p=1.) ,Zoom(min_zoom=0.9, max_zoom=1.3), Brightness(max_lighting=0.5), Flip() , Contrast() , Resize(resize)]
comp = setup_aug_tfms(tfms)
 return DataBlock(blocks=(ImageBlock, CategoryBlock),
get_items=lambda p: get_image_files(p),
get_y=partial(get_label, labels),
splitter=RandomSplitter() ,
item_tfms=[Resize(presize)],
batch_tfms=[*comp, Normalize.from_stats(*imagenet_stats)] ).dataloaders(path/'train_images',bs=bs,num_workers=8 )
crew_name_Avy_Kaufman=train.loc[train['crew_name_Avy Kaufman']==1,]
crew_name_Robert_Rodriguez=train.loc[train['crew_name_Robert Rodriguez']==1,]
crew_name_Deborah_Aquila=train.loc[train['crew_name_Deborah Aquila']==1,]
crew_name_James_Newton_Howard=train.loc[train['crew_name_James Newton Howard']==1,]
crew_name_Mary_Vernieu=train.loc[train['crew_name_Mary Vernieu']==1,]
crew_name_Steven_Spielberg=train.loc[train['crew_name_Steven Spielberg']==1,]
crew_name_Luc_Besson=train.loc[train['crew_name_Luc Besson']==1,]
crew_name_Jerry_Goldsmith=train.loc[train['crew_name_Jerry Goldsmith']==1,]
crew_name_Francine_Maisler=train.loc[train['crew_name_Francine Maisler']==1,]
crew_name_Tricia_Wood=train.loc[train['crew_name_Tricia Wood']==1,]
crew_name_Avy_Kaufman_revenue=crew_name_Avy_Kaufman.mean() ['revenue']
crew_name_Robert_Rodriguez_revenue=crew_name_Robert_Rodriguez.mean() ['revenue']
crew_name_Deborah_Aquila_revenue=crew_name_Deborah_Aquila.mean() ['revenue']
crew_name_James_Newton_Howard_revenue=crew_name_James_Newton_Howard.mean() ['revenue']
crew_name_Mary_Vernieu_revenue=crew_name_Mary_Vernieu.mean() ['revenue']
crew_name_Steven_Spielberg_revenue=crew_name_Steven_Spielberg.mean() ['revenue']
crew_name_Luc_Besson_revenue=crew_name_Luc_Besson.mean() ['revenue']
crew_name_Jerry_Goldsmith_revenue=crew_name_Jerry_Goldsmith.mean() ['revenue']
crew_name_Francine_Maisler_revenue=crew_name_Francine_Maisler.mean() ['revenue']
crew_name_Tricia_Wood_revenue=crew_name_Tricia_Wood.mean() ['revenue']
crew_revenue_concat = pd.Series([crew_name_Avy_Kaufman_revenue,crew_name_Robert_Rodriguez_revenue,crew_name_Deborah_Aquila_revenue,crew_name_James_Newton_Howard_revenue,
crew_name_Mary_Vernieu_revenue,crew_name_Steven_Spielberg_revenue,crew_name_Luc_Besson_revenue,crew_name_Jerry_Goldsmith_revenue,
crew_name_Francine_Maisler_revenue,crew_name_Tricia_Wood_revenue])
 crew_revenue_concat.index=['Avy Kaufman','Robert Rodriguez','Deborah Aquila','James Newton Howard','Mary Vernieu','Steven Spielberg','Luc Besson','Jerry Goldsmith',
'Francine Maisler','Tricia Wood']
fig = plt.figure(figsize=(13, 7))
crew_revenue_concat.sort_values(ascending=True ).plot(kind='barh',title='Mean Revenue(100 million dollars)by Top 10 Most Common Crew')
 plt.xlabel('Revenue(100 million dollars)' )<feature_engineering>
dls = get_data(labels,bs=128, presize=384)
 learn = cnn_learner(dls, models.resnet50, metrics=[accuracy], pretrained=False)
 test_files = get_image_files(path/'test_images')
 predictions = []
for fold in range(3):
learn.load(f'/kaggle/input/resnet50/models/resnet50-full-fold_{fold}')
 test_dl = dls.test_dl(test_files)
 preds = learn.tta(dl=test_dl, n=6)
 predictions.append(preds[0] )
list_of_crew_jobs = list(train['crew'].apply(lambda x: [i['job'] for i in x] if x != {} else [] ).values)
 list_of_crew_genders = list(train['crew'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 list_of_crew_departments = list(train['crew'].apply(lambda x: [i['department'] for i in x] if x != {} else [] ).values)
 train['genders_0'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['crew_character_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 top_crew_jobs = [m[0] for m in Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15)]
for j in top_crew_jobs:
train['jobs_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
top_crew_departments = [m[0] for m in Counter([i for j in list_of_crew_departments for i in j] ).most_common(15)]
for j in top_crew_departments:
train['departments_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
test['genders_0'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['crew_character_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 for j in top_crew_jobs:
test['jobs_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
for j in top_crew_departments:
test['departments_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
train = train.drop(['crew'], axis=1)
 test = test.drop(['crew'], axis=1 )<feature_engineering>
preds = torch.argmax(torch.mean(torch.stack(predictions), dim=0), dim=1)
 test_fn = map(lambda x: x.name, test_files )
<feature_engineering><EOS> 
submission = pd.DataFrame({'image_id': test_fn, 'label': preds})
 submission.to_csv('submission.csv', index=False )