Kaggle competition: Histopathologic Cancer Detection
new_data=new_data.drop(['original_language'],axis=1 )<count_missing_values>
print('fast.ai version:{}'.format(fastai.__version__))
sum(new_data['geres_name'].isna() )<data_type_conversions>
show_doc(ImageDataBunch.from_csv )
new_data['geres_name']=new_data['geres_name'].fillna('Unknow' )<feature_engineering>
param_transforms = {
'flip_vert':True,
'max_rotate':90,
'max_zoom':1,
'max_warp':None,
'max_lighting':None,
}
params = {'path': Path('.. /input/histopathologic-cancer-detection'),
'csv_labels':'train_labels.csv',
'ds_tfms':get_transforms(**param_transforms),
'suffix':'.tif',
'folder':'train',
'test':'test',
'size':32,
'bs':128,
'num_workers':0,
}
data = ImageDataBunch.from_csv(**params ).normalize(imagenet_stats)
 data
new_data['geres_name_count']=new_data['geres_name'].apply(lambda x:len(x))<groupby>
data.show_batch(rows=3 )
ge_name_count_rev=new_data.loc[train.index].groupby(['geres_name_count'] ).revenue.median().sort_values()<drop_column>
learn = create_cnn(data, models.resnet101, metrics=accuracy, path='.', model_dir='.' )
new_data=new_data.drop('homepage_end',axis=1 )<feature_engineering>
learn.load('.. /input/dataset/histopathologic_cancer_detection' )
new_data['original_and_new']=new_data['original_and_new'].apply(lambda x: 1 if x else 0 )<define_variables>
learn.unfreeze()
learn.fit_one_cycle(16,slice(1e-6,1e-4))
list_geres_name=['Comedy','Thriller','Action','Drama','Romance']<feature_engineering>
learn.save('histopathologic_cancer_detection' )
for i in list_geres_name:
new_data['geres_name'+'_'+i]=new_data['geres_name'].apply(lambda x: 1 if i in x else 0 )<drop_column>
try:
DatasetType
except:
new_data=new_data.drop('geres_name',axis=1 )<data_type_conversions>
try:
preds, _ = learn.TTA(ds_type=DatasetType.Test)
 print('Predict w/ test time augument done.')
 except:
preds, _ = learn.get_preds(DatasetType.Test )
new_data['production_countries']=new_data['production_countries'].fillna('QQ' )<feature_engineering>
ids = [file.stem for file in data.test_ds.x.items]
ids[:5]
new_data['production_countries'].loc[new_data['production_countries']=='Unknow']['production_countries']='QQ'<count_unique_values>
df=pd.read_csv('.. /input/histopathologic-cancer-detection/sample_submission.csv' ).set_index('id')
 df.head()
list_pro_coun=list(new_data['production_countries'])
 d = Counter([j for i in list_pro_coun for j in i] ).most_common(5)
 d<feature_engineering>
df.loc[ids,'label'] = preds[:,1].numpy()
df = df.reset_index()
df.head()
<feature_engineering><EOS> 
df.to_csv('submission.csv', index=False )