Kaggle competition: SIIM-ACR Pneumothorax Segmentation
<SOS> metric: SIIMDice Kaggle data source: siim-acr-pneumothorax-segmentation<prepare_x_and_y>
import torchvision
from fastai.vision import *
from fastai.metrics import error_rate
from fastai import *
import cv2 as cv
import numpy as np
import pandas as pd
import fastai
X = tourney_result.drop('result', axis=1)
 y = tourney_result.result<init_hyperparams>
os.listdir('.. /input')
 !tar -xf /kaggle/input/make-pneumothorax-classifer-data/classifier_data.tar.gz -C.
 !tar -xf /kaggle/input/make-pneumothorax-classifer-data/test_data.tar.gz -C .
params = {'num_leaves': 470,
'min_child_weight': 0.034,
'feature_fraction': 0.379,
'bagging_fraction': 0.418,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'logloss',
"verbosity": -1,
'reg_alpha': 0.3899,
'reg_lambda': 0.648,
'random_state': 47,
}<prepare_x_and_y>
tfms = get_transforms(do_flip=True, flip_vert=False, max_lighting=0.1, max_zoom=1.05,
max_warp=0.,
xtra_tfms=[rand_crop() , rand_zoom(1, 1.5),
symmetric_warp(magnitude=(-0.2, 0.2)) ])
 path = '/kaggle/working/classifier_data'
path_test = '/kaggle/working/test'
data =(ImageList.from_folder(path)
.split_by_rand_pct(seed=seed)
.label_from_folder() 
.transform(tfms, size=224)
.databunch().normalize(imagenet_stats))
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(test_df.shape[0])
 y_oof = np.zeros(X.shape[0])
 feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
print('Fold:',fold_n+1)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
y_preds += clf.predict(test_df)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()<load_from_csv>
data.show_batch(rows=3, figsize=(12,9))
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = y_preds
submission_df<save_to_csv>
len(data.classes),data.c
submission_df.to_csv('submission.csv', index=False )<save_to_csv>
!pip install pretrainedmodels
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(10, 6))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(7), x='average', y='feature');
plt.title('7 TOP feature importance over {} folds average'.format(folds.n_splits)) ;<set_options>
class FocalLoss(nn.Module):
def __init__(self, alpha=1., gamma=2.) :
super().__init__()
self.alpha = alpha
self.gamma = gamma
def forward(self, inputs, targets, **kwargs):
CE_loss = nn.CrossEntropyLoss(reduction='none' )(inputs, targets)
 pt = torch.exp(-CE_loss)
 F_loss = self.alpha *(( 1-pt)**self.gamma)* CE_loss
return F_loss.mean()
warnings.filterwarnings('ignore' )<load_from_csv>
def resnext50_32x4d(pretrained=False):
pretrained = 'imagenet' if pretrained else None
model = pretrainedmodels.se_resnext50_32x4d(pretrained=pretrained)
 return nn.Sequential(*list(model.children()))
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 dataset = pd.concat([train, test], ignore_index = True)
 PassengerId = test['PassengerId']<count_missing_values>
learn = cnn_learner(data, resnext50_32x4d, pretrained=True, cut=-2,
split_on=lambda m:(m[0][3], m[1]),
metrics=[accuracy])
 learn.loss_fn = FocalLoss()
dataset = dataset.fillna(np.nan)
 dataset.isnull().sum()<feature_engineering>
learn.fit_one_cycle(32, max_lr=slice(2e-2), wd=1e-5 )
dataset['Title'] = dataset['Name'].apply(lambda x:x.split(',')[1].split('.')[0].strip())
 Title_Dict = {}
Title_Dict.update(dict.fromkeys(['Capt', 'Col', 'Major', 'Dr', 'Rev'], 'Officer'))
Title_Dict.update(dict.fromkeys(['Don', 'Sir', 'the Countess', 'Dona', 'Lady'], 'Royalty'))
Title_Dict.update(dict.fromkeys(['Mme', 'Ms', 'Mrs'], 'Mrs'))
Title_Dict.update(dict.fromkeys(['Mlle', 'Miss'], 'Miss'))
Title_Dict.update(dict.fromkeys(['Mr'], 'Mr'))
Title_Dict.update(dict.fromkeys(['Master','Jonkheer'], 'Master'))
dataset['Title'] = dataset['Title'].map(Title_Dict)
 sns.barplot(x="Title", y="Survived", data=dataset, palette='Set3' )<predict_on_test>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
age = dataset[['Age','Pclass','Sex','Title']]
age = pd.get_dummies(age)
 known_age = age[age.Age.notnull() ].values
null_age = age[age.Age.isnull() ].values
x = known_age[:, 1:]
y = known_age[:, 0]
rf = RandomForestRegressor(n_jobs=-1)
 rf.fit(x, y)
 predictedAge = rf.predict(null_age[:, 1:])
 dataset.loc[(dataset.Age.isnull()),'Age'] = predictedAge<filter>
learn.load('resnext50_32x4d_1');
learn.unfreeze() ;
learn = learn.clip_grad() ;
dataset[dataset['Embarked'].isnull() ]<feature_engineering>
lr = [3e-3/100, 3e-3/20, 3e-3/10]
learn.fit_one_cycle(36, lr, wd=1e-7)
C = dataset[(dataset['Embarked']=='C')&(dataset['Pclass'] == 1)]['Fare'].median()
print(C)
 S = dataset[(dataset['Embarked']=='S')&(dataset['Pclass'] == 1)]['Fare'].median()
print(S)
 Q = dataset[(dataset['Embarked']=='S')&(dataset['Pclass'] == 1)]['Fare'].median()
print(Q)
 dataset['Embarked'] = dataset['Embarked'].fillna('C' )<filter>
learn.save('resnext50_32x4d_2');
dataset[dataset['Fare'].isnull() ]<feature_engineering>
SZ = 224
cutout_frac = 0.20
p_cutout = 0.75
cutout_sz = round(SZ*cutout_frac)
 cutout_tfm = cutout(n_holes=(1,1), length=(cutout_sz, cutout_sz), p=p_cutout)
 tfms = get_transforms(do_flip=True, max_rotate=15, flip_vert=False, max_lighting=0.1,
max_zoom=1.05, max_warp=0.,
xtra_tfms=[rand_crop() , rand_zoom(1, 1.5),
symmetric_warp(magnitude=(-0.2, 0.2)) , cutout_tfm] )
fare=dataset[(dataset['Embarked'] == "S")&(dataset['Pclass'] == 3)].Fare.median()
dataset['Fare']=dataset['Fare'].fillna(fare )<feature_engineering>
data =(ImageList.from_folder(path)
.split_by_rand_pct(seed=seed)
.label_from_folder() 
.transform(tfms, size=224)
.databunch().normalize(imagenet_stats))
learn.data = data
learn.bs = 32
data.train_ds[0][0].shape
dataset['Surname']=dataset['Name'].apply(lambda x:x.split(',')[0].strip())
 Surname_Count = dict(dataset['Surname'].value_counts())
 dataset['FamilyGroup'] = dataset['Surname'].apply(lambda x:Surname_Count[x])
 Female_Child_Group=dataset.loc[(dataset['FamilyGroup']>=2)&(( dataset['Age']<=12)|(dataset['Sex']=='female')) ]
Male_Adult_Group=dataset.loc[(dataset['FamilyGroup']>=2)&(dataset['Age']>12)&(dataset['Sex']=='male')]<create_dataframe>
learn.loss_func = FocalLoss()
Female_Child=pd.DataFrame(Female_Child_Group.groupby('Surname')['Survived'].mean().value_counts())
 Female_Child.columns=['GroupCount']
Female_Child<create_dataframe>
learn.fit_one_cycle(24, slice(3e-3), wd=5e-6 )
Male_Adult=pd.DataFrame(Male_Adult_Group.groupby('Surname')['Survived'].mean().value_counts())
 Male_Adult.columns=['GroupCount']
Male_Adult<groupby>
learn.save('resnext50_32x4d_3');
learn.load('resnext50_32x4d_3');
Female_Child_Group=Female_Child_Group.groupby('Surname')['Survived'].mean()
Dead_List=set(Female_Child_Group[Female_Child_Group.apply(lambda x:x==0)].index)
 print(Dead_List)
 Male_Adult_List=Male_Adult_Group.groupby('Surname')['Survived'].mean()
Survived_List=set(Male_Adult_List[Male_Adult_List.apply(lambda x:x==1)].index)
 print(Survived_List )<feature_engineering>
learn.unfreeze() ;
learn = learn.clip_grad() ;
train=dataset.loc[dataset['Survived'].notnull() ]
test=dataset.loc[dataset['Survived'].isnull() ]
test.loc[(test['Surname'].apply(lambda x:x in Dead_List)) ,'Sex'] = 'male'
test.loc[(test['Surname'].apply(lambda x:x in Dead_List)) ,'Age'] = 60
test.loc[(test['Surname'].apply(lambda x:x in Dead_List)) ,'Title'] = 'Mr'
test.loc[(test['Surname'].apply(lambda x:x in Survived_List)) ,'Sex'] = 'female'
test.loc[(test['Surname'].apply(lambda x:x in Survived_List)) ,'Age'] = 5
test.loc[(test['Surname'].apply(lambda x:x in Survived_List)) ,'Title'] = 'Miss'<prepare_x_and_y>
lr = [1e-3/200, 1e-3/20, 1e-3/10]
learn.fit_one_cycle(32, lr )
dataset = pd.concat([train, test])
 dataset=dataset[['Survived','Pclass','Sex','Age','Fare','Embarked','Title','FamilyLabel','Deck','TicketGroup']]
dataset=pd.get_dummies(dataset)
 trainset=dataset[dataset['Survived'].notnull() ]
testset=dataset[dataset['Survived'].isnull() ].drop('Survived',axis=1)
 X = trainset.values[:,1:]
Y = trainset.values[:,0]<train_on_grid>
learn.save('resnext50_32x4d_4');
learn.load('resnext50_32x4d_4');
pipe=Pipeline([('select',SelectKBest(k=20)) , 
('classify', RandomForestClassifier(random_state = 10, max_features = 'sqrt')) ])
 param_test = {'classify__n_estimators':list(range(20,50,2)) ,
'classify__max_depth':list(range(3,60,3)) }
gsearch = GridSearchCV(estimator = pipe, param_grid = param_test, scoring='accuracy', cv=10)
 gsearch.fit(X,Y)
 print(gsearch.best_params_, gsearch.best_score_ )<train_model>
learn.export('/kaggle/working/fastai_resnet.pkl');
select = SelectKBest(k = 20)
 clf = RandomForestClassifier(random_state = 10, warm_start = True,
n_estimators = 30,
max_depth = 6,
max_features = 'sqrt')
 pipeline = make_pipeline(select, clf)
 pipeline.fit(X, Y )<compute_test_metric>
learn = load_learner('/kaggle/working/','fastai_resnet.pkl', ImageList.from_folder(path_test))
preds,_ = learn.get_preds(ds_type=DatasetType.Test)
 cls_pred = F.softmax(preds,1 ).argmax(1 ).cpu().numpy()
cv_score = model_selection.cross_val_score(pipeline, X, Y, cv= 10)
 print("CV Score : Mean - %.7g | Std - %.7g " %(np.mean(cv_score), np.std(cv_score)) )<save_to_csv>
paths = list(map(str,list(learn.data.test_ds.x.items)))
 all_test_paths = [p.split('/')[-1][:-4] for p in paths]
df_preds = pd.DataFrame()
df_preds['test_paths'] = all_test_paths
df_preds['class_pred'] = cls_pred
df_preds.set_index('test_paths',inplace=True )
predictions = pipeline.predict(testset)
 submission = pd.DataFrame({"PassengerId": PassengerId, "Survived": predictions.astype(np.int32)})
 submission.to_csv("submission.csv", index=False )<set_options>
no_dis_idx = df_preds[df_preds.class_pred==1].index
print(len(no_dis_idx))
%matplotlib inline 
<load_from_csv>
sub = pd.read_csv('/kaggle/input/unet-plus-plus-with-efficientnet-encoder/submission.csv'
,index_col='ImageId')
 sub.head()
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<count_missing_values>
sub.loc[no_dis_idx] = ' -1'
<categorify><EOS> 
sub.to_csv('sub_classifier_correction.csv' )