Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
warnings.filterwarnings('ignore')
 seed = 12345
np.random.seed(seed)
 random.seed(seed)
 pd.set_option('display.expand_frame_repr', False)
 pd.set_option('max_colwidth', -1)
 plt.style.use('ggplot' )
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 full = train.append(test)
 full.sort_values('PassengerId', inplace=True)
 full.reset_index(drop=True, inplace=True )
test.to_csv('submission.csv', index=False )<import_modules>
full[['Last','Full']] = full['Name'].str.split(r", ",expand=True)
 full[['Title','Rest']] = full['Full'].str.split(r'(?<=\.) \s',n=1, expand=True)
 full[['First', 'Parenthesis', 'blank']] = full['Rest'].str.split(r"\((.*)\)", expand=True)
 full['blank'].unique()
full = full.drop('blank', axis=1)
 full[['drop', 'Parenthesis2', 'blank']] = full['Rest'].str.split(r'\(" (.*)"\)', expand=True)
 full['blank'].unique()
full = full.drop(['blank','drop'], axis=1)
 full[['First', 'Quote', 'blank']] = full['First'].str.split(r"" (.*)"", expand=True)
 full['Quote'].replace('', None, inplace=True)
 full['blank'].unique()
full = full.drop('blank', axis=1)
 full['First'] = full['First'].str.strip()
full['Last'] = full['Last'].str.strip()
full['Title'] = full['Title'].str.strip()
full.drop(['Full', 'Rest'], axis=1, inplace=True)
 full['hasNickname'] = ~full['Quote'].isnull()
full['hasParenthesis'] = ~full['Parenthesis'].isnull()
print(full.hasNickname.value_counts())
 print(full.hasParenthesis.value_counts())
 full = full.drop(['Parenthesis','Parenthesis2', 'Quote'], axis='columns' )
import numpy as np
import pandas as pd
import os
from fastai.vision.all import * 
<load_from_csv>
cab_dum = full['Cabin'].str.get_dummies(' ')
 cab_occupancy =cab_dum.sum(axis=0)
 occ_multiple = cab_occupancy[cab_occupancy.values>1].index
cab_cSum = cab_dum.sum(axis=1)
 cab_multiple = cab_cSum[cab_cSum.values>1].index
full_cab = pd.concat([full, cab_dum.loc[:, occ_multiple]], axis=1)
 df_temp = pd.DataFrame([])
 for c in occ_multiple:
n = full[full_cab[c]>0].Cabin.nunique()
if n>1:
df_temp = df_temp.append(full[full_cab[c]>0])
 df_temp[['Cabin','Embarked', 'Fare', 'Pclass','Name', 'Ticket']].head(20 )
cassavaPath = '.. /input/cassava-leaf-disease-classification/'
cassavaOutputPath = './'
cassavaModelPath = '.. /input/cassavleafdiseaseclassificationpretrained/'
df = pd.read_csv(cassavaPath+'train.csv')
 df['label'] = df['label'].astype(str)
 df.head()<count_values>
full.Cabin = full.Cabin.str.replace('F ', 'F' )
df['label'].value_counts()<load_pretrained>
temp = full.groupby('Ticket' ).Cabin.nunique().sort_values(ascending=False)
 print(temp.value_counts() )
dls = dblock.dataloaders(df )<define_variables>
temp = full[full.groupby('Ticket' ).Cabin.transform(lambda x: x.fillna('' ).nunique())>1]
temp.sort_values(by=['Ticket','Cabin'], inplace=True)
 temp[['Cabin','Embarked', 'Fare', 'Pclass','Name', 'Ticket']].head(10 )
dls.show_batch(nrows=3, ncols=3 )<train_model>
full['Embarked'] = full.Embarked.fillna(full.Embarked.mode() [0])
 full['ppFare'] = full.groupby(['Pclass', 'Embarked'] ).ppFare.transform(lambda x: x.fillna(x.median()))
FareToFill = full.ppFare * full.ppTicket
full.loc[full.Fare.isna() , 'Fare'] = FareToFill[full.Fare.isna() ]
<train_model>
full['nFamily'] = full.groupby(['Last', 'Ticket_Type', 'Embarked'] ).Last.transform('count')
 to_impute = full[['Age', 'ppFare', 'Parch', 'Pclass', 'SibSp', 'hasNickname', 'hasParenthesis', 'ppTicket', 'nFamily']]
dummy = pd.get_dummies(data=full[['Title','Embarked', 'Sex','Ticket_Type']])
 to_impute = pd.concat([to_impute,dummy], axis=1)
 col = to_impute.columns
imp = IterativeImputer(estimator=ExtraTreesRegressor(n_estimators=100), max_iter=50, verbose=2)
 imp.fit(to_impute)
 imputed = pd.DataFrame(imp.transform(to_impute))
imputed.columns = col
imputed['PassengerId'] = full['PassengerId']
imputed['Age']=np.round(imputed['Age'], decimals=1)
 full.update(imputed[['PassengerId','Age']])
 full.info()
<load_pretrained>
full['Title'] = full['Title'].replace(['Don.', 'Rev.', 'Jonkheer.', 'Sir.'], 'Honor')
 full['Title'] = full['Title'].replace(['Dr.', 'Col.', 'Major.', 'Capt.'], 'Profession')
 full['Title'] = full['Title'].replace(['Ms.', 'Mlle.', 'the Countess.', 'Lady.'], 'Miss.')
 full['Title'] = full['Title'].replace(['Mme.', 'Dona.'], 'Mrs.')
 full.Title.value_counts()
full['Cabin_Level'] = full.groupby(['ppFare', 'Embarked', 'Pclass'] ).Cabin_Level.transform(lambda x: x.replace('T', x.mode() [0]))
full['Cabin_Level'].cat.remove_unused_categories(inplace=True)
 print(full.Cabin_Level.value_counts())
 full['Cabin_Level'] = full['Cabin_Level'].replace('G', 'F')
 full['Cabin_Level'].cat.remove_unused_categories(inplace=True)
learn = load_learner(cassavaModelPath+'CassavaDiseaseModelResnet34.pkl')
 learn.export(cassavaOutputPath+'CassavaDiseaseModelResnet34.pkl')
 learn = load_learner(cassavaOutputPath+'CassavaDiseaseModelResnet34.pkl' )<save_to_csv>
full['isMarried'] = full.Title.isin(['Mrs.', 'Mme.', 'Dona.'])
 full['FamilySize'] = full ['SibSp'] + full['Parch'] + 1
full['CabinGroup'] = full['Cabin_Level'].replace(['A','B'], 'AB')
 full['CabinGroup'] = full['CabinGroup'].replace(['C','F'], 'CF')
 full['CabinGroup'] = full['CabinGroup'].replace(['D','E'], 'DE')
 full.CabinGroup.value_counts()
full.loc[(full['FamilySize']==1)&(full['ppTicket']==1)&(full['nFamily']==1), 'GroupCat'] = 'single'
full.loc[(full['FamilySize']==2)|(full['ppTicket']==2)|(full['nFamily']==2), 'GroupCat'] = 'couple'
full.loc[(full['FamilySize'].isin([3,4])) |(full['ppTicket'].isin([3,4])) |(full['nFamily'].isin([3,4])) , 'GroupCat'] = 'small'
full.loc[(full['FamilySize']>4)|(full['ppTicket']>4)|(full['nFamily']>4), 'GroupCat'] = 'large'
full['Pclass_cat'] = full['Pclass'].astype('category' )
print('Computing predictions...')
 test_files = get_image_files(cassavaPath+'test_images')
 predictions_ResultArray = [None for tempX in range(len(test_files)) ]
predictions_InputArray = [None for tempX in range(len(test_files)) ]
for test_idx in range(0,len(test_files)) :
predictions = learn.predict(test_files[test_idx])
 head, tail = os.path.split(test_files[test_idx])
 predictions_ResultArray[test_idx] = int(predictions[0])
 predictions_InputArray[test_idx] = tail
np.savetxt(cassavaOutputPath+'submission.csv', np.rec.fromarrays([predictions_InputArray, predictions_ResultArray]), fmt=['%s', '%d'], delimiter=',', header='image_id,label', comments='')
 !head submission.csv<load_from_csv>
full2 = full[['PassengerId','Survived', 'Age','Pclass_cat', 'Embarked', 'Sex', 'Title', 'Cabin_Level', 'ppTicket', 'ppFare', 'Survival_Rate', 'isMarried', 'hasSurvivalInfo', 'FamilySize', 'Parch', 'SibSp', 'nFamily', 'GroupCat']]
dummyVars = full2.select_dtypes(include=['object','bool','category'] ).columns
full3 = pd.get_dummies(full2, columns = dummyVars, drop_first=True)
 train = full3[~full3['Survived'].isna() ]
test = full3[full3['Survived'].isna() ]
X_train = train.drop(["Survived","PassengerId"],axis=1)
 Y_train = train['Survived'].astype(int)
 X_test = test.drop(["Survived","PassengerId"],axis=1 )
df_sample_submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 df_exact_submission = pd.read_csv(cassavaOutputPath+'submission.csv')
 print(df_sample_submission.compare(df_exact_submission))
print(df_sample_submission.equals(df_exact_submission)) 
<install_modules>
clf_ET = ExtraTreesClassifier(random_state=0, bootstrap=True, oob_score=True)
 sss = model_selection.StratifiedShuffleSplit(n_splits=10, test_size=0.33, random_state= 0)
 sss.get_n_splits(X_train, Y_train)
 parameters = {'n_estimators' : np.r_[10:210:10],
'max_depth': np.r_[1:6]
}
grid = model_selection.GridSearchCV(clf_ET, param_grid=parameters, scoring = 'accuracy', cv = sss, return_train_score=True, n_jobs=4, verbose=2)
 grid.fit(X_train,Y_train )
%%time
!cp.. /input/rapids/rapids.0.11.0 /opt/conda/envs/rapids.tar.gz
!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz
sys.path = ["/opt/conda/envs/rapids/lib"] + ["/opt/conda/envs/rapids/lib/python3.6"] + ["/opt/conda/envs/rapids/lib/python3.6/site-packages"] + sys.path
!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/<import_modules>
p = a[a.param_max_depth==4].sort_values(by='mean_test_score', ascending=False ).iloc[0]
print("Best mean test score: %f using %s" %(p.mean_test_score, p.params))
bst_ET = grid.best_estimator_
bst_ET.set_params(**p.params)
 bst_ET.fit(X_train,Y_train)
 pred_ET = bst_ET.predict(X_test)
 test['Survived'] = pred_ET.astype(int)
 submission = test[['PassengerId', 'Survived']]
submission.PassengerId = submission.PassengerId.astype(int)
 submission.sort_values(by=['PassengerId'], inplace=True)
 submission.to_csv('pred_final.csv', index=False)
 submission.head(10 )