Kaggle competition: DSNet: fastai Hackathon
train_init_df.isnull().sum()<drop_column>
from fastai.vision import *
from fastai.metrics import error_rate
train_init_df=train_init_df.drop(['Cabin','PassengerId'],axis=1 )<sort_values>
train_arr = []
for file in glob.glob(".. /input/train/train/*/*"):
train_arr.append({"name": file, "label": file.split("/")[-2]})
 df = pd.DataFrame(train_arr )
train_init_df[["Pclass","Survived"]].groupby(["Pclass"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )<sort_values>
test_df = pd.read_csv(f".. /input/sample_submission.csv" )
train_init_df[["Sex","Survived"]].groupby(["Sex"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )<sort_values>
df["label"].value_counts()
train_init_df[["SibSp","Survived"]].groupby(["SibSp"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )<sort_values>
init_notebook_mode(connected=True)
train_init_df[["Parch","Survived"]].groupby(["Parch"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )<sort_values>
!pip install imagesize
train_init_df[["Embarked","Survived"]].groupby(["Embarked"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )<compute_test_metric>
df["width"] = 0
df["height"] = 0
df["aspect_ratio"] = 0.0
for idx, row in df.iterrows() :
width, height = imagesize.get(row["name"])
 df.at[idx, "width"] = width
df.at[idx, "height"] = height
df.at[idx, "aspect_ratio"] = float(height)/ float(width )
def checkNameSalutations(name):
salutation = re.findall(r'(\w{2,})\.',name)
 return salutation[0]<feature_engineering>
path = Path(".. /input")
 SEED = 24
tfms = get_transforms(do_flip=True, max_rotate=10, max_zoom=1.3, max_lighting=0.4, max_warp=0.25, xtra_tfms=[rgb_randomize(channel=0, thresh=0.9, p=0.1),rgb_randomize(channel=2, thresh=0.9, p=0.1),rgb_randomize(channel=2, thresh=0.9, p=0.1)])
 data = ImageDataBunch.from_folder(path/"train",valid_pct=0.2, ds_tfms=tfms, size=128, bs=64, seed=SEED ).normalize(imagenet_stats )
train_init_df['Salutation']=train_init_df['Name'].apply(checkNameSalutations)
 train_init_df.head()<sort_values>
learn = cnn_learner(data, models.resnet34, metrics=[accuracy],model_dir="/tmp/model/" )
train_init_df[["Salutation","Survived"]].groupby(["Salutation"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )<categorify>
lr=5e-2
learn.fit_one_cycle(15,slice(lr))
def binaryGender(val):
if val=='male':
return 0
elif val=='female':
return 1<feature_engineering>
learn.save('stage1' )
train_init_df['Sex']= train_init_df['Sex'].apply(binaryGender)
 train_init_df.head()<sort_values>
learn.load('stage1' )
train_init_df[["Sex","Survived"]].groupby(["Sex"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )<compute_test_metric>
learn.fit_one_cycle(10, slice(7e-6,(7e-6)/10))
def checkAplphabetInTicket(ticket):
alpha=re.findall(r'(\w{1,}\D)',ticket)
 if len(alpha)!=0:
return 1
else:
return 0<feature_engineering>
learn.save('stage-2' )
train_init_df['Alpha_ticket']= train_init_df['Ticket'].apply(checkAplphabetInTicket)
 train_init_df.head()<count_values>
path = Path(".. /input")
 SEED = 24
tfms = get_transforms(do_flip=True, max_rotate=10, max_zoom=1.3, max_lighting=0.4, max_warp=0.25, xtra_tfms=[rgb_randomize(channel=0, thresh=0.9, p=0.1),rgb_randomize(channel=2, thresh=0.9, p=0.1),rgb_randomize(channel=2, thresh=0.9, p=0.1)])
 data = ImageDataBunch.from_folder(path/"train",valid_pct=0.2, ds_tfms=tfms, size=256, bs=64, seed=SEED ).normalize(imagenet_stats )
train_init_df['Alpha_ticket'].value_counts()<sort_values>
learn.data = data
train_init_df[["Alpha_ticket","Survived"]].groupby(["Alpha_ticket"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )<feature_engineering>
lr=7e-3
train_init_df["Salutation"] = train_init_df["Salutation"].replace(["Lady","Countess","Capt","Col","Don","Dr","Major","Rev","Sir","Jonkheer","Dona"],"other")
 train_init_df["Salutation"] = [0 if i == "Master" else 1 if i == "Miss" or i == "Ms" or i == "Mlle" or i == "Mrs" else 2 if i == "Mr" else 3 for i in train_init_df["Salutation"]]
train_init_df["Salutation"].head(20 )<drop_column>
learn.fit_one_cycle(15, slice(lr))
train_init_df.drop(labels= ["Name"], axis = 1, inplace = True)
<drop_column>
learn.save('stage-3' )
train_init_df.drop(labels= ["Ticket"], axis = 1, inplace = True )<count_missing_values>
learn.load('stage-3' )
train_init_df.isnull().sum()<feature_engineering>
learn.fit_one_cycle(10, slice(1e-5, 1e-4))
train_init_df["Embarked"] = train_init_df["Embarked"].fillna("C")
 train_init_df[train_init_df["Embarked"].isnull() ]<feature_engineering>
path = ".. /input"
test_df = pd.read_csv(f"{path}/sample_submission.csv")
 sub_df = pd.read_csv(f"{path}/sample_submission.csv")
 data.add_test(ImageList.from_df(test_df, path, folder="test/test"))
index_nan_age = list(train_init_df["Age"][train_init_df["Age"].isnull() ].index)
 for i in index_nan_age:
age_pred = train_init_df["Age"][(( train_init_df["SibSp"] == train_init_df.iloc[i]["SibSp"])&(train_init_df["Parch"] == train_init_df.iloc[i]["Parch"])&(train_init_df["Pclass"] == train_init_df.iloc[i]["Pclass"])) ].median()
age_med = train_init_df["Age"].median()
if not np.isnan(age_pred):
train_init_df["Age"].iloc[i] = age_pred
else:
train_init_df["Age"].iloc[i] = age_med<categorify>
test_probs, _ = learn.get_preds(ds_type=DatasetType.Test)
 test_preds = [data.classes[pred] for pred in np.argmax(test_probs.numpy() , axis=-1)]
<categorify><EOS> 
sub_df = pd.read_csv(f"{path}/sample_submission.csv")
 sub_df.predicted_class = test_preds
sub_df.to_csv("submission.csv", index=False )