Kaggle competition: Titanic - Machine Learning from Disaster
epochs = 2
for epoch in range(epochs):
total_loss = 0
accuracy = 0
model.train()
print("epoch {} start!".format(epoch+1))
for iter_num,(input_ids, segment_ids, attention_masks, labels)in tqdm(enumerate(train_dl),total = len(train_dl)) :
optimizer.zero_grad()
if cuda:
input_ids, segment_ids, attention_masks, labels =\
input_ids.cuda() , segment_ids.cuda() , attention_masks.cuda() , labels.cuda()
outputs = model(input_ids = input_ids,
token_type_ids = segment_ids,
attention_mask = attention_masks)
 pred_proba = outputs.softmax(dim=1)[:,1]
pred =(pred_proba>=0.5 ).type(torch.int)
 loss = criterion(outputs,labels)
 loss.backward()
optimizer.step()
total_loss += loss.item()
accuracy +=(pred==labels ).sum().item()
if(iter_num+1)% 50 == 0:
size = batch_size*(iter_num+1)
 print("{} iter loss:{:.4f} accuracy:{:.4f}".format(
 iter_num+1,total_loss/(iter_num+1),accuracy/size))
total_loss /= len(train_dl)
 accuracy /= len(train_ds)
 val_total_loss = 0
val_accuracy = 0
model.eval()
for input_ids, segment_ids, attention_masks, labels in tqdm(val_dl):
if cuda:
input_ids, segment_ids, attention_masks, labels =\
input_ids.cuda() , segment_ids.cuda() , attention_masks.cuda() , labels.cuda()
with torch.no_grad() :
outputs = model(input_ids = input_ids,
token_type_ids = segment_ids,
attention_mask = attention_masks)
 pred_proba = outputs.softmax(dim=1)[:,1]
pred =(pred_proba>=0.5 ).type(torch.int)
 loss = criterion(outputs,labels)
 val_total_loss += loss.item()
val_accuracy +=(pred==labels ).sum().item()
val_total_loss /= len(val_dl)
 val_accuracy /= len(val_ds)
 print("epoch{} total loss:{:.4f}, accuracy:{:.4f}, val_total loss:{:.4f}, val_accuracy:{:.4f}"\ 
.format(epoch+1,total_loss,accuracy,val_total_loss,val_accuracy)) 
<create_dataframe>
init_notebook_mode(connected=True)
test_ds = ReviewDataset(texts=test_df["Body"].values)
 test_dl = torch.utils.data.DataLoader(
 test_ds, batch_size=8, shuffle=False )<categorify>
warnings.filterwarnings('ignore' )
lists = []
for input_ids, segment_ids, attention_masks in tqdm(test_dl):
if cuda:
input_ids, segment_ids, attention_masks =\
input_ids.cuda() , segment_ids.cuda() , attention_masks.cuda()
outputs = model(input_ids = input_ids,
token_type_ids = segment_ids,
attention_mask = attention_masks)
 pred_proba = outputs.softmax(dim=1)[:,1]
lists += list(pred_proba.cpu().detach().numpy() )<save_to_csv>
train_df=pd.read_csv("/kaggle/input/titanic/train.csv")
 test_df = pd.read_csv("/kaggle/input/titanic/test.csv" )
test_df["Label"]=lists
submission_df=test_df[["Id","Label"]]
submission_df.to_csv("sample_submission.csv",index=False )<install_modules>
train_len = len(train_df)
 combined = train_df.append(test_df,ignore_index=True)
 combined.fillna(np.nan )
!pip install mecab-python3==0.996.2<load_pretrained>
combined.isnull().sum()
tokenizer = BertJapaneseTokenizer.from_pretrained("bert-base-japanese")
 for i,item in enumerate(tokenizer.vocab.items()):
print(item)
 if i > 20:
break<choose_model_class>
combined.groupby(['Pclass','Sex'])['Age'].mean()
cuda = torch.cuda.is_available()
bert = BertModel.from_pretrained("bert-base-japanese")
 model = BertClassification(bert)
 torch.backends.cudnn.benchmark = True
if cuda:
model.cuda()
optimizer = optim.Adam(model.parameters() ,lr = 4e-4 ,betas=(0.9, 0.999))
criterion = nn.CrossEntropyLoss()<import_modules>
combined['AgeGroup'] = 'adult'
combined.loc[combined['Name'].str.contains('Master'),'AgeGroup'] = "child"
combined.loc[combined['Age'] <= 14.0,'AgeGroup'] = "child"
combined.loc[(combined['Age'].isnull())&(combined['Name'].str.contains('Miss')) &(combined['Parch'] != 0),'AgeGroup'] = "child"
import numpy as np
import pandas as pd<load_from_csv>
combined[combined['Age'].notnull() ].groupby(['Pclass','Sex','AgeGroup'])['Age'].mean()
train_data = pd.read_csv('/kaggle/input/syde522/train.csv' )<load_pretrained>
def Age(cols):
Age=cols[0]
Pclass=cols[1]
Sex=cols[2]
AgeGroup=cols[3]
if pd.isnull(Age):
if Pclass==1:
if Sex=="male":
if AgeGroup=='adult':
return 42
else:
return 7
elif Sex=="female":
if AgeGroup=='adult':
return 37
else:
return 8
elif Pclass==2:
if Sex=="male":
if AgeGroup=='adult':
return 33
else:
return 4
elif Sex=="female":
if AgeGroup=='adult':
return 31
else:
return 7
elif Pclass==3:
if Sex=="male":
if AgeGroup=='adult':
return 29
else:
return 7
elif Sex=="female":
if AgeGroup=='adult':
return 27
else:
return 5
else:
return Age
combined["Age"]=combined[["Age","Pclass","Sex","AgeGroup"]].apply(Age,axis=1 )
CNNmodel = VGG16(weights=None,input_shape=(150, 150, 3), include_top=False, pooling='max',)
 CNNmodel.load_weights('/kaggle/input/vgg16-weights/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5')
 img_path = '/kaggle/input/syde522/train/train/0011.png'
img = image.load_img(img_path, target_size=(150,150))
def getFeatures(img):
x = image.img_to_array(img)
 x = np.expand_dims(x, axis=0)
 x = preprocess_input(x)
 featur = CNNmodel.predict(x)
 return featur.reshape(1, 512)
<categorify>
def AgeBand(col):
Age=col[0]
if Age <=7:
return "0-7"
elif Age <=14:
return "8-14"
elif Age <=21:
return "15-21"
elif Age <= 28:
return "22-28"
elif Age <= 35:
return "29-35"
elif Age <= 42:
return "36-42"
elif Age <= 49:
return "43-49"
elif Age <= 56:
return "50-56"
elif Age <= 63:
return "57-63"
else:
return ">=64"
combined["AgeBand"]=combined[["Age"]].apply(AgeBand,axis=1 )
features=np.zeros(( 1,512))
labels=np.array('init')
 for idx,(_, entry)in enumerate(train_data.iterrows()):
img=imread(entry.train_file)
 feature = getFeatures(img)
 features=np.append(features,feature,axis=0)
 labels=np.append(labels,entry.Category)
 labels=labels.reshape(features[:,1].size)
 features=np.delete(features, 0, 0)
 labels=np.delete(labels, 0, 0)
 print(features.shape)
 print(labels.shape)
 features =(preprocessing.normalize(features))
neigh = KNeighborsClassifier(n_neighbors=7)
 neigh.fit(features, labels)
<save_to_csv>
combined.groupby(['Pclass','Embarked'])['PassengerId'].count()
def predictor(imgID):
img=imread('/kaggle/input/syde522/test/test/'+imgID)
 feature = getFeatures(img)
 feature = preprocessing.normalize(feature)
 prediction=neigh.predict(feature)
 return prediction[0]
test_files = glob.glob('/kaggle/input/syde522/test/test/*.png')
 test_file_id = [os.path.basename(test_file)for test_file in test_files]
test_submission = pd.DataFrame({'Id': test_file_id, 'Category': [predictor(test_img_id)for test_img_id in test_file_id]})
 test_submission.to_csv('submission.csv', index=False )<install_modules>
combined[combined['Embarked'].isnull() ]['Embarked'] = combined['Embarked'].mode()
!pip install category_encoders<load_from_csv>
ticketCount = combined.groupby('Ticket')['PassengerId'].count().reset_index()
ticketCount.rename(columns={'PassengerId':'Count on Ticket'},inplace=True)
 combined = combined.merge(ticketCount, on="Ticket",how="left" )
train = pd.read_csv("/kaggle/input/heroz-internal-competition/train.csv")
 test = pd.read_csv("/kaggle/input/heroz-internal-competition/test_features.csv")
 ans = pd.read_csv("/kaggle/input/heroz-internal-competition/sample_submission.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape)
 print("ans shape : ", ans.shape)
 train<count_missing_values>
combined['Diff'] = combined['FamilySize'] - combined['Count on Ticket']
combined['Family Status'] = combined.apply(lambda x:"Has Family On Same Ticket" if(x['FamilySize'] - x['Count on Ticket'])<= 0 else "Family Not on same ticket",axis=1)
train.isnull().sum()<count_missing_values>
combined['Family Status'] = combined.apply(lambda x:"Is Alone" if(x['FamilySize']==1)&(x['Count on Ticket']==1)else x['Family Status'],axis=1 )
test.isnull().sum()<feature_engineering>
combined['Cabin Class'] = 'No Cabin'
combined['Cabin Class'] = combined.apply(lambda x: "No Cabin" if pd.isna(x["Cabin"])else x["Cabin"][0] , axis=1)
train['reviews_per_month'] = train['reviews_per_month'].fillna(0)
 test['reviews_per_month'] = test['reviews_per_month'].fillna(0 )<feature_engineering>
tickcab = combined[combined['Cabin Class'] != 'No Cabin'][['Ticket','Cabin Class']].drop_duplicates()
tickcab = tickcab.rename(columns={'Cabin Class':'CabNam'})
 combined = combined.merge(tickcab,how="left",on="Ticket")
 combined['CabNam'].fillna('No Cabin')
 combined['Cabin Class'] = combined.apply(lambda x:x['Cabin Class'] if x['Cabin Class'] != 'No Cabin' else x['CabNam'],axis=1)
 combined.drop(columns=['CabNam'],inplace=True)
 combined.drop_duplicates(inplace=True)
train['last_review'] = train['last_review'].fillna('2019-12-31')
 test['last_review'] = test['last_review'].fillna('2019-12-31' )<feature_engineering>
combined["Fare"] = combined["Fare"].map(lambda i: np.log(i)if i > 0 else 0)
 combined["Fare"] = combined["Fare"]/combined['Count on Ticket']
train['last_review'] = pd.to_datetime(train['last_review'])
 test['last_review'] = pd.to_datetime(test['last_review'])
 train['year'] = train['last_review'].dt.year-2010
test['year'] = test['last_review'].dt.year-2010
train['month'] = train['last_review'].dt.month-1
test['month'] = test['last_review'].dt.month-1<feature_engineering>
companion = pd.pivot_table(combined, values='PassengerId',index=['Ticket'],columns=['AgeGroup'], aggfunc="count" ).reset_index().fillna(0)
 companion.columns = ['Ticket','No.of Adult Companion', 'No.of Child Companion']
combined = combined.merge(companion, on='Ticket',how='left' )
train['latitude'] = train['latitude']-40
train['longitude'] = train['longitude']+75
test['latitude'] = test['latitude']-40
test['longitude'] = test['longitude']+75
train['waru'] = train['latitude'] / train['longitude']
train['kakeru'] = train['latitude'] * train['longitude']
test['waru'] = test['latitude'] / test['longitude']
test['kakeru'] = test['latitude'] * test['longitude']<feature_engineering>
combined.loc[combined['AgeGroup']=='adult','No.of Adult Companion'] = combined.loc[combined['AgeGroup']=='adult','No.of Adult Companion'] - 1
combined.loc[combined['AgeGroup']=='child','No.of Child Companion'] = combined.loc[combined['AgeGroup']=='child','No.of Child Companion'] - 1
combined['Companion'] = 'Adult & Child Companion'
combined['Companion'] = combined.apply(lambda x:'Only Adult Companion' if(x['No.of Adult Companion'] > 0)&(x['No.of Child Companion']==0)else x['Companion'],axis=1)
 combined['Companion'] = combined.apply(lambda x:'Only Child Companion' if(x['No.of Adult Companion'] == 0)&(x['No.of Child Companion']>0)else x['Companion'],axis=1)
 combined['Companion'] = combined.apply(lambda x:'No Companion' if(x['No.of Adult Companion'] == 0)&(x['No.of Child Companion']==0)else x['Companion'],axis=1 )
train['minimum_nights_log'] = np.log1p(train['minimum_nights'])
 test['minimum_nights_log'] = np.log1p(test['minimum_nights'])
 train['calculated_host_listings_count_log'] = np.log1p(train['calculated_host_listings_count'])
 test['calculated_host_listings_count_log'] = np.log1p(test['calculated_host_listings_count'])
 train['availability_365_log'] = np.log1p(train['availability_365'])
 test['availability_365_log'] = np.log1p(test['availability_365'])
 train['number_of_reviews_log'] = np.log1p(train['number_of_reviews'])
 test['number_of_reviews_log'] = np.log1p(test['number_of_reviews'])
 train['reviews_per_month_log'] = np.log1p(train['reviews_per_month'])
 test['reviews_per_month_log'] = np.log1p(test['reviews_per_month'] )<define_variables>
combined[combined["Survived"].notnull() ].groupby(['AgeGroup','Companion'])['PassengerId'].count()
target = 'price'
list_cols = ['neighbourhood_group', 'neighbourhood', 'room_type',]
target_columns = ['neighbourhood_group', 'neighbourhood', 'latitude', 'longitude', 'room_type',
'minimum_nights', 'number_of_reviews', 'reviews_per_month', 'calculated_host_listings_count', 'availability_365', 'waru', 'kakeru', 'minimum_nights_log',
'calculated_host_listings_count_log', 'availability_365_log',
'number_of_reviews_log', 'reviews_per_month_log', 'year', 'month']
ce_ohe = ce.OneHotEncoder(cols=list_cols)
 train_onehot = ce_ohe.fit_transform(train[target_columns] )<count_missing_values>
combined[combined["Survived"].notnull() ].groupby(['AgeGroup','Companion'])['Survived'].mean()
train_onehot.isnull().sum()<categorify>
train_copy = combined[:train_len]
test_copy = combined[train_len:].reset_index(drop=True)
 test_copy.drop(columns=['Survived'],inplace=True )
test_onehot = ce_ohe.transform(test[target_columns])
<categorify>
combined.drop(columns=['PassengerId','Name','Age', 'AgeGroup','SibSp','Parch','Ticket','Cabin','Count on Ticket','Diff','No.of Adult Companion','No.of Child Companion'],inplace=True )
pca2 = PCA(n_components=5)
 pca2_results = pca2.fit_transform(train_onehot)
 train_onehot['pca0']=pca2_results[:,0]
train_onehot['pca1']=pca2_results[:,1]
train_onehot['pca2']=pca2_results[:,2]
train_onehot['pca3']=pca2_results[:,3]
train_onehot['pca4']=pca2_results[:,4]
pca2_results = pca2.transform(test_onehot)
 test_onehot['pca0']=pca2_results[:,0]
test_onehot['pca1']=pca2_results[:,1]
test_onehot['pca2']=pca2_results[:,2]
test_onehot['pca3']=pca2_results[:,3]
test_onehot['pca4']=pca2_results[:,4]<prepare_x_and_y>
combined = pd.get_dummies(combined, columns = ["Sex","Embarked","AgeBand","Family Status","Cabin Class","Companion"],drop_first=True )
y_train = np.log1p(train["price"] )<split>
train = combined[:train_len]
test = combined[train_len:]
test.drop(columns=['Survived'],inplace=True )
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 s = list(kf.split(train_onehot, y_train))<choose_model_class>
X = train.iloc[:,1:]
y = train.iloc[:,0]
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=0 )
gbk_models = []
for train_i, val_i in s:
gbk = GradientBoostingRegressor()
gbk.fit(train_onehot.iloc[train_i], y_train.iloc[train_i])
 y_pred = gbk.predict(train_onehot.iloc[val_i])
 acc_gbk = round(mean_squared_error(np.expm1(y_pred), np.expm1(y_train[val_i])))
 print(acc_gbk)
 gbk_models.append(gbk )<train_model>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, y_train)
 pred_train = decision_tree.predict(X_train)
 pred=decision_tree.predict(X_test)
 pred_train_df=pd.DataFrame({"Actual":y_train,"Pred":pred_train})
 pred_df=pd.DataFrame({"Actual":y_test,"Pred":pred})
 cm=confusion_matrix(y_test,pred)
 cm
mlp_models = []
for train_i, val_i in s:
mlp = MLPRegressor(max_iter=100, hidden_layer_sizes=(100,100),
activation='relu', learning_rate_init=0.01)
 mlp.fit(train_onehot.iloc[train_i], y_train.iloc[train_i])
 y_pred = mlp.predict(train_onehot.iloc[val_i])
 acc_gbk = round(mean_squared_error(np.expm1(y_pred), np.expm1(y_train[val_i])))
 print(acc_gbk)
 mlp_models.append(mlp )<find_best_model_class>
report=classification_report(y_test,pred)
 print("Decision Tree report
",report )
rfc_models = []
for train_i, val_i in s:
rfc = RandomForestRegressor()
rfc.fit(train_onehot.iloc[train_i], y_train.iloc[train_i])
 y_pred = rfc.predict(train_onehot.iloc[val_i])
 acc_gbk = round(mean_squared_error(np.expm1(y_pred), np.expm1(y_train[val_i])))
 print(acc_gbk)
 rfc_models.append(rfc )<predict_on_test>
rfc=ensemble.RandomForestClassifier(max_depth=6,random_state=0,n_estimators=64)
 rfc.fit(X_train, y_train)
 pred_train = rfc.predict(X_train)
 pred=rfc.predict(X_test)
 pred_train_df=pd.DataFrame({"Actual":y_train,"Pred":pred_train})
 pred_df=pd.DataFrame({"Actual":y_test,"Pred":pred})
 cm=confusion_matrix(y_test,pred)
 print(cm)
 report=classification_report(y_test,pred)
 print("Random Forest report
",report )
models = gbk_models + rfc_models
preds = np.array([np.expm1(model.predict(test_onehot)) for model in models])
 preds = preds.mean(axis=0)
 preds = np.where(preds < 0 , 0, preds )<save_to_csv>
y_test_rfc = rfc.predict(test ).astype(int)
 test_out = pd.concat([test_copy['PassengerId'],pd.Series(y_test_rfc,name="Survived")],axis=1)
 test_out['Survived'] = test_out['Survived'].astype('int')
 test_out.to_csv('submission.csv',index=False )